diff options
Diffstat (limited to 'contrib/llvm/tools/clang/lib/CodeGen')
72 files changed, 9517 insertions, 4688 deletions
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h index 530a7ef..ac31dfd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h @@ -142,6 +142,8 @@ namespace swiftcall { llvm::Type *eltTy, unsigned elts) const; + virtual bool isSwiftErrorInRegister() const = 0; + static bool classof(const ABIInfo *info) { return info->supportsSwift(); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp index 165b6dd..d2ce6ea 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp @@ -14,24 +14,31 @@ #include "clang/Frontend/CodeGenOptions.h" #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/Utils.h" +#include "clang/Lex/HeaderSearchOptions.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Bitcode/BitcodeWriterPass.h" -#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/Verifier.h" +#include "llvm/LTO/LTOBackend.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Object/ModuleSummaryIndexObjectFile.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/Timer.h" @@ -39,7 +46,9 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Transforms/Coroutines.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/ObjCARC.h" @@ -54,6 +63,7 @@ namespace { class EmitAssemblyHelper { DiagnosticsEngine &Diags; + const HeaderSearchOptions &HSOpts; const CodeGenOptions &CodeGenOpts; const clang::TargetOptions &TargetOpts; const LangOptions &LangOpts; @@ -74,8 +84,7 @@ private: /// Set LLVM command line options passed through -backend-option. void setCommandLineOpts(); - void CreatePasses(legacy::PassManager &MPM, legacy::FunctionPassManager &FPM, - ModuleSummaryIndex *ModuleSummary); + void CreatePasses(legacy::PassManager &MPM, legacy::FunctionPassManager &FPM); /// Generates the TargetMachine. /// Leaves TM unchanged if it is unable to create the target machine. @@ -94,11 +103,14 @@ private: raw_pwrite_stream &OS); public: - EmitAssemblyHelper(DiagnosticsEngine &_Diags, const CodeGenOptions &CGOpts, + EmitAssemblyHelper(DiagnosticsEngine &_Diags, + const HeaderSearchOptions &HeaderSearchOpts, + const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, const LangOptions &LOpts, Module *M) - : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts), - TheModule(M), CodeGenerationTime("Code Generation Time") {} + : Diags(_Diags), HSOpts(HeaderSearchOpts), CodeGenOpts(CGOpts), + TargetOpts(TOpts), LangOpts(LOpts), TheModule(M), + CodeGenerationTime("codegen", "Code Generation Time") {} ~EmitAssemblyHelper() { if (CodeGenOpts.DisableFree) @@ -109,6 +121,9 @@ public: void EmitAssembly(BackendAction Action, std::unique_ptr<raw_pwrite_stream> OS); + + void EmitAssemblyWithNewPassManager(BackendAction Action, + std::unique_ptr<raw_pwrite_stream> OS); }; // We need this wrapper to access LangOpts and CGOpts from extension functions @@ -147,17 +162,6 @@ static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder, PM.add(createAddDiscriminatorsPass()); } -static void addCleanupPassesForSampleProfiler( - const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { - // instcombine is needed before sample profile annotation because it converts - // certain function calls to be inlinable. simplifycfg and sroa are needed - // before instcombine for necessary preparation. E.g. load store is eliminated - // properly so that instcombine will not introduce unecessary liverange. - PM.add(createCFGSimplificationPass()); - PM.add(createSROAPass()); - PM.add(createInstructionCombiningPass()); -} - static void addBoundsCheckingPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createBoundsCheckingPass()); @@ -174,8 +178,11 @@ static void addSanitizerCoveragePass(const PassManagerBuilder &Builder, Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls; Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB; Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp; + Opts.TraceDiv = CGOpts.SanitizeCoverageTraceDiv; + Opts.TraceGep = CGOpts.SanitizeCoverageTraceGep; Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters; Opts.TracePC = CGOpts.SanitizeCoverageTracePC; + Opts.TracePCGuard = CGOpts.SanitizeCoverageTracePCGuard; PM.add(createSanitizerCoverageModulePass(Opts)); } @@ -205,7 +212,9 @@ static void addMemorySanitizerPass(const PassManagerBuilder &Builder, const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); - PM.add(createMemorySanitizerPass(CGOpts.SanitizeMemoryTrackOrigins)); + int TrackOrigins = CGOpts.SanitizeMemoryTrackOrigins; + bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Memory); + PM.add(createMemorySanitizerPass(TrackOrigins, Recover)); // MemorySanitizer inserts complex instrumentation that mostly follows // the logic of the original code, but operates on "shadow" values. @@ -263,6 +272,9 @@ static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple, case CodeGenOptions::Accelerate: TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate); break; + case CodeGenOptions::SVML: + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML); + break; default: break; } @@ -281,47 +293,34 @@ static void addSymbolRewriterPass(const CodeGenOptions &Opts, } void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, - legacy::FunctionPassManager &FPM, - ModuleSummaryIndex *ModuleSummary) { + legacy::FunctionPassManager &FPM) { + // Handle disabling of all LLVM passes, where we want to preserve the + // internal module before any optimization. if (CodeGenOpts.DisableLLVMPasses) return; - unsigned OptLevel = CodeGenOpts.OptimizationLevel; - CodeGenOptions::InliningMethod Inlining = CodeGenOpts.getInlining(); - - // Handle disabling of LLVM optimization, where we want to preserve the - // internal module before any optimization. - if (CodeGenOpts.DisableLLVMOpts) { - OptLevel = 0; - Inlining = CodeGenOpts.NoInlining; - } - PassManagerBuilderWrapper PMBuilder(CodeGenOpts, LangOpts); - // Figure out TargetLibraryInfo. + // Figure out TargetLibraryInfo. This needs to be added to MPM and FPM + // manually (and not via PMBuilder), since some passes (eg. InstrProfiling) + // are inserted before PMBuilder ones - they'd get the default-constructed + // TLI with an unknown target otherwise. Triple TargetTriple(TheModule->getTargetTriple()); - PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); + std::unique_ptr<TargetLibraryInfoImpl> TLII( + createTLII(TargetTriple, CodeGenOpts)); - switch (Inlining) { - case CodeGenOptions::NoInlining: - break; - case CodeGenOptions::NormalInlining: - case CodeGenOptions::OnlyHintInlining: { - PMBuilder.Inliner = - createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize); - break; - } - case CodeGenOptions::OnlyAlwaysInlining: - // Respect always_inline. - if (OptLevel == 0) - // Do not insert lifetime intrinsics at -O0. - PMBuilder.Inliner = createAlwaysInlinerPass(false); - else - PMBuilder.Inliner = createAlwaysInlinerPass(); - break; + // At O0 and O1 we only run the always inliner which is more efficient. At + // higher optimization levels we run the normal inliner. + if (CodeGenOpts.OptimizationLevel <= 1) { + bool InsertLifetimeIntrinsics = (CodeGenOpts.OptimizationLevel != 0 && + !CodeGenOpts.DisableLifetimeMarkers); + PMBuilder.Inliner = createAlwaysInlinerLegacyPass(InsertLifetimeIntrinsics); + } else { + PMBuilder.Inliner = createFunctionInliningPass( + CodeGenOpts.OptimizationLevel, CodeGenOpts.OptimizeSize); } - PMBuilder.OptLevel = OptLevel; + PMBuilder.OptLevel = CodeGenOpts.OptimizationLevel; PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize; PMBuilder.BBVectorize = CodeGenOpts.VectorizeBB; PMBuilder.SLPVectorize = CodeGenOpts.VectorizeSLP; @@ -333,13 +332,7 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO; PMBuilder.RerollLoops = CodeGenOpts.RerollLoops; - // If we are performing a ThinLTO importing compile, invoke the LTO - // pipeline and pass down the in-memory module summary index. - if (ModuleSummary) { - PMBuilder.ModuleSummary = ModuleSummary; - PMBuilder.populateThinLTOPassManager(MPM); - return; - } + MPM.add(new TargetLibraryInfoWrapperPass(*TLII)); // Add target-specific passes that need to run as early as possible. if (TM) @@ -413,6 +406,9 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, addDataFlowSanitizerPass); } + if (LangOpts.CoroutinesTS) + addCoroutinePassesToExtensionPoints(PMBuilder); + if (LangOpts.Sanitize.hasOneOf(SanitizerKind::Efficiency)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addEfficiencySanitizerPass); @@ -421,6 +417,7 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, } // Set up the per-function pass manager. + FPM.add(new TargetLibraryInfoWrapperPass(*TLII)); if (CodeGenOpts.VerifyModule) FPM.add(createVerifierPass()); @@ -453,20 +450,17 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM, MPM.add(createInstrProfilingLegacyPass(Options)); } if (CodeGenOpts.hasProfileIRInstr()) { + PMBuilder.EnablePGOInstrGen = true; if (!CodeGenOpts.InstrProfileOutput.empty()) PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput; else - PMBuilder.PGOInstrGen = "default.profraw"; + PMBuilder.PGOInstrGen = "default_%m.profraw"; } if (CodeGenOpts.hasProfileIRUse()) PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath; - if (!CodeGenOpts.SampleProfileFile.empty()) { - MPM.add(createPruneEHPass()); - MPM.add(createSampleProfileLoaderPass(CodeGenOpts.SampleProfileFile)); - PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, - addCleanupPassesForSampleProfiler); - } + if (!CodeGenOpts.SampleProfileFile.empty()) + PMBuilder.PGOSampleUse = CodeGenOpts.SampleProfileFile; PMBuilder.populateFunctionPassManager(FPM); PMBuilder.populateModulePassManager(MPM); @@ -517,28 +511,35 @@ void EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { // Keep this synced with the equivalent code in tools/driver/cc1as_main.cpp. llvm::Optional<llvm::Reloc::Model> RM; - if (CodeGenOpts.RelocationModel == "static") { - RM = llvm::Reloc::Static; - } else if (CodeGenOpts.RelocationModel == "pic") { - RM = llvm::Reloc::PIC_; - } else { - assert(CodeGenOpts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - RM = llvm::Reloc::DynamicNoPIC; - } - - CodeGenOpt::Level OptLevel = CodeGenOpt::Default; + RM = llvm::StringSwitch<llvm::Reloc::Model>(CodeGenOpts.RelocationModel) + .Case("static", llvm::Reloc::Static) + .Case("pic", llvm::Reloc::PIC_) + .Case("ropi", llvm::Reloc::ROPI) + .Case("rwpi", llvm::Reloc::RWPI) + .Case("ropi-rwpi", llvm::Reloc::ROPI_RWPI) + .Case("dynamic-no-pic", llvm::Reloc::DynamicNoPIC); + assert(RM.hasValue() && "invalid PIC model!"); + + CodeGenOpt::Level OptLevel; switch (CodeGenOpts.OptimizationLevel) { - default: break; - case 0: OptLevel = CodeGenOpt::None; break; - case 3: OptLevel = CodeGenOpt::Aggressive; break; + default: + llvm_unreachable("Invalid optimization level!"); + case 0: + OptLevel = CodeGenOpt::None; + break; + case 1: + OptLevel = CodeGenOpt::Less; + break; + case 2: + OptLevel = CodeGenOpt::Default; + break; // O2/Os/Oz + case 3: + OptLevel = CodeGenOpt::Aggressive; + break; } llvm::TargetOptions Options; - if (!TargetOpts.Reciprocals.empty()) - Options.Reciprocals = TargetRecip(TargetOpts.Reciprocals); - Options.ThreadModel = llvm::StringSwitch<llvm::ThreadModel::Model>(CodeGenOpts.ThreadModel) .Case("posix", llvm::ThreadModel::POSIX) @@ -601,9 +602,18 @@ void EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack; Options.MCOptions.MCIncrementalLinkerCompatible = CodeGenOpts.IncrementalLinkerCompatible; + Options.MCOptions.MCPIECopyRelocations = CodeGenOpts.PIECopyRelocations; Options.MCOptions.MCFatalWarnings = CodeGenOpts.FatalWarnings; Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose; + Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments; Options.MCOptions.ABIName = TargetOpts.ABI; + for (const auto &Entry : HSOpts.UserEntries) + if (!Entry.IsFramework && + (Entry.Group == frontend::IncludeDirGroup::Quoted || + Entry.Group == frontend::IncludeDirGroup::Angled || + Entry.Group == frontend::IncludeDirGroup::System)) + Options.MCOptions.IASSearchPaths.push_back( + Entry.IgnoreSysRoot ? Entry.Path : HSOpts.Sysroot + Entry.Path); TM.reset(TheTarget->createTargetMachine(Triple, TargetOpts.CPU, FeaturesStr, Options, RM, CM, OptLevel)); @@ -659,26 +669,6 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, if (TM) TheModule->setDataLayout(TM->createDataLayout()); - // If we are performing a ThinLTO importing compile, load the function - // index into memory and pass it into CreatePasses, which will add it - // to the PassManagerBuilder and invoke LTO passes. - std::unique_ptr<ModuleSummaryIndex> ModuleSummary; - if (!CodeGenOpts.ThinLTOIndexFile.empty()) { - ErrorOr<std::unique_ptr<ModuleSummaryIndex>> IndexOrErr = - llvm::getModuleSummaryIndexForFile( - CodeGenOpts.ThinLTOIndexFile, [&](const DiagnosticInfo &DI) { - TheModule->getContext().diagnose(DI); - }); - if (std::error_code EC = IndexOrErr.getError()) { - std::string Error = EC.message(); - errs() << "Error loading index file '" << CodeGenOpts.ThinLTOIndexFile - << "': " << Error << "\n"; - return; - } - ModuleSummary = std::move(IndexOrErr.get()); - assert(ModuleSummary && "Expected non-empty module summary index"); - } - legacy::PassManager PerModulePasses; PerModulePasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); @@ -687,7 +677,7 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, PerFunctionPasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); - CreatePasses(PerModulePasses, PerFunctionPasses, ModuleSummary.get()); + CreatePasses(PerModulePasses, PerFunctionPasses); legacy::PassManager CodeGenPasses; CodeGenPasses.add( @@ -740,15 +730,252 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, } } +static PassBuilder::OptimizationLevel mapToLevel(const CodeGenOptions &Opts) { + switch (Opts.OptimizationLevel) { + default: + llvm_unreachable("Invalid optimization level!"); + + case 1: + return PassBuilder::O1; + + case 2: + switch (Opts.OptimizeSize) { + default: + llvm_unreachable("Invalide optimization level for size!"); + + case 0: + return PassBuilder::O2; + + case 1: + return PassBuilder::Os; + + case 2: + return PassBuilder::Oz; + } + + case 3: + return PassBuilder::O3; + } +} + +/// A clean version of `EmitAssembly` that uses the new pass manager. +/// +/// Not all features are currently supported in this system, but where +/// necessary it falls back to the legacy pass manager to at least provide +/// basic functionality. +/// +/// This API is planned to have its functionality finished and then to replace +/// `EmitAssembly` at some point in the future when the default switches. +void EmitAssemblyHelper::EmitAssemblyWithNewPassManager( + BackendAction Action, std::unique_ptr<raw_pwrite_stream> OS) { + TimeRegion Region(llvm::TimePassesIsEnabled ? &CodeGenerationTime : nullptr); + setCommandLineOpts(); + + // The new pass manager always makes a target machine available to passes + // during construction. + CreateTargetMachine(/*MustCreateTM*/ true); + if (!TM) + // This will already be diagnosed, just bail. + return; + TheModule->setDataLayout(TM->createDataLayout()); + + PassBuilder PB(TM.get()); + + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + // Register the AA manager first so that our version is the one used. + FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + + // Register all the basic analyses with the managers. + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + ModulePassManager MPM; + + if (!CodeGenOpts.DisableLLVMPasses) { + if (CodeGenOpts.OptimizationLevel == 0) { + // Build a minimal pipeline based on the semantics required by Clang, + // which is just that always inlining occurs. + MPM.addPass(AlwaysInlinerPass()); + } else { + // Otherwise, use the default pass pipeline. We also have to map our + // optimization levels into one of the distinct levels used to configure + // the pipeline. + PassBuilder::OptimizationLevel Level = mapToLevel(CodeGenOpts); + + MPM = PB.buildPerModuleDefaultPipeline(Level); + } + } + + // FIXME: We still use the legacy pass manager to do code generation. We + // create that pass manager here and use it as needed below. + legacy::PassManager CodeGenPasses; + bool NeedCodeGen = false; + + // Append any output we need to the pass manager. + switch (Action) { + case Backend_EmitNothing: + break; + + case Backend_EmitBC: + MPM.addPass(BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists, + CodeGenOpts.EmitSummaryIndex, + CodeGenOpts.EmitSummaryIndex)); + break; + + case Backend_EmitLL: + MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists)); + break; + + case Backend_EmitAssembly: + case Backend_EmitMCNull: + case Backend_EmitObj: + NeedCodeGen = true; + CodeGenPasses.add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); + if (!AddEmitPasses(CodeGenPasses, Action, *OS)) + // FIXME: Should we handle this error differently? + return; + break; + } + + // Before executing passes, print the final values of the LLVM options. + cl::PrintOptionValues(); + + // Now that we have all of the passes ready, run them. + { + PrettyStackTraceString CrashInfo("Optimizer"); + MPM.run(*TheModule, MAM); + } + + // Now if needed, run the legacy PM for codegen. + if (NeedCodeGen) { + PrettyStackTraceString CrashInfo("Code generation"); + CodeGenPasses.run(*TheModule); + } +} + +static void runThinLTOBackend(ModuleSummaryIndex *CombinedIndex, Module *M, + std::unique_ptr<raw_pwrite_stream> OS) { + StringMap<std::map<GlobalValue::GUID, GlobalValueSummary *>> + ModuleToDefinedGVSummaries; + CombinedIndex->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); + + // We can simply import the values mentioned in the combined index, since + // we should only invoke this using the individual indexes written out + // via a WriteIndexesThinBackend. + FunctionImporter::ImportMapTy ImportList; + for (auto &GlobalList : *CombinedIndex) { + auto GUID = GlobalList.first; + assert(GlobalList.second.size() == 1 && + "Expected individual combined index to have one summary per GUID"); + auto &Summary = GlobalList.second[0]; + // Skip the summaries for the importing module. These are included to + // e.g. record required linkage changes. + if (Summary->modulePath() == M->getModuleIdentifier()) + continue; + // Doesn't matter what value we plug in to the map, just needs an entry + // to provoke importing by thinBackend. + ImportList[Summary->modulePath()][GUID] = 1; + } + + std::vector<std::unique_ptr<llvm::MemoryBuffer>> OwnedImports; + MapVector<llvm::StringRef, llvm::BitcodeModule> ModuleMap; + + for (auto &I : ImportList) { + ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MBOrErr = + llvm::MemoryBuffer::getFile(I.first()); + if (!MBOrErr) { + errs() << "Error loading imported file '" << I.first() + << "': " << MBOrErr.getError().message() << "\n"; + return; + } + + Expected<std::vector<BitcodeModule>> BMsOrErr = + getBitcodeModuleList(**MBOrErr); + if (!BMsOrErr) { + handleAllErrors(BMsOrErr.takeError(), [&](ErrorInfoBase &EIB) { + errs() << "Error loading imported file '" << I.first() + << "': " << EIB.message() << '\n'; + }); + return; + } + + // The bitcode file may contain multiple modules, we want the one with a + // summary. + bool FoundModule = false; + for (BitcodeModule &BM : *BMsOrErr) { + Expected<bool> HasSummary = BM.hasSummary(); + if (HasSummary && *HasSummary) { + ModuleMap.insert({I.first(), BM}); + FoundModule = true; + break; + } + } + if (!FoundModule) { + errs() << "Error loading imported file '" << I.first() + << "': Could not find module summary\n"; + return; + } + + OwnedImports.push_back(std::move(*MBOrErr)); + } + auto AddStream = [&](size_t Task) { + return llvm::make_unique<lto::NativeObjectStream>(std::move(OS)); + }; + lto::Config Conf; + if (Error E = thinBackend( + Conf, 0, AddStream, *M, *CombinedIndex, ImportList, + ModuleToDefinedGVSummaries[M->getModuleIdentifier()], ModuleMap)) { + handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { + errs() << "Error running ThinLTO backend: " << EIB.message() << '\n'; + }); + } +} + void clang::EmitBackendOutput(DiagnosticsEngine &Diags, + const HeaderSearchOptions &HeaderOpts, const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, - const LangOptions &LOpts, const llvm::DataLayout &TDesc, - Module *M, BackendAction Action, + const LangOptions &LOpts, + const llvm::DataLayout &TDesc, Module *M, + BackendAction Action, std::unique_ptr<raw_pwrite_stream> OS) { - EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M); + if (!CGOpts.ThinLTOIndexFile.empty()) { + // If we are performing a ThinLTO importing compile, load the function index + // into memory and pass it into runThinLTOBackend, which will run the + // function importer and invoke LTO passes. + Expected<std::unique_ptr<ModuleSummaryIndex>> IndexOrErr = + llvm::getModuleSummaryIndexForFile(CGOpts.ThinLTOIndexFile); + if (!IndexOrErr) { + logAllUnhandledErrors(IndexOrErr.takeError(), errs(), + "Error loading index file '" + + CGOpts.ThinLTOIndexFile + "': "); + return; + } + std::unique_ptr<ModuleSummaryIndex> CombinedIndex = std::move(*IndexOrErr); + // A null CombinedIndex means we should skip ThinLTO compilation + // (LLVM will optionally ignore empty index files, returning null instead + // of an error). + bool DoThinLTOBackend = CombinedIndex != nullptr; + if (DoThinLTOBackend) { + runThinLTOBackend(CombinedIndex.get(), M, std::move(OS)); + return; + } + } + + EmitAssemblyHelper AsmHelper(Diags, HeaderOpts, CGOpts, TOpts, LOpts, M); - AsmHelper.EmitAssembly(Action, std::move(OS)); + if (CGOpts.ExperimentalNewPassManager) + AsmHelper.EmitAssemblyWithNewPassManager(Action, std::move(OS)); + else + AsmHelper.EmitAssembly(Action, std::move(OS)); // Verify clang's TargetInfo DataLayout against the LLVM TargetMachine's // DataLayout. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp index 7b747c1..9287e46 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp @@ -11,13 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenFunction.h" #include "CGCall.h" #include "CGRecordLayout.h" +#include "CodeGenFunction.h" #include "CodeGenModule.h" #include "clang/AST/ASTContext.h" #include "clang/CodeGen/CGFunctionInfo.h" -#include "llvm/ADT/StringExtras.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Operator.h" @@ -308,7 +307,8 @@ static RValue emitAtomicLibcall(CodeGenFunction &CGF, CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); - return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); + auto callee = CGCallee::forDirect(fn); + return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); } /// Does a store of the given IR type modify the full expected width? diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp index e3658ab..b250b9a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp @@ -16,6 +16,7 @@ #include "CGObjCRuntime.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantBuilder.h" #include "clang/AST/DeclObjC.h" #include "llvm/ADT/SmallSet.h" #include "llvm/IR/CallSite.h" @@ -77,63 +78,63 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, const CGBlockInfo &blockInfo) { ASTContext &C = CGM.getContext(); - llvm::Type *ulong = CGM.getTypes().ConvertType(C.UnsignedLongTy); - llvm::Type *i8p = nullptr; + llvm::IntegerType *ulong = + cast<llvm::IntegerType>(CGM.getTypes().ConvertType(C.UnsignedLongTy)); + llvm::PointerType *i8p = nullptr; if (CGM.getLangOpts().OpenCL) i8p = llvm::Type::getInt8PtrTy( CGM.getLLVMContext(), C.getTargetAddressSpace(LangAS::opencl_constant)); else - i8p = CGM.getTypes().ConvertType(C.VoidPtrTy); + i8p = CGM.VoidPtrTy; - SmallVector<llvm::Constant*, 6> elements; + ConstantInitBuilder builder(CGM); + auto elements = builder.beginStruct(); // reserved - elements.push_back(llvm::ConstantInt::get(ulong, 0)); + elements.addInt(ulong, 0); // Size // FIXME: What is the right way to say this doesn't fit? We should give // a user diagnostic in that case. Better fix would be to change the // API to size_t. - elements.push_back(llvm::ConstantInt::get(ulong, - blockInfo.BlockSize.getQuantity())); + elements.addInt(ulong, blockInfo.BlockSize.getQuantity()); // Optional copy/dispose helpers. if (blockInfo.NeedsCopyDispose) { // copy_func_helper_decl - elements.push_back(buildCopyHelper(CGM, blockInfo)); + elements.add(buildCopyHelper(CGM, blockInfo)); // destroy_func_decl - elements.push_back(buildDisposeHelper(CGM, blockInfo)); + elements.add(buildDisposeHelper(CGM, blockInfo)); } // Signature. Mandatory ObjC-style method descriptor @encode sequence. std::string typeAtEncoding = CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr()); - elements.push_back(llvm::ConstantExpr::getBitCast( + elements.add(llvm::ConstantExpr::getBitCast( CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer(), i8p)); // GC layout. if (C.getLangOpts().ObjC1) { if (CGM.getLangOpts().getGC() != LangOptions::NonGC) - elements.push_back(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo)); + elements.add(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo)); else - elements.push_back(CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo)); + elements.add(CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo)); } else - elements.push_back(llvm::Constant::getNullValue(i8p)); - - llvm::Constant *init = llvm::ConstantStruct::getAnon(elements); + elements.addNullPointer(i8p); unsigned AddrSpace = 0; if (C.getLangOpts().OpenCL) AddrSpace = C.getTargetAddressSpace(LangAS::opencl_constant); + llvm::GlobalVariable *global = - new llvm::GlobalVariable(CGM.getModule(), init->getType(), true, - llvm::GlobalValue::InternalLinkage, - init, "__block_descriptor_tmp", nullptr, - llvm::GlobalValue::NotThreadLocal, - AddrSpace); + elements.finishAndCreateGlobal("__block_descriptor_tmp", + CGM.getPointerAlign(), + /*constant*/ true, + llvm::GlobalValue::InternalLinkage, + AddrSpace); return llvm::ConstantExpr::getBitCast(global, CGM.getBlockDescriptorType()); } @@ -188,9 +189,6 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, }; */ -/// The number of fields in a block header. -const unsigned BlockHeaderSize = 5; - namespace { /// A chunk of data that we actually have to capture in the block. struct BlockLayoutChunk { @@ -199,13 +197,14 @@ namespace { Qualifiers::ObjCLifetime Lifetime; const BlockDecl::Capture *Capture; // null for 'this' llvm::Type *Type; + QualType FieldType; BlockLayoutChunk(CharUnits align, CharUnits size, Qualifiers::ObjCLifetime lifetime, const BlockDecl::Capture *capture, - llvm::Type *type) + llvm::Type *type, QualType fieldType) : Alignment(align), Size(size), Lifetime(lifetime), - Capture(capture), Type(type) {} + Capture(capture), Type(type), FieldType(fieldType) {} /// Tell the block info that this chunk has the given field index. void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) { @@ -213,8 +212,8 @@ namespace { info.CXXThisIndex = index; info.CXXThisOffset = offset; } else { - info.Captures.insert({Capture->getVariable(), - CGBlockInfo::Capture::makeIndex(index, offset)}); + auto C = CGBlockInfo::Capture::makeIndex(index, offset, FieldType); + info.Captures.insert({Capture->getVariable(), C}); } } }; @@ -317,8 +316,6 @@ static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info, elementTypes.push_back(CGM.IntTy); elementTypes.push_back(CGM.VoidPtrTy); elementTypes.push_back(CGM.getBlockDescriptorType()); - - assert(elementTypes.size() == BlockHeaderSize); } /// Compute the layout of the given block. Attempts to lay the block @@ -363,7 +360,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, layout.push_back(BlockLayoutChunk(tinfo.second, tinfo.first, Qualifiers::OCL_None, - nullptr, llvmType)); + nullptr, llvmType, thisType)); } // Next, all the block captures. @@ -380,7 +377,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, layout.push_back(BlockLayoutChunk(align, CGM.getPointerSize(), Qualifiers::OCL_None, &CI, - CGM.VoidPtrTy)); + CGM.VoidPtrTy, variable->getType())); continue; } @@ -436,6 +433,14 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, } QualType VT = variable->getType(); + + // If the variable is captured by an enclosing block or lambda expression, + // use the type of the capture field. + if (CGF->BlockInfo && CI.isNested()) + VT = CGF->BlockInfo->getCapture(variable).fieldType(); + else if (auto *FD = CGF->LambdaCaptureFields.lookup(variable)) + VT = FD->getType(); + CharUnits size = C.getTypeSizeInChars(VT); CharUnits align = C.getDeclAlign(variable); @@ -444,7 +449,8 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, llvm::Type *llvmType = CGM.getTypes().ConvertTypeForMem(VT); - layout.push_back(BlockLayoutChunk(align, size, lifetime, &CI, llvmType)); + layout.push_back( + BlockLayoutChunk(align, size, lifetime, &CI, llvmType, VT)); } // If that was everything, we're done here. @@ -680,6 +686,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) { // If the block has no captures, we won't have a pre-computed // layout for it. if (!blockExpr->getBlockDecl()->hasCaptures()) { + if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(blockExpr)) + return Block; CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName()); computeBlockInfo(CGM, this, blockInfo); blockInfo.BlockExpression = blockExpr; @@ -775,7 +783,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Ignore constant captures. if (capture.isConstant()) continue; - QualType type = variable->getType(); + QualType type = capture.fieldType(); // This will be a [[type]]*, except that a byref entry will just be // an i8**. @@ -965,25 +973,24 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, const BlockPointerType *BPT = E->getCallee()->getType()->getAs<BlockPointerType>(); - llvm::Value *Callee = EmitScalarExpr(E->getCallee()); + llvm::Value *BlockPtr = EmitScalarExpr(E->getCallee()); // Get a pointer to the generic block literal. llvm::Type *BlockLiteralTy = llvm::PointerType::getUnqual(CGM.getGenericBlockLiteralType()); // Bitcast the callee to a block literal. - llvm::Value *BlockLiteral = - Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal"); + BlockPtr = Builder.CreateBitCast(BlockPtr, BlockLiteralTy, "block.literal"); // Get the function pointer from the literal. llvm::Value *FuncPtr = - Builder.CreateStructGEP(CGM.getGenericBlockLiteralType(), BlockLiteral, 3); + Builder.CreateStructGEP(CGM.getGenericBlockLiteralType(), BlockPtr, 3); - BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy); + BlockPtr = Builder.CreateBitCast(BlockPtr, VoidPtrTy); // Add the block literal. CallArgList Args; - Args.add(RValue::get(BlockLiteral), getContext().VoidPtrTy); + Args.add(RValue::get(BlockPtr), getContext().VoidPtrTy); QualType FnType = BPT->getPointeeType(); @@ -1003,8 +1010,11 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, llvm::Type *BlockFTyPtr = llvm::PointerType::getUnqual(BlockFTy); Func = Builder.CreateBitCast(Func, BlockFTyPtr); + // Prepare the callee. + CGCallee Callee(CGCalleeInfo(), Func); + // And call the block. - return EmitCall(FnInfo, Func, ReturnValue, Args); + return EmitCall(FnInfo, Callee, ReturnValue, Args); } Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable, @@ -1033,18 +1043,27 @@ Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable, variable->getName()); } - if (auto refType = variable->getType()->getAs<ReferenceType>()) { + if (auto refType = capture.fieldType()->getAs<ReferenceType>()) addr = EmitLoadOfReference(addr, refType); - } return addr; } +void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE, + llvm::Constant *Addr) { + bool Ok = EmittedGlobalBlocks.insert(std::make_pair(BE, Addr)).second; + (void)Ok; + assert(Ok && "Trying to replace an already-existing global block!"); +} + llvm::Constant * -CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *blockExpr, - const char *name) { - CGBlockInfo blockInfo(blockExpr->getBlockDecl(), name); - blockInfo.BlockExpression = blockExpr; +CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE, + StringRef Name) { + if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE)) + return Block; + + CGBlockInfo blockInfo(BE->getBlockDecl(), Name); + blockInfo.BlockExpression = BE; // Compute information about the layout, etc., of this block. computeBlockInfo(*this, nullptr, blockInfo); @@ -1067,43 +1086,46 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, const CGBlockInfo &blockInfo, llvm::Constant *blockFn) { assert(blockInfo.CanBeGlobal); + // Callers should detect this case on their own: calling this function + // generally requires computing layout information, which is a waste of time + // if we've already emitted this block. + assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) && + "Refusing to re-emit a global block."); // Generate the constants for the block literal initializer. - llvm::Constant *fields[BlockHeaderSize]; + ConstantInitBuilder builder(CGM); + auto fields = builder.beginStruct(); // isa - fields[0] = CGM.getNSConcreteGlobalBlock(); + fields.add(CGM.getNSConcreteGlobalBlock()); // __flags BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE; if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET; - fields[1] = llvm::ConstantInt::get(CGM.IntTy, flags.getBitMask()); + fields.addInt(CGM.IntTy, flags.getBitMask()); // Reserved - fields[2] = llvm::Constant::getNullValue(CGM.IntTy); + fields.addInt(CGM.IntTy, 0); // Function - fields[3] = blockFn; + fields.add(blockFn); // Descriptor - fields[4] = buildBlockDescriptor(CGM, blockInfo); - - llvm::Constant *init = llvm::ConstantStruct::getAnon(fields); + fields.add(buildBlockDescriptor(CGM, blockInfo)); - llvm::GlobalVariable *literal = - new llvm::GlobalVariable(CGM.getModule(), - init->getType(), - /*constant*/ true, - llvm::GlobalVariable::InternalLinkage, - init, - "__block_literal_global"); - literal->setAlignment(blockInfo.BlockAlign.getQuantity()); + llvm::Constant *literal = + fields.finishAndCreateGlobal("__block_literal_global", + blockInfo.BlockAlign, + /*constant*/ true); // Return a constant of the appropriately-casted type. - llvm::Type *requiredType = + llvm::Type *RequiredType = CGM.getTypes().ConvertType(blockInfo.getBlockExpr()->getType()); - return llvm::ConstantExpr::getBitCast(literal, requiredType); + llvm::Constant *Result = + llvm::ConstantExpr::getBitCast(literal, RequiredType); + CGM.setAddrOfGlobalBlock(blockInfo.BlockExpression, Result); + return Result; } void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, @@ -1939,7 +1961,7 @@ static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo, generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator); generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator); - T *copy = new (CGM.getContext()) T(std::move(generator)); + T *copy = new (CGM.getContext()) T(std::forward<T>(generator)); CGM.ByrefHelpersCache.InsertNode(copy, insertPos); return copy; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h index 1edabef..80e255f 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h @@ -25,10 +25,8 @@ #include "clang/AST/ExprObjC.h" #include "clang/AST/Type.h" #include "clang/Basic/TargetInfo.h" -#include "llvm/IR/Module.h" namespace llvm { -class Module; class Constant; class Function; class GlobalValue; @@ -40,10 +38,8 @@ class LLVMContext; } namespace clang { - namespace CodeGen { -class CodeGenModule; class CGBlockInfo; // Flags stored in __block variables. @@ -163,6 +159,11 @@ public: EHScopeStack::stable_iterator Cleanup; CharUnits::QuantityType Offset; + /// Type of the capture field. Normally, this is identical to the type of + /// the capture's VarDecl, but can be different if there is an enclosing + /// lambda. + QualType FieldType; + public: bool isIndex() const { return (Data & 1) != 0; } bool isConstant() const { return !isIndex(); } @@ -189,10 +190,16 @@ public: return reinterpret_cast<llvm::Value*>(Data); } - static Capture makeIndex(unsigned index, CharUnits offset) { + QualType fieldType() const { + return FieldType; + } + + static Capture makeIndex(unsigned index, CharUnits offset, + QualType FieldType) { Capture v; v.Data = (index << 1) | 1; v.Offset = offset.getQuantity(); + v.FieldType = FieldType; return v; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h b/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h index 027435d..42f9a42 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h @@ -102,11 +102,6 @@ public: assert(Addr->getType()->getPointerElementType() == Ty); return CreateAlignedLoad(Addr, Align.getQuantity(), Name); } - llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align, - bool IsVolatile, - const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr, Align.getQuantity(), IsVolatile, Name); - } // Note that we intentionally hide the CreateStore APIs that don't // take an alignment. @@ -124,19 +119,6 @@ public: // FIXME: these "default-aligned" APIs should be removed, // but I don't feel like fixing all the builtin code right now. - llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, - const llvm::Twine &Name = "") { - return CGBuilderBaseTy::CreateLoad(Addr, false, Name); - } - llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, - const char *Name) { - return CGBuilderBaseTy::CreateLoad(Addr, false, Name); - } - llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, bool IsVolatile, - const llvm::Twine &Name = "") { - return CGBuilderBaseTy::CreateLoad(Addr, IsVolatile, Name); - } - llvm::StoreInst *CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile = false) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp index a5fc531..b3d02f1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp @@ -11,13 +11,15 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenFunction.h" #include "CGCXXABI.h" #include "CGObjCRuntime.h" +#include "CGOpenCLRuntime.h" +#include "CodeGenFunction.h" #include "CodeGenModule.h" #include "TargetInfo.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Decl.h" +#include "clang/Analysis/Analyses/OSLog.h" #include "clang/Basic/TargetBuiltins.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" @@ -33,10 +35,15 @@ using namespace clang; using namespace CodeGen; using namespace llvm; +static +int64_t clamp(int64_t Value, int64_t Low, int64_t High) { + return std::min(High, std::max(Low, Value)); +} + /// getBuiltinLibFunction - Given a builtin id for a function like /// "__builtin_fabsf", return a Function* for "fabsf". -llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, - unsigned BuiltinID) { +llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, + unsigned BuiltinID) { assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); // Get the name, skip over the __builtin_ prefix (if necessary). @@ -302,10 +309,10 @@ static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { return CGF.Builder.CreateICmpSLT(V, Zero); } -static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, - const CallExpr *E, llvm::Value *calleeValue) { - return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, - ReturnValueSlot(), Fn); +static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, + const CallExpr *E, llvm::Constant *calleeValue) { + CGCallee callee = CGCallee::forDirect(calleeValue, FD); + return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot()); } /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* @@ -462,6 +469,119 @@ CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, return Builder.CreateCall(F, {EmitScalarExpr(E), CI}); } +// Many of MSVC builtins are on both x64 and ARM; to avoid repeating code, we +// handle them here. +enum class CodeGenFunction::MSVCIntrin { + _BitScanForward, + _BitScanReverse, + _InterlockedAnd, + _InterlockedDecrement, + _InterlockedExchange, + _InterlockedExchangeAdd, + _InterlockedExchangeSub, + _InterlockedIncrement, + _InterlockedOr, + _InterlockedXor, +}; + +Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, + const CallExpr *E) { + switch (BuiltinID) { + case MSVCIntrin::_BitScanForward: + case MSVCIntrin::_BitScanReverse: { + Value *ArgValue = EmitScalarExpr(E->getArg(1)); + + llvm::Type *ArgType = ArgValue->getType(); + llvm::Type *IndexType = + EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType(); + llvm::Type *ResultType = ConvertType(E->getType()); + + Value *ArgZero = llvm::Constant::getNullValue(ArgType); + Value *ResZero = llvm::Constant::getNullValue(ResultType); + Value *ResOne = llvm::ConstantInt::get(ResultType, 1); + + BasicBlock *Begin = Builder.GetInsertBlock(); + BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn); + Builder.SetInsertPoint(End); + PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result"); + + Builder.SetInsertPoint(Begin); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero); + BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn); + Builder.CreateCondBr(IsZero, End, NotZero); + Result->addIncoming(ResZero, Begin); + + Builder.SetInsertPoint(NotZero); + Address IndexAddress = EmitPointerWithAlignment(E->getArg(0)); + + if (BuiltinID == MSVCIntrin::_BitScanForward) { + Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); + Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()}); + ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false); + Builder.CreateStore(ZeroCount, IndexAddress, false); + } else { + unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth(); + Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1); + + Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); + Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()}); + ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false); + Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount); + Builder.CreateStore(Index, IndexAddress, false); + } + Builder.CreateBr(End); + Result->addIncoming(ResOne, NotZero); + + Builder.SetInsertPoint(End); + return Result; + } + case MSVCIntrin::_InterlockedAnd: + return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E); + case MSVCIntrin::_InterlockedExchange: + return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E); + case MSVCIntrin::_InterlockedExchangeAdd: + return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E); + case MSVCIntrin::_InterlockedExchangeSub: + return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E); + case MSVCIntrin::_InterlockedOr: + return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E); + case MSVCIntrin::_InterlockedXor: + return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E); + + case MSVCIntrin::_InterlockedDecrement: { + llvm::Type *IntTy = ConvertType(E->getType()); + AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( + AtomicRMWInst::Sub, + EmitScalarExpr(E->getArg(0)), + ConstantInt::get(IntTy, 1), + llvm::AtomicOrdering::SequentiallyConsistent); + return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1)); + } + case MSVCIntrin::_InterlockedIncrement: { + llvm::Type *IntTy = ConvertType(E->getType()); + AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( + AtomicRMWInst::Add, + EmitScalarExpr(E->getArg(0)), + ConstantInt::get(IntTy, 1), + llvm::AtomicOrdering::SequentiallyConsistent); + return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1)); + } + } + llvm_unreachable("Incorrect MSVC intrinsic!"); +} + +namespace { +// ARC cleanup for __builtin_os_log_format +struct CallObjCArcUse final : EHScopeStack::Cleanup { + CallObjCArcUse(llvm::Value *object) : object(object) {} + llvm::Value *object; + + void Emit(CodeGenFunction &CGF, Flags flags) override { + CGF.EmitARCIntrinsicUse(object); + } +}; +} + RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue) { @@ -681,6 +801,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, "cast"); return RValue::get(Result); } + case Builtin::BI__popcnt16: + case Builtin::BI__popcnt: + case Builtin::BI__popcnt64: case Builtin::BI__builtin_popcount: case Builtin::BI__builtin_popcountl: case Builtin::BI__builtin_popcountll: { @@ -696,6 +819,58 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, "cast"); return RValue::get(Result); } + case Builtin::BI_rotr8: + case Builtin::BI_rotr16: + case Builtin::BI_rotr: + case Builtin::BI_lrotr: + case Builtin::BI_rotr64: { + Value *Val = EmitScalarExpr(E->getArg(0)); + Value *Shift = EmitScalarExpr(E->getArg(1)); + + llvm::Type *ArgType = Val->getType(); + Shift = Builder.CreateIntCast(Shift, ArgType, false); + unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth(); + Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth); + Value *ArgZero = llvm::Constant::getNullValue(ArgType); + + Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1); + Shift = Builder.CreateAnd(Shift, Mask); + Value *LeftShift = Builder.CreateSub(ArgTypeSize, Shift); + + Value *RightShifted = Builder.CreateLShr(Val, Shift); + Value *LeftShifted = Builder.CreateShl(Val, LeftShift); + Value *Rotated = Builder.CreateOr(LeftShifted, RightShifted); + + Value *ShiftIsZero = Builder.CreateICmpEQ(Shift, ArgZero); + Value *Result = Builder.CreateSelect(ShiftIsZero, Val, Rotated); + return RValue::get(Result); + } + case Builtin::BI_rotl8: + case Builtin::BI_rotl16: + case Builtin::BI_rotl: + case Builtin::BI_lrotl: + case Builtin::BI_rotl64: { + Value *Val = EmitScalarExpr(E->getArg(0)); + Value *Shift = EmitScalarExpr(E->getArg(1)); + + llvm::Type *ArgType = Val->getType(); + Shift = Builder.CreateIntCast(Shift, ArgType, false); + unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth(); + Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth); + Value *ArgZero = llvm::Constant::getNullValue(ArgType); + + Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1); + Shift = Builder.CreateAnd(Shift, Mask); + Value *RightShift = Builder.CreateSub(ArgTypeSize, Shift); + + Value *LeftShifted = Builder.CreateShl(Val, Shift); + Value *RightShifted = Builder.CreateLShr(Val, RightShift); + Value *Rotated = Builder.CreateOr(LeftShifted, RightShifted); + + Value *ShiftIsZero = Builder.CreateICmpEQ(Shift, ArgZero); + Value *Result = Builder.CreateSelect(ShiftIsZero, Val, Rotated); + return RValue::get(Result); + } case Builtin::BI__builtin_unpredictable: { // Always return the argument of __builtin_unpredictable. LLVM does not // handle this builtin. Metadata for this builtin should be added directly @@ -789,8 +964,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, SanitizerScope SanScope(this); EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), SanitizerKind::Unreachable), - "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), - None); + SanitizerHandler::BuiltinUnreachable, + EmitCheckSourceLocation(E->getExprLoc()), None); } else Builder.CreateUnreachable(); @@ -851,6 +1026,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); } + case Builtin::BIfinite: + case Builtin::BI__finite: + case Builtin::BIfinitef: + case Builtin::BI__finitef: + case Builtin::BIfinitel: + case Builtin::BI__finitel: case Builtin::BI__builtin_isinf: case Builtin::BI__builtin_isfinite: { // isinf(x) --> fabs(x) == infinity @@ -963,8 +1144,29 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, case Builtin::BI_alloca: case Builtin::BI__builtin_alloca: { Value *Size = EmitScalarExpr(E->getArg(0)); - return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); + const TargetInfo &TI = getContext().getTargetInfo(); + // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__. + unsigned SuitableAlignmentInBytes = + CGM.getContext() + .toCharUnitsFromBits(TI.getSuitableAlign()) + .getQuantity(); + AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size); + AI->setAlignment(SuitableAlignmentInBytes); + return RValue::get(AI); + } + + case Builtin::BI__builtin_alloca_with_align: { + Value *Size = EmitScalarExpr(E->getArg(0)); + Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1)); + auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue); + unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue(); + unsigned AlignmentInBytes = + CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity(); + AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size); + AI->setAlignment(AlignmentInBytes); + return RValue::get(AI); } + case Builtin::BIbzero: case Builtin::BI__builtin_bzero: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); @@ -987,6 +1189,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Dest.getPointer()); } + case Builtin::BI__builtin_char_memchr: + BuiltinID = Builtin::BI__builtin_memchr; + break; + case Builtin::BI__builtin___memcpy_chk: { // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. llvm::APSInt Size, DstSize; @@ -1085,6 +1291,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); return RValue::get(Builder.CreateCall(F, Depth)); } + case Builtin::BI_ReturnAddress: { + Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); + return RValue::get(Builder.CreateCall(F, Builder.getInt32(0))); + } case Builtin::BI__builtin_frame_address: { Value *Depth = CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); @@ -1390,7 +1600,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args); llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); - return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); + return EmitCall(FuncInfo, CGCallee::forDirect(Func), + ReturnValueSlot(), Args); } case Builtin::BI__atomic_test_and_set: { @@ -1905,12 +2116,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, const CallExpr *Call = cast<CallExpr>(E->getArg(0)); const Expr *Chain = E->getArg(1); return EmitCall(Call->getCallee()->getType(), - EmitScalarExpr(Call->getCallee()), Call, ReturnValue, - Call->getCalleeDecl(), EmitScalarExpr(Chain)); + EmitCallee(Call->getCallee()), Call, ReturnValue, + EmitScalarExpr(Chain)); } + case Builtin::BI_InterlockedExchange8: + case Builtin::BI_InterlockedExchange16: case Builtin::BI_InterlockedExchange: case Builtin::BI_InterlockedExchangePointer: - return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); + return RValue::get( + EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E)); case Builtin::BI_InterlockedCompareExchangePointer: { llvm::Type *RTy; llvm::IntegerType *IntType = @@ -1938,7 +2152,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 0), RTy)); } - case Builtin::BI_InterlockedCompareExchange: { + case Builtin::BI_InterlockedCompareExchange8: + case Builtin::BI_InterlockedCompareExchange16: + case Builtin::BI_InterlockedCompareExchange: + case Builtin::BI_InterlockedCompareExchange64: { AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(2)), @@ -1948,42 +2165,44 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, CXI->setVolatile(true); return RValue::get(Builder.CreateExtractValue(CXI, 0)); } - case Builtin::BI_InterlockedIncrement: { - llvm::Type *IntTy = ConvertType(E->getType()); - AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( - AtomicRMWInst::Add, - EmitScalarExpr(E->getArg(0)), - ConstantInt::get(IntTy, 1), - llvm::AtomicOrdering::SequentiallyConsistent); - RMWI->setVolatile(true); - return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1))); - } - case Builtin::BI_InterlockedDecrement: { - llvm::Type *IntTy = ConvertType(E->getType()); - AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( - AtomicRMWInst::Sub, - EmitScalarExpr(E->getArg(0)), - ConstantInt::get(IntTy, 1), - llvm::AtomicOrdering::SequentiallyConsistent); - RMWI->setVolatile(true); - return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1))); - } - case Builtin::BI_InterlockedExchangeAdd: { - AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( - AtomicRMWInst::Add, - EmitScalarExpr(E->getArg(0)), - EmitScalarExpr(E->getArg(1)), - llvm::AtomicOrdering::SequentiallyConsistent); - RMWI->setVolatile(true); - return RValue::get(RMWI); - } + case Builtin::BI_InterlockedIncrement16: + case Builtin::BI_InterlockedIncrement: + return RValue::get( + EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E)); + case Builtin::BI_InterlockedDecrement16: + case Builtin::BI_InterlockedDecrement: + return RValue::get( + EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E)); + case Builtin::BI_InterlockedAnd8: + case Builtin::BI_InterlockedAnd16: + case Builtin::BI_InterlockedAnd: + return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E)); + case Builtin::BI_InterlockedExchangeAdd8: + case Builtin::BI_InterlockedExchangeAdd16: + case Builtin::BI_InterlockedExchangeAdd: + return RValue::get( + EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E)); + case Builtin::BI_InterlockedExchangeSub8: + case Builtin::BI_InterlockedExchangeSub16: + case Builtin::BI_InterlockedExchangeSub: + return RValue::get( + EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E)); + case Builtin::BI_InterlockedOr8: + case Builtin::BI_InterlockedOr16: + case Builtin::BI_InterlockedOr: + return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E)); + case Builtin::BI_InterlockedXor8: + case Builtin::BI_InterlockedXor16: + case Builtin::BI_InterlockedXor: + return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E)); case Builtin::BI__readfsdword: { llvm::Type *IntTy = ConvertType(E->getType()); Value *IntToPtr = Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), llvm::PointerType::get(IntTy, 257)); - LoadInst *Load = - Builder.CreateDefaultAlignedLoad(IntToPtr, /*isVolatile=*/true); + LoadInst *Load = Builder.CreateAlignedLoad( + IntTy, IntToPtr, getContext().getTypeAlignInChars(E->getType())); + Load->setVolatile(true); return RValue::get(Load); } @@ -2004,7 +2223,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Attribute::ReturnsTwice); llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), - "_setjmpex", ReturnsTwiceAttr); + "_setjmpex", ReturnsTwiceAttr, /*Local=*/true); llvm::Value *Buf = Builder.CreateBitOrPointerCast( EmitScalarExpr(E->getArg(0)), Int8PtrTy); llvm::Value *FrameAddr = @@ -2029,7 +2248,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), - "_setjmp3", ReturnsTwiceAttr); + "_setjmp3", ReturnsTwiceAttr, /*Local=*/true); llvm::Value *Count = ConstantInt::get(IntTy, 0); llvm::Value *Args[] = {Buf, Count}; CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); @@ -2037,7 +2256,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), - "_setjmp", ReturnsTwiceAttr); + "_setjmp", ReturnsTwiceAttr, /*Local=*/true); llvm::Value *FrameAddr = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), ConstantInt::get(Int32Ty, 0)); @@ -2057,11 +2276,47 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, break; } + case Builtin::BI__builtin_coro_size: { + auto & Context = getContext(); + auto SizeTy = Context.getSizeType(); + auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); + Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T); + return RValue::get(Builder.CreateCall(F)); + } + + case Builtin::BI__builtin_coro_id: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_id); + case Builtin::BI__builtin_coro_promise: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise); + case Builtin::BI__builtin_coro_resume: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume); + case Builtin::BI__builtin_coro_frame: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame); + case Builtin::BI__builtin_coro_free: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_free); + case Builtin::BI__builtin_coro_destroy: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy); + case Builtin::BI__builtin_coro_done: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_done); + case Builtin::BI__builtin_coro_alloc: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc); + case Builtin::BI__builtin_coro_begin: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin); + case Builtin::BI__builtin_coro_end: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_end); + case Builtin::BI__builtin_coro_suspend: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend); + case Builtin::BI__builtin_coro_param: + return EmitCoroutineIntrinsic(E, Intrinsic::coro_param); + // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions case Builtin::BIread_pipe: case Builtin::BIwrite_pipe: { Value *Arg0 = EmitScalarExpr(E->getArg(0)), *Arg1 = EmitScalarExpr(E->getArg(1)); + CGOpenCLRuntime OpenCLRT(CGM); + Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0)); + Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0)); // Type of the generic packet parameter. unsigned GenericAS = @@ -2075,19 +2330,21 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, : "__write_pipe_2"; // Creating a generic function type to be able to call with any builtin or // user defined type. - llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy}; + llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty}; llvm::FunctionType *FTy = llvm::FunctionType::get( Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy); - return RValue::get(Builder.CreateCall( - CGM.CreateRuntimeFunction(FTy, Name), {Arg0, BCast})); + return RValue::get( + Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), + {Arg0, BCast, PacketSize, PacketAlign})); } else { assert(4 == E->getNumArgs() && "Illegal number of parameters to pipe function"); const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4" : "__write_pipe_4"; - llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy}; + llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy, + Int32Ty, Int32Ty}; Value *Arg2 = EmitScalarExpr(E->getArg(2)), *Arg3 = EmitScalarExpr(E->getArg(3)); llvm::FunctionType *FTy = llvm::FunctionType::get( @@ -2098,7 +2355,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, if (Arg2->getType() != Int32Ty) Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty); return RValue::get(Builder.CreateCall( - CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1, Arg2, BCast})); + CGM.CreateRuntimeFunction(FTy, Name), + {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign})); } } // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write @@ -2127,9 +2385,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Arg0 = EmitScalarExpr(E->getArg(0)), *Arg1 = EmitScalarExpr(E->getArg(1)); llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy); + CGOpenCLRuntime OpenCLRT(CGM); + Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0)); + Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0)); // Building the generic function prototype. - llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty}; + llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty}; llvm::FunctionType *FTy = llvm::FunctionType::get( ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false); // We know the second argument is an integer type, but we may need to cast @@ -2137,7 +2398,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, if (Arg1->getType() != Int32Ty) Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty); return RValue::get( - Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); + Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), + {Arg0, Arg1, PacketSize, PacketAlign})); } // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write // functions @@ -2163,15 +2425,19 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Arg0 = EmitScalarExpr(E->getArg(0)), *Arg1 = EmitScalarExpr(E->getArg(1)); + CGOpenCLRuntime OpenCLRT(CGM); + Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0)); + Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0)); // Building the generic function prototype. - llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType()}; + llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty}; llvm::FunctionType *FTy = llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()), llvm::ArrayRef<llvm::Type *>(ArgTys), false); return RValue::get( - Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); + Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), + {Arg0, Arg1, PacketSize, PacketAlign})); } // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions case Builtin::BIget_pipe_num_packets: @@ -2184,12 +2450,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Building the generic function prototype. Value *Arg0 = EmitScalarExpr(E->getArg(0)); - llvm::Type *ArgTys[] = {Arg0->getType()}; + CGOpenCLRuntime OpenCLRT(CGM); + Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0)); + Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0)); + llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty}; llvm::FunctionType *FTy = llvm::FunctionType::get( Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); - return RValue::get( - Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0})); + return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), + {Arg0, PacketSize, PacketAlign})); } // OpenCL v2.0 s6.13.9 - Address space qualifier functions. @@ -2258,17 +2527,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, std::vector<llvm::Type *> ArgTys = {QueueTy, IntTy, RangeTy, Int8PtrTy, IntTy}; - // Add the variadics. - for (unsigned I = 4; I < NumArgs; ++I) { - llvm::Value *ArgSize = EmitScalarExpr(E->getArg(I)); - unsigned TypeSizeInBytes = - getContext() - .getTypeSizeInChars(E->getArg(I)->getType()) - .getQuantity(); - Args.push_back(TypeSizeInBytes < 4 - ? Builder.CreateZExt(ArgSize, Int32Ty) - : ArgSize); - } + // Each of the following arguments specifies the size of the corresponding + // argument passed to the enqueued block. + for (unsigned I = 4/*Position of the first size arg*/; I < NumArgs; ++I) + Args.push_back( + Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy)); llvm::FunctionType *FTy = llvm::FunctionType::get( Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), true); @@ -2279,29 +2542,26 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Any calls now have event arguments passed. if (NumArgs >= 7) { llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy); - unsigned AS4 = - E->getArg(4)->getType()->isArrayType() - ? E->getArg(4)->getType().getAddressSpace() - : E->getArg(4)->getType()->getPointeeType().getAddressSpace(); - llvm::Type *EventPtrAS4Ty = - EventTy->getPointerTo(CGM.getContext().getTargetAddressSpace(AS4)); - unsigned AS5 = - E->getArg(5)->getType()->getPointeeType().getAddressSpace(); - llvm::Type *EventPtrAS5Ty = - EventTy->getPointerTo(CGM.getContext().getTargetAddressSpace(AS5)); - - llvm::Value *NumEvents = EmitScalarExpr(E->getArg(3)); + llvm::Type *EventPtrTy = EventTy->getPointerTo( + CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic)); + + llvm::Value *NumEvents = + Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty); llvm::Value *EventList = E->getArg(4)->getType()->isArrayType() ? EmitArrayToPointerDecay(E->getArg(4)).getPointer() : EmitScalarExpr(E->getArg(4)); llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5)); + // Convert to generic address space. + EventList = Builder.CreatePointerCast(EventList, EventPtrTy); + ClkEvent = Builder.CreatePointerCast(ClkEvent, EventPtrTy); llvm::Value *Block = Builder.CreateBitCast(EmitScalarExpr(E->getArg(6)), Int8PtrTy); - std::vector<llvm::Type *> ArgTys = { - QueueTy, Int32Ty, RangeTy, Int32Ty, - EventPtrAS4Ty, EventPtrAS5Ty, Int8PtrTy}; + std::vector<llvm::Type *> ArgTys = {QueueTy, Int32Ty, RangeTy, + Int32Ty, EventPtrTy, EventPtrTy, + Int8PtrTy}; + std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents, EventList, ClkEvent, Block}; @@ -2320,17 +2580,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, ArgTys.push_back(Int32Ty); Name = "__enqueue_kernel_events_vaargs"; - // Add the variadics. - for (unsigned I = 7; I < NumArgs; ++I) { - llvm::Value *ArgSize = EmitScalarExpr(E->getArg(I)); - unsigned TypeSizeInBytes = - getContext() - .getTypeSizeInChars(E->getArg(I)->getType()) - .getQuantity(); - Args.push_back(TypeSizeInBytes < 4 - ? Builder.CreateZExt(ArgSize, Int32Ty) - : ArgSize); - } + // Each of the following arguments specifies the size of the corresponding + // argument passed to the enqueued block. + for (unsigned I = 7/*Position of the first size arg*/; I < NumArgs; ++I) + Args.push_back( + Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy)); + llvm::FunctionType *FTy = llvm::FunctionType::get( Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), true); return RValue::get( @@ -2373,6 +2628,76 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Fall through - it's already mapped to the intrinsic by GCCBuiltin. break; } + case Builtin::BI__builtin_os_log_format: { + assert(E->getNumArgs() >= 2 && + "__builtin_os_log_format takes at least 2 arguments"); + analyze_os_log::OSLogBufferLayout Layout; + analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout); + Address BufAddr = EmitPointerWithAlignment(E->getArg(0)); + // Ignore argument 1, the format string. It is not currently used. + CharUnits Offset; + Builder.CreateStore( + Builder.getInt8(Layout.getSummaryByte()), + Builder.CreateConstByteGEP(BufAddr, Offset++, "summary")); + Builder.CreateStore( + Builder.getInt8(Layout.getNumArgsByte()), + Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs")); + + llvm::SmallVector<llvm::Value *, 4> RetainableOperands; + for (const auto &Item : Layout.Items) { + Builder.CreateStore( + Builder.getInt8(Item.getDescriptorByte()), + Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor")); + Builder.CreateStore( + Builder.getInt8(Item.getSizeByte()), + Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize")); + Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset); + if (const Expr *TheExpr = Item.getExpr()) { + Addr = Builder.CreateElementBitCast( + Addr, ConvertTypeForMem(TheExpr->getType())); + // Check if this is a retainable type. + if (TheExpr->getType()->isObjCRetainableType()) { + assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar && + "Only scalar can be a ObjC retainable type"); + llvm::Value *SV = EmitScalarExpr(TheExpr, /*Ignore*/ false); + RValue RV = RValue::get(SV); + LValue LV = MakeAddrLValue(Addr, TheExpr->getType()); + EmitStoreThroughLValue(RV, LV); + // Check if the object is constant, if not, save it in + // RetainableOperands. + if (!isa<Constant>(SV)) + RetainableOperands.push_back(SV); + } else { + EmitAnyExprToMem(TheExpr, Addr, Qualifiers(), /*isInit*/ true); + } + } else { + Addr = Builder.CreateElementBitCast(Addr, Int32Ty); + Builder.CreateStore( + Builder.getInt32(Item.getConstValue().getQuantity()), Addr); + } + Offset += Item.size(); + } + + // Push a clang.arc.use cleanup for each object in RetainableOperands. The + // cleanup will cause the use to appear after the final log call, keeping + // the object valid while it’s held in the log buffer. Note that if there’s + // a release cleanup on the object, it will already be active; since + // cleanups are emitted in reverse order, the use will occur before the + // object is released. + if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount && + CGM.getCodeGenOpts().OptimizationLevel != 0) + for (llvm::Value *object : RetainableOperands) + pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), object); + + return RValue::get(BufAddr.getPointer()); + } + + case Builtin::BI__builtin_os_log_format_buffer_size: { + analyze_os_log::OSLogBufferLayout Layout; + analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout); + return RValue::get(ConstantInt::get(ConvertType(E->getType()), + Layout.size().getQuantity())); + } } // If this is an alias for a lib function (e.g. __builtin_sin), emit @@ -2385,7 +2710,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // If this is a predefined lib function (e.g. malloc), emit the call // using exactly the normal call path. if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) - return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); + return emitLibraryCall(*this, FD, E, + cast<llvm::Constant>(EmitScalarExpr(E->getCallee()))); // Check that a call to a target specific builtin has the correct target // features. @@ -2397,14 +2723,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // See if we have a target specific intrinsic. const char *Name = getContext().BuiltinInfo.getName(BuiltinID); Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; - if (const char *Prefix = - llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { - IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); + StringRef Prefix = + llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch()); + if (!Prefix.empty()) { + IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name); // NOTE we dont need to perform a compatibility flag check here since the // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. if (IntrinsicID == Intrinsic::not_intrinsic) - IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); + IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name); } if (IntrinsicID != Intrinsic::not_intrinsic) { @@ -3871,7 +4198,7 @@ static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, if (SysReg.empty()) { const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); - SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); + SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString(); } llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; @@ -3995,9 +4322,9 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, } if (BuiltinID == ARM::BI__builtin_arm_rbit) { - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), - EmitScalarExpr(E->getArg(0)), - "rbit"); + llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall( + CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit"); } if (BuiltinID == ARM::BI__clear_cache) { @@ -4120,19 +4447,21 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, QualType Ty = E->getType(); llvm::Type *RealResTy = ConvertType(Ty); - llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), - getContext().getTypeSize(Ty)); - LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); + llvm::Type *PtrTy = llvm::IntegerType::get( + getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo(); + LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy); Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex, - LoadAddr->getType()); + PtrTy); Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); if (RealResTy->isPointerTy()) return Builder.CreateIntToPtr(Val, RealResTy); else { + llvm::Type *IntResTy = llvm::IntegerType::get( + getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy)); Val = Builder.CreateTruncOrBitCast(Val, IntResTy); return Builder.CreateBitCast(Val, RealResTy); } @@ -4173,7 +4502,10 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, if (StoreVal->getType()->isPointerTy()) StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); else { - StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); + llvm::Type *IntTy = llvm::IntegerType::get( + getLLVMContext(), + CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType())); + StoreVal = Builder.CreateBitCast(StoreVal, IntTy); StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); } @@ -4184,6 +4516,41 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); } + switch (BuiltinID) { + case ARM::BI__iso_volatile_load8: + case ARM::BI__iso_volatile_load16: + case ARM::BI__iso_volatile_load32: + case ARM::BI__iso_volatile_load64: { + Value *Ptr = EmitScalarExpr(E->getArg(0)); + QualType ElTy = E->getArg(0)->getType()->getPointeeType(); + CharUnits LoadSize = getContext().getTypeSizeInChars(ElTy); + llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), + LoadSize.getQuantity() * 8); + Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); + llvm::LoadInst *Load = + Builder.CreateAlignedLoad(Ptr, LoadSize); + Load->setVolatile(true); + return Load; + } + case ARM::BI__iso_volatile_store8: + case ARM::BI__iso_volatile_store16: + case ARM::BI__iso_volatile_store32: + case ARM::BI__iso_volatile_store64: { + Value *Ptr = EmitScalarExpr(E->getArg(0)); + Value *Value = EmitScalarExpr(E->getArg(1)); + QualType ElTy = E->getArg(0)->getType()->getPointeeType(); + CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); + llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), + StoreSize.getQuantity() * 8); + Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); + llvm::StoreInst *Store = + Builder.CreateAlignedStore(Value, Ptr, + StoreSize); + Store->setVolatile(true); + return Store; + } + } + if (BuiltinID == ARM::BI__builtin_arm_clrex) { Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); return Builder.CreateCall(F); @@ -4397,6 +4764,29 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], Ops[3], Ops[4], Ops[5]}); } + case ARM::BI_BitScanForward: + case ARM::BI_BitScanForward64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E); + case ARM::BI_BitScanReverse: + case ARM::BI_BitScanReverse64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E); + + case ARM::BI_InterlockedAnd64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E); + case ARM::BI_InterlockedExchange64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E); + case ARM::BI_InterlockedExchangeAdd64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E); + case ARM::BI_InterlockedExchangeSub64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E); + case ARM::BI_InterlockedOr64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E); + case ARM::BI_InterlockedXor64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E); + case ARM::BI_InterlockedDecrement64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E); + case ARM::BI_InterlockedIncrement64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E); } // Get the last argument, which specifies the vector type. @@ -4840,14 +5230,14 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, "rbit of unusual size!"); llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); return Builder.CreateCall( - CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); + CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit"); } if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { assert((getContext().getTypeSize(E->getType()) == 64) && "rbit of unusual size!"); llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); return Builder.CreateCall( - CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); + CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit"); } if (BuiltinID == AArch64::BI__clear_cache) { @@ -4889,19 +5279,21 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, QualType Ty = E->getType(); llvm::Type *RealResTy = ConvertType(Ty); - llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), - getContext().getTypeSize(Ty)); - LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); + llvm::Type *PtrTy = llvm::IntegerType::get( + getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo(); + LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy); Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr, - LoadAddr->getType()); + PtrTy); Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); if (RealResTy->isPointerTy()) return Builder.CreateIntToPtr(Val, RealResTy); + llvm::Type *IntResTy = llvm::IntegerType::get( + getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy)); Val = Builder.CreateTruncOrBitCast(Val, IntResTy); return Builder.CreateBitCast(Val, RealResTy); } @@ -4940,7 +5332,10 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, if (StoreVal->getType()->isPointerTy()) StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); else { - StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); + llvm::Type *IntTy = llvm::IntegerType::get( + getLLVMContext(), + CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType())); + StoreVal = Builder.CreateBitCast(StoreVal, IntTy); StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); } @@ -5065,9 +5460,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, switch (BuiltinID) { default: break; case NEON::BI__builtin_neon_vldrq_p128: { - llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); + llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128); + llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0); Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); - return Builder.CreateDefaultAlignedLoad(Ptr); + return Builder.CreateAlignedLoad(Int128Ty, Ptr, + CharUnits::fromQuantity(16)); } case NEON::BI__builtin_neon_vstrq_p128: { llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); @@ -6240,27 +6637,37 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); } case NEON::BI__builtin_neon_vld1_v: - case NEON::BI__builtin_neon_vld1q_v: + case NEON::BI__builtin_neon_vld1q_v: { Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); - return Builder.CreateDefaultAlignedLoad(Ops[0]); + auto Alignment = CharUnits::fromQuantity( + BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16); + return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment); + } case NEON::BI__builtin_neon_vst1_v: case NEON::BI__builtin_neon_vst1q_v: Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); Ops[1] = Builder.CreateBitCast(Ops[1], VTy); return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); case NEON::BI__builtin_neon_vld1_lane_v: - case NEON::BI__builtin_neon_vld1q_lane_v: + case NEON::BI__builtin_neon_vld1q_lane_v: { Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ty = llvm::PointerType::getUnqual(VTy->getElementType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); + auto Alignment = CharUnits::fromQuantity( + BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16); + Ops[0] = + Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment); return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); + } case NEON::BI__builtin_neon_vld1_dup_v: case NEON::BI__builtin_neon_vld1q_dup_v: { Value *V = UndefValue::get(Ty); Ty = llvm::PointerType::getUnqual(VTy->getElementType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); + auto Alignment = CharUnits::fromQuantity( + BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16); + Ops[0] = + Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment); llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); return EmitNeonSplat(Ops[0], CI); @@ -6620,6 +7027,26 @@ static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, return CGF.Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]); } +static Value *EmitX86SubVectorBroadcast(CodeGenFunction &CGF, + SmallVectorImpl<Value *> &Ops, + llvm::Type *DstTy, + unsigned SrcSizeInBits, + unsigned Align) { + // Load the subvector. + Ops[0] = CGF.Builder.CreateAlignedLoad(Ops[0], Align); + + // Create broadcast mask. + unsigned NumDstElts = DstTy->getVectorNumElements(); + unsigned NumSrcElts = SrcSizeInBits / DstTy->getScalarSizeInBits(); + + SmallVector<uint32_t, 8> Mask; + for (unsigned i = 0; i != NumDstElts; i += NumSrcElts) + for (unsigned j = 0; j != NumSrcElts; ++j) + Mask.push_back(j); + + return CGF.Builder.CreateShuffleVector(Ops[0], Ops[0], Mask, "subvecbcst"); +} + static Value *EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1) { @@ -6676,6 +7103,18 @@ static Value *EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, std::max(NumElts, 8U))); } +static Value *EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred, + ArrayRef<Value *> Ops) { + Value *Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); + Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Ops[1]); + + if (Ops.size() == 2) + return Res; + + assert(Ops.size() == 4); + return EmitX86Select(CGF, Ops[3], Res, Ops[2]); +} + Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E) { if (BuiltinID == X86::BI__builtin_ms_va_start || @@ -6860,6 +7299,25 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Value *F = CGM.getIntrinsic(Intrinsic::prefetch); return Builder.CreateCall(F, {Address, RW, Locality, Data}); } + case X86::BI_mm_clflush: { + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush), + Ops[0]); + } + case X86::BI_mm_lfence: { + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence)); + } + case X86::BI_mm_mfence: { + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence)); + } + case X86::BI_mm_sfence: { + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence)); + } + case X86::BI_mm_pause: { + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause)); + } + case X86::BI__rdtsc: { + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc)); + } case X86::BI__builtin_ia32_undef128: case X86::BI__builtin_ia32_undef256: case X86::BI__builtin_ia32_undef512: @@ -6872,12 +7330,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, case X86::BI__builtin_ia32_vec_ext_v2si: return Builder.CreateExtractElement(Ops[0], llvm::ConstantInt::get(Ops[1]->getType(), 0)); + case X86::BI_mm_setcsr: case X86::BI__builtin_ia32_ldmxcsr: { Address Tmp = CreateMemTemp(E->getArg(0)->getType()); Builder.CreateStore(Ops[0], Tmp); return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); } + case X86::BI_mm_getcsr: case X86::BI__builtin_ia32_stmxcsr: { Address Tmp = CreateMemTemp(E->getType()); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), @@ -6944,6 +7404,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, case X86::BI__builtin_ia32_storeups512_mask: return EmitX86MaskedStore(*this, Ops, 1); + case X86::BI__builtin_ia32_storess128_mask: + case X86::BI__builtin_ia32_storesd128_mask: { + return EmitX86MaskedStore(*this, Ops, 16); + } case X86::BI__builtin_ia32_movdqa32store128_mask: case X86::BI__builtin_ia32_movdqa64store128_mask: case X86::BI__builtin_ia32_storeaps128_mask: @@ -6980,6 +7444,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, case X86::BI__builtin_ia32_loaddqudi512_mask: return EmitX86MaskedLoad(*this, Ops, 1); + case X86::BI__builtin_ia32_loadss128_mask: + case X86::BI__builtin_ia32_loadsd128_mask: + return EmitX86MaskedLoad(*this, Ops, 16); + case X86::BI__builtin_ia32_loadaps128_mask: case X86::BI__builtin_ia32_loadaps256_mask: case X86::BI__builtin_ia32_loadaps512_mask: @@ -6996,6 +7464,13 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity(); return EmitX86MaskedLoad(*this, Ops, Align); } + + case X86::BI__builtin_ia32_vbroadcastf128_pd256: + case X86::BI__builtin_ia32_vbroadcastf128_ps256: { + llvm::Type *DstTy = ConvertType(E->getType()); + return EmitX86SubVectorBroadcast(*this, Ops, DstTy, 128, 1); + } + case X86::BI__builtin_ia32_storehps: case X86::BI__builtin_ia32_storelps: { llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); @@ -7015,8 +7490,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } case X86::BI__builtin_ia32_palignr128: case X86::BI__builtin_ia32_palignr256: - case X86::BI__builtin_ia32_palignr128_mask: - case X86::BI__builtin_ia32_palignr256_mask: case X86::BI__builtin_ia32_palignr512_mask: { unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); @@ -7059,36 +7532,26 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } case X86::BI__builtin_ia32_movnti: - case X86::BI__builtin_ia32_movnti64: { - llvm::MDNode *Node = llvm::MDNode::get( - getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); - - // Convert the type of the pointer to a pointer to the stored type. - Value *BC = Builder.CreateBitCast(Ops[0], - llvm::PointerType::getUnqual(Ops[1]->getType()), - "cast"); - StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); - SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); - - // No alignment for scalar intrinsic store. - SI->setAlignment(1); - return SI; - } + case X86::BI__builtin_ia32_movnti64: case X86::BI__builtin_ia32_movntsd: case X86::BI__builtin_ia32_movntss: { llvm::MDNode *Node = llvm::MDNode::get( getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); + Value *Ptr = Ops[0]; + Value *Src = Ops[1]; + // Extract the 0'th element of the source vector. - Value *Scl = Builder.CreateExtractElement(Ops[1], (uint64_t)0, "extract"); + if (BuiltinID == X86::BI__builtin_ia32_movntsd || + BuiltinID == X86::BI__builtin_ia32_movntss) + Src = Builder.CreateExtractElement(Src, (uint64_t)0, "extract"); // Convert the type of the pointer to a pointer to the stored type. - Value *BC = Builder.CreateBitCast(Ops[0], - llvm::PointerType::getUnqual(Scl->getType()), - "cast"); + Value *BC = Builder.CreateBitCast( + Ptr, llvm::PointerType::getUnqual(Src->getType()), "cast"); // Unaligned nontemporal store of the scalar value. - StoreInst *SI = Builder.CreateDefaultAlignedStore(Scl, BC); + StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC); SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); SI->setAlignment(1); return SI; @@ -7182,43 +7645,58 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Ops[1]); } - // TODO: Handle 64/512-bit vector widths of min/max. case X86::BI__builtin_ia32_pmaxsb128: case X86::BI__builtin_ia32_pmaxsw128: case X86::BI__builtin_ia32_pmaxsd128: + case X86::BI__builtin_ia32_pmaxsq128_mask: case X86::BI__builtin_ia32_pmaxsb256: case X86::BI__builtin_ia32_pmaxsw256: - case X86::BI__builtin_ia32_pmaxsd256: { - Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Ops[1]); - return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); - } + case X86::BI__builtin_ia32_pmaxsd256: + case X86::BI__builtin_ia32_pmaxsq256_mask: + case X86::BI__builtin_ia32_pmaxsb512_mask: + case X86::BI__builtin_ia32_pmaxsw512_mask: + case X86::BI__builtin_ia32_pmaxsd512_mask: + case X86::BI__builtin_ia32_pmaxsq512_mask: + return EmitX86MinMax(*this, ICmpInst::ICMP_SGT, Ops); case X86::BI__builtin_ia32_pmaxub128: case X86::BI__builtin_ia32_pmaxuw128: case X86::BI__builtin_ia32_pmaxud128: + case X86::BI__builtin_ia32_pmaxuq128_mask: case X86::BI__builtin_ia32_pmaxub256: case X86::BI__builtin_ia32_pmaxuw256: - case X86::BI__builtin_ia32_pmaxud256: { - Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_UGT, Ops[0], Ops[1]); - return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); - } + case X86::BI__builtin_ia32_pmaxud256: + case X86::BI__builtin_ia32_pmaxuq256_mask: + case X86::BI__builtin_ia32_pmaxub512_mask: + case X86::BI__builtin_ia32_pmaxuw512_mask: + case X86::BI__builtin_ia32_pmaxud512_mask: + case X86::BI__builtin_ia32_pmaxuq512_mask: + return EmitX86MinMax(*this, ICmpInst::ICMP_UGT, Ops); case X86::BI__builtin_ia32_pminsb128: case X86::BI__builtin_ia32_pminsw128: case X86::BI__builtin_ia32_pminsd128: + case X86::BI__builtin_ia32_pminsq128_mask: case X86::BI__builtin_ia32_pminsb256: case X86::BI__builtin_ia32_pminsw256: - case X86::BI__builtin_ia32_pminsd256: { - Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SLT, Ops[0], Ops[1]); - return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); - } + case X86::BI__builtin_ia32_pminsd256: + case X86::BI__builtin_ia32_pminsq256_mask: + case X86::BI__builtin_ia32_pminsb512_mask: + case X86::BI__builtin_ia32_pminsw512_mask: + case X86::BI__builtin_ia32_pminsd512_mask: + case X86::BI__builtin_ia32_pminsq512_mask: + return EmitX86MinMax(*this, ICmpInst::ICMP_SLT, Ops); case X86::BI__builtin_ia32_pminub128: case X86::BI__builtin_ia32_pminuw128: case X86::BI__builtin_ia32_pminud128: + case X86::BI__builtin_ia32_pminuq128_mask: case X86::BI__builtin_ia32_pminub256: case X86::BI__builtin_ia32_pminuw256: - case X86::BI__builtin_ia32_pminud256: { - Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_ULT, Ops[0], Ops[1]); - return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); - } + case X86::BI__builtin_ia32_pminud256: + case X86::BI__builtin_ia32_pminuq256_mask: + case X86::BI__builtin_ia32_pminub512_mask: + case X86::BI__builtin_ia32_pminuw512_mask: + case X86::BI__builtin_ia32_pminud512_mask: + case X86::BI__builtin_ia32_pminuq512_mask: + return EmitX86MinMax(*this, ICmpInst::ICMP_ULT, Ops); // 3DNow! case X86::BI__builtin_ia32_pswapdsf: @@ -7363,6 +7841,87 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6); case X86::BI__builtin_ia32_cmpordsd: return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7); + + case X86::BI__emul: + case X86::BI__emulu: { + llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64); + bool isSigned = (BuiltinID == X86::BI__emul); + Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned); + Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned); + return Builder.CreateMul(LHS, RHS, "", !isSigned, isSigned); + } + case X86::BI__mulh: + case X86::BI__umulh: + case X86::BI_mul128: + case X86::BI_umul128: { + llvm::Type *ResType = ConvertType(E->getType()); + llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); + + bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128); + Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned); + Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned); + + Value *MulResult, *HigherBits; + if (IsSigned) { + MulResult = Builder.CreateNSWMul(LHS, RHS); + HigherBits = Builder.CreateAShr(MulResult, 64); + } else { + MulResult = Builder.CreateNUWMul(LHS, RHS); + HigherBits = Builder.CreateLShr(MulResult, 64); + } + HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned); + + if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh) + return HigherBits; + + Address HighBitsAddress = EmitPointerWithAlignment(E->getArg(2)); + Builder.CreateStore(HigherBits, HighBitsAddress); + return Builder.CreateIntCast(MulResult, ResType, IsSigned); + } + + case X86::BI__faststorefence: { + return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, + llvm::CrossThread); + } + case X86::BI_ReadWriteBarrier: + case X86::BI_ReadBarrier: + case X86::BI_WriteBarrier: { + return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, + llvm::SingleThread); + } + case X86::BI_BitScanForward: + case X86::BI_BitScanForward64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E); + case X86::BI_BitScanReverse: + case X86::BI_BitScanReverse64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E); + + case X86::BI_InterlockedAnd64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E); + case X86::BI_InterlockedExchange64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E); + case X86::BI_InterlockedExchangeAdd64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E); + case X86::BI_InterlockedExchangeSub64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E); + case X86::BI_InterlockedOr64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E); + case X86::BI_InterlockedXor64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E); + case X86::BI_InterlockedDecrement64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E); + case X86::BI_InterlockedIncrement64: + return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E); + + case X86::BI_AddressOfReturnAddress: { + Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress); + return Builder.CreateCall(F); + } + case X86::BI__stosb: { + // We treat __stosb as a volatile memset - it may not generate "rep stosb" + // instruction, but it will create a memset that won't be optimized away. + return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], 1, true); + } } } @@ -7384,7 +7943,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, case PPC::BI__builtin_ppc_get_timebase: return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter)); - // vec_ld, vec_lvsl, vec_lvsr + // vec_ld, vec_xl_be, vec_lvsl, vec_lvsr case PPC::BI__builtin_altivec_lvx: case PPC::BI__builtin_altivec_lvxl: case PPC::BI__builtin_altivec_lvebx: @@ -7394,11 +7953,19 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, case PPC::BI__builtin_altivec_lvsr: case PPC::BI__builtin_vsx_lxvd2x: case PPC::BI__builtin_vsx_lxvw4x: + case PPC::BI__builtin_vsx_lxvd2x_be: + case PPC::BI__builtin_vsx_lxvw4x_be: + case PPC::BI__builtin_vsx_lxvl: + case PPC::BI__builtin_vsx_lxvll: { - Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); - - Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); - Ops.pop_back(); + if(BuiltinID == PPC::BI__builtin_vsx_lxvl || + BuiltinID == PPC::BI__builtin_vsx_lxvll){ + Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy); + }else { + Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); + Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); + Ops.pop_back(); + } switch (BuiltinID) { default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); @@ -7429,12 +7996,24 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, case PPC::BI__builtin_vsx_lxvw4x: ID = Intrinsic::ppc_vsx_lxvw4x; break; + case PPC::BI__builtin_vsx_lxvd2x_be: + ID = Intrinsic::ppc_vsx_lxvd2x_be; + break; + case PPC::BI__builtin_vsx_lxvw4x_be: + ID = Intrinsic::ppc_vsx_lxvw4x_be; + break; + case PPC::BI__builtin_vsx_lxvl: + ID = Intrinsic::ppc_vsx_lxvl; + break; + case PPC::BI__builtin_vsx_lxvll: + ID = Intrinsic::ppc_vsx_lxvll; + break; } llvm::Function *F = CGM.getIntrinsic(ID); return Builder.CreateCall(F, Ops, ""); } - // vec_st + // vec_st, vec_xst_be case PPC::BI__builtin_altivec_stvx: case PPC::BI__builtin_altivec_stvxl: case PPC::BI__builtin_altivec_stvebx: @@ -7442,10 +8021,19 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, case PPC::BI__builtin_altivec_stvewx: case PPC::BI__builtin_vsx_stxvd2x: case PPC::BI__builtin_vsx_stxvw4x: + case PPC::BI__builtin_vsx_stxvd2x_be: + case PPC::BI__builtin_vsx_stxvw4x_be: + case PPC::BI__builtin_vsx_stxvl: + case PPC::BI__builtin_vsx_stxvll: { - Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); - Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); - Ops.pop_back(); + if(BuiltinID == PPC::BI__builtin_vsx_stxvl || + BuiltinID == PPC::BI__builtin_vsx_stxvll ){ + Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); + }else { + Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); + Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); + Ops.pop_back(); + } switch (BuiltinID) { default: llvm_unreachable("Unsupported st intrinsic!"); @@ -7470,6 +8058,18 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, case PPC::BI__builtin_vsx_stxvw4x: ID = Intrinsic::ppc_vsx_stxvw4x; break; + case PPC::BI__builtin_vsx_stxvd2x_be: + ID = Intrinsic::ppc_vsx_stxvd2x_be; + break; + case PPC::BI__builtin_vsx_stxvw4x_be: + ID = Intrinsic::ppc_vsx_stxvw4x_be; + break; + case PPC::BI__builtin_vsx_stxvl: + ID = Intrinsic::ppc_vsx_stxvl; + break; + case PPC::BI__builtin_vsx_stxvll: + ID = Intrinsic::ppc_vsx_stxvll; + break; } llvm::Function *F = CGM.getIntrinsic(ID); return Builder.CreateCall(F, Ops, ""); @@ -7494,6 +8094,25 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); return Builder.CreateCall(F, {X, Undef}); } + case PPC::BI__builtin_altivec_vctzb: + case PPC::BI__builtin_altivec_vctzh: + case PPC::BI__builtin_altivec_vctzw: + case PPC::BI__builtin_altivec_vctzd: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); + Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); + return Builder.CreateCall(F, {X, Undef}); + } + case PPC::BI__builtin_altivec_vpopcntb: + case PPC::BI__builtin_altivec_vpopcnth: + case PPC::BI__builtin_altivec_vpopcntw: + case PPC::BI__builtin_altivec_vpopcntd: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); + return Builder.CreateCall(F, X); + } // Copy sign case PPC::BI__builtin_vsx_xvcpsgnsp: case PPC::BI__builtin_vsx_xvcpsgndp: { @@ -7581,6 +8200,85 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, llvm_unreachable("Unknown FMA operation"); return nullptr; // Suppress no-return warning } + + case PPC::BI__builtin_vsx_insertword: { + llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw); + + // Third argument is a compile time constant int. It must be clamped to + // to the range [0, 12]. + ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]); + assert(ArgCI && + "Third arg to xxinsertw intrinsic must be constant integer"); + const int64_t MaxIndex = 12; + int64_t Index = clamp(ArgCI->getSExtValue(), 0, MaxIndex); + + // The builtin semantics don't exactly match the xxinsertw instructions + // semantics (which ppc_vsx_xxinsertw follows). The builtin extracts the + // word from the first argument, and inserts it in the second argument. The + // instruction extracts the word from its second input register and inserts + // it into its first input register, so swap the first and second arguments. + std::swap(Ops[0], Ops[1]); + + // Need to cast the second argument from a vector of unsigned int to a + // vector of long long. + Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2)); + + if (getTarget().isLittleEndian()) { + // Create a shuffle mask of (1, 0) + Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1), + ConstantInt::get(Int32Ty, 0) + }; + Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts); + + // Reverse the double words in the vector we will extract from. + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); + Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ShuffleMask); + + // Reverse the index. + Index = MaxIndex - Index; + } + + // Intrinsic expects the first arg to be a vector of int. + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); + Ops[2] = ConstantInt::getSigned(Int32Ty, Index); + return Builder.CreateCall(F, Ops); + } + + case PPC::BI__builtin_vsx_extractuword: { + llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw); + + // Intrinsic expects the first argument to be a vector of doublewords. + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); + + // The second argument is a compile time constant int that needs to + // be clamped to the range [0, 12]. + ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]); + assert(ArgCI && + "Second Arg to xxextractuw intrinsic must be a constant integer!"); + const int64_t MaxIndex = 12; + int64_t Index = clamp(ArgCI->getSExtValue(), 0, MaxIndex); + + if (getTarget().isLittleEndian()) { + // Reverse the index. + Index = MaxIndex - Index; + Ops[1] = ConstantInt::getSigned(Int32Ty, Index); + + // Emit the call, then reverse the double words of the results vector. + Value *Call = Builder.CreateCall(F, Ops); + + // Create a shuffle mask of (1, 0) + Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1), + ConstantInt::get(Int32Ty, 0) + }; + Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts); + + Value *ShuffleCall = Builder.CreateShuffleVector(Call, Call, ShuffleMask); + return ShuffleCall; + } else { + Ops[1] = ConstantInt::getSigned(Int32Ty, Index); + return Builder.CreateCall(F, Ops); + } + } } } @@ -7625,45 +8323,73 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); } + + case AMDGPU::BI__builtin_amdgcn_ds_swizzle: + return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_ds_swizzle); case AMDGPU::BI__builtin_amdgcn_div_fixup: case AMDGPU::BI__builtin_amdgcn_div_fixupf: + case AMDGPU::BI__builtin_amdgcn_div_fixuph: return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_div_fixup); case AMDGPU::BI__builtin_amdgcn_trig_preop: case AMDGPU::BI__builtin_amdgcn_trig_preopf: return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop); case AMDGPU::BI__builtin_amdgcn_rcp: case AMDGPU::BI__builtin_amdgcn_rcpf: + case AMDGPU::BI__builtin_amdgcn_rcph: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp); case AMDGPU::BI__builtin_amdgcn_rsq: case AMDGPU::BI__builtin_amdgcn_rsqf: + case AMDGPU::BI__builtin_amdgcn_rsqh: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); case AMDGPU::BI__builtin_amdgcn_rsq_clamp: case AMDGPU::BI__builtin_amdgcn_rsq_clampf: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp); case AMDGPU::BI__builtin_amdgcn_sinf: + case AMDGPU::BI__builtin_amdgcn_sinh: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin); case AMDGPU::BI__builtin_amdgcn_cosf: + case AMDGPU::BI__builtin_amdgcn_cosh: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos); case AMDGPU::BI__builtin_amdgcn_log_clampf: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp); case AMDGPU::BI__builtin_amdgcn_ldexp: case AMDGPU::BI__builtin_amdgcn_ldexpf: + case AMDGPU::BI__builtin_amdgcn_ldexph: return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); case AMDGPU::BI__builtin_amdgcn_frexp_mant: - case AMDGPU::BI__builtin_amdgcn_frexp_mantf: { + case AMDGPU::BI__builtin_amdgcn_frexp_mantf: + case AMDGPU::BI__builtin_amdgcn_frexp_manth: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant); - } case AMDGPU::BI__builtin_amdgcn_frexp_exp: case AMDGPU::BI__builtin_amdgcn_frexp_expf: { - return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_exp); + Value *Src0 = EmitScalarExpr(E->getArg(0)); + Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp, + { Builder.getInt32Ty(), Src0->getType() }); + return Builder.CreateCall(F, Src0); + } + case AMDGPU::BI__builtin_amdgcn_frexp_exph: { + Value *Src0 = EmitScalarExpr(E->getArg(0)); + Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp, + { Builder.getInt16Ty(), Src0->getType() }); + return Builder.CreateCall(F, Src0); } case AMDGPU::BI__builtin_amdgcn_fract: case AMDGPU::BI__builtin_amdgcn_fractf: + case AMDGPU::BI__builtin_amdgcn_fracth: return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract); case AMDGPU::BI__builtin_amdgcn_lerp: return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_lerp); + case AMDGPU::BI__builtin_amdgcn_uicmp: + case AMDGPU::BI__builtin_amdgcn_uicmpl: + case AMDGPU::BI__builtin_amdgcn_sicmp: + case AMDGPU::BI__builtin_amdgcn_sicmpl: + return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_icmp); + case AMDGPU::BI__builtin_amdgcn_fcmp: + case AMDGPU::BI__builtin_amdgcn_fcmpf: + return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_fcmp); case AMDGPU::BI__builtin_amdgcn_class: case AMDGPU::BI__builtin_amdgcn_classf: + case AMDGPU::BI__builtin_amdgcn_classh: return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class); case AMDGPU::BI__builtin_amdgcn_read_exec: { @@ -7951,7 +8677,13 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, Ptr->getType()}), {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())}); }; - + auto MakeScopedAtomic = [&](unsigned IntrinsicID) { + Value *Ptr = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall( + CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(), + Ptr->getType()}), + {Ptr, EmitScalarExpr(E->getArg(1))}); + }; switch (BuiltinID) { case NVPTX::BI__nvvm_atom_add_gen_i: case NVPTX::BI__nvvm_atom_add_gen_l: @@ -8070,6 +8802,109 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, case NVPTX::BI__nvvm_ldg_d: case NVPTX::BI__nvvm_ldg_d2: return MakeLdg(Intrinsic::nvvm_ldg_global_f); + + case NVPTX::BI__nvvm_atom_cta_add_gen_i: + case NVPTX::BI__nvvm_atom_cta_add_gen_l: + case NVPTX::BI__nvvm_atom_cta_add_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_add_gen_i: + case NVPTX::BI__nvvm_atom_sys_add_gen_l: + case NVPTX::BI__nvvm_atom_sys_add_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_add_gen_f: + case NVPTX::BI__nvvm_atom_cta_add_gen_d: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta); + case NVPTX::BI__nvvm_atom_sys_add_gen_f: + case NVPTX::BI__nvvm_atom_sys_add_gen_d: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys); + case NVPTX::BI__nvvm_atom_cta_xchg_gen_i: + case NVPTX::BI__nvvm_atom_cta_xchg_gen_l: + case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_xchg_gen_i: + case NVPTX::BI__nvvm_atom_sys_xchg_gen_l: + case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_max_gen_i: + case NVPTX::BI__nvvm_atom_cta_max_gen_ui: + case NVPTX::BI__nvvm_atom_cta_max_gen_l: + case NVPTX::BI__nvvm_atom_cta_max_gen_ul: + case NVPTX::BI__nvvm_atom_cta_max_gen_ll: + case NVPTX::BI__nvvm_atom_cta_max_gen_ull: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_max_gen_i: + case NVPTX::BI__nvvm_atom_sys_max_gen_ui: + case NVPTX::BI__nvvm_atom_sys_max_gen_l: + case NVPTX::BI__nvvm_atom_sys_max_gen_ul: + case NVPTX::BI__nvvm_atom_sys_max_gen_ll: + case NVPTX::BI__nvvm_atom_sys_max_gen_ull: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_min_gen_i: + case NVPTX::BI__nvvm_atom_cta_min_gen_ui: + case NVPTX::BI__nvvm_atom_cta_min_gen_l: + case NVPTX::BI__nvvm_atom_cta_min_gen_ul: + case NVPTX::BI__nvvm_atom_cta_min_gen_ll: + case NVPTX::BI__nvvm_atom_cta_min_gen_ull: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_min_gen_i: + case NVPTX::BI__nvvm_atom_sys_min_gen_ui: + case NVPTX::BI__nvvm_atom_sys_min_gen_l: + case NVPTX::BI__nvvm_atom_sys_min_gen_ul: + case NVPTX::BI__nvvm_atom_sys_min_gen_ll: + case NVPTX::BI__nvvm_atom_sys_min_gen_ull: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_inc_gen_ui: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta); + case NVPTX::BI__nvvm_atom_cta_dec_gen_ui: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_inc_gen_ui: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys); + case NVPTX::BI__nvvm_atom_sys_dec_gen_ui: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_and_gen_i: + case NVPTX::BI__nvvm_atom_cta_and_gen_l: + case NVPTX::BI__nvvm_atom_cta_and_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_and_gen_i: + case NVPTX::BI__nvvm_atom_sys_and_gen_l: + case NVPTX::BI__nvvm_atom_sys_and_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_or_gen_i: + case NVPTX::BI__nvvm_atom_cta_or_gen_l: + case NVPTX::BI__nvvm_atom_cta_or_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_or_gen_i: + case NVPTX::BI__nvvm_atom_sys_or_gen_l: + case NVPTX::BI__nvvm_atom_sys_or_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_xor_gen_i: + case NVPTX::BI__nvvm_atom_cta_xor_gen_l: + case NVPTX::BI__nvvm_atom_cta_xor_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta); + case NVPTX::BI__nvvm_atom_sys_xor_gen_i: + case NVPTX::BI__nvvm_atom_sys_xor_gen_l: + case NVPTX::BI__nvvm_atom_sys_xor_gen_ll: + return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys); + case NVPTX::BI__nvvm_atom_cta_cas_gen_i: + case NVPTX::BI__nvvm_atom_cta_cas_gen_l: + case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: { + Value *Ptr = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall( + CGM.getIntrinsic( + Intrinsic::nvvm_atomic_cas_gen_i_cta, + {Ptr->getType()->getPointerElementType(), Ptr->getType()}), + {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))}); + } + case NVPTX::BI__nvvm_atom_sys_cas_gen_i: + case NVPTX::BI__nvvm_atom_sys_cas_gen_l: + case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: { + Value *Ptr = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall( + CGM.getIntrinsic( + Intrinsic::nvvm_atomic_cas_gen_i_sys, + {Ptr->getType()->getPointerElementType(), Ptr->getType()}), + {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))}); + } default: return nullptr; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDABuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDABuiltin.cpp index ea3b888..44dd003 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDABuiltin.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDABuiltin.cpp @@ -99,6 +99,12 @@ CodeGenFunction::EmitCUDADevicePrintfCallExpr(const CallExpr *E, llvm::SmallVector<llvm::Type *, 8> ArgTypes; for (unsigned I = 1, NumArgs = Args.size(); I < NumArgs; ++I) ArgTypes.push_back(Args[I].RV.getScalarVal()->getType()); + + // Using llvm::StructType is correct only because printf doesn't accept + // aggregates. If we had to handle aggregates here, we'd have to manually + // compute the offsets within the alloca -- we wouldn't be able to assume + // that the alignment of the llvm type was the same as the alignment of the + // clang type. llvm::Type *AllocaTy = llvm::StructType::create(ArgTypes, "printf_args"); llvm::Value *Alloca = CreateTempAlloca(AllocaTy); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp index 6a04d4e..83febcb 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp @@ -15,6 +15,7 @@ #include "CGCUDARuntime.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantBuilder.h" #include "clang/AST/Decl.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" @@ -29,7 +30,8 @@ namespace { class CGNVCUDARuntime : public CGCUDARuntime { private: - llvm::Type *IntTy, *SizeTy, *VoidTy; + llvm::IntegerType *IntTy, *SizeTy; + llvm::Type *VoidTy; llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy; /// Convenience reference to LLVM Context @@ -55,10 +57,18 @@ private: /// where the C code specifies const char*. llvm::Constant *makeConstantString(const std::string &Str, const std::string &Name = "", + const std::string &SectionName = "", unsigned Alignment = 0) { llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), llvm::ConstantInt::get(SizeTy, 0)}; auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + llvm::GlobalVariable *GV = + cast<llvm::GlobalVariable>(ConstStr.getPointer()); + if (!SectionName.empty()) + GV->setSection(SectionName); + if (Alignment) + GV->setAlignment(Alignment); + return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(), ConstStr.getPointer(), Zeros); } @@ -87,9 +97,9 @@ CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) CodeGen::CodeGenTypes &Types = CGM.getTypes(); ASTContext &Ctx = CGM.getContext(); - IntTy = Types.ConvertType(Ctx.IntTy); - SizeTy = Types.ConvertType(Ctx.getSizeType()); - VoidTy = llvm::Type::getVoidTy(Context); + IntTy = CGM.IntTy; + SizeTy = CGM.SizeTy; + VoidTy = CGM.VoidTy; CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy)); VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy)); @@ -118,37 +128,28 @@ void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF, void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args) { - // Build the argument value list and the argument stack struct type. - SmallVector<llvm::Value *, 16> ArgValues; - std::vector<llvm::Type *> ArgTypes; - for (FunctionArgList::const_iterator I = Args.begin(), E = Args.end(); - I != E; ++I) { - llvm::Value *V = CGF.GetAddrOfLocalVar(*I).getPointer(); - ArgValues.push_back(V); - assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType"); - ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType()); - } - llvm::StructType *ArgStackTy = llvm::StructType::get(Context, ArgTypes); - - llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); - - // Emit the calls to cudaSetupArgument + // Emit a call to cudaSetupArgument for each arg in Args. llvm::Constant *cudaSetupArgFn = getSetupArgumentFn(); - for (unsigned I = 0, E = Args.size(); I != E; ++I) { - llvm::Value *Args[3]; - llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next"); - Args[0] = CGF.Builder.CreatePointerCast(ArgValues[I], VoidPtrTy); - Args[1] = CGF.Builder.CreateIntCast( - llvm::ConstantExpr::getSizeOf(ArgTypes[I]), - SizeTy, false); - Args[2] = CGF.Builder.CreateIntCast( - llvm::ConstantExpr::getOffsetOf(ArgStackTy, I), - SizeTy, false); + llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); + CharUnits Offset = CharUnits::Zero(); + for (const VarDecl *A : Args) { + CharUnits TyWidth, TyAlign; + std::tie(TyWidth, TyAlign) = + CGM.getContext().getTypeInfoInChars(A->getType()); + Offset = Offset.alignTo(TyAlign); + llvm::Value *Args[] = { + CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(), + VoidPtrTy), + llvm::ConstantInt::get(SizeTy, TyWidth.getQuantity()), + llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), + }; llvm::CallSite CS = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args); llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0); llvm::Value *CSZero = CGF.Builder.CreateICmpEQ(CS.getInstruction(), Zero); + llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next"); CGF.Builder.CreateCondBr(CSZero, NextBlock, EndBlock); CGF.EmitBlock(NextBlock); + Offset += TyWidth; } // Emit the call to cudaLaunch @@ -290,18 +291,29 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { continue; } - // Create initialized wrapper structure that points to the loaded GPU binary - llvm::Constant *Values[] = { - llvm::ConstantInt::get(IntTy, 0x466243b1), // Fatbin wrapper magic. - llvm::ConstantInt::get(IntTy, 1), // Fatbin version. - makeConstantString(GpuBinaryOrErr.get()->getBuffer(), "", 16), // Data. - llvm::ConstantPointerNull::get(VoidPtrTy)}; // Unused in fatbin v1. - llvm::GlobalVariable *FatbinWrapper = new llvm::GlobalVariable( - TheModule, FatbinWrapperTy, true, llvm::GlobalValue::InternalLinkage, - llvm::ConstantStruct::get(FatbinWrapperTy, Values), - "__cuda_fatbin_wrapper"); + const char *FatbinConstantName = + CGM.getTriple().isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin"; // NVIDIA's cuobjdump looks for fatbins in this section. - FatbinWrapper->setSection(".nvFatBinSegment"); + const char *FatbinSectionName = + CGM.getTriple().isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment"; + + // Create initialized wrapper structure that points to the loaded GPU binary + ConstantInitBuilder Builder(CGM); + auto Values = Builder.beginStruct(FatbinWrapperTy); + // Fatbin wrapper magic. + Values.addInt(IntTy, 0x466243b1); + // Fatbin version. + Values.addInt(IntTy, 1); + // Data. + Values.add(makeConstantString(GpuBinaryOrErr.get()->getBuffer(), + "", FatbinConstantName, 8)); + // Unused in fatbin v1. + Values.add(llvm::ConstantPointerNull::get(VoidPtrTy)); + llvm::GlobalVariable *FatbinWrapper = + Values.finishAndCreateGlobal("__cuda_fatbin_wrapper", + CGM.getPointerAlign(), + /*constant*/ true); + FatbinWrapper->setSection(FatbinSectionName); // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper); llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp index 014a5db..1936f9f 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.cpp @@ -36,16 +36,7 @@ RValue CGCUDARuntime::EmitCUDAKernelCallExpr(CodeGenFunction &CGF, eval.begin(CGF); CGF.EmitBlock(ConfigOKBlock); - - const Decl *TargetDecl = nullptr; - if (const ImplicitCastExpr *CE = dyn_cast<ImplicitCastExpr>(E->getCallee())) { - if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CE->getSubExpr())) { - TargetDecl = DRE->getDecl(); - } - } - - llvm::Value *Callee = CGF.EmitScalarExpr(E->getCallee()); - CGF.EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue, TargetDecl); + CGF.EmitSimpleCallExpr(E, ReturnValue); CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp index 40f1bc4..59010f4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp @@ -134,6 +134,11 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, llvm::GlobalValue::LinkageTypes TargetLinkage = getFunctionLinkage(TargetDecl); + // available_externally definitions aren't real definitions, so we cannot + // create an alias to one. + if (TargetLinkage == llvm::GlobalValue::AvailableExternallyLinkage) + return true; + // Check if we have it already. StringRef MangledName = getMangledName(AliasDecl); llvm::GlobalValue *Entry = GetGlobalValue(MangledName); @@ -156,14 +161,7 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, // Instead of creating as alias to a linkonce_odr, replace all of the uses // of the aliasee. - if (llvm::GlobalValue::isDiscardableIfUnused(Linkage) && - (TargetLinkage != llvm::GlobalValue::AvailableExternallyLinkage || - !TargetDecl.getDecl()->hasAttr<AlwaysInlineAttr>())) { - // FIXME: An extern template instantiation will create functions with - // linkage "AvailableExternally". In libc++, some classes also define - // members with attribute "AlwaysInline" and expect no reference to - // be generated. It is desirable to reenable this optimisation after - // corresponding LLVM changes. + if (llvm::GlobalValue::isDiscardableIfUnused(Linkage)) { addReplacement(MangledName, Aliasee); return false; } @@ -220,7 +218,7 @@ llvm::Function *CodeGenModule::codegenCXXStructor(const CXXMethodDecl *MD, getTypes().arrangeCXXStructorDeclaration(MD, Type); auto *Fn = cast<llvm::Function>( getAddrOfCXXStructor(MD, Type, &FnInfo, /*FnType=*/nullptr, - /*DontDefer=*/true, /*IsForDefinition=*/true)); + /*DontDefer=*/true, ForDefinition)); GlobalDecl GD; if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD)) { @@ -241,7 +239,8 @@ llvm::Function *CodeGenModule::codegenCXXStructor(const CXXMethodDecl *MD, llvm::Constant *CodeGenModule::getAddrOfCXXStructor( const CXXMethodDecl *MD, StructorType Type, const CGFunctionInfo *FnInfo, - llvm::FunctionType *FnType, bool DontDefer, bool IsForDefinition) { + llvm::FunctionType *FnType, bool DontDefer, + ForDefinition_t IsForDefinition) { GlobalDecl GD; if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) { GD = GlobalDecl(CD, toCXXCtorType(Type)); @@ -260,10 +259,10 @@ llvm::Constant *CodeGenModule::getAddrOfCXXStructor( /*isThunk=*/false, /*ExtraAttrs=*/llvm::AttributeSet(), IsForDefinition); } -static llvm::Value *BuildAppleKextVirtualCall(CodeGenFunction &CGF, - GlobalDecl GD, - llvm::Type *Ty, - const CXXRecordDecl *RD) { +static CGCallee BuildAppleKextVirtualCall(CodeGenFunction &CGF, + GlobalDecl GD, + llvm::Type *Ty, + const CXXRecordDecl *RD) { assert(!CGF.CGM.getTarget().getCXXABI().isMicrosoft() && "No kext in Microsoft ABI"); GD = GD.getCanonicalDecl(); @@ -273,22 +272,26 @@ static llvm::Value *BuildAppleKextVirtualCall(CodeGenFunction &CGF, VTable = CGF.Builder.CreateBitCast(VTable, Ty); assert(VTable && "BuildVirtualCall = kext vtbl pointer is null"); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); - uint64_t AddressPoint = - CGM.getItaniumVTableContext().getVTableLayout(RD) - .getAddressPoint(BaseSubobject(RD, CharUnits::Zero())); - VTableIndex += AddressPoint; + const VTableLayout &VTLayout = CGM.getItaniumVTableContext().getVTableLayout(RD); + VTableLayout::AddressPointLocation AddressPoint = + VTLayout.getAddressPoint(BaseSubobject(RD, CharUnits::Zero())); + VTableIndex += VTLayout.getVTableOffset(AddressPoint.VTableIndex) + + AddressPoint.AddressPointIndex; llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt"); - return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.PointerAlignInBytes); + llvm::Value *VFunc = + CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.PointerAlignInBytes); + CGCallee Callee(GD.getDecl(), VFunc); + return Callee; } /// BuildAppleKextVirtualCall - This routine is to support gcc's kext ABI making /// indirect call to virtual functions. It makes the call through indexing /// into the vtable. -llvm::Value * +CGCallee CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD, - NestedNameSpecifier *Qual, - llvm::Type *Ty) { + NestedNameSpecifier *Qual, + llvm::Type *Ty) { assert((Qual->getKind() == NestedNameSpecifier::TypeSpec) && "BuildAppleKextVirtualCall - bad Qual kind"); @@ -306,21 +309,15 @@ CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD, /// BuildVirtualCall - This routine makes indirect vtable call for /// call to virtual destructors. It returns 0 if it could not do it. -llvm::Value * +CGCallee CodeGenFunction::BuildAppleKextVirtualDestructorCall( const CXXDestructorDecl *DD, CXXDtorType Type, const CXXRecordDecl *RD) { - const auto *MD = cast<CXXMethodDecl>(DD); - // FIXME. Dtor_Base dtor is always direct!! - // It need be somehow inline expanded into the caller. - // -O does that. But need to support -O0 as well. - if (MD->isVirtual() && Type != Dtor_Base) { - // Compute the function type we're calling. - const CGFunctionInfo &FInfo = CGM.getTypes().arrangeCXXStructorDeclaration( - DD, StructorType::Complete); - llvm::Type *Ty = CGM.getTypes().GetFunctionType(FInfo); - return ::BuildAppleKextVirtualCall(*this, GlobalDecl(DD, Type), Ty, RD); - } - return nullptr; + assert(DD->isVirtual() && Type != Dtor_Base); + // Compute the function type we're calling. + const CGFunctionInfo &FInfo = CGM.getTypes().arrangeCXXStructorDeclaration( + DD, StructorType::Complete); + llvm::Type *Ty = CGM.getTypes().GetFunctionType(FInfo); + return ::BuildAppleKextVirtualCall(*this, GlobalDecl(DD, Type), Ty, RD); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp index e4da447..df75a7d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp @@ -73,7 +73,7 @@ CGCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) { return CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()); } -llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer( +CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer( CodeGenFunction &CGF, const Expr *E, Address This, llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, const MemberPointerType *MPT) { @@ -86,7 +86,8 @@ llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer( cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl()); llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType( CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr)); - return llvm::Constant::getNullValue(FTy->getPointerTo()); + llvm::Constant *FnPtr = llvm::Constant::getNullValue(FTy->getPointerTo()); + return CGCallee::forDirect(FnPtr, FPT); } llvm::Value * diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h index 9e10ec0..d53fd4c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h @@ -35,6 +35,7 @@ class FieldDecl; class MangleContext; namespace CodeGen { +class CGCallee; class CodeGenFunction; class CodeGenModule; struct CatchTypeInfo; @@ -154,7 +155,7 @@ public: /// Load a member function from an object and a member function /// pointer. Apply the this-adjustment and set 'This' to the /// adjusted value. - virtual llvm::Value *EmitLoadOfMemberFunctionPointer( + virtual CGCallee EmitLoadOfMemberFunctionPointer( CodeGenFunction &CGF, const Expr *E, Address This, llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, const MemberPointerType *MPT); @@ -403,11 +404,11 @@ public: CharUnits VPtrOffset) = 0; /// Build a virtual function pointer in the ABI-specific way. - virtual llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, - GlobalDecl GD, - Address This, - llvm::Type *Ty, - SourceLocation Loc) = 0; + virtual CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, + GlobalDecl GD, + Address This, + llvm::Type *Ty, + SourceLocation Loc) = 0; /// Emit the ABI-specific virtual destructor call. virtual llvm::Value * diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp index 242b596..c7c61e0 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp @@ -29,6 +29,7 @@ #include "clang/CodeGen/SwiftCallingConv.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/CallSite.h" @@ -47,6 +48,7 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) { default: return llvm::CallingConv::C; case CC_X86StdCall: return llvm::CallingConv::X86_StdCall; case CC_X86FastCall: return llvm::CallingConv::X86_FastCall; + case CC_X86RegCall: return llvm::CallingConv::X86_RegCall; case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall; case CC_X86_64Win64: return llvm::CallingConv::X86_64_Win64; case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV; @@ -172,6 +174,9 @@ static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) { if (D->hasAttr<FastCallAttr>()) return CC_X86FastCall; + if (D->hasAttr<RegCallAttr>()) + return CC_X86RegCall; + if (D->hasAttr<ThisCallAttr>()) return CC_X86ThisCall; @@ -388,15 +393,13 @@ CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) { // When declaring a function without a prototype, always use a // non-variadic type. - if (isa<FunctionNoProtoType>(FTy)) { - CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>(); + if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) { return arrangeLLVMFunctionInfo( noProto->getReturnType(), /*instanceMethod=*/false, /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All); } - assert(isa<FunctionProtoType>(FTy)); - return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>(), FD); + return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>(), FD); } /// Arrange the argument and result information for the declaration or @@ -1647,6 +1650,8 @@ void CodeGenModule::ConstructAttributeList( FuncAttrs.addAttribute(llvm::Attribute::NoReturn); if (TargetDecl->hasAttr<NoDuplicateAttr>()) FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate); + if (TargetDecl->hasAttr<ConvergentAttr>()) + FuncAttrs.addAttribute(llvm::Attribute::Convergent); if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) { AddAttributesFromFunctionProtoType( @@ -1676,6 +1681,14 @@ void CodeGenModule::ConstructAttributeList( HasAnyX86InterruptAttr = TargetDecl->hasAttr<AnyX86InterruptAttr>(); HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>(); + if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) { + Optional<unsigned> NumElemsParam; + // alloc_size args are base-1, 0 means not present. + if (unsigned N = AllocSize->getNumElemsParam()) + NumElemsParam = N - 1; + FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam() - 1, + NumElemsParam); + } } // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed. @@ -1722,6 +1735,16 @@ void CodeGenModule::ConstructAttributeList( FuncAttrs.addAttribute("less-precise-fpmad", llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD)); + + if (!CodeGenOpts.FPDenormalMode.empty()) + FuncAttrs.addAttribute("denormal-fp-math", + CodeGenOpts.FPDenormalMode); + + FuncAttrs.addAttribute("no-trapping-math", + llvm::toStringRef(CodeGenOpts.NoTrappingMath)); + + // TODO: Are these all needed? + // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags. FuncAttrs.addAttribute("no-infs-fp-math", llvm::toStringRef(CodeGenOpts.NoInfsFPMath)); FuncAttrs.addAttribute("no-nans-fp-math", @@ -1734,6 +1757,15 @@ void CodeGenModule::ConstructAttributeList( llvm::utostr(CodeGenOpts.SSPBufferSize)); FuncAttrs.addAttribute("no-signed-zeros-fp-math", llvm::toStringRef(CodeGenOpts.NoSignedZeros)); + FuncAttrs.addAttribute( + "correctly-rounded-divide-sqrt-fp-math", + llvm::toStringRef(CodeGenOpts.CorrectlyRoundedDivSqrt)); + + // TODO: Reciprocal estimate codegen options should apply to instructions? + std::vector<std::string> &Recips = getTarget().getTargetOpts().Reciprocals; + if (!Recips.empty()) + FuncAttrs.addAttribute("reciprocal-estimates", + llvm::join(Recips.begin(), Recips.end(), ",")); if (CodeGenOpts.StackRealignment) FuncAttrs.addAttribute("stackrealign"); @@ -1794,6 +1826,9 @@ void CodeGenModule::ConstructAttributeList( // them). LLVM will remove this attribute where it safely can. FuncAttrs.addAttribute(llvm::Attribute::Convergent); + // Exceptions aren't supported in CUDA device code. + FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); + // Respect -fcuda-flush-denormals-to-zero. if (getLangOpts().CUDADeviceFlushDenormalsToZero) FuncAttrs.addAttribute("nvptx-f32ftz", "true"); @@ -2299,13 +2334,6 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, if (isPromoted) V = emitArgumentDemotion(*this, Arg, V); - if (const CXXMethodDecl *MD = - dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl)) { - if (MD->isVirtual() && Arg == CXXABIThisDecl) - V = CGM.getCXXABI(). - adjustThisParameterInVirtualFunctionPrologue(*this, CurGD, V); - } - // Because of merging of function types from multiple decls it is // possible for the type of an argument to not match the corresponding // type in the function type. Since we are codegening the callee @@ -2465,7 +2493,7 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, // result is in a BasicBlock and is therefore an Instruction. llvm::Instruction *generator = cast<llvm::Instruction>(result); - SmallVector<llvm::Instruction*,4> insnsToKill; + SmallVector<llvm::Instruction *, 4> InstsToKill; // Look for: // %generator = bitcast %type1* %generator2 to %type2* @@ -2478,7 +2506,7 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, if (generator->getNextNode() != bitcast) return nullptr; - insnsToKill.push_back(bitcast); + InstsToKill.push_back(bitcast); } // Look for: @@ -2511,27 +2539,26 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, assert(isa<llvm::CallInst>(prev)); assert(cast<llvm::CallInst>(prev)->getCalledValue() == CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker); - insnsToKill.push_back(prev); + InstsToKill.push_back(prev); } } else { return nullptr; } result = call->getArgOperand(0); - insnsToKill.push_back(call); + InstsToKill.push_back(call); // Keep killing bitcasts, for sanity. Note that we no longer care // about precise ordering as long as there's exactly one use. while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) { if (!bitcast->hasOneUse()) break; - insnsToKill.push_back(bitcast); + InstsToKill.push_back(bitcast); result = bitcast->getOperand(0); } // Delete all the unnecessary instructions, from latest to earliest. - for (SmallVectorImpl<llvm::Instruction*>::iterator - i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i) - (*i)->eraseFromParent(); + for (auto *I : InstsToKill) + I->eraseFromParent(); // Do the fused retain/autorelease if we were asked to. if (doRetainAutorelease) @@ -2841,7 +2868,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, EmitCheckSourceLocation(RetNNAttr->getLocation()), }; EmitCheck(std::make_pair(Cond, SanitizerKind::ReturnsNonnullAttribute), - "nonnull_return", StaticData, None); + SanitizerHandler::NonnullReturn, StaticData, None); } } Ret = Builder.CreateRet(RV); @@ -2863,13 +2890,13 @@ static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, // FIXME: Generate IR in one pass, rather than going back and fixing up these // placeholders. llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty); - llvm::Value *Placeholder = - llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo()); - Placeholder = CGF.Builder.CreateDefaultAlignedLoad(Placeholder); + llvm::Type *IRPtrTy = IRTy->getPointerTo(); + llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo()); // FIXME: When we generate this IR in one pass, we shouldn't need // this win32-specific alignment hack. CharUnits Align = CharUnits::fromQuantity(4); + Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align); return AggValueSlot::forAddr(Address(Placeholder, Align), Ty.getQualifiers(), @@ -2891,22 +2918,36 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, assert(!isInAllocaArgument(CGM.getCXXABI(), type) && "cannot emit delegate call arguments for inalloca arguments!"); + // GetAddrOfLocalVar returns a pointer-to-pointer for references, + // but the argument needs to be the original pointer. + if (type->isReferenceType()) { + args.add(RValue::get(Builder.CreateLoad(local)), type); + + // In ARC, move out of consumed arguments so that the release cleanup + // entered by StartFunction doesn't cause an over-release. This isn't + // optimal -O0 code generation, but it should get cleaned up when + // optimization is enabled. This also assumes that delegate calls are + // performed exactly once for a set of arguments, but that should be safe. + } else if (getLangOpts().ObjCAutoRefCount && + param->hasAttr<NSConsumedAttr>() && + type->isObjCRetainableType()) { + llvm::Value *ptr = Builder.CreateLoad(local); + auto null = + llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType())); + Builder.CreateStore(null, local); + args.add(RValue::get(ptr), type); + // For the most part, we just need to load the alloca, except that // aggregate r-values are actually pointers to temporaries. - if (type->isReferenceType()) - args.add(RValue::get(Builder.CreateLoad(local)), type); - else + } else { args.add(convertTempToRValue(local, type, loc), type); + } } static bool isProvablyNull(llvm::Value *addr) { return isa<llvm::ConstantPointerNull>(addr); } -static bool isProvablyNonNull(llvm::Value *addr) { - return isa<llvm::AllocaInst>(addr); -} - /// Emit the actual writing-back of a writeback. static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback) { @@ -2919,7 +2960,7 @@ static void emitWriteback(CodeGenFunction &CGF, // If the argument wasn't provably non-null, we need to null check // before doing the store. - bool provablyNonNull = isProvablyNonNull(srcAddr.getPointer()); + bool provablyNonNull = llvm::isKnownNonNull(srcAddr.getPointer()); if (!provablyNonNull) { llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); contBB = CGF.createBasicBlock("icr.done"); @@ -3059,7 +3100,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // If the address is *not* known to be non-null, we need to switch. llvm::Value *finalArgument; - bool provablyNonNull = isProvablyNonNull(srcAddr.getPointer()); + bool provablyNonNull = llvm::isKnownNonNull(srcAddr.getPointer()); if (provablyNonNull) { finalArgument = temp.getPointer(); } else { @@ -3130,7 +3171,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, } void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) { - assert(!StackBase && !StackCleanup.isValid()); + assert(!StackBase); // Save the stack. llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave); @@ -3167,13 +3208,14 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, llvm::ConstantInt::get(Int32Ty, ArgNo + 1), }; EmitCheck(std::make_pair(Cond, SanitizerKind::NonnullAttribute), - "nonnull_arg", StaticData, None); + SanitizerHandler::NonnullArg, StaticData, None); } void CodeGenFunction::EmitCallArgs( CallArgList &Args, ArrayRef<QualType> ArgTypes, llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, - const FunctionDecl *CalleeDecl, unsigned ParamsToSkip) { + const FunctionDecl *CalleeDecl, unsigned ParamsToSkip, + EvaluationOrder Order) { assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin())); auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg) { @@ -3191,10 +3233,18 @@ void CodeGenFunction::EmitCallArgs( }; // We *have* to evaluate arguments from right to left in the MS C++ ABI, - // because arguments are destroyed left to right in the callee. - if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { - // Insert a stack save if we're going to need any inalloca args. - bool HasInAllocaArgs = false; + // because arguments are destroyed left to right in the callee. As a special + // case, there are certain language constructs that require left-to-right + // evaluation, and in those cases we consider the evaluation order requirement + // to trump the "destruction order is reverse construction order" guarantee. + bool LeftToRight = + CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee() + ? Order == EvaluationOrder::ForceLeftToRight + : Order != EvaluationOrder::ForceRightToLeft; + + // Insert a stack save if we're going to need any inalloca args. + bool HasInAllocaArgs = false; + if (CGM.getTarget().getCXXABI().isMicrosoft()) { for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end(); I != E && !HasInAllocaArgs; ++I) HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I); @@ -3202,30 +3252,24 @@ void CodeGenFunction::EmitCallArgs( assert(getTarget().getTriple().getArch() == llvm::Triple::x86); Args.allocateArgumentMemory(*this); } + } - // Evaluate each argument. - size_t CallArgsStart = Args.size(); - for (int I = ArgTypes.size() - 1; I >= 0; --I) { - CallExpr::const_arg_iterator Arg = ArgRange.begin() + I; - MaybeEmitImplicitObjectSize(I, *Arg); - EmitCallArg(Args, *Arg, ArgTypes[I]); - EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], (*Arg)->getExprLoc(), - CalleeDecl, ParamsToSkip + I); - } + // Evaluate each argument in the appropriate order. + size_t CallArgsStart = Args.size(); + for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) { + unsigned Idx = LeftToRight ? I : E - I - 1; + CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx; + if (!LeftToRight) MaybeEmitImplicitObjectSize(Idx, *Arg); + EmitCallArg(Args, *Arg, ArgTypes[Idx]); + EmitNonNullArgCheck(Args.back().RV, ArgTypes[Idx], (*Arg)->getExprLoc(), + CalleeDecl, ParamsToSkip + Idx); + if (LeftToRight) MaybeEmitImplicitObjectSize(Idx, *Arg); + } + if (!LeftToRight) { // Un-reverse the arguments we just evaluated so they match up with the LLVM // IR function. std::reverse(Args.begin() + CallArgsStart, Args.end()); - return; - } - - for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) { - CallExpr::const_arg_iterator Arg = ArgRange.begin() + I; - assert(Arg != ArgRange.end()); - EmitCallArg(Args, *Arg, ArgTypes[I]); - EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], (*Arg)->getExprLoc(), - CalleeDecl, ParamsToSkip + I); - MaybeEmitImplicitObjectSize(I, *Arg); } } @@ -3267,7 +3311,7 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, if (const ObjCIndirectCopyRestoreExpr *CRE = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) { assert(getLangOpts().ObjCAutoRefCount); - assert(getContext().hasSameType(E->getType(), type)); + assert(getContext().hasSameUnqualifiedType(E->getType(), type)); return emitWritebackArg(*this, args, CRE); } @@ -3505,21 +3549,22 @@ void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old, } RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, - llvm::Value *Callee, + const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &CallArgs, - CGCalleeInfo CalleeInfo, llvm::Instruction **callOrInvoke) { // FIXME: We no longer need the types from CallArgs; lift up and simplify. + assert(Callee.isOrdinary()); + // Handle struct-return functions by passing a pointer to the // location that we would like to return into. QualType RetTy = CallInfo.getReturnType(); const ABIArgInfo &RetAI = CallInfo.getReturnInfo(); - llvm::FunctionType *IRFuncTy = - cast<llvm::FunctionType>( - cast<llvm::PointerType>(Callee->getType())->getElementType()); + llvm::FunctionType *IRFuncTy = Callee.getFunctionType(); + + // 1. Set up the arguments. // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! @@ -3579,6 +3624,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address swiftErrorTemp = Address::invalid(); Address swiftErrorArg = Address::invalid(); + // Translate all of the arguments as necessary to match the IR lowering. assert(CallInfo.arg_size() == CallArgs.size() && "Mismatch between function signature & arguments."); unsigned ArgNo = 0; @@ -3826,6 +3872,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } + llvm::Value *CalleePtr = Callee.getFunctionPointer(); + + // If we're using inalloca, set up that argument. if (ArgMemory.isValid()) { llvm::Value *Arg = ArgMemory.getPointer(); if (CallInfo.isVariadic()) { @@ -3833,10 +3882,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // end up with a variadic prototype and an inalloca call site. In such // cases, we can't do any parameter mismatch checks. Give up and bitcast // the callee. - unsigned CalleeAS = - cast<llvm::PointerType>(Callee->getType())->getAddressSpace(); - Callee = Builder.CreateBitCast( - Callee, getTypes().GetFunctionType(CallInfo)->getPointerTo(CalleeAS)); + unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace(); + auto FnTy = getTypes().GetFunctionType(CallInfo)->getPointerTo(CalleeAS); + CalleePtr = Builder.CreateBitCast(CalleePtr, FnTy); } else { llvm::Type *LastParamTy = IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1); @@ -3860,39 +3908,57 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg; } - if (!CallArgs.getCleanupsToDeactivate().empty()) - deactivateArgCleanupsBeforeCall(*this, CallArgs); + // 2. Prepare the function pointer. + + // If the callee is a bitcast of a non-variadic function to have a + // variadic function pointer type, check to see if we can remove the + // bitcast. This comes up with unprototyped functions. + // + // This makes the IR nicer, but more importantly it ensures that we + // can inline the function at -O0 if it is marked always_inline. + auto simplifyVariadicCallee = [](llvm::Value *Ptr) -> llvm::Value* { + llvm::FunctionType *CalleeFT = + cast<llvm::FunctionType>(Ptr->getType()->getPointerElementType()); + if (!CalleeFT->isVarArg()) + return Ptr; + + llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr); + if (!CE || CE->getOpcode() != llvm::Instruction::BitCast) + return Ptr; + + llvm::Function *OrigFn = dyn_cast<llvm::Function>(CE->getOperand(0)); + if (!OrigFn) + return Ptr; + + llvm::FunctionType *OrigFT = OrigFn->getFunctionType(); + + // If the original type is variadic, or if any of the component types + // disagree, we cannot remove the cast. + if (OrigFT->isVarArg() || + OrigFT->getNumParams() != CalleeFT->getNumParams() || + OrigFT->getReturnType() != CalleeFT->getReturnType()) + return Ptr; + + for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i) + if (OrigFT->getParamType(i) != CalleeFT->getParamType(i)) + return Ptr; + + return OrigFn; + }; + CalleePtr = simplifyVariadicCallee(CalleePtr); - // If the callee is a bitcast of a function to a varargs pointer to function - // type, check to see if we can remove the bitcast. This handles some cases - // with unprototyped functions. - if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee)) - if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) { - llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType()); - llvm::FunctionType *CurFT = - cast<llvm::FunctionType>(CurPT->getElementType()); - llvm::FunctionType *ActualFT = CalleeF->getFunctionType(); - - if (CE->getOpcode() == llvm::Instruction::BitCast && - ActualFT->getReturnType() == CurFT->getReturnType() && - ActualFT->getNumParams() == CurFT->getNumParams() && - ActualFT->getNumParams() == IRCallArgs.size() && - (CurFT->isVarArg() || !ActualFT->isVarArg())) { - bool ArgsMatch = true; - for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i) - if (ActualFT->getParamType(i) != CurFT->getParamType(i)) { - ArgsMatch = false; - break; - } + // 3. Perform the actual call. - // Strip the cast if we can get away with it. This is a nice cleanup, - // but also allows us to inline the function at -O0 if it is marked - // always_inline. - if (ArgsMatch) - Callee = CalleeF; - } - } + // Deactivate any cleanups that we're supposed to do immediately before + // the call. + if (!CallArgs.getCleanupsToDeactivate().empty()) + deactivateArgCleanupsBeforeCall(*this, CallArgs); + // Assert that the arguments we computed match up. The IR verifier + // will catch this, but this is a common enough source of problems + // during IRGen changes that it's way better for debugging to catch + // it ourselves here. +#ifndef NDEBUG assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg()); for (unsigned i = 0; i < IRCallArgs.size(); ++i) { // Inalloca argument can have different type. @@ -3902,75 +3968,106 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (i < IRFuncTy->getNumParams()) assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i)); } +#endif + // Compute the calling convention and attributes. unsigned CallingConv; CodeGen::AttributeListType AttributeList; - CGM.ConstructAttributeList(Callee->getName(), CallInfo, CalleeInfo, + CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo, + Callee.getAbstractInfo(), AttributeList, CallingConv, /*AttrOnCallSite=*/true); llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(), AttributeList); + // Apply some call-site-specific attributes. + // TODO: work this into building the attribute set. + + // Apply always_inline to all calls within flatten functions. + // FIXME: should this really take priority over __try, below? + if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() && + !(Callee.getAbstractInfo().getCalleeDecl() && + Callee.getAbstractInfo().getCalleeDecl()->hasAttr<NoInlineAttr>())) { + Attrs = + Attrs.addAttribute(getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::AlwaysInline); + } + + // Disable inlining inside SEH __try blocks. + if (isSEHTryScope()) { + Attrs = + Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoInline); + } + + // Decide whether to use a call or an invoke. bool CannotThrow; if (currentFunctionUsesSEHTry()) { - // SEH cares about asynchronous exceptions, everything can "throw." + // SEH cares about asynchronous exceptions, so everything can "throw." CannotThrow = false; } else if (isCleanupPadScope() && EHPersonality::get(*this).isMSVCXXPersonality()) { // The MSVC++ personality will implicitly terminate the program if an - // exception is thrown. An unwind edge cannot be reached. + // exception is thrown during a cleanup outside of a try/catch. + // We don't need to model anything in IR to get this behavior. CannotThrow = true; } else { - // Otherwise, nowunind callsites will never throw. + // Otherwise, nounwind call sites will never throw. CannotThrow = Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::NoUnwind); } llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest(); SmallVector<llvm::OperandBundleDef, 1> BundleList; - getBundlesForFunclet(Callee, CurrentFuncletPad, BundleList); + getBundlesForFunclet(CalleePtr, CurrentFuncletPad, BundleList); + // Emit the actual call/invoke instruction. llvm::CallSite CS; if (!InvokeDest) { - CS = Builder.CreateCall(Callee, IRCallArgs, BundleList); + CS = Builder.CreateCall(CalleePtr, IRCallArgs, BundleList); } else { llvm::BasicBlock *Cont = createBasicBlock("invoke.cont"); - CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs, + CS = Builder.CreateInvoke(CalleePtr, Cont, InvokeDest, IRCallArgs, BundleList); EmitBlock(Cont); } + llvm::Instruction *CI = CS.getInstruction(); if (callOrInvoke) - *callOrInvoke = CS.getInstruction(); - - if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() && - !CS.hasFnAttr(llvm::Attribute::NoInline)) - Attrs = - Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, - llvm::Attribute::AlwaysInline); - - // Disable inlining inside SEH __try blocks. - if (isSEHTryScope()) - Attrs = - Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoInline); + *callOrInvoke = CI; + // Apply the attributes and calling convention. CS.setAttributes(Attrs); CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv)); + // Apply various metadata. + + if (!CI->getType()->isVoidTy()) + CI->setName("call"); + // Insert instrumentation or attach profile metadata at indirect call sites. // For more details, see the comment before the definition of // IPVK_IndirectCallTarget in InstrProfData.inc. if (!CS.getCalledFunction()) PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget, - CS.getInstruction(), Callee); + CI, CalleePtr); // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC // optimizer it can aggressively ignore unwind edges. if (CGM.getLangOpts().ObjCAutoRefCount) - AddObjCARCExceptionMetadata(CS.getInstruction()); + AddObjCARCExceptionMetadata(CI); + + // Suppress tail calls if requested. + if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) { + const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl(); + if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>()) + Call->setTailCallKind(llvm::CallInst::TCK_NoTail); + } + + // 4. Finish the call. // If the call doesn't return, finish the basic block and clear the - // insertion point; this allows the rest of IRgen to discard + // insertion point; this allows the rest of IRGen to discard // unreachable code. if (CS.doesNotReturn()) { if (UnusedReturnSize) @@ -3989,18 +4086,14 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, return GetUndefRValue(RetTy); } - llvm::Instruction *CI = CS.getInstruction(); - if (!CI->getType()->isVoidTy()) - CI->setName("call"); - // Perform the swifterror writeback. if (swiftErrorTemp.isValid()) { llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp); Builder.CreateStore(errorResult, swiftErrorArg); } - // Emit any writebacks immediately. Arguably this should happen - // after any return-value munging. + // Emit any call-associated writebacks immediately. Arguably this + // should happen after any return-value munging. if (CallArgs.hasWritebacks()) emitWritebacks(*this, CallArgs); @@ -4008,12 +4101,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // lexical order, so deactivate it and run it manually here. CallArgs.freeArgumentMemory(*this); - if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) { - const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); - if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>()) - Call->setTailCallKind(llvm::CallInst::TCK_NoTail); - } - + // Extract the return value. RValue Ret = [&] { switch (RetAI.getKind()) { case ABIArgInfo::CoerceAndExpand: { @@ -4110,8 +4198,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm_unreachable("Unhandled ABIArgInfo::Kind"); } (); - const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); - + // Emit the assume_aligned check on the return value. + const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl(); if (Ret.isScalar() && TargetDecl) { if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) { llvm::Value *OffsetValue = nullptr; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h index 2ebd09b..031ce83 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h @@ -19,7 +19,6 @@ #include "EHScopeStack.h" #include "clang/AST/CanonicalType.h" #include "clang/AST/Type.h" -#include "llvm/ADT/FoldingSet.h" #include "llvm/IR/Value.h" // FIXME: Restructure so we don't have to expose so much stuff. @@ -42,6 +41,134 @@ namespace clang { namespace CodeGen { typedef SmallVector<llvm::AttributeSet, 8> AttributeListType; + /// Abstract information about a function or function prototype. + class CGCalleeInfo { + /// \brief The function prototype of the callee. + const FunctionProtoType *CalleeProtoTy; + /// \brief The function declaration of the callee. + const Decl *CalleeDecl; + + public: + explicit CGCalleeInfo() : CalleeProtoTy(nullptr), CalleeDecl(nullptr) {} + CGCalleeInfo(const FunctionProtoType *calleeProtoTy, const Decl *calleeDecl) + : CalleeProtoTy(calleeProtoTy), CalleeDecl(calleeDecl) {} + CGCalleeInfo(const FunctionProtoType *calleeProtoTy) + : CalleeProtoTy(calleeProtoTy), CalleeDecl(nullptr) {} + CGCalleeInfo(const Decl *calleeDecl) + : CalleeProtoTy(nullptr), CalleeDecl(calleeDecl) {} + + const FunctionProtoType *getCalleeFunctionProtoType() const { + return CalleeProtoTy; + } + const Decl *getCalleeDecl() const { return CalleeDecl; } + }; + + /// All available information about a concrete callee. + class CGCallee { + enum class SpecialKind : uintptr_t { + Invalid, + Builtin, + PseudoDestructor, + + Last = PseudoDestructor + }; + + struct BuiltinInfoStorage { + const FunctionDecl *Decl; + unsigned ID; + }; + struct PseudoDestructorInfoStorage { + const CXXPseudoDestructorExpr *Expr; + }; + + SpecialKind KindOrFunctionPointer; + union { + CGCalleeInfo AbstractInfo; + BuiltinInfoStorage BuiltinInfo; + PseudoDestructorInfoStorage PseudoDestructorInfo; + }; + + explicit CGCallee(SpecialKind kind) : KindOrFunctionPointer(kind) {} + + CGCallee(const FunctionDecl *builtinDecl, unsigned builtinID) + : KindOrFunctionPointer(SpecialKind::Builtin) { + BuiltinInfo.Decl = builtinDecl; + BuiltinInfo.ID = builtinID; + } + + public: + CGCallee() : KindOrFunctionPointer(SpecialKind::Invalid) {} + + /// Construct a callee. Call this constructor directly when this + /// isn't a direct call. + CGCallee(const CGCalleeInfo &abstractInfo, llvm::Value *functionPtr) + : KindOrFunctionPointer(SpecialKind(uintptr_t(functionPtr))) { + AbstractInfo = abstractInfo; + assert(functionPtr && "configuring callee without function pointer"); + assert(functionPtr->getType()->isPointerTy()); + assert(functionPtr->getType()->getPointerElementType()->isFunctionTy()); + } + + static CGCallee forBuiltin(unsigned builtinID, + const FunctionDecl *builtinDecl) { + CGCallee result(SpecialKind::Builtin); + result.BuiltinInfo.Decl = builtinDecl; + result.BuiltinInfo.ID = builtinID; + return result; + } + + static CGCallee forPseudoDestructor(const CXXPseudoDestructorExpr *E) { + CGCallee result(SpecialKind::PseudoDestructor); + result.PseudoDestructorInfo.Expr = E; + return result; + } + + static CGCallee forDirect(llvm::Constant *functionPtr, + const CGCalleeInfo &abstractInfo = CGCalleeInfo()) { + return CGCallee(abstractInfo, functionPtr); + } + + bool isBuiltin() const { + return KindOrFunctionPointer == SpecialKind::Builtin; + } + const FunctionDecl *getBuiltinDecl() const { + assert(isBuiltin()); + return BuiltinInfo.Decl; + } + unsigned getBuiltinID() const { + assert(isBuiltin()); + return BuiltinInfo.ID; + } + + bool isPseudoDestructor() const { + return KindOrFunctionPointer == SpecialKind::PseudoDestructor; + } + const CXXPseudoDestructorExpr *getPseudoDestructorExpr() const { + assert(isPseudoDestructor()); + return PseudoDestructorInfo.Expr; + } + + bool isOrdinary() const { + return uintptr_t(KindOrFunctionPointer) > uintptr_t(SpecialKind::Last); + } + const CGCalleeInfo &getAbstractInfo() const { + assert(isOrdinary()); + return AbstractInfo; + } + llvm::Value *getFunctionPointer() const { + assert(isOrdinary()); + return reinterpret_cast<llvm::Value*>(uintptr_t(KindOrFunctionPointer)); + } + llvm::FunctionType *getFunctionType() const { + return cast<llvm::FunctionType>( + getFunctionPointer()->getType()->getPointerElementType()); + } + void setFunctionPointer(llvm::Value *functionPtr) { + assert(isOrdinary()); + KindOrFunctionPointer = SpecialKind(uintptr_t(functionPtr)); + } + }; + struct CallArg { RValue RV; QualType Ty; @@ -82,10 +209,19 @@ namespace CodeGen { push_back(CallArg(rvalue, type, needscopy)); } + /// Add all the arguments from another CallArgList to this one. After doing + /// this, the old CallArgList retains its list of arguments, but must not + /// be used to emit a call. void addFrom(const CallArgList &other) { insert(end(), other.begin(), other.end()); Writebacks.insert(Writebacks.end(), other.Writebacks.begin(), other.Writebacks.end()); + CleanupsToDeactivate.insert(CleanupsToDeactivate.end(), + other.CleanupsToDeactivate.begin(), + other.CleanupsToDeactivate.end()); + assert(!(StackBase && other.StackBase) && "can't merge stackbases"); + if (!StackBase) + StackBase = other.StackBase; } void addWriteback(LValue srcLV, Address temporary, @@ -133,11 +269,6 @@ namespace CodeGen { /// The stacksave call. It dominates all of the argument evaluation. llvm::CallInst *StackBase; - - /// The iterator pointing to the stack restore cleanup. We manually run and - /// deactivate this cleanup after the call in the unexceptional case because - /// it doesn't run in the normal order. - EHScopeStack::stable_iterator StackCleanup; }; /// FunctionArgList - Type for representing both the decl and type diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp index 7ed891f..05d0567 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp @@ -562,105 +562,6 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, isBaseVirtual); } -static void EmitAggMemberInitializer(CodeGenFunction &CGF, - LValue LHS, - Expr *Init, - Address ArrayIndexVar, - QualType T, - ArrayRef<VarDecl *> ArrayIndexes, - unsigned Index) { - if (Index == ArrayIndexes.size()) { - LValue LV = LHS; - - if (ArrayIndexVar.isValid()) { - // If we have an array index variable, load it and use it as an offset. - // Then, increment the value. - llvm::Value *Dest = LHS.getPointer(); - llvm::Value *ArrayIndex = CGF.Builder.CreateLoad(ArrayIndexVar); - Dest = CGF.Builder.CreateInBoundsGEP(Dest, ArrayIndex, "destaddress"); - llvm::Value *Next = llvm::ConstantInt::get(ArrayIndex->getType(), 1); - Next = CGF.Builder.CreateAdd(ArrayIndex, Next, "inc"); - CGF.Builder.CreateStore(Next, ArrayIndexVar); - - // Update the LValue. - CharUnits EltSize = CGF.getContext().getTypeSizeInChars(T); - CharUnits Align = LV.getAlignment().alignmentOfArrayElement(EltSize); - LV.setAddress(Address(Dest, Align)); - } - - switch (CGF.getEvaluationKind(T)) { - case TEK_Scalar: - CGF.EmitScalarInit(Init, /*decl*/ nullptr, LV, false); - break; - case TEK_Complex: - CGF.EmitComplexExprIntoLValue(Init, LV, /*isInit*/ true); - break; - case TEK_Aggregate: { - AggValueSlot Slot = - AggValueSlot::forLValue(LV, - AggValueSlot::IsDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased); - - CGF.EmitAggExpr(Init, Slot); - break; - } - } - - return; - } - - const ConstantArrayType *Array = CGF.getContext().getAsConstantArrayType(T); - assert(Array && "Array initialization without the array type?"); - Address IndexVar = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]); - - // Initialize this index variable to zero. - llvm::Value* Zero - = llvm::Constant::getNullValue(IndexVar.getElementType()); - CGF.Builder.CreateStore(Zero, IndexVar); - - // Start the loop with a block that tests the condition. - llvm::BasicBlock *CondBlock = CGF.createBasicBlock("for.cond"); - llvm::BasicBlock *AfterFor = CGF.createBasicBlock("for.end"); - - CGF.EmitBlock(CondBlock); - - llvm::BasicBlock *ForBody = CGF.createBasicBlock("for.body"); - // Generate: if (loop-index < number-of-elements) fall to the loop body, - // otherwise, go to the block after the for-loop. - uint64_t NumElements = Array->getSize().getZExtValue(); - llvm::Value *Counter = CGF.Builder.CreateLoad(IndexVar); - llvm::Value *NumElementsPtr = - llvm::ConstantInt::get(Counter->getType(), NumElements); - llvm::Value *IsLess = CGF.Builder.CreateICmpULT(Counter, NumElementsPtr, - "isless"); - - // If the condition is true, execute the body. - CGF.Builder.CreateCondBr(IsLess, ForBody, AfterFor); - - CGF.EmitBlock(ForBody); - llvm::BasicBlock *ContinueBlock = CGF.createBasicBlock("for.inc"); - - // Inside the loop body recurse to emit the inner loop or, eventually, the - // constructor call. - EmitAggMemberInitializer(CGF, LHS, Init, ArrayIndexVar, - Array->getElementType(), ArrayIndexes, Index + 1); - - CGF.EmitBlock(ContinueBlock); - - // Emit the increment of the loop counter. - llvm::Value *NextVal = llvm::ConstantInt::get(Counter->getType(), 1); - Counter = CGF.Builder.CreateLoad(IndexVar); - NextVal = CGF.Builder.CreateAdd(Counter, NextVal, "inc"); - CGF.Builder.CreateStore(NextVal, IndexVar); - - // Finally, branch back up to the condition for the next iteration. - CGF.EmitBranch(CondBlock); - - // Emit the fall-through block. - CGF.EmitBlock(AfterFor, true); -} - static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { auto *CD = dyn_cast<CXXConstructorDecl>(D); if (!(CD && CD->isCopyOrMoveConstructor()) && @@ -744,14 +645,11 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, } } - ArrayRef<VarDecl *> ArrayIndexes; - if (MemberInit->getNumArrayIndices()) - ArrayIndexes = MemberInit->getArrayIndices(); - CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit(), ArrayIndexes); + CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit()); } void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS, - Expr *Init, ArrayRef<VarDecl *> ArrayIndexes) { + Expr *Init) { QualType FieldType = Field->getType(); switch (getEvaluationKind(FieldType)) { case TEK_Scalar: @@ -766,30 +664,13 @@ void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS, EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true); break; case TEK_Aggregate: { - Address ArrayIndexVar = Address::invalid(); - if (ArrayIndexes.size()) { - // The LHS is a pointer to the first object we'll be constructing, as - // a flat array. - QualType BaseElementTy = getContext().getBaseElementType(FieldType); - llvm::Type *BasePtr = ConvertType(BaseElementTy); - BasePtr = llvm::PointerType::getUnqual(BasePtr); - Address BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), BasePtr); - LHS = MakeAddrLValue(BaseAddrPtr, BaseElementTy); - - // Create an array index that will be used to walk over all of the - // objects we're constructing. - ArrayIndexVar = CreateMemTemp(getContext().getSizeType(), "object.index"); - llvm::Value *Zero = - llvm::Constant::getNullValue(ArrayIndexVar.getElementType()); - Builder.CreateStore(Zero, ArrayIndexVar); - - // Emit the block variables for the array indices, if any. - for (unsigned I = 0, N = ArrayIndexes.size(); I != N; ++I) - EmitAutoVarDecl(*ArrayIndexes[I]); - } - - EmitAggMemberInitializer(*this, LHS, Init, ArrayIndexVar, FieldType, - ArrayIndexes, 0); + AggValueSlot Slot = + AggValueSlot::forLValue(LHS, + AggValueSlot::IsDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased); + EmitAggExpr(Init, Slot); + break; } } @@ -2146,10 +2027,12 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, *this, D, Type, ForVirtualBase, Delegating, Args); // Emit the call. - llvm::Value *Callee = CGM.getAddrOfCXXStructor(D, getFromCtorType(Type)); + llvm::Constant *CalleePtr = + CGM.getAddrOfCXXStructor(D, getFromCtorType(Type)); const CGFunctionInfo &Info = - CGM.getTypes().arrangeCXXConstructorCall(Args, D, Type, ExtraArgs); - EmitCall(Info, Callee, ReturnValueSlot(), Args, D); + CGM.getTypes().arrangeCXXConstructorCall(Args, D, Type, ExtraArgs); + CGCallee Callee = CGCallee::forDirect(CalleePtr, D); + EmitCall(Info, Callee, ReturnValueSlot(), Args); // Generate vtable assumptions if we're constructing a complete object // with a vtable. We don't do this for base subobjects for two reasons: @@ -2765,8 +2648,8 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::MDString::get(CGM.getLLVMContext(), "all-vtables")); llvm::Value *ValidVtable = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables}); - EmitCheck(std::make_pair(TypeTest, M), "cfi_check_fail", StaticData, - {CastedVTable, ValidVtable}); + EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail, + StaticData, {CastedVTable, ValidVtable}); } bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) { @@ -2798,38 +2681,13 @@ llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad( llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1); EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall), - "cfi_check_fail", nullptr, nullptr); + SanitizerHandler::CFICheckFail, nullptr, nullptr); return Builder.CreateBitCast( Builder.CreateExtractValue(CheckedLoad, 0), cast<llvm::PointerType>(VTable->getType())->getElementType()); } -// FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do -// quite what we want. -static const Expr *skipNoOpCastsAndParens(const Expr *E) { - while (true) { - if (const ParenExpr *PE = dyn_cast<ParenExpr>(E)) { - E = PE->getSubExpr(); - continue; - } - - if (const CastExpr *CE = dyn_cast<CastExpr>(E)) { - if (CE->getCastKind() == CK_NoOp) { - E = CE->getSubExpr(); - continue; - } - } - if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) { - if (UO->getOpcode() == UO_Extension) { - E = UO->getSubExpr(); - continue; - } - } - return E; - } -} - bool CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, const CXXMethodDecl *MD) { @@ -2838,31 +2696,41 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, if (getLangOpts().AppleKext) return false; - // If the most derived class is marked final, we know that no subclass can - // override this member function and so we can devirtualize it. For example: - // - // struct A { virtual void f(); } - // struct B final : A { }; - // - // void f(B *b) { - // b->f(); - // } - // - const CXXRecordDecl *MostDerivedClassDecl = Base->getBestDynamicClassType(); - if (MostDerivedClassDecl->hasAttr<FinalAttr>()) - return true; - // If the member function is marked 'final', we know that it can't be - // overridden and can therefore devirtualize it. + // overridden and can therefore devirtualize it unless it's pure virtual. if (MD->hasAttr<FinalAttr>()) + return !MD->isPure(); + + // If the base expression (after skipping derived-to-base conversions) is a + // class prvalue, then we can devirtualize. + Base = Base->getBestDynamicClassTypeExpr(); + if (Base->isRValue() && Base->getType()->isRecordType()) + return true; + + // If we don't even know what we would call, we can't devirtualize. + const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType(); + if (!BestDynamicDecl) + return false; + + // There may be a method corresponding to MD in a derived class. + const CXXMethodDecl *DevirtualizedMethod = + MD->getCorrespondingMethodInClass(BestDynamicDecl); + + // If that method is pure virtual, we can't devirtualize. If this code is + // reached, the result would be UB, not a direct call to the derived class + // function, and we can't assume the derived class function is defined. + if (DevirtualizedMethod->isPure()) + return false; + + // If that method is marked final, we can devirtualize it. + if (DevirtualizedMethod->hasAttr<FinalAttr>()) return true; // Similarly, if the class itself is marked 'final' it can't be overridden // and we can therefore devirtualize the member function call. - if (MD->getParent()->hasAttr<FinalAttr>()) + if (BestDynamicDecl->hasAttr<FinalAttr>()) return true; - Base = skipNoOpCastsAndParens(Base); if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Base)) { if (const VarDecl *VD = dyn_cast<VarDecl>(DRE->getDecl())) { // This is a record decl. We know the type and can devirtualize it. @@ -2879,17 +2747,15 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, if (const ValueDecl *VD = dyn_cast<ValueDecl>(ME->getMemberDecl())) return VD->getType()->isRecordType(); - // We can always devirtualize calls on temporary object expressions. - if (isa<CXXConstructExpr>(Base)) - return true; - - // And calls on bound temporaries. - if (isa<CXXBindTemporaryExpr>(Base)) - return true; - - // Check if this is a call expr that returns a record type. - if (const CallExpr *CE = dyn_cast<CallExpr>(Base)) - return CE->getCallReturnType(getContext())->isRecordType(); + // Likewise for calls on an object accessed by a (non-reference) pointer to + // member access. + if (auto *BO = dyn_cast<BinaryOperator>(Base)) { + if (BO->isPtrMemOp()) { + auto *MPT = BO->getRHS()->getType()->castAs<MemberPointerType>(); + if (MPT->getPointeeType()->isRecordType()) + return true; + } + } // We can't devirtualize the call. return false; @@ -2901,7 +2767,7 @@ void CodeGenFunction::EmitForwardingCallToLambda( // Get the address of the call operator. const CGFunctionInfo &calleeFnInfo = CGM.getTypes().arrangeCXXMethodDeclaration(callOperator); - llvm::Value *callee = + llvm::Constant *calleePtr = CGM.GetAddrOfFunction(GlobalDecl(callOperator), CGM.getTypes().GetFunctionType(calleeFnInfo)); @@ -2920,8 +2786,8 @@ void CodeGenFunction::EmitForwardingCallToLambda( // variadic arguments. // Now emit our call. - RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, - callArgs, callOperator); + auto callee = CGCallee::forDirect(calleePtr, callOperator); + RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs); // If necessary, copy the returned value into the slot. if (!resultType->isVoidType() && returnSlot.isNull()) diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp index b3278b3..3666858 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp @@ -445,7 +445,7 @@ CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old, for (size_t I = OldLifetimeExtendedSize, E = LifetimeExtendedCleanupStack.size(); I != E; /**/) { // Alignment should be guaranteed by the vptrs in the individual cleanups. - assert((I % llvm::alignOf<LifetimeExtendedCleanupHeader>() == 0) && + assert((I % alignof(LifetimeExtendedCleanupHeader) == 0) && "misaligned cleanup stack entry"); LifetimeExtendedCleanupHeader &Header = diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h index 98d01b1..105c562 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h @@ -427,8 +427,7 @@ public: // EHCleanupScope ought to have alignment equal to that -- not more // (would be misaligned by the stack allocator), and not less (would // break the appended classes). -static_assert(llvm::AlignOf<EHCleanupScope>::Alignment == - EHScopeStack::ScopeStackAlignment, +static_assert(alignof(EHCleanupScope) == EHScopeStack::ScopeStackAlignment, "EHCleanupScope expected alignment"); /// An exceptions scope which filters exceptions thrown through it. @@ -617,6 +616,8 @@ struct EHPersonality { static const EHPersonality GNU_C_SJLJ; static const EHPersonality GNU_C_SEH; static const EHPersonality GNU_ObjC; + static const EHPersonality GNU_ObjC_SJLJ; + static const EHPersonality GNU_ObjC_SEH; static const EHPersonality GNUstep_ObjC; static const EHPersonality GNU_ObjCXX; static const EHPersonality NeXT_ObjC; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCoroutine.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCoroutine.cpp new file mode 100644 index 0000000..2fdb127 --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCoroutine.cpp @@ -0,0 +1,116 @@ +//===----- CGCoroutine.cpp - Emit LLVM Code for C++ coroutines ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This contains code dealing with C++ code generation of coroutines. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenFunction.h" +#include "clang/AST/StmtCXX.h" + +using namespace clang; +using namespace CodeGen; + +namespace clang { +namespace CodeGen { + +struct CGCoroData { + // Stores the llvm.coro.id emitted in the function so that we can supply it + // as the first argument to coro.begin, coro.alloc and coro.free intrinsics. + // Note: llvm.coro.id returns a token that cannot be directly expressed in a + // builtin. + llvm::CallInst *CoroId = nullptr; + // If coro.id came from the builtin, remember the expression to give better + // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by + // EmitCoroutineBody. + CallExpr const *CoroIdExpr = nullptr; +}; +} +} + +clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {} +CodeGenFunction::CGCoroInfo::~CGCoroInfo() {} + +static void createCoroData(CodeGenFunction &CGF, + CodeGenFunction::CGCoroInfo &CurCoro, + llvm::CallInst *CoroId, + CallExpr const *CoroIdExpr = nullptr) { + if (CurCoro.Data) { + if (CurCoro.Data->CoroIdExpr) + CGF.CGM.Error(CoroIdExpr->getLocStart(), + "only one __builtin_coro_id can be used in a function"); + else if (CoroIdExpr) + CGF.CGM.Error(CoroIdExpr->getLocStart(), + "__builtin_coro_id shall not be used in a C++ coroutine"); + else + llvm_unreachable("EmitCoroutineBodyStatement called twice?"); + + return; + } + + CurCoro.Data = std::unique_ptr<CGCoroData>(new CGCoroData); + CurCoro.Data->CoroId = CoroId; + CurCoro.Data->CoroIdExpr = CoroIdExpr; +} + +void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { + auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); + auto &TI = CGM.getContext().getTargetInfo(); + unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); + + auto *CoroId = Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::coro_id), + {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); + createCoroData(*this, CurCoro, CoroId); + + EmitScalarExpr(S.getAllocate()); + // FIXME: Emit the rest of the coroutine. + EmitStmt(S.getDeallocate()); +} + +// Emit coroutine intrinsic and patch up arguments of the token type. +RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, + unsigned int IID) { + SmallVector<llvm::Value *, 8> Args; + switch (IID) { + default: + break; + // The following three intrinsics take a token parameter referring to a token + // returned by earlier call to @llvm.coro.id. Since we cannot represent it in + // builtins, we patch it up here. + case llvm::Intrinsic::coro_alloc: + case llvm::Intrinsic::coro_begin: + case llvm::Intrinsic::coro_free: { + if (CurCoro.Data && CurCoro.Data->CoroId) { + Args.push_back(CurCoro.Data->CoroId); + break; + } + CGM.Error(E->getLocStart(), "this builtin expect that __builtin_coro_id has" + " been used earlier in this function"); + // Fallthrough to the next case to add TokenNone as the first argument. + } + // @llvm.coro.suspend takes a token parameter. Add token 'none' as the first + // argument. + case llvm::Intrinsic::coro_suspend: + Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext())); + break; + } + for (auto &Arg : E->arguments()) + Args.push_back(EmitScalarExpr(Arg)); + + llvm::Value *F = CGM.getIntrinsic(IID); + llvm::CallInst *Call = Builder.CreateCall(F, Args); + + // If we see @llvm.coro.id remember it in the CoroData. We will update + // coro.alloc, coro.begin and coro.free intrinsics to refer to it. + if (IID == llvm::Intrinsic::coro_id) { + createCoroData(*this, CurCoro, Call, E); + } + return RValue::get(Call); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp index 0607a51..12a6803 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp @@ -13,9 +13,9 @@ #include "CGDebugInfo.h" #include "CGBlocks.h" -#include "CGRecordLayout.h" #include "CGCXXABI.h" #include "CGObjCRuntime.h" +#include "CGRecordLayout.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "clang/AST/ASTContext.h" @@ -31,6 +31,7 @@ #include "clang/Lex/HeaderSearchOptions.h" #include "clang/Lex/ModuleMap.h" #include "clang/Lex/PreprocessorOptions.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/Constants.h" @@ -40,10 +41,24 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/MD5.h" #include "llvm/Support/Path.h" using namespace clang; using namespace clang::CodeGen; +static uint32_t getTypeAlignIfRequired(const Type *Ty, const ASTContext &Ctx) { + auto TI = Ctx.getTypeInfo(Ty); + return TI.AlignIsRequired ? TI.Align : 0; +} + +static uint32_t getTypeAlignIfRequired(QualType Ty, const ASTContext &Ctx) { + return getTypeAlignIfRequired(Ty.getTypePtr(), Ctx); +} + +static uint32_t getDeclAlignIfRequired(const Decl *D, const ASTContext &Ctx) { + return D->hasAttr<AlignedAttr>() ? D->getMaxAlignment() : 0; +} + CGDebugInfo::CGDebugInfo(CodeGenModule &CGM) : CGM(CGM), DebugKind(CGM.getCodeGenOpts().getDebugInfo()), DebugTypeExtRefs(CGM.getCodeGenOpts().DebugTypeExtRefs), @@ -306,11 +321,36 @@ StringRef CGDebugInfo::getClassName(const RecordDecl *RD) { return StringRef(); } +llvm::DIFile::ChecksumKind +CGDebugInfo::computeChecksum(FileID FID, SmallString<32> &Checksum) const { + Checksum.clear(); + + if (!CGM.getCodeGenOpts().EmitCodeView) + return llvm::DIFile::CSK_None; + + SourceManager &SM = CGM.getContext().getSourceManager(); + bool Invalid; + llvm::MemoryBuffer *MemBuffer = SM.getBuffer(FID, &Invalid); + if (Invalid) + return llvm::DIFile::CSK_None; + + llvm::MD5 Hash; + llvm::MD5::MD5Result Result; + + Hash.update(MemBuffer->getBuffer()); + Hash.final(Result); + + Hash.stringifyResult(Result, Checksum); + return llvm::DIFile::CSK_MD5; +} + llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (!Loc.isValid()) // If Location is not valid then use main input file. return DBuilder.createFile(remapDIPath(TheCU->getFilename()), - remapDIPath(TheCU->getDirectory())); + remapDIPath(TheCU->getDirectory()), + TheCU->getFile()->getChecksumKind(), + TheCU->getFile()->getChecksum()); SourceManager &SM = CGM.getContext().getSourceManager(); PresumedLoc PLoc = SM.getPresumedLoc(Loc); @@ -318,7 +358,9 @@ llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty()) // If the location is not valid then use main input file. return DBuilder.createFile(remapDIPath(TheCU->getFilename()), - remapDIPath(TheCU->getDirectory())); + remapDIPath(TheCU->getDirectory()), + TheCU->getFile()->getChecksumKind(), + TheCU->getFile()->getChecksum()); // Cache the results. const char *fname = PLoc.getFilename(); @@ -330,8 +372,13 @@ llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { return cast<llvm::DIFile>(V); } + SmallString<32> Checksum; + llvm::DIFile::ChecksumKind CSKind = + computeChecksum(SM.getFileID(Loc), Checksum); + llvm::DIFile *F = DBuilder.createFile(remapDIPath(PLoc.getFilename()), - remapDIPath(getCurrentDirname())); + remapDIPath(getCurrentDirname()), + CSKind, Checksum); DIFileCache[fname].reset(F); return F; @@ -339,7 +386,9 @@ llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { llvm::DIFile *CGDebugInfo::getOrCreateMainFile() { return DBuilder.createFile(remapDIPath(TheCU->getFilename()), - remapDIPath(TheCU->getDirectory())); + remapDIPath(TheCU->getDirectory()), + TheCU->getFile()->getChecksumKind(), + TheCU->getFile()->getChecksum()); } std::string CGDebugInfo::remapDIPath(StringRef Path) const { @@ -382,6 +431,8 @@ StringRef CGDebugInfo::getCurrentDirname() { } void CGDebugInfo::CreateCompileUnit() { + SmallString<32> Checksum; + llvm::DIFile::ChecksumKind CSKind = llvm::DIFile::CSK_None; // Should we be asking the SourceManager for the main file name, instead of // accepting it as an argument? This just causes the main file name to @@ -408,6 +459,7 @@ void CGDebugInfo::CreateCompileUnit() { llvm::sys::path::append(MainFileDirSS, MainFileName); MainFileName = MainFileDirSS.str(); } + CSKind = computeChecksum(SM.getMainFileID(), Checksum); } llvm::dwarf::SourceLanguage LangTag; @@ -452,9 +504,12 @@ void CGDebugInfo::CreateCompileUnit() { // Create new compile unit. // FIXME - Eliminate TheCU. TheCU = DBuilder.createCompileUnit( - LangTag, remapDIPath(MainFileName), remapDIPath(getCurrentDirname()), + LangTag, DBuilder.createFile(remapDIPath(MainFileName), + remapDIPath(getCurrentDirname()), CSKind, + Checksum), Producer, LO.Optimize, CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers, - CGM.getCodeGenOpts().SplitDwarfFile, EmissionKind, 0 /* DWOid */); + CGM.getCodeGenOpts().SplitDwarfFile, EmissionKind, 0 /* DWOid */, + CGM.getCodeGenOpts().SplitDwarfInlining); } llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { @@ -494,14 +549,14 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { auto *ISATy = DBuilder.createPointerType(ClassTy, Size); - ObjTy = - DBuilder.createStructType(TheCU, "objc_object", getOrCreateMainFile(), - 0, 0, 0, 0, nullptr, llvm::DINodeArray()); + ObjTy = DBuilder.createStructType( + TheCU, "objc_object", getOrCreateMainFile(), 0, 0, 0, + llvm::DINode::FlagZero, nullptr, llvm::DINodeArray()); DBuilder.replaceArrays( - ObjTy, - DBuilder.getOrCreateArray(&*DBuilder.createMemberType( - ObjTy, "isa", getOrCreateMainFile(), 0, Size, 0, 0, 0, ISATy))); + ObjTy, DBuilder.getOrCreateArray(&*DBuilder.createMemberType( + ObjTy, "isa", getOrCreateMainFile(), 0, Size, 0, 0, + llvm::DINode::FlagZero, ISATy))); return ObjTy; } case BuiltinType::ObjCSel: { @@ -518,9 +573,8 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { SingletonId); #include "clang/Basic/OpenCLImageTypes.def" case BuiltinType::OCLSampler: - return DBuilder.createBasicType( - "opencl_sampler_t", CGM.getContext().getTypeSize(BT), - CGM.getContext().getTypeAlign(BT), llvm::dwarf::DW_ATE_unsigned); + return getOrCreateStructPtrType("opencl_sampler_t", + OCLSamplerDITy); case BuiltinType::OCLEvent: return getOrCreateStructPtrType("opencl_event_t", OCLEventDITy); case BuiltinType::OCLClkEvent: @@ -594,21 +648,19 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { BTName = BT->getName(CGM.getLangOpts()); break; } - // Bit size, align and offset of the type. + // Bit size and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(BT); - uint64_t Align = CGM.getContext().getTypeAlign(BT); - return DBuilder.createBasicType(BTName, Size, Align, Encoding); + return DBuilder.createBasicType(BTName, Size, Encoding); } llvm::DIType *CGDebugInfo::CreateType(const ComplexType *Ty) { - // Bit size, align and offset of the type. + // Bit size and offset of the type. llvm::dwarf::TypeKind Encoding = llvm::dwarf::DW_ATE_complex_float; if (Ty->isComplexIntegerType()) Encoding = llvm::dwarf::DW_ATE_lo_user; uint64_t Size = CGM.getContext().getTypeSize(Ty); - uint64_t Align = CGM.getContext().getTypeAlign(Ty); - return DBuilder.createBasicType("complex", Size, Align, Encoding); + return DBuilder.createBasicType("complex", Size, Encoding); } llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty, @@ -721,13 +773,7 @@ CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty, StringRef RDName = getClassName(RD); uint64_t Size = 0; - uint64_t Align = 0; - - const RecordDecl *D = RD->getDefinition(); - if (D && D->isCompleteDefinition()) { - Size = CGM.getContext().getTypeSize(Ty); - Align = CGM.getContext().getTypeAlign(Ty); - } + uint32_t Align = 0; // Create the type. SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); @@ -749,7 +795,7 @@ llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, // because that does not return the correct value for references. unsigned AS = CGM.getContext().getTargetAddressSpace(PointeeTy); uint64_t Size = CGM.getTarget().getPointerWidth(AS); - uint64_t Align = CGM.getContext().getTypeAlign(Ty); + auto Align = getTypeAlignIfRequired(Ty, CGM.getContext()); if (Tag == llvm::dwarf::DW_TAG_reference_type || Tag == llvm::dwarf::DW_TAG_rvalue_reference_type) @@ -776,7 +822,7 @@ llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty, SmallVector<llvm::Metadata *, 8> EltTys; QualType FType; uint64_t FieldSize, FieldOffset; - unsigned FieldAlign; + uint32_t FieldAlign; llvm::DINodeArray Elements; FieldOffset = 0; @@ -787,7 +833,7 @@ llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty, Elements = DBuilder.getOrCreateArray(EltTys); EltTys.clear(); - unsigned Flags = llvm::DINode::FlagAppleBlock; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagAppleBlock; unsigned LineNo = 0; auto *EltTy = @@ -811,9 +857,9 @@ llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty, FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy); FieldSize = CGM.getContext().getTypeSize(Ty); FieldAlign = CGM.getContext().getTypeAlign(Ty); - EltTys.push_back(DBuilder.createMemberType(Unit, "__descriptor", nullptr, LineNo, - FieldSize, FieldAlign, FieldOffset, - 0, DescTy)); + EltTys.push_back(DBuilder.createMemberType( + Unit, "__descriptor", nullptr, LineNo, FieldSize, FieldAlign, FieldOffset, + llvm::DINode::FlagZero, DescTy)); FieldOffset += FieldSize; Elements = DBuilder.getOrCreateArray(EltTys); @@ -893,6 +939,7 @@ static unsigned getDwarfCC(CallingConv CC) { case CC_Swift: case CC_PreserveMost: case CC_PreserveAll: + case CC_X86RegCall: return 0; } return 0; @@ -917,14 +964,15 @@ llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty, } llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); - return DBuilder.createSubroutineType(EltTypeArray, 0, + return DBuilder.createSubroutineType(EltTypeArray, llvm::DINode::FlagZero, getDwarfCC(Ty->getCallConv())); } /// Convert an AccessSpecifier into the corresponding DINode flag. /// As an optimization, return 0 if the access specifier equals the /// default for the containing type. -static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { +static llvm::DINode::DIFlags getAccessFlag(AccessSpecifier Access, + const RecordDecl *RD) { AccessSpecifier Default = clang::AS_none; if (RD && RD->isClass()) Default = clang::AS_private; @@ -932,7 +980,7 @@ static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { Default = clang::AS_public; if (Access == Default) - return 0; + return llvm::DINode::FlagZero; switch (Access) { case clang::AS_private: @@ -942,7 +990,7 @@ static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { case clang::AS_public: return llvm::DINode::FlagPublic; case clang::AS_none: - return 0; + return llvm::DINode::FlagZero; } llvm_unreachable("unexpected access enumerator"); } @@ -964,21 +1012,20 @@ llvm::DIType *CGDebugInfo::createBitFieldType(const FieldDecl *BitFieldDecl, CGM.getTypes().getCGRecordLayout(RD).getBitFieldInfo(BitFieldDecl); uint64_t SizeInBits = BitFieldInfo.Size; assert(SizeInBits > 0 && "found named 0-width bitfield"); - unsigned AlignInBits = CGM.getContext().getTypeAlign(Ty); uint64_t StorageOffsetInBits = CGM.getContext().toBits(BitFieldInfo.StorageOffset); uint64_t OffsetInBits = StorageOffsetInBits + BitFieldInfo.Offset; - unsigned Flags = getAccessFlag(BitFieldDecl->getAccess(), RD); + llvm::DINode::DIFlags Flags = getAccessFlag(BitFieldDecl->getAccess(), RD); return DBuilder.createBitFieldMemberType( - RecordTy, Name, File, Line, SizeInBits, AlignInBits, OffsetInBits, - StorageOffsetInBits, Flags, DebugType); + RecordTy, Name, File, Line, SizeInBits, OffsetInBits, StorageOffsetInBits, + Flags, DebugType); } llvm::DIType * CGDebugInfo::createFieldType(StringRef name, QualType type, SourceLocation loc, AccessSpecifier AS, uint64_t offsetInBits, - llvm::DIFile *tunit, llvm::DIScope *scope, - const RecordDecl *RD) { + uint32_t AlignInBits, llvm::DIFile *tunit, + llvm::DIScope *scope, const RecordDecl *RD) { llvm::DIType *debugType = getOrCreateType(type, tunit); // Get the location for the field. @@ -986,16 +1033,17 @@ CGDebugInfo::createFieldType(StringRef name, QualType type, SourceLocation loc, unsigned line = getLineNumber(loc); uint64_t SizeInBits = 0; - unsigned AlignInBits = 0; + auto Align = AlignInBits; if (!type->isIncompleteArrayType()) { TypeInfo TI = CGM.getContext().getTypeInfo(type); SizeInBits = TI.Width; - AlignInBits = TI.Align; + if (!Align) + Align = getTypeAlignIfRequired(type, CGM.getContext()); } - unsigned flags = getAccessFlag(AS, RD); + llvm::DINode::DIFlags flags = getAccessFlag(AS, RD); return DBuilder.createMemberType(scope, name, file, line, SizeInBits, - AlignInBits, offsetInBits, flags, debugType); + Align, offsetInBits, flags, debugType); } void CGDebugInfo::CollectRecordLambdaFields( @@ -1017,9 +1065,10 @@ void CGDebugInfo::CollectRecordLambdaFields( VarDecl *V = C.getCapturedVar(); StringRef VName = V->getName(); llvm::DIFile *VUnit = getOrCreateFile(Loc); + auto Align = getDeclAlignIfRequired(V, CGM.getContext()); llvm::DIType *FieldType = createFieldType( VName, Field->getType(), Loc, Field->getAccess(), - layout.getFieldOffset(fieldno), VUnit, RecordTy, CXXDecl); + layout.getFieldOffset(fieldno), Align, VUnit, RecordTy, CXXDecl); elements.push_back(FieldType); } else if (C.capturesThis()) { // TODO: Need to handle 'this' in some way by probably renaming the @@ -1060,9 +1109,10 @@ CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, llvm::DIType *RecordTy, } } - unsigned Flags = getAccessFlag(Var->getAccess(), RD); + llvm::DINode::DIFlags Flags = getAccessFlag(Var->getAccess(), RD); + auto Align = getDeclAlignIfRequired(Var, CGM.getContext()); llvm::DIDerivedType *GV = DBuilder.createStaticMemberType( - RecordTy, VName, VUnit, LineNumber, VTy, Flags, C); + RecordTy, VName, VUnit, LineNumber, VTy, Flags, C, Align); StaticDataMemberCache[Var->getCanonicalDecl()].reset(GV); return GV; } @@ -1082,14 +1132,26 @@ void CGDebugInfo::CollectRecordNormalField( if (field->isBitField()) { FieldType = createBitFieldType(field, RecordTy, RD); } else { + auto Align = getDeclAlignIfRequired(field, CGM.getContext()); FieldType = createFieldType(name, type, field->getLocation(), field->getAccess(), - OffsetInBits, tunit, RecordTy, RD); + OffsetInBits, Align, tunit, RecordTy, RD); } elements.push_back(FieldType); } +void CGDebugInfo::CollectRecordNestedRecord( + const RecordDecl *RD, SmallVectorImpl<llvm::Metadata *> &elements) { + QualType Ty = CGM.getContext().getTypeDeclType(RD); + // Injected class names are not considered nested records. + if (isa<InjectedClassNameType>(Ty)) + return; + SourceLocation Loc = RD->getLocation(); + llvm::DIType *nestedType = getOrCreateType(Ty, getOrCreateFile(Loc)); + elements.push_back(nestedType); +} + void CGDebugInfo::CollectRecordFields( const RecordDecl *record, llvm::DIFile *tunit, SmallVectorImpl<llvm::Metadata *> &elements, @@ -1101,6 +1163,10 @@ void CGDebugInfo::CollectRecordFields( else { const ASTRecordLayout &layout = CGM.getContext().getASTRecordLayout(record); + // Debug info for nested records is included in the member list only for + // CodeView. + bool IncludeNestedRecords = CGM.getCodeGenOpts().EmitCodeView; + // Field number for non-static fields. unsigned fieldNo = 0; @@ -1126,7 +1192,10 @@ void CGDebugInfo::CollectRecordFields( // Bump field number for next field. ++fieldNo; - } + } else if (const auto *nestedRec = dyn_cast<CXXRecordDecl>(I)) + if (IncludeNestedRecords && !nestedRec->isImplicit() && + nestedRec->getDeclContext() == record) + CollectRecordNestedRecord(nestedRec, elements); } } @@ -1162,7 +1231,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType( QualType PointeeTy = ThisPtrTy->getPointeeType(); unsigned AS = CGM.getContext().getTargetAddressSpace(PointeeTy); uint64_t Size = CGM.getTarget().getPointerWidth(AS); - uint64_t Align = CGM.getContext().getTypeAlign(ThisPtrTy); + auto Align = getTypeAlignIfRequired(ThisPtrTy, CGM.getContext()); llvm::DIType *PointeeType = getOrCreateType(PointeeTy, Unit); llvm::DIType *ThisPtrType = DBuilder.createPointerType(PointeeType, Size, Align); @@ -1185,7 +1254,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType( llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; if (Func->getExtProtoInfo().RefQualifier == RQ_LValue) Flags |= llvm::DINode::FlagLValueReference; if (Func->getExtProtoInfo().RefQualifier == RQ_RValue) @@ -1236,7 +1305,7 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction( llvm::DIType *ContainingType = nullptr; unsigned Virtuality = 0; unsigned VIndex = 0; - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; int ThisAdjustment = 0; if (Method->isVirtual()) { @@ -1347,13 +1416,33 @@ void CGDebugInfo::CollectCXXMemberFunctions( void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile *Unit, SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy) { - const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); - for (const auto &BI : RD->bases()) { - unsigned BFlags = 0; - uint64_t BaseOffset; + llvm::DenseSet<CanonicalDeclPtr<const CXXRecordDecl>> SeenTypes; + CollectCXXBasesAux(RD, Unit, EltTys, RecordTy, RD->bases(), SeenTypes, + llvm::DINode::FlagZero); + + // If we are generating CodeView debug info, we also need to emit records for + // indirect virtual base classes. + if (CGM.getCodeGenOpts().EmitCodeView) { + CollectCXXBasesAux(RD, Unit, EltTys, RecordTy, RD->vbases(), SeenTypes, + llvm::DINode::FlagIndirectVirtualBase); + } +} +void CGDebugInfo::CollectCXXBasesAux( + const CXXRecordDecl *RD, llvm::DIFile *Unit, + SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy, + const CXXRecordDecl::base_class_const_range &Bases, + llvm::DenseSet<CanonicalDeclPtr<const CXXRecordDecl>> &SeenTypes, + llvm::DINode::DIFlags StartingFlags) { + const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); + for (const auto &BI : Bases) { const auto *Base = cast<CXXRecordDecl>(BI.getType()->getAs<RecordType>()->getDecl()); + if (!SeenTypes.insert(Base).second) + continue; + auto *BaseTy = getOrCreateType(BI.getType(), Unit); + llvm::DINode::DIFlags BFlags = StartingFlags; + uint64_t BaseOffset; if (BI.isVirtual()) { if (CGM.getTarget().getCXXABI().isItaniumFamily()) { @@ -1368,15 +1457,15 @@ void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile *Unit, BaseOffset = 4 * CGM.getMicrosoftVTableContext().getVBTableIndex(RD, Base); } - BFlags = llvm::DINode::FlagVirtual; + BFlags |= llvm::DINode::FlagVirtual; } else BaseOffset = CGM.getContext().toBits(RL.getBaseClassOffset(Base)); // FIXME: Inconsistent units for BaseOffset. It is in bytes when // BI->isVirtual() and bits when not. BFlags |= getAccessFlag(BI.getAccessSpecifier(), RD); - llvm::DIType *DTy = DBuilder.createInheritance( - RecordTy, getOrCreateType(BI.getType(), Unit), BaseOffset, BFlags); + llvm::DIType *DTy = + DBuilder.createInheritance(RecordTy, BaseTy, BaseOffset, BFlags); EltTys.push_back(DTy); } } @@ -1531,22 +1620,56 @@ StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) { } void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile *Unit, - SmallVectorImpl<llvm::Metadata *> &EltTys) { + SmallVectorImpl<llvm::Metadata *> &EltTys, + llvm::DICompositeType *RecordTy) { + // If this class is not dynamic then there is not any vtable info to collect. + if (!RD->isDynamicClass()) + return; + + // Don't emit any vtable shape or vptr info if this class doesn't have an + // extendable vfptr. This can happen if the class doesn't have virtual + // methods, or in the MS ABI if those virtual methods only come from virtually + // inherited bases. const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); + if (!RL.hasExtendableVFPtr()) + return; - // If there is a primary base then it will hold vtable info. + // CodeView needs to know how large the vtable of every dynamic class is, so + // emit a special named pointer type into the element list. The vptr type + // points to this type as well. + llvm::DIType *VPtrTy = nullptr; + bool NeedVTableShape = CGM.getCodeGenOpts().EmitCodeView && + CGM.getTarget().getCXXABI().isMicrosoft(); + if (NeedVTableShape) { + uint64_t PtrWidth = + CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy); + const VTableLayout &VFTLayout = + CGM.getMicrosoftVTableContext().getVFTableLayout(RD, CharUnits::Zero()); + unsigned VSlotCount = + VFTLayout.vtable_components().size() - CGM.getLangOpts().RTTIData; + unsigned VTableWidth = PtrWidth * VSlotCount; + + // Create a very wide void* type and insert it directly in the element list. + llvm::DIType *VTableType = + DBuilder.createPointerType(nullptr, VTableWidth, 0, "__vtbl_ptr_type"); + EltTys.push_back(VTableType); + + // The vptr is a pointer to this special vtable type. + VPtrTy = DBuilder.createPointerType(VTableType, PtrWidth); + } + + // If there is a primary base then the artificial vptr member lives there. if (RL.getPrimaryBase()) return; - // If this class is not dynamic then there is not any vtable info to collect. - if (!RD->isDynamicClass()) - return; + if (!VPtrTy) + VPtrTy = getOrCreateVTablePtrType(Unit); unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy); - llvm::DIType *VPTR = DBuilder.createMemberType( + llvm::DIType *VPtrMember = DBuilder.createMemberType( Unit, getVTableName(RD), Unit, 0, Size, 0, 0, - llvm::DINode::FlagArtificial, getOrCreateVTablePtrType(Unit)); - EltTys.push_back(VPTR); + llvm::DINode::FlagArtificial, VPtrTy); + EltTys.push_back(VPtrMember); } llvm::DIType *CGDebugInfo::getOrCreateRecordType(QualType RTy, @@ -1591,23 +1714,6 @@ void CGDebugInfo::completeType(const RecordDecl *RD) { completeRequiredType(RD); } -void CGDebugInfo::completeRequiredType(const RecordDecl *RD) { - if (DebugKind <= codegenoptions::DebugLineTablesOnly) - return; - - if (const auto *CXXDecl = dyn_cast<CXXRecordDecl>(RD)) - if (CXXDecl->isDynamicClass()) - return; - - if (DebugTypeExtRefs && RD->isFromASTFile()) - return; - - QualType Ty = CGM.getContext().getRecordType(RD); - llvm::DIType *T = getTypeOrNull(Ty); - if (T && T->isForwardDecl()) - completeClassData(RD); -} - void CGDebugInfo::completeClassData(const RecordDecl *RD) { if (DebugKind <= codegenoptions::DebugLineTablesOnly) return; @@ -1633,21 +1739,37 @@ static bool hasExplicitMemberDefinition(CXXRecordDecl::method_iterator I, /// Does a type definition exist in an imported clang module? static bool isDefinedInClangModule(const RecordDecl *RD) { + // Only definitions that where imported from an AST file come from a module. if (!RD || !RD->isFromASTFile()) return false; + // Anonymous entities cannot be addressed. Treat them as not from module. if (!RD->isExternallyVisible() && RD->getName().empty()) return false; if (auto *CXXDecl = dyn_cast<CXXRecordDecl>(RD)) { - assert(CXXDecl->isCompleteDefinition() && "incomplete record definition"); - if (CXXDecl->getTemplateSpecializationKind() != TSK_Undeclared) - // Make sure the instantiation is actually in a module. - if (CXXDecl->field_begin() != CXXDecl->field_end()) - return CXXDecl->field_begin()->isFromASTFile(); + if (!CXXDecl->isCompleteDefinition()) + return false; + auto TemplateKind = CXXDecl->getTemplateSpecializationKind(); + if (TemplateKind != TSK_Undeclared) { + // This is a template, check the origin of the first member. + if (CXXDecl->field_begin() == CXXDecl->field_end()) + return TemplateKind == TSK_ExplicitInstantiationDeclaration; + if (!CXXDecl->field_begin()->isFromASTFile()) + return false; + } } - return true; } +/// Return true if the class or any of its methods are marked dllimport. +static bool isClassOrMethodDLLImport(const CXXRecordDecl *RD) { + if (RD->hasAttr<DLLImportAttr>()) + return true; + for (const CXXMethodDecl *MD : RD->methods()) + if (MD->hasAttr<DLLImportAttr>()) + return true; + return false; +} + static bool shouldOmitDefinition(codegenoptions::DebugInfoKind DebugKind, bool DebugTypeExtRefs, const RecordDecl *RD, const LangOptions &LangOpts) { @@ -1668,7 +1790,14 @@ static bool shouldOmitDefinition(codegenoptions::DebugInfoKind DebugKind, if (!CXXDecl) return false; - if (CXXDecl->hasDefinition() && CXXDecl->isDynamicClass()) + // Only emit complete debug info for a dynamic class when its vtable is + // emitted. However, Microsoft debuggers don't resolve type information + // across DLL boundaries, so skip this optimization if the class or any of its + // methods are marked dllimport. This isn't a complete solution, since objects + // without any dllimport methods can be used in one DLL and constructed in + // another, but it is the current behavior of LimitedDebugInfo. + if (CXXDecl->hasDefinition() && CXXDecl->isDynamicClass() && + !isClassOrMethodDLLImport(CXXDecl)) return true; TemplateSpecializationKind Spec = TSK_Undeclared; @@ -1683,6 +1812,16 @@ static bool shouldOmitDefinition(codegenoptions::DebugInfoKind DebugKind, return false; } +void CGDebugInfo::completeRequiredType(const RecordDecl *RD) { + if (shouldOmitDefinition(DebugKind, DebugTypeExtRefs, RD, CGM.getLangOpts())) + return; + + QualType Ty = CGM.getContext().getRecordType(RD); + llvm::DIType *T = getTypeOrNull(Ty); + if (T && T->isForwardDecl()) + completeClassData(RD); +} + llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0))); @@ -1732,7 +1871,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { const auto *CXXDecl = dyn_cast<CXXRecordDecl>(RD); if (CXXDecl) { CollectCXXBases(CXXDecl, DefUnit, EltTys, FwdDecl); - CollectVTableInfo(CXXDecl, DefUnit, EltTys); + CollectVTableInfo(CXXDecl, DefUnit, EltTys, FwdDecl); } // Collect data fields (including static variables and any initializers). @@ -1760,6 +1899,18 @@ llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectType *Ty, return getOrCreateType(Ty->getBaseType(), Unit); } +llvm::DIType *CGDebugInfo::CreateType(const ObjCTypeParamType *Ty, + llvm::DIFile *Unit) { + // Ignore protocols. + SourceLocation Loc = Ty->getDecl()->getLocation(); + + // Use Typedefs to represent ObjCTypeParamType. + return DBuilder.createTypedef( + getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit), + Ty->getDecl()->getName(), getOrCreateFile(Loc), getLineNumber(Loc), + getDeclContextDescriptor(Ty->getDecl())); +} + /// \return true if Getter has the default name for the property PD. static bool hasDefaultGetterName(const ObjCPropertyDecl *PD, const ObjCMethodDecl *Getter) { @@ -1860,10 +2011,11 @@ CGDebugInfo::getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod, // but LLVM detects skeleton CUs by looking for a non-zero DWO id. uint64_t Signature = Mod.getSignature() ? Mod.getSignature() : ~1ULL; llvm::DIBuilder DIB(CGM.getModule()); - DIB.createCompileUnit(TheCU->getSourceLanguage(), Mod.getModuleName(), - Mod.getPath(), TheCU->getProducer(), true, - StringRef(), 0, Mod.getASTFile(), - llvm::DICompileUnit::FullDebug, Signature); + DIB.createCompileUnit(TheCU->getSourceLanguage(), + DIB.createFile(Mod.getModuleName(), Mod.getPath()), + TheCU->getProducer(), true, StringRef(), 0, + Mod.getASTFile(), llvm::DICompileUnit::FullDebug, + Signature); DIB.finalize(); } llvm::DIModule *Parent = @@ -1887,9 +2039,9 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(Ty); - uint64_t Align = CGM.getContext().getTypeAlign(Ty); + auto Align = getTypeAlignIfRequired(Ty, CGM.getContext()); - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; if (ID->getImplementation()) Flags |= llvm::DINode::FlagObjcClassComplete; @@ -1915,7 +2067,8 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, if (!SClassTy) return nullptr; - llvm::DIType *InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0); + llvm::DIType *InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, + llvm::DINode::FlagZero); EltTys.push_back(InhTag); } @@ -1970,7 +2123,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned FieldLine = getLineNumber(Field->getLocation()); QualType FType = Field->getType(); uint64_t FieldSize = 0; - unsigned FieldAlign = 0; + uint32_t FieldAlign = 0; if (!FType->isIncompleteArrayType()) { @@ -1978,7 +2131,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, FieldSize = Field->isBitField() ? Field->getBitWidthValue(CGM.getContext()) : CGM.getContext().getTypeSize(FType); - FieldAlign = CGM.getContext().getTypeAlign(FType); + FieldAlign = getTypeAlignIfRequired(FType, CGM.getContext()); } uint64_t FieldOffset; @@ -1997,7 +2150,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, FieldOffset = RL.getFieldOffset(FieldNo); } - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; if (Field->getAccessControl() == ObjCIvarDecl::Protected) Flags = llvm::DINode::FlagProtected; else if (Field->getAccessControl() == ObjCIvarDecl::Private) @@ -2052,33 +2205,33 @@ llvm::DIType *CGDebugInfo::CreateType(const VectorType *Ty, llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscript); uint64_t Size = CGM.getContext().getTypeSize(Ty); - uint64_t Align = CGM.getContext().getTypeAlign(Ty); + auto Align = getTypeAlignIfRequired(Ty, CGM.getContext()); return DBuilder.createVectorType(Size, Align, ElementTy, SubscriptArray); } llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) { uint64_t Size; - uint64_t Align; + uint32_t Align; // FIXME: make getTypeAlign() aware of VLAs and incomplete array types if (const auto *VAT = dyn_cast<VariableArrayType>(Ty)) { Size = 0; - Align = - CGM.getContext().getTypeAlign(CGM.getContext().getBaseElementType(VAT)); + Align = getTypeAlignIfRequired(CGM.getContext().getBaseElementType(VAT), + CGM.getContext()); } else if (Ty->isIncompleteArrayType()) { Size = 0; if (Ty->getElementType()->isIncompleteType()) Align = 0; else - Align = CGM.getContext().getTypeAlign(Ty->getElementType()); + Align = getTypeAlignIfRequired(Ty->getElementType(), CGM.getContext()); } else if (Ty->isIncompleteType()) { Size = 0; Align = 0; } else { // Size and align of the whole array, not the element type. Size = CGM.getContext().getTypeSize(Ty); - Align = CGM.getContext().getTypeAlign(Ty); + Align = getTypeAlignIfRequired(Ty, CGM.getContext()); } // Add the dimensions of the array. FIXME: This loses CV qualifiers from @@ -2097,6 +2250,13 @@ llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) { int64_t Count = -1; // Count == -1 is an unbounded array. if (const auto *CAT = dyn_cast<ConstantArrayType>(Ty)) Count = CAT->getSize().getZExtValue(); + else if (const auto *VAT = dyn_cast<VariableArrayType>(Ty)) { + if (Expr *Size = VAT->getSizeExpr()) { + llvm::APSInt V; + if (Size->EvaluateAsInt(V, CGM.getContext())) + Count = V.getExtValue(); + } + } // FIXME: Verify this is right for VLAs. Subscripts.push_back(DBuilder.getOrCreateSubrange(0, Count)); @@ -2123,7 +2283,7 @@ llvm::DIType *CGDebugInfo::CreateType(const RValueReferenceType *Ty, llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty, llvm::DIFile *U) { - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; uint64_t Size = 0; if (!Ty->isIncompleteType()) { @@ -2163,9 +2323,8 @@ llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty, } llvm::DIType *CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile *U) { - // Ignore the atomic wrapping - // FIXME: What is the correct representation? - return getOrCreateType(Ty->getValueType(), U); + auto *FromTy = getOrCreateType(Ty->getValueType(), U); + return DBuilder.createQualifiedType(llvm::dwarf::DW_TAG_atomic_type, FromTy); } llvm::DIType* CGDebugInfo::CreateType(const PipeType *Ty, @@ -2177,10 +2336,10 @@ llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); uint64_t Size = 0; - uint64_t Align = 0; + uint32_t Align = 0; if (!ED->getTypeForDecl()->isIncompleteType()) { Size = CGM.getContext().getTypeSize(ED->getTypeForDecl()); - Align = CGM.getContext().getTypeAlign(ED->getTypeForDecl()); + Align = getDeclAlignIfRequired(ED, CGM.getContext()); } SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); @@ -2220,10 +2379,10 @@ llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) { llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); uint64_t Size = 0; - uint64_t Align = 0; + uint32_t Align = 0; if (!ED->getTypeForDecl()->isIncompleteType()) { Size = CGM.getContext().getTypeSize(ED->getTypeForDecl()); - Align = CGM.getContext().getTypeAlign(ED->getTypeForDecl()); + Align = getDeclAlignIfRequired(ED, CGM.getContext()); } SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); @@ -2292,12 +2451,18 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { case Type::SubstTemplateTypeParm: T = cast<SubstTemplateTypeParmType>(T)->getReplacementType(); break; - case Type::Auto: + case Type::Auto: { QualType DT = cast<AutoType>(T)->getDeducedType(); assert(!DT.isNull() && "Undeduced types shouldn't reach here."); T = DT; break; } + case Type::Adjusted: + case Type::Decayed: + // Decayed and adjusted types use the adjusted type in LLVM and DWARF. + T = cast<AdjustedType>(T)->getAdjustedType(); + break; + } assert(T != LastT && "Type unwrapping failed to unwrap!"); (void)LastT; @@ -2406,6 +2571,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { return CreateType(cast<ObjCObjectPointerType>(Ty), Unit); case Type::ObjCObject: return CreateType(cast<ObjCObjectType>(Ty), Unit); + case Type::ObjCTypeParam: + return CreateType(cast<ObjCTypeParamType>(Ty), Unit); case Type::ObjCInterface: return CreateType(cast<ObjCInterfaceType>(Ty), Unit); case Type::Builtin: @@ -2414,11 +2581,6 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { return CreateType(cast<ComplexType>(Ty)); case Type::Pointer: return CreateType(cast<PointerType>(Ty), Unit); - case Type::Adjusted: - case Type::Decayed: - // Decayed and adjusted types use the adjusted type in LLVM and DWARF. - return CreateType( - cast<PointerType>(cast<AdjustedType>(Ty)->getAdjustedType()), Unit); case Type::BlockPointer: return CreateType(cast<BlockPointerType>(Ty), Unit); case Type::Typedef: @@ -2454,6 +2616,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { case Type::Auto: case Type::Attributed: + case Type::Adjusted: + case Type::Decayed: case Type::Elaborated: case Type::Paren: case Type::SubstTemplateTypeParm: @@ -2518,13 +2682,13 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) { return getOrCreateRecordFwdDecl(Ty, RDContext); uint64_t Size = CGM.getContext().getTypeSize(Ty); - uint64_t Align = CGM.getContext().getTypeAlign(Ty); + auto Align = getDeclAlignIfRequired(D, CGM.getContext()); SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); llvm::DICompositeType *RealDecl = DBuilder.createReplaceableCompositeType( - getTagForRecord(RD), RDName, RDContext, DefUnit, Line, 0, Size, Align, 0, - FullName); + getTagForRecord(RD), RDName, RDContext, DefUnit, Line, 0, Size, Align, + llvm::DINode::FlagZero, FullName); // Elements of composite types usually have back to the type, creating // uniquing cycles. Distinct nodes are more efficient. @@ -2587,9 +2751,10 @@ llvm::DIType *CGDebugInfo::CreateMemberType(llvm::DIFile *Unit, QualType FType, StringRef Name, uint64_t *Offset) { llvm::DIType *FieldTy = CGDebugInfo::getOrCreateType(FType, Unit); uint64_t FieldSize = CGM.getContext().getTypeSize(FType); - unsigned FieldAlign = CGM.getContext().getTypeAlign(FType); - llvm::DIType *Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize, - FieldAlign, *Offset, 0, FieldTy); + auto FieldAlign = getTypeAlignIfRequired(FType, CGM.getContext()); + llvm::DIType *Ty = + DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize, FieldAlign, + *Offset, llvm::DINode::FlagZero, FieldTy); *Offset += FieldSize; return Ty; } @@ -2599,7 +2764,7 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, StringRef &LinkageName, llvm::DIScope *&FDContext, llvm::DINodeArray &TParamsArray, - unsigned &Flags) { + llvm::DINode::DIFlags &Flags) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); Name = getFunctionName(FD); // Use mangled name as linkage name for C/C++ functions. @@ -2624,6 +2789,9 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, llvm::DIScope *Mod = getParentModuleOrNull(RDecl); FDContext = getContextDescriptor(RDecl, Mod ? Mod : TheCU); } + // Check if it is a noreturn-marked function + if (FD->isNoReturn()) + Flags |= llvm::DINode::FlagNoReturn; // Collect template parameters. TParamsArray = CollectFunctionTemplateParams(FD, Unit); } @@ -2680,7 +2848,7 @@ llvm::DISubprogram * CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) { llvm::DINodeArray TParamsArray; StringRef Name, LinkageName; - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; SourceLocation Loc = FD->getLocation(); llvm::DIFile *Unit = getOrCreateFile(Loc); llvm::DIScope *DContext = Unit; @@ -2717,9 +2885,10 @@ CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) { unsigned Line = getLineNumber(Loc); collectVarDeclProps(VD, Unit, Line, T, Name, LinkageName, DContext); + auto Align = getDeclAlignIfRequired(VD, CGM.getContext()); auto *GV = DBuilder.createTempGlobalVariableFwdDecl( DContext, Name, LinkageName, Unit, Line, getOrCreateType(T, Unit), - !VD->isExternallyVisible(), nullptr, nullptr); + !VD->isExternallyVisible(), nullptr, Align); FwdDeclReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(cast<VarDecl>(VD->getCanonicalDecl())), @@ -2737,8 +2906,12 @@ llvm::DINode *CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { getOrCreateFile(TD->getLocation())); auto I = DeclCache.find(D->getCanonicalDecl()); - if (I != DeclCache.end()) - return dyn_cast_or_null<llvm::DINode>(I->second); + if (I != DeclCache.end()) { + auto N = I->second; + if (auto *GVE = dyn_cast_or_null<llvm::DIGlobalVariableExpression>(N)) + return GVE->getVariable(); + return dyn_cast_or_null<llvm::DINode>(N); + } // No definition for now. Emit a forward definition that might be // merged with a potential upcoming definition. @@ -2834,7 +3007,8 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, Elts.push_back(DBuilder.createUnspecifiedParameter()); llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); - return DBuilder.createSubroutineType(EltTypeArray, 0, getDwarfCC(CC)); + return DBuilder.createSubroutineType(EltTypeArray, llvm::DINode::FlagZero, + getDwarfCC(CC)); } // Handle variadic function types; they need an additional @@ -2848,7 +3022,8 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, EltTys.push_back(getOrCreateType(ParamType, F)); EltTys.push_back(DBuilder.createUnspecifiedParameter()); llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); - return DBuilder.createSubroutineType(EltTypeArray, 0, getDwarfCC(CC)); + return DBuilder.createSubroutineType(EltTypeArray, llvm::DINode::FlagZero, + getDwarfCC(CC)); } return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F)); @@ -2866,7 +3041,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, const Decl *D = GD.getDecl(); bool HasDecl = (D != nullptr); - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; llvm::DIFile *Unit = getOrCreateFile(Loc); llvm::DIScope *FDContext = Unit; llvm::DINodeArray TParamsArray; @@ -2899,9 +3074,8 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, if (!HasDecl || D->isImplicit()) { Flags |= llvm::DINode::FlagArtificial; - // Artificial functions without a location should not silently reuse CurLoc. - if (Loc.isInvalid()) - CurLoc = SourceLocation(); + // Artificial functions should not silently reuse CurLoc. + CurLoc = SourceLocation(); } unsigned LineNo = getLineNumber(Loc); unsigned ScopeLine = getLineNumber(ScopeLoc); @@ -2939,7 +3113,7 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, if (!D) return; - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; llvm::DIFile *Unit = getOrCreateFile(Loc); llvm::DIScope *FDContext = getDeclContextDescriptor(D); llvm::DINodeArray TParamsArray; @@ -3042,7 +3216,7 @@ llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, SmallVector<llvm::Metadata *, 5> EltTys; QualType FType; uint64_t FieldSize, FieldOffset; - unsigned FieldAlign; + uint32_t FieldAlign; llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); QualType Type = VD->getType(); @@ -3096,13 +3270,14 @@ llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, *XOffset = FieldOffset; FieldTy = DBuilder.createMemberType(Unit, VD->getName(), Unit, 0, FieldSize, - FieldAlign, FieldOffset, 0, FieldTy); + FieldAlign, FieldOffset, + llvm::DINode::FlagZero, FieldTy); EltTys.push_back(FieldTy); FieldOffset += FieldSize; llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); - unsigned Flags = llvm::DINode::FlagBlockByrefStruct; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagBlockByrefStruct; return DBuilder.createStructType(Unit, "", Unit, 0, FieldOffset, 0, Flags, nullptr, Elements); @@ -3142,9 +3317,12 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::Value *Storage, Column = getColumnNumber(VD->getLocation()); } SmallVector<int64_t, 9> Expr; - unsigned Flags = 0; + llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; if (VD->isImplicit()) Flags |= llvm::DINode::FlagArtificial; + + auto Align = getDeclAlignIfRequired(VD, CGM.getContext()); + // If this is the first argument and it is implicit then // give it an object pointer flag. // FIXME: There has to be a better way to do this, but for static @@ -3179,7 +3357,7 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::Value *Storage, ? DBuilder.createParameterVariable(Scope, VD->getName(), *ArgNo, Unit, Line, Ty) : DBuilder.createAutoVariable(Scope, VD->getName(), Unit, - Line, Ty); + Line, Ty, Align); // Insert an llvm.dbg.declare into the current block. DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), @@ -3209,9 +3387,10 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::Value *Storage, continue; // Use VarDecl's Tag, Scope and Line number. + auto FieldAlign = getDeclAlignIfRequired(Field, CGM.getContext()); auto *D = DBuilder.createAutoVariable( Scope, FieldName, Unit, Line, FieldTy, CGM.getLangOpts().Optimize, - Flags | llvm::DINode::FlagArtificial); + Flags | llvm::DINode::FlagArtificial, FieldAlign); // Insert an llvm.dbg.declare into the current block. DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), @@ -3222,13 +3401,13 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::Value *Storage, } // Create the descriptor for the variable. - auto *D = - ArgNo - ? DBuilder.createParameterVariable(Scope, Name, *ArgNo, Unit, Line, - Ty, CGM.getLangOpts().Optimize, - Flags) - : DBuilder.createAutoVariable(Scope, Name, Unit, Line, Ty, - CGM.getLangOpts().Optimize, Flags); + auto *D = ArgNo + ? DBuilder.createParameterVariable( + Scope, Name, *ArgNo, Unit, Line, Ty, + CGM.getLangOpts().Optimize, Flags) + : DBuilder.createAutoVariable(Scope, Name, Unit, Line, Ty, + CGM.getLangOpts().Optimize, Flags, + Align); // Insert an llvm.dbg.declare into the current block. DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), @@ -3307,9 +3486,10 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable( } // Create the descriptor for the variable. + auto Align = getDeclAlignIfRequired(VD, CGM.getContext()); auto *D = DBuilder.createAutoVariable( cast<llvm::DILocalScope>(LexicalBlockStack.back()), VD->getName(), Unit, - Line, Ty); + Line, Ty, false, llvm::DINode::FlagZero, Align); // Insert an llvm.dbg.declare into the current block. auto DL = llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back()); @@ -3438,17 +3618,19 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, llvm::DIType *fieldType; if (capture->isByRef()) { TypeInfo PtrInfo = C.getTypeInfo(C.VoidPtrTy); + auto Align = PtrInfo.AlignIsRequired ? PtrInfo.Align : 0; // FIXME: this creates a second copy of this type! uint64_t xoffset; fieldType = EmitTypeForVarWithBlocksAttr(variable, &xoffset); fieldType = DBuilder.createPointerType(fieldType, PtrInfo.Width); - fieldType = - DBuilder.createMemberType(tunit, name, tunit, line, PtrInfo.Width, - PtrInfo.Align, offsetInBits, 0, fieldType); + fieldType = DBuilder.createMemberType(tunit, name, tunit, line, + PtrInfo.Width, Align, offsetInBits, + llvm::DINode::FlagZero, fieldType); } else { + auto Align = getDeclAlignIfRequired(variable, CGM.getContext()); fieldType = createFieldType(name, variable->getType(), loc, AS_public, - offsetInBits, tunit, tunit); + offsetInBits, Align, tunit, tunit); } fields.push_back(fieldType); } @@ -3459,14 +3641,14 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, llvm::DINodeArray fieldsArray = DBuilder.getOrCreateArray(fields); - llvm::DIType *type = DBuilder.createStructType( - tunit, typeName.str(), tunit, line, - CGM.getContext().toBits(block.BlockSize), - CGM.getContext().toBits(block.BlockAlign), 0, nullptr, fieldsArray); + llvm::DIType *type = + DBuilder.createStructType(tunit, typeName.str(), tunit, line, + CGM.getContext().toBits(block.BlockSize), 0, + llvm::DINode::FlagZero, nullptr, fieldsArray); type = DBuilder.createPointerType(type, CGM.PointerWidthInBits); // Get overall information about the block. - unsigned flags = llvm::DINode::FlagArtificial; + llvm::DINode::DIFlags flags = llvm::DINode::FlagArtificial; auto *scope = cast<llvm::DILocalScope>(LexicalBlockStack.back()); // Create the descriptor for the parameter. @@ -3505,10 +3687,10 @@ CGDebugInfo::getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D) { return CreateRecordStaticField(D, Ctxt, cast<RecordDecl>(DC)); } -llvm::DIGlobalVariable *CGDebugInfo::CollectAnonRecordDecls( +llvm::DIGlobalVariableExpression *CGDebugInfo::CollectAnonRecordDecls( const RecordDecl *RD, llvm::DIFile *Unit, unsigned LineNo, StringRef LinkageName, llvm::GlobalVariable *Var, llvm::DIScope *DContext) { - llvm::DIGlobalVariable *GV = nullptr; + llvm::DIGlobalVariableExpression *GVE = nullptr; for (const auto *Field : RD->fields()) { llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); @@ -3517,16 +3699,17 @@ llvm::DIGlobalVariable *CGDebugInfo::CollectAnonRecordDecls( // Ignore unnamed fields, but recurse into anonymous records. if (FieldName.empty()) { if (const auto *RT = dyn_cast<RecordType>(Field->getType())) - GV = CollectAnonRecordDecls(RT->getDecl(), Unit, LineNo, LinkageName, + GVE = CollectAnonRecordDecls(RT->getDecl(), Unit, LineNo, LinkageName, Var, DContext); continue; } // Use VarDecl's Tag, Scope and Line number. - GV = DBuilder.createGlobalVariable(DContext, FieldName, LinkageName, Unit, - LineNo, FieldTy, - Var->hasLocalLinkage(), Var, nullptr); + GVE = DBuilder.createGlobalVariableExpression( + DContext, FieldName, LinkageName, Unit, LineNo, FieldTy, + Var->hasLocalLinkage()); + Var->addDebugInfo(GVE); } - return GV; + return GVE; } void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, @@ -3534,6 +3717,14 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, assert(DebugKind >= codegenoptions::LimitedDebugInfo); if (D->hasAttr<NoDebugAttr>()) return; + + // If we already created a DIGlobalVariable for this declaration, just attach + // it to the llvm::GlobalVariable. + auto Cached = DeclCache.find(D->getCanonicalDecl()); + if (Cached != DeclCache.end()) + return Var->addDebugInfo( + cast<llvm::DIGlobalVariableExpression>(Cached->second)); + // Create global variable debug descriptor. llvm::DIFile *Unit = nullptr; llvm::DIScope *DContext = nullptr; @@ -3544,7 +3735,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, // Attempt to store one global variable for the declaration - even if we // emit a lot of fields. - llvm::DIGlobalVariable *GV = nullptr; + llvm::DIGlobalVariableExpression *GVE = nullptr; // If this is an anonymous union then we'll want to emit a global // variable for each member of the anonymous union so that it's possible @@ -3553,21 +3744,23 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, const RecordDecl *RD = T->castAs<RecordType>()->getDecl(); assert(RD->isAnonymousStructOrUnion() && "unnamed non-anonymous struct or union?"); - GV = CollectAnonRecordDecls(RD, Unit, LineNo, LinkageName, Var, DContext); + GVE = CollectAnonRecordDecls(RD, Unit, LineNo, LinkageName, Var, DContext); } else { - GV = DBuilder.createGlobalVariable( + auto Align = getDeclAlignIfRequired(D, CGM.getContext()); + GVE = DBuilder.createGlobalVariableExpression( DContext, DeclName, LinkageName, Unit, LineNo, getOrCreateType(T, Unit), - Var->hasLocalLinkage(), Var, - getOrCreateStaticDataMemberDeclarationOrNull(D)); + Var->hasLocalLinkage(), /*Expr=*/nullptr, + getOrCreateStaticDataMemberDeclarationOrNull(D), Align); + Var->addDebugInfo(GVE); } - DeclCache[D->getCanonicalDecl()].reset(GV); + DeclCache[D->getCanonicalDecl()].reset(GVE); } -void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, - llvm::Constant *Init) { +void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, const APValue &Init) { assert(DebugKind >= codegenoptions::LimitedDebugInfo); if (VD->hasAttr<NoDebugAttr>()) return; + auto Align = getDeclAlignIfRequired(VD, CGM.getContext()); // Create the descriptor for the variable. llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); StringRef Name = VD->getName(); @@ -3604,9 +3797,20 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, auto &GV = DeclCache[VD]; if (GV) return; - GV.reset(DBuilder.createGlobalVariable( + llvm::DIExpression *InitExpr = nullptr; + if (CGM.getContext().getTypeSize(VD->getType()) <= 64) { + // FIXME: Add a representation for integer constants wider than 64 bits. + if (Init.isInt()) + InitExpr = + DBuilder.createConstantValueExpression(Init.getInt().getExtValue()); + else if (Init.isFloat()) + InitExpr = DBuilder.createConstantValueExpression( + Init.getFloat().bitcastToAPInt().getZExtValue()); + } + GV.reset(DBuilder.createGlobalVariableExpression( DContext, Name, StringRef(), Unit, getLineNumber(VD->getLocation()), Ty, - true, Init, getOrCreateStaticDataMemberDeclarationOrNull(VarD))); + true, InitExpr, getOrCreateStaticDataMemberDeclarationOrNull(VarD), + Align)); } llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) { @@ -3620,8 +3824,8 @@ void CGDebugInfo::EmitUsingDirective(const UsingDirectiveDecl &UD) { if (CGM.getCodeGenOpts().getDebugInfo() < codegenoptions::LimitedDebugInfo) return; const NamespaceDecl *NSDecl = UD.getNominatedNamespace(); - if (!NSDecl->isAnonymousNamespace() || - CGM.getCodeGenOpts().DebugExplicitImport) { + if (!NSDecl->isAnonymousNamespace() || + CGM.getCodeGenOpts().DebugExplicitImport) { DBuilder.createImportedModule( getCurrentContextDescriptor(cast<Decl>(UD.getDeclContext())), getOrCreateNameSpace(NSDecl), @@ -3700,8 +3904,8 @@ CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) { unsigned LineNo = getLineNumber(NSDecl->getLocation()); llvm::DIFile *FileD = getOrCreateFile(NSDecl->getLocation()); llvm::DIScope *Context = getDeclContextDescriptor(NSDecl); - llvm::DINamespace *NS = - DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo); + llvm::DINamespace *NS = DBuilder.createNameSpace( + Context, NSDecl->getName(), FileD, LineNo, NSDecl->isInline()); NameSpaceCache[NSDecl].reset(NS); return NS; } @@ -3750,6 +3954,8 @@ void CGDebugInfo::finalize() { else Repl = it->second; + if (auto *GVE = dyn_cast_or_null<llvm::DIGlobalVariableExpression>(Repl)) + Repl = GVE->getVariable(); DBuilder.replaceTemporary(std::move(FwdDecl), cast<llvm::MDNode>(Repl)); } @@ -3770,3 +3976,12 @@ void CGDebugInfo::EmitExplicitCastType(QualType Ty) { // Don't ignore in case of explicit cast where it is referenced indirectly. DBuilder.retainType(DieTy); } + +llvm::DebugLoc CGDebugInfo::SourceLocToDebugLoc(SourceLocation Loc) { + if (LexicalBlockStack.empty()) + return llvm::DebugLoc(); + + llvm::MDNode *Scope = LexicalBlockStack.back(); + return llvm::DebugLoc::get( + getLineNumber(Loc), getColumnNumber(Loc), Scope); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h index 366dd81..ac2e8dd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h @@ -15,12 +15,14 @@ #define LLVM_CLANG_LIB_CODEGEN_CGDEBUGINFO_H #include "CGBuilder.h" +#include "clang/AST/DeclCXX.h" #include "clang/AST/Expr.h" #include "clang/AST/ExternalASTSource.h" #include "clang/AST/Type.h" #include "clang/Basic/SourceLocation.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Optional.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfo.h" @@ -32,7 +34,6 @@ class MDNode; } namespace clang { -class CXXMethodDecl; class ClassTemplateSpecializationDecl; class GlobalDecl; class ModuleMap; @@ -67,6 +68,7 @@ class CGDebugInfo { #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \ llvm::DIType *SingletonId = nullptr; #include "clang/Basic/OpenCLImageTypes.def" + llvm::DIType *OCLSamplerDITy = nullptr; llvm::DIType *OCLEventDITy = nullptr; llvm::DIType *OCLClkEventDITy = nullptr; llvm::DIType *OCLQueueDITy = nullptr; @@ -96,8 +98,7 @@ class CGDebugInfo { /// List of interfaces we want to keep even if orphaned. std::vector<void *> RetainedTypes; - /// Cache of forward declared types to RAUW at the end of - /// compilation. + /// Cache of forward declared types to RAUW at the end of compilation. std::vector<std::pair<const TagType *, llvm::TrackingMDRef>> ReplaceMap; /// Cache of replaceable forward declarations (functions and @@ -155,6 +156,8 @@ class CGDebugInfo { llvm::DIFile *F); /// Get Objective-C object type. llvm::DIType *CreateType(const ObjCObjectType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const ObjCTypeParamType *Ty, llvm::DIFile *Unit); + llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F); llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F); llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F); @@ -216,6 +219,15 @@ class CGDebugInfo { SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy); + /// Helper function for CollectCXXBases. + /// Adds debug info entries for types in Bases that are not in SeenTypes. + void CollectCXXBasesAux(const CXXRecordDecl *RD, llvm::DIFile *Unit, + SmallVectorImpl<llvm::Metadata *> &EltTys, + llvm::DIType *RecordTy, + const CXXRecordDecl::base_class_const_range &Bases, + llvm::DenseSet<CanonicalDeclPtr<const CXXRecordDecl>> &SeenTypes, + llvm::DINode::DIFlags StartingFlags); + /// A helper function to collect template parameters. llvm::DINodeArray CollectTemplateParams(const TemplateParameterList *TPList, ArrayRef<TemplateArgument> TAList, @@ -233,9 +245,19 @@ class CGDebugInfo { llvm::DIType *createFieldType(StringRef name, QualType type, SourceLocation loc, AccessSpecifier AS, + uint64_t offsetInBits, + uint32_t AlignInBits, + llvm::DIFile *tunit, llvm::DIScope *scope, + const RecordDecl *RD = nullptr); + + llvm::DIType *createFieldType(StringRef name, QualType type, + SourceLocation loc, AccessSpecifier AS, uint64_t offsetInBits, llvm::DIFile *tunit, llvm::DIScope *scope, - const RecordDecl *RD = nullptr); + const RecordDecl *RD = nullptr) { + return createFieldType(name, type, loc, AS, offsetInBits, 0, tunit, scope, + RD); + } /// Create new bit field member. llvm::DIType *createBitFieldType(const FieldDecl *BitFieldDecl, @@ -254,6 +276,8 @@ class CGDebugInfo { llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, llvm::DIType *RecordTy, const RecordDecl *RD); + void CollectRecordNestedRecord(const RecordDecl *RD, + SmallVectorImpl<llvm::Metadata *> &E); void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, llvm::DICompositeType *RecordTy); @@ -261,7 +285,8 @@ class CGDebugInfo { /// If the C++ class has vtable info then insert appropriate debug /// info entry in EltTys vector. void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile *F, - SmallVectorImpl<llvm::Metadata *> &EltTys); + SmallVectorImpl<llvm::Metadata *> &EltTys, + llvm::DICompositeType *RecordTy); /// @} /// Create a new lexical block node and push it on the stack. @@ -295,6 +320,9 @@ public: /// ignored. void setLocation(SourceLocation Loc); + // Converts a SourceLocation to a DebugLoc + llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Loc); + /// Emit metadata to indicate a change in line/column information in /// the source file. If the location is invalid, the previous /// location will be reused. @@ -350,8 +378,8 @@ public: /// Emit information about a global variable. void EmitGlobalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl); - /// Emit global variable's debug info. - void EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init); + /// Emit a constant global variable's debug info. + void EmitGlobalVariable(const ValueDecl *VD, const APValue &Init); /// Emit C++ using directive. void EmitUsingDirective(const UsingDirectiveDecl &UD); @@ -414,6 +442,10 @@ private: /// Remap a given path with the current debug prefix map std::string remapDIPath(StringRef) const; + /// Compute the file checksum debug info for input file ID. + llvm::DIFile::ChecksumKind computeChecksum(FileID FID, + SmallString<32> &Checksum) const; + /// Get the file debug info descriptor for the input location. llvm::DIFile *getOrCreateFile(SourceLocation Loc); @@ -468,14 +500,14 @@ private: llvm::DIGlobalVariable * getGlobalVariableForwardDeclaration(const VarDecl *VD); - /// \brief Return a global variable that represents one of the - /// collection of global variables created for an anonmyous union. + /// Return a global variable that represents one of the collection of global + /// variables created for an anonmyous union. /// /// Recursively collect all of the member fields of a global /// anonymous decl and create static variables for them. The first /// time this is called it needs to be on a union and then from /// there we can have additional unnamed fields. - llvm::DIGlobalVariable * + llvm::DIGlobalVariableExpression * CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile *Unit, unsigned LineNo, StringRef LinkageName, llvm::GlobalVariable *Var, llvm::DIScope *DContext); @@ -514,7 +546,7 @@ private: StringRef &Name, StringRef &LinkageName, llvm::DIScope *&FDContext, llvm::DINodeArray &TParamsArray, - unsigned &Flags); + llvm::DINode::DIFlags &Flags); /// Collect various properties of a VarDecl. void collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit, diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp index 89407cd..0a88b23 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp @@ -13,6 +13,7 @@ #include "CodeGenFunction.h" #include "CGBlocks.h" +#include "CGCXXABI.h" #include "CGCleanup.h" #include "CGDebugInfo.h" #include "CGOpenCLRuntime.h" @@ -77,6 +78,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) { case Decl::PragmaDetectMismatch: case Decl::AccessSpec: case Decl::LinkageSpec: + case Decl::Export: case Decl::ObjCPropertyImpl: case Decl::FileScopeAsm: case Decl::Friend: @@ -87,6 +89,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) { case Decl::UsingShadow: case Decl::ConstructorUsingShadow: case Decl::ObjCTypeParam: + case Decl::Binding: llvm_unreachable("Declaration should not be in declstmts!"); case Decl::Function: // void X(); case Decl::Record: // struct/union/class X; @@ -110,15 +113,25 @@ void CodeGenFunction::EmitDecl(const Decl &D) { if (CGDebugInfo *DI = getDebugInfo()) DI->EmitUsingDecl(cast<UsingDecl>(D)); return; + case Decl::UsingPack: + for (auto *Using : cast<UsingPackDecl>(D).expansions()) + EmitDecl(*Using); + return; case Decl::UsingDirective: // using namespace X; [C++] if (CGDebugInfo *DI = getDebugInfo()) DI->EmitUsingDirective(cast<UsingDirectiveDecl>(D)); return; - case Decl::Var: { + case Decl::Var: + case Decl::Decomposition: { const VarDecl &VD = cast<VarDecl>(D); assert(VD.isLocalVarDecl() && "Should not see file-scope variables inside a function!"); - return EmitVarDecl(VD); + EmitVarDecl(VD); + if (auto *DD = dyn_cast<DecompositionDecl>(&VD)) + for (auto *B : DD->bindings()) + if (auto *HD = B->getHoldingVar()) + EmitVarDecl(*HD); + return; } case Decl::OMPDeclareReduction: @@ -298,7 +311,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D, if (!Init) { if (!getLangOpts().CPlusPlus) CGM.ErrorUnsupported(D.getInit(), "constant l-value expression"); - else if (Builder.GetInsertBlock()) { + else if (HaveInsertPoint()) { // Since we have a static initializer, this global variable can't // be constant. GV->setConstant(false); @@ -339,7 +352,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D, GV->setConstant(CGM.isTypeConstant(D.getType(), true)); GV->setInitializer(Init); - if (hasNontrivialDestruction(D.getType())) { + if (hasNontrivialDestruction(D.getType()) && HaveInsertPoint()) { // We have a constant initializer, but a nontrivial destructor. We still // need to perform a guarded "initialization" in order to register the // destructor. @@ -526,7 +539,8 @@ namespace { CallArgList Args; Args.add(RValue::get(Arg), CGF.getContext().getPointerType(Var.getType())); - CGF.EmitCall(FnInfo, CleanupFn, ReturnValueSlot(), Args); + auto Callee = CGCallee::forDirect(CleanupFn); + CGF.EmitCall(FnInfo, Callee, ReturnValueSlot(), Args); } }; } // end anonymous namespace @@ -698,7 +712,7 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, } auto ty = cast<llvm::PointerType>(tempLV.getAddress().getElementType()); - llvm::Value *zero = llvm::ConstantPointerNull::get(ty); + llvm::Value *zero = CGM.getNullPointer(ty, tempLV.getType()); // If __weak, we want to use a barrier under certain conditions. if (lifetime == Qualifiers::OCL_Weak) @@ -765,37 +779,6 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, EmitStoreOfScalar(value, lvalue, /* isInitialization */ true); } -/// EmitScalarInit - Initialize the given lvalue with the given object. -void CodeGenFunction::EmitScalarInit(llvm::Value *init, LValue lvalue) { - Qualifiers::ObjCLifetime lifetime = lvalue.getObjCLifetime(); - if (!lifetime) - return EmitStoreThroughLValue(RValue::get(init), lvalue, true); - - switch (lifetime) { - case Qualifiers::OCL_None: - llvm_unreachable("present but none"); - - case Qualifiers::OCL_ExplicitNone: - // nothing to do - break; - - case Qualifiers::OCL_Strong: - init = EmitARCRetain(lvalue.getType(), init); - break; - - case Qualifiers::OCL_Weak: - // Initialize and then skip the primitive store. - EmitARCInitWeak(lvalue.getAddress(), init); - return; - - case Qualifiers::OCL_Autoreleasing: - init = EmitARCRetainAutorelease(lvalue.getType(), init); - break; - } - - EmitStoreOfScalar(init, lvalue, /* isInitialization */ true); -} - /// canEmitInitWithFewStoresAfterMemset - Decide whether we can emit the /// non-zero parts of the specified initializer with equal or fewer than /// NumStores scalar stores. @@ -907,29 +890,12 @@ void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) { EmitAutoVarCleanups(emission); } -/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time -/// markers. -static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts, - const LangOptions &LangOpts) { - // Asan uses markers for use-after-scope checks. - if (CGOpts.SanitizeAddressUseAfterScope) - return true; - - // Disable lifetime markers in msan builds. - // FIXME: Remove this when msan works with lifetime markers. - if (LangOpts.Sanitize.has(SanitizerKind::Memory)) - return false; - - // For now, only in optimized builds. - return CGOpts.OptimizationLevel != 0; -} - /// Emit a lifetime.begin marker if some criteria are satisfied. /// \return a pointer to the temporary size Value if a marker was emitted, null /// otherwise llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size, llvm::Value *Addr) { - if (!shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), getLangOpts())) + if (!ShouldEmitLifetimeMarkers) return nullptr; llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size); @@ -986,8 +952,12 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // If the variable's a const type, and it's neither an NRVO // candidate nor a __block variable and has no mutable members, // emit it as a global instead. - if (CGM.getCodeGenOpts().MergeAllConstants && !NRVO && !isByRef && - CGM.isTypeConstant(Ty, true)) { + // Exception is if a variable is located in non-constant address space + // in OpenCL. + if ((!getLangOpts().OpenCL || + Ty.getAddressSpace() == LangAS::opencl_constant) && + (CGM.getCodeGenOpts().MergeAllConstants && !NRVO && !isByRef && + CGM.isTypeConstant(Ty, true))) { EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage); // Signal this condition to later callbacks. @@ -1049,12 +1019,18 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { bool IsMSCatchParam = D.isExceptionVariable() && getTarget().getCXXABI().isMicrosoft(); - // Emit a lifetime intrinsic if meaningful. There's no point - // in doing this if we don't have a valid insertion point (?). + // Emit a lifetime intrinsic if meaningful. There's no point in doing this + // if we don't have a valid insertion point (?). if (HaveInsertPoint() && !IsMSCatchParam) { - uint64_t size = CGM.getDataLayout().getTypeAllocSize(allocaTy); - emission.SizeForLifetimeMarkers = - EmitLifetimeStart(size, address.getPointer()); + // goto or switch-case statements can break lifetime into several + // regions which need more efforts to handle them correctly. PR28267 + // This is rare case, but it's better just omit intrinsics than have + // them incorrectly placed. + if (!Bypasses.IsBypassed(&D)) { + uint64_t size = CGM.getDataLayout().getTypeAllocSize(allocaTy); + emission.SizeForLifetimeMarkers = + EmitLifetimeStart(size, address.getPointer()); + } } else { assert(!emission.useLifetimeMarkers()); } @@ -1257,10 +1233,16 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { // Otherwise, create a temporary global with the initializer then // memcpy from the global to the alloca. std::string Name = getStaticDeclName(CGM, D); + unsigned AS = 0; + if (getLangOpts().OpenCL) { + AS = CGM.getContext().getTargetAddressSpace(LangAS::opencl_constant); + BP = llvm::PointerType::getInt8PtrTy(getLLVMContext(), AS); + } llvm::GlobalVariable *GV = new llvm::GlobalVariable(CGM.getModule(), constant->getType(), true, llvm::GlobalValue::PrivateLinkage, - constant, Name); + constant, Name, nullptr, + llvm::GlobalValue::NotThreadLocal, AS); GV->setAlignment(Loc.getAlignment().getQuantity()); GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); @@ -1762,6 +1744,24 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, setBlockContextParameter(IPD, ArgNo, Arg.getDirectValue()); return; } + + // Apply any prologue 'this' adjustments required by the ABI. Be careful to + // handle the case where 'this' is passed indirectly as part of an inalloca + // struct. + if (const CXXMethodDecl *MD = + dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl)) { + if (MD->isVirtual() && IPD == CXXABIThisDecl) { + llvm::Value *This = Arg.isIndirect() + ? Builder.CreateLoad(Arg.getIndirectAddress()) + : Arg.getDirectValue(); + This = CGM.getCXXABI().adjustThisParameterInVirtualFunctionPrologue( + *this, CurGD, This); + if (Arg.isIndirect()) + Builder.CreateStore(This, Arg.getIndirectAddress()); + else + Arg = ParamValue::forDirect(This); + } + } } Address DeclPtr = Address::invalid(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp index 89d142e..f56e182 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp @@ -121,13 +121,15 @@ static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D, /// constant from this point onwards. static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D, llvm::Constant *Addr) { - // Don't emit the intrinsic if we're not optimizing. + // Do not emit the intrinsic if we're not optimizing. if (!CGF.CGM.getCodeGenOpts().OptimizationLevel) return; // Grab the llvm.invariant.start intrinsic. llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start; - llvm::Constant *InvariantStart = CGF.CGM.getIntrinsic(InvStartID); + // Overloaded address space type. + llvm::Type *ObjectPtr[1] = {CGF.Int8PtrTy}; + llvm::Constant *InvariantStart = CGF.CGM.getIntrinsic(InvStartID, ObjectPtr); // Emit a call with the size in bytes of the object. CharUnits WidthChars = CGF.getContext().getTypeSizeInChars(D.getType()); @@ -235,7 +237,8 @@ void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD, llvm::FunctionType::get(IntTy, dtorStub->getType(), false); llvm::Constant *atexit = - CGM.CreateRuntimeFunction(atexitTy, "atexit"); + CGM.CreateRuntimeFunction(atexitTy, "atexit", llvm::AttributeSet(), + /*Local=*/true); if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(atexit)) atexitFn->setDoesNotThrow(); @@ -350,9 +353,6 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, if (D->getTLSKind()) { // FIXME: Should we support init_priority for thread_local? - // FIXME: Ideally, initialization of instantiated thread_local static data - // members of class templates should not trigger initialization of other - // entities in the TU. // FIXME: We only need to register one __cxa_thread_atexit function for the // entire TU. CXXThreadLocalInits.push_back(Fn); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp index 4a7dc42..f908bf2 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp @@ -97,6 +97,10 @@ EHPersonality::GNU_CPlusPlus_SEH = { "__gxx_personality_seh0", nullptr }; const EHPersonality EHPersonality::GNU_ObjC = {"__gnu_objc_personality_v0", "objc_exception_throw"}; const EHPersonality +EHPersonality::GNU_ObjC_SJLJ = {"__gnu_objc_personality_sj0", "objc_exception_throw"}; +const EHPersonality +EHPersonality::GNU_ObjC_SEH = {"__gnu_objc_personality_seh0", "objc_exception_throw"}; +const EHPersonality EHPersonality::GNU_ObjCXX = { "__gnustep_objcxx_personality_v0", nullptr }; const EHPersonality EHPersonality::GNUstep_ObjC = { "__gnustep_objc_personality_v0", nullptr }; @@ -137,6 +141,10 @@ static const EHPersonality &getObjCPersonality(const llvm::Triple &T, // fallthrough case ObjCRuntime::GCC: case ObjCRuntime::ObjFW: + if (L.SjLjExceptions) + return EHPersonality::GNU_ObjC_SJLJ; + else if (useLibGCCSEHPersonality(T)) + return EHPersonality::GNU_ObjC_SEH; return EHPersonality::GNU_ObjC; } llvm_unreachable("bad runtime kind"); @@ -221,10 +229,9 @@ const EHPersonality &EHPersonality::get(CodeGenFunction &CGF) { static llvm::Constant *getPersonalityFn(CodeGenModule &CGM, const EHPersonality &Personality) { - llvm::Constant *Fn = - CGM.CreateRuntimeFunction(llvm::FunctionType::get(CGM.Int32Ty, true), - Personality.PersonalityFn); - return Fn; + return CGM.CreateRuntimeFunction(llvm::FunctionType::get(CGM.Int32Ty, true), + Personality.PersonalityFn, + llvm::AttributeSet(), /*Local=*/true); } static llvm::Constant *getOpaquePersonalityFn(CodeGenModule &CGM, @@ -698,6 +705,10 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() { return nullptr; } + // CUDA device code doesn't have exceptions. + if (LO.CUDA && LO.CUDAIsDevice) + return nullptr; + // Check the innermost scope for a cached landing pad. If this is // a non-EH cleanup, we'll check enclosing scopes in EmitLandingPad. llvm::BasicBlock *LP = EHStack.begin()->getCachedLandingPad(); @@ -1429,7 +1440,8 @@ struct PerformSEHFinally final : EHScopeStack::Cleanup { const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeBuiltinFunctionCall(Context.VoidTy, Args); - CGF.EmitCall(FnInfo, OutlinedFinally, ReturnValueSlot(), Args); + auto Callee = CGCallee::forDirect(OutlinedFinally); + CGF.EmitCall(FnInfo, Callee, ReturnValueSlot(), Args); } }; } // end anonymous namespace diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp index 5f3b290..e5e34a5 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp @@ -24,6 +24,7 @@ #include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" #include "clang/AST/DeclObjC.h" +#include "clang/AST/NSAPI.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringExtras.h" @@ -36,6 +37,8 @@ #include "llvm/Support/Path.h" #include "llvm/Transforms/Utils/SanitizerStats.h" +#include <string> + using namespace clang; using namespace CodeGen; @@ -601,13 +604,14 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, } if (Checks.size() > 0) { + // Make sure we're not losing information. Alignment needs to be a power of + // 2 + assert(!AlignVal || (uint64_t)1 << llvm::Log2_64(AlignVal) == AlignVal); llvm::Constant *StaticData[] = { - EmitCheckSourceLocation(Loc), - EmitCheckTypeDescriptor(Ty), - llvm::ConstantInt::get(SizeTy, AlignVal), - llvm::ConstantInt::get(Int8Ty, TCK) - }; - EmitCheck(Checks, "type_mismatch", StaticData, Ptr); + EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty), + llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2_64(AlignVal) : 1), + llvm::ConstantInt::get(Int8Ty, TCK)}; + EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData, Ptr); } // If possible, check that the vptr indicates that there is a subobject of @@ -675,7 +679,8 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, }; llvm::Value *DynamicData[] = { Ptr, Hash }; EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr), - "dynamic_type_cache_miss", StaticData, DynamicData); + SanitizerHandler::DynamicTypeCacheMiss, StaticData, + DynamicData); } } @@ -708,6 +713,8 @@ static bool isFlexibleArrayMemberExpr(const Expr *E) { DeclContext::decl_iterator(const_cast<FieldDecl *>(FD))); return ++FI == FD->getParent()->field_end(); } + } else if (const auto *IRE = dyn_cast<ObjCIvarRefExpr>(E)) { + return IRE->getDecl()->getNextIvar() == nullptr; } return false; @@ -763,8 +770,8 @@ void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, }; llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal) : Builder.CreateICmpULE(IndexVal, BoundVal); - EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds), "out_of_bounds", - StaticData, Index); + EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds), + SanitizerHandler::OutOfBounds, StaticData, Index); } @@ -1180,10 +1187,10 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) { // This should probably fire even for if (isa<VarDecl>(value)) { if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value))) - EmitDeclRefExprDbgValue(refExpr, C); + EmitDeclRefExprDbgValue(refExpr, result.Val); } else { assert(isa<EnumConstantDecl>(value)); - EmitDeclRefExprDbgValue(refExpr, C); + EmitDeclRefExprDbgValue(refExpr, result.Val); } // If we emitted a reference constant, we need to dereference that. @@ -1217,11 +1224,10 @@ static bool hasBooleanRepresentation(QualType Ty) { static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, llvm::APInt &Min, llvm::APInt &End, - bool StrictEnums) { + bool StrictEnums, bool IsBool) { const EnumType *ET = Ty->getAs<EnumType>(); bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums && ET && !ET->getDecl()->isFixed(); - bool IsBool = hasBooleanRepresentation(Ty); if (!IsBool && !IsRegularCPlusPlusEnum) return false; @@ -1251,8 +1257,8 @@ static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) { llvm::APInt Min, End; - if (!getRangeForType(*this, Ty, Min, End, - CGM.getCodeGenOpts().StrictEnums)) + if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums, + hasBooleanRepresentation(Ty))) return nullptr; llvm::MDBuilder MDHelper(getLLVMContext()); @@ -1311,14 +1317,15 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, false /*ConvertTypeToTag*/); } - bool NeedsBoolCheck = - SanOpts.has(SanitizerKind::Bool) && hasBooleanRepresentation(Ty); + bool IsBool = hasBooleanRepresentation(Ty) || + NSAPI(CGM.getContext()).isObjCBOOLType(Ty); + bool NeedsBoolCheck = SanOpts.has(SanitizerKind::Bool) && IsBool; bool NeedsEnumCheck = SanOpts.has(SanitizerKind::Enum) && Ty->getAs<EnumType>(); if (NeedsBoolCheck || NeedsEnumCheck) { SanitizerScope SanScope(this); llvm::APInt Min, End; - if (getRangeForType(*this, Ty, Min, End, true)) { + if (getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool)) { --End; llvm::Value *Check; if (!Min) @@ -1336,8 +1343,8 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, EmitCheckTypeDescriptor(Ty) }; SanitizerMask Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; - EmitCheck(std::make_pair(Check, Kind), "load_invalid_value", StaticArgs, - EmitCheckValue(Load)); + EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue, + StaticArgs, EmitCheckValue(Load)); } } else if (CGM.getCodeGenOpts().OptimizationLevel > 0) if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) @@ -1627,11 +1634,19 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, break; case Qualifiers::OCL_Strong: + if (isInit) { + Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal())); + break; + } EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true); return; case Qualifiers::OCL_Weak: - EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true); + if (isInit) + // Initialize and then skip the primitive store. + EmitARCInitWeak(Dst.getAddress(), Src.getScalarVal()); + else + EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true); return; case Qualifiers::OCL_Autoreleasing: @@ -2015,9 +2030,14 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, return LV; } -static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, - const Expr *E, const FunctionDecl *FD) { - llvm::Value *V = CGF.CGM.GetAddrOfFunction(FD); +static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM, + const FunctionDecl *FD) { + if (FD->hasAttr<WeakRefAttr>()) { + ConstantAddress aliasee = CGM.GetWeakRefReference(FD); + return aliasee.getPointer(); + } + + llvm::Constant *V = CGM.GetAddrOfFunction(FD); if (!FD->hasPrototype()) { if (const FunctionProtoType *Proto = FD->getType()->getAs<FunctionProtoType>()) { @@ -2025,11 +2045,18 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, // isn't the same as the type of a use. Correct for this with a // bitcast. QualType NoProtoType = - CGF.getContext().getFunctionNoProtoType(Proto->getReturnType()); - NoProtoType = CGF.getContext().getPointerType(NoProtoType); - V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType)); + CGM.getContext().getFunctionNoProtoType(Proto->getReturnType()); + NoProtoType = CGM.getContext().getPointerType(NoProtoType); + V = llvm::ConstantExpr::getBitCast(V, + CGM.getTypes().ConvertType(NoProtoType)); } } + return V; +} + +static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, + const Expr *E, const FunctionDecl *FD) { + llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, FD); CharUnits Alignment = CGF.getContext().getDeclAlign(FD); return CGF.MakeAddrLValue(V, E->getType(), Alignment, AlignmentSource::Decl); } @@ -2205,6 +2232,12 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { if (const auto *FD = dyn_cast<FunctionDecl>(ND)) return EmitFunctionDeclLValue(*this, E, FD); + // FIXME: While we're emitting a binding from an enclosing scope, all other + // DeclRefExprs we see should be implicitly treated as if they also refer to + // an enclosing scope. + if (const auto *BD = dyn_cast<BindingDecl>(ND)) + return EmitLValue(BD->getBinding()); + llvm_unreachable("Unhandled DeclRefExpr"); } @@ -2291,9 +2324,19 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) { StringRef NameItems[] = { PredefinedExpr::getIdentTypeName(E->getIdentType()), FnName}; std::string GVName = llvm::join(NameItems, NameItems + 2, "."); - if (CurCodeDecl && isa<BlockDecl>(CurCodeDecl)) { - auto C = CGM.GetAddrOfConstantCString(FnName, GVName.c_str()); - return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); + if (auto *BD = dyn_cast<BlockDecl>(CurCodeDecl)) { + std::string Name = SL->getString(); + if (!Name.empty()) { + unsigned Discriminator = + CGM.getCXXABI().getMangleContext().getBlockId(BD, true); + if (Discriminator) + Name += "_" + Twine(Discriminator + 1).str(); + auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str()); + return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); + } else { + auto C = CGM.GetAddrOfConstantCString(FnName, GVName.c_str()); + return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); + } } auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName); return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); @@ -2461,17 +2504,35 @@ static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) { } } +namespace { +struct SanitizerHandlerInfo { + char const *const Name; + unsigned Version; +}; +} + +const SanitizerHandlerInfo SanitizerHandlers[] = { +#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version}, + LIST_SANITIZER_CHECKS +#undef SANITIZER_CHECK +}; + static void emitCheckHandlerCall(CodeGenFunction &CGF, llvm::FunctionType *FnType, ArrayRef<llvm::Value *> FnArgs, - StringRef CheckName, + SanitizerHandler CheckHandler, CheckRecoverableKind RecoverKind, bool IsFatal, llvm::BasicBlock *ContBB) { assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable); bool NeedsAbortSuffix = IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable; - std::string FnName = ("__ubsan_handle_" + CheckName + - (NeedsAbortSuffix ? "_abort" : "")).str(); + const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler]; + const StringRef CheckName = CheckInfo.Name; + std::string FnName = + ("__ubsan_handle_" + CheckName + + (CheckInfo.Version ? "_v" + llvm::utostr(CheckInfo.Version) : "") + + (NeedsAbortSuffix ? "_abort" : "")) + .str(); bool MayReturn = !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable; @@ -2485,7 +2546,8 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, llvm::Value *Fn = CGF.CGM.CreateRuntimeFunction( FnType, FnName, llvm::AttributeSet::get(CGF.getLLVMContext(), - llvm::AttributeSet::FunctionIndex, B)); + llvm::AttributeSet::FunctionIndex, B), + /*Local=*/true); llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs); if (!MayReturn) { HandlerCall->setDoesNotReturn(); @@ -2497,10 +2559,13 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, void CodeGenFunction::EmitCheck( ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, - StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, + SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs) { assert(IsSanitizerScope); assert(Checked.size() > 0); + assert(CheckHandler >= 0 && + CheckHandler < sizeof(SanitizerHandlers) / sizeof(*SanitizerHandlers)); + const StringRef CheckName = SanitizerHandlers[CheckHandler].Name; llvm::Value *FatalCond = nullptr; llvm::Value *RecoverableCond = nullptr; @@ -2580,7 +2645,7 @@ void CodeGenFunction::EmitCheck( if (!FatalCond || !RecoverableCond) { // Simple case: we need to generate a single handler call, either // fatal, or non-fatal. - emitCheckHandlerCall(*this, FnType, Args, CheckName, RecoverKind, + emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, (FatalCond != nullptr), Cont); } else { // Emit two handler calls: first one for set of unrecoverable checks, @@ -2590,10 +2655,10 @@ void CodeGenFunction::EmitCheck( llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName); Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB); EmitBlock(FatalHandlerBB); - emitCheckHandlerCall(*this, FnType, Args, CheckName, RecoverKind, true, + emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true, NonFatalHandlerBB); EmitBlock(NonFatalHandlerBB); - emitCheckHandlerCall(*this, FnType, Args, CheckName, RecoverKind, false, + emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false, Cont); } @@ -2718,7 +2783,7 @@ void CodeGenFunction::EmitCfiCheckFail() { llvm::Value *Cond = Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind)); if (CGM.getLangOpts().Sanitize.has(Mask)) - EmitCheck(std::make_pair(Cond, Mask), "cfi_check_fail", {}, + EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {}, {Data, Addr, ValidVtable}); else EmitTrapCheck(Cond); @@ -2753,10 +2818,11 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) { llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) { llvm::CallInst *TrapCall = Builder.CreateCall(CGM.getIntrinsic(IntrID)); - if (!CGM.getCodeGenOpts().TrapFuncName.empty()) - TrapCall->addAttribute(llvm::AttributeSet::FunctionIndex, - "trap-func-name", - CGM.getCodeGenOpts().TrapFuncName); + if (!CGM.getCodeGenOpts().TrapFuncName.empty()) { + auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name", + CGM.getCodeGenOpts().TrapFuncName); + TrapCall->addAttribute(llvm::AttributeSet::FunctionIndex, A); + } return TrapCall; } @@ -2869,13 +2935,30 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, bool Accessed) { - // The index must always be an integer, which is not an aggregate. Emit it. - llvm::Value *Idx = EmitScalarExpr(E->getIdx()); - QualType IdxTy = E->getIdx()->getType(); - bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType(); + // The index must always be an integer, which is not an aggregate. Emit it + // in lexical order (this complexity is, sadly, required by C++17). + llvm::Value *IdxPre = + (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr; + auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * { + auto *Idx = IdxPre; + if (E->getLHS() != E->getIdx()) { + assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS"); + Idx = EmitScalarExpr(E->getIdx()); + } - if (SanOpts.has(SanitizerKind::ArrayBounds)) - EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed); + QualType IdxTy = E->getIdx()->getType(); + bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType(); + + if (SanOpts.has(SanitizerKind::ArrayBounds)) + EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed); + + // Extend or truncate the index type to 32 or 64-bits. + if (Promote && Idx->getType() != IntPtrTy) + Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom"); + + return Idx; + }; + IdxPre = nullptr; // If the base is a vector type, then we are forming a vector element lvalue // with this subscript. @@ -2883,6 +2966,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, !isa<ExtVectorElementExpr>(E->getBase())) { // Emit the vector as an lvalue to get its address. LValue LHS = EmitLValue(E->getBase()); + auto *Idx = EmitIdxAfterBase(/*Promote*/false); assert(LHS.isSimple() && "Can only subscript lvalue vectors here!"); return LValue::MakeVectorElt(LHS.getAddress(), Idx, E->getBase()->getType(), @@ -2891,13 +2975,10 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // All the other cases basically behave like simple offsetting. - // Extend or truncate the index type to 32 or 64-bits. - if (Idx->getType() != IntPtrTy) - Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom"); - // Handle the extvector case we ignored above. if (isa<ExtVectorElementExpr>(E->getBase())) { LValue LV = EmitLValue(E->getBase()); + auto *Idx = EmitIdxAfterBase(/*Promote*/true); Address Addr = EmitExtVectorElementLValue(LV); QualType EltType = LV.getType()->castAs<VectorType>()->getElementType(); @@ -2913,6 +2994,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // it. It needs to be emitted first in case it's what captures // the VLA bounds. Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource); + auto *Idx = EmitIdxAfterBase(/*Promote*/true); // The element count here is the total number of non-VLA elements. llvm::Value *numElements = getVLASize(vla).first; @@ -2932,14 +3014,16 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){ // Indexing over an interface, as in "NSString *P; P[4];" - CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT); - llvm::Value *InterfaceSizeVal = - llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());; - - llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal); // Emit the base pointer. Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource); + auto *Idx = EmitIdxAfterBase(/*Promote*/true); + + CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT); + llvm::Value *InterfaceSizeVal = + llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity()); + + llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal); // We don't necessarily build correct LLVM struct types for ObjC // interfaces, so we can't rely on GEP to do this scaling @@ -2971,6 +3055,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true); else ArrayLV = EmitLValue(Array); + auto *Idx = EmitIdxAfterBase(/*Promote*/true); // Propagate the alignment from the array itself to the result. Addr = emitArraySubscriptGEP(*this, ArrayLV.getAddress(), @@ -2981,6 +3066,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, } else { // The base must be a pointer; emit it with an estimate of its alignment. Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource); + auto *Idx = EmitIdxAfterBase(/*Promote*/true); Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(), !getLangOpts().isSignedOverflowDefined()); } @@ -3463,7 +3549,7 @@ LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) { return EmitAggExprToLValue(E); // An lvalue initializer list must be initializing a reference. - assert(E->getNumInits() == 1 && "reference init with multiple values"); + assert(E->isTransparent() && "non-transparent glvalue init list"); return EmitLValue(E->getInit(0)); } @@ -3602,6 +3688,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_ARCExtendBlockObject: case CK_CopyAndAutoreleaseBlockObject: case CK_AddressSpaceConversion: + case CK_IntToOCLSampler: return EmitUnsupportedLValue(E, "unexpected cast lvalue"); case CK_Dependent: @@ -3695,6 +3782,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { ConvertType(E->getType())); return MakeAddrLValue(V, E->getType(), LV.getAlignmentSource()); } + case CK_ZeroToOCLQueue: + llvm_unreachable("NULL to OpenCL queue lvalue cast is not valid"); case CK_ZeroToOCLEvent: llvm_unreachable("NULL to OpenCL event lvalue cast is not valid"); } @@ -3743,70 +3832,86 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E)) return EmitCUDAKernelCallExpr(CE, ReturnValue); - const Decl *TargetDecl = E->getCalleeDecl(); - if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { - if (unsigned builtinID = FD->getBuiltinID()) - return EmitBuiltinExpr(FD, builtinID, E, ReturnValue); - } - if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E)) - if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(TargetDecl)) + if (const CXXMethodDecl *MD = + dyn_cast_or_null<CXXMethodDecl>(CE->getCalleeDecl())) return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue); - if (const auto *PseudoDtor = - dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) { - QualType DestroyedType = PseudoDtor->getDestroyedType(); - if (DestroyedType.hasStrongOrWeakObjCLifetime()) { - // Automatic Reference Counting: - // If the pseudo-expression names a retainable object with weak or - // strong lifetime, the object shall be released. - Expr *BaseExpr = PseudoDtor->getBase(); - Address BaseValue = Address::invalid(); - Qualifiers BaseQuals; - - // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar. - if (PseudoDtor->isArrow()) { - BaseValue = EmitPointerWithAlignment(BaseExpr); - const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>(); - BaseQuals = PTy->getPointeeType().getQualifiers(); - } else { - LValue BaseLV = EmitLValue(BaseExpr); - BaseValue = BaseLV.getAddress(); - QualType BaseTy = BaseExpr->getType(); - BaseQuals = BaseTy.getQualifiers(); - } + CGCallee callee = EmitCallee(E->getCallee()); - switch (DestroyedType.getObjCLifetime()) { - case Qualifiers::OCL_None: - case Qualifiers::OCL_ExplicitNone: - case Qualifiers::OCL_Autoreleasing: - break; + if (callee.isBuiltin()) { + return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(), + E, ReturnValue); + } - case Qualifiers::OCL_Strong: - EmitARCRelease(Builder.CreateLoad(BaseValue, - PseudoDtor->getDestroyedType().isVolatileQualified()), - ARCPreciseLifetime); - break; + if (callee.isPseudoDestructor()) { + return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr()); + } - case Qualifiers::OCL_Weak: - EmitARCDestroyWeak(BaseValue); - break; - } - } else { - // C++ [expr.pseudo]p1: - // The result shall only be used as the operand for the function call - // operator (), and the result of such a call has type void. The only - // effect is the evaluation of the postfix-expression before the dot or - // arrow. - EmitScalarExpr(E->getCallee()); + return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue); +} + +/// Emit a CallExpr without considering whether it might be a subclass. +RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E, + ReturnValueSlot ReturnValue) { + CGCallee Callee = EmitCallee(E->getCallee()); + return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue); +} + +static CGCallee EmitDirectCallee(CodeGenFunction &CGF, const FunctionDecl *FD) { + if (auto builtinID = FD->getBuiltinID()) { + return CGCallee::forBuiltin(builtinID, FD); + } + + llvm::Constant *calleePtr = EmitFunctionDeclPointer(CGF.CGM, FD); + return CGCallee::forDirect(calleePtr, FD); +} + +CGCallee CodeGenFunction::EmitCallee(const Expr *E) { + E = E->IgnoreParens(); + + // Look through function-to-pointer decay. + if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) { + if (ICE->getCastKind() == CK_FunctionToPointerDecay || + ICE->getCastKind() == CK_BuiltinFnToFnPtr) { + return EmitCallee(ICE->getSubExpr()); } - return RValue::get(nullptr); + // Resolve direct calls. + } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) { + if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) { + return EmitDirectCallee(*this, FD); + } + } else if (auto ME = dyn_cast<MemberExpr>(E)) { + if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) { + EmitIgnoredExpr(ME->getBase()); + return EmitDirectCallee(*this, FD); + } + + // Look through template substitutions. + } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) { + return EmitCallee(NTTP->getReplacement()); + + // Treat pseudo-destructor calls differently. + } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) { + return CGCallee::forPseudoDestructor(PDE); } - llvm::Value *Callee = EmitScalarExpr(E->getCallee()); - return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue, - TargetDecl); + // Otherwise, we have an indirect reference. + llvm::Value *calleePtr; + QualType functionType; + if (auto ptrType = E->getType()->getAs<PointerType>()) { + calleePtr = EmitScalarExpr(E); + functionType = ptrType->getPointeeType(); + } else { + functionType = E->getType(); + calleePtr = EmitLValue(E).getPointer(); + } + assert(functionType->isFunctionType()); + CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), + E->getReferencedDeclOfCallee()); + CGCallee callee(calleeInfo, calleePtr); + return callee; } LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) { @@ -3982,22 +4087,15 @@ LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) { AlignmentSource::Decl); } -RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, +RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee, const CallExpr *E, ReturnValueSlot ReturnValue, - CGCalleeInfo CalleeInfo, llvm::Value *Chain) { + llvm::Value *Chain) { // Get the actual function type. The callee type will always be a pointer to // function type or a block pointer type. assert(CalleeType->isFunctionPointerType() && "Call must have function pointer type!"); - // Preserve the non-canonical function type because things like exception - // specifications disappear in the canonical type. That information is useful - // to drive the generation of more accurate code for this call later on. - const FunctionProtoType *NonCanonicalFTP = CalleeType->getAs<PointerType>() - ->getPointeeType() - ->getAs<FunctionProtoType>(); - - const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); + const Decl *TargetDecl = OrigCallee.getAbstractInfo().getCalleeDecl(); if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) // We can only guarantee that a function is called from the correct @@ -4015,6 +4113,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, const auto *FnType = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType()); + CGCallee Callee = OrigCallee; + if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) && (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { if (llvm::Constant *PrefixSig = @@ -4029,8 +4129,10 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, llvm::StructType *PrefixStructTy = llvm::StructType::get( CGM.getLLVMContext(), PrefixStructTyElems, /*isPacked=*/true); + llvm::Value *CalleePtr = Callee.getFunctionPointer(); + llvm::Value *CalleePrefixStruct = Builder.CreateBitCast( - Callee, llvm::PointerType::getUnqual(PrefixStructTy)); + CalleePtr, llvm::PointerType::getUnqual(PrefixStructTy)); llvm::Value *CalleeSigPtr = Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0); llvm::Value *CalleeSig = @@ -4053,7 +4155,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, EmitCheckTypeDescriptor(CalleeType) }; EmitCheck(std::make_pair(CalleeRTTIMatch, SanitizerKind::Function), - "function_type_mismatch", StaticData, Callee); + SanitizerHandler::FunctionTypeMismatch, StaticData, CalleePtr); Builder.CreateBr(Cont); EmitBlock(Cont); @@ -4070,7 +4172,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0)); llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD); - llvm::Value *CastedCallee = Builder.CreateBitCast(Callee, Int8PtrTy); + llvm::Value *CalleePtr = Callee.getFunctionPointer(); + llvm::Value *CastedCallee = Builder.CreateBitCast(CalleePtr, Int8PtrTy); llvm::Value *TypeTest = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedCallee, TypeId}); @@ -4085,7 +4188,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, CastedCallee, StaticData); } else { EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall), - "cfi_check_fail", StaticData, + SanitizerHandler::CFICheckFail, StaticData, {CastedCallee, llvm::UndefValue::get(IntPtrTy)}); } } @@ -4094,8 +4197,35 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, if (Chain) Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), CGM.getContext().VoidPtrTy); + + // C++17 requires that we evaluate arguments to a call using assignment syntax + // right-to-left, and that we evaluate arguments to certain other operators + // left-to-right. Note that we allow this to override the order dictated by + // the calling convention on the MS ABI, which means that parameter + // destruction order is not necessarily reverse construction order. + // FIXME: Revisit this based on C++ committee response to unimplementability. + EvaluationOrder Order = EvaluationOrder::Default; + if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) { + if (OCE->isAssignmentOp()) + Order = EvaluationOrder::ForceRightToLeft; + else { + switch (OCE->getOperator()) { + case OO_LessLess: + case OO_GreaterGreater: + case OO_AmpAmp: + case OO_PipePipe: + case OO_Comma: + case OO_ArrowStar: + Order = EvaluationOrder::ForceLeftToRight; + break; + default: + break; + } + } + } + EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(), - E->getDirectCallee(), /*ParamsToSkip*/ 0); + E->getDirectCallee(), /*ParamsToSkip*/ 0, Order); const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( Args, FnType, /*isChainCall=*/Chain); @@ -4123,11 +4253,13 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, if (isa<FunctionNoProtoType>(FnType) || Chain) { llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo); CalleeTy = CalleeTy->getPointerTo(); - Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast"); + + llvm::Value *CalleePtr = Callee.getFunctionPointer(); + CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast"); + Callee.setFunctionPointer(CalleePtr); } - return EmitCall(FnInfo, Callee, ReturnValue, Args, - CGCalleeInfo(NonCanonicalFTP, TargetDecl)); + return EmitCall(FnInfo, Callee, ReturnValue, Args); } LValue CodeGenFunction:: diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp index 6d18843..0092447 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp @@ -164,6 +164,8 @@ public: void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO); void VisitChooseExpr(const ChooseExpr *CE); void VisitInitListExpr(InitListExpr *E); + void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, + llvm::Value *outerBegin = nullptr); void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E); void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing. void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) { @@ -749,7 +751,9 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { case CK_CopyAndAutoreleaseBlockObject: case CK_BuiltinFnToFnPtr: case CK_ZeroToOCLEvent: + case CK_ZeroToOCLQueue: case CK_AddressSpaceConversion: + case CK_IntToOCLSampler: llvm_unreachable("cast kind invalid for aggregate types"); } } @@ -1049,7 +1053,8 @@ static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) { return true; // (int*)0 - Null pointer expressions. if (const CastExpr *ICE = dyn_cast<CastExpr>(E)) - return ICE->getCastKind() == CK_NullToPointer; + return ICE->getCastKind() == CK_NullToPointer && + CGF.getTypes().isPointerZeroInitializable(E->getType()); // '\0' if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E)) return CL->getValue() == 0; @@ -1144,15 +1149,15 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { if (E->hadArrayRangeDesignator()) CGF.ErrorUnsupported(E, "GNU array range designator extension"); + if (E->isTransparent()) + return Visit(E->getInit(0)); + AggValueSlot Dest = EnsureSlot(E->getType()); LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); // Handle initialization of an array. if (E->getType()->isArrayType()) { - if (E->isStringLiteralInit()) - return Visit(E->getInit(0)); - QualType elementType = CGF.getContext().getAsArrayType(E->getType())->getElementType(); @@ -1161,16 +1166,6 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { return; } - if (E->getType()->isAtomicType()) { - // An _Atomic(T) object can be list-initialized from an expression - // of the same type. - assert(E->getNumInits() == 1 && - CGF.getContext().hasSameUnqualifiedType(E->getInit(0)->getType(), - E->getType()) && - "unexpected list initialization for atomic object"); - return Visit(E->getInit(0)); - } - assert(E->getType()->isRecordType() && "Only support structs/unions here!"); // Do struct initialization; this code just sets each individual member @@ -1316,6 +1311,98 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { cleanupDominator->eraseFromParent(); } +void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, + llvm::Value *outerBegin) { + // Emit the common subexpression. + CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr()); + + Address destPtr = EnsureSlot(E->getType()).getAddress(); + uint64_t numElements = E->getArraySize().getZExtValue(); + + if (!numElements) + return; + + // destPtr is an array*. Construct an elementType* by drilling down a level. + llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); + llvm::Value *indices[] = {zero, zero}; + llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getPointer(), indices, + "arrayinit.begin"); + + // Prepare to special-case multidimensional array initialization: we avoid + // emitting multiple destructor loops in that case. + if (!outerBegin) + outerBegin = begin; + ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr()); + + QualType elementType = + CGF.getContext().getAsArrayType(E->getType())->getElementType(); + CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); + CharUnits elementAlign = + destPtr.getAlignment().alignmentOfArrayElement(elementSize); + + llvm::BasicBlock *entryBB = Builder.GetInsertBlock(); + llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body"); + + // Jump into the body. + CGF.EmitBlock(bodyBB); + llvm::PHINode *index = + Builder.CreatePHI(zero->getType(), 2, "arrayinit.index"); + index->addIncoming(zero, entryBB); + llvm::Value *element = Builder.CreateInBoundsGEP(begin, index); + + // Prepare for a cleanup. + QualType::DestructionKind dtorKind = elementType.isDestructedType(); + EHScopeStack::stable_iterator cleanup; + if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) { + if (outerBegin->getType() != element->getType()) + outerBegin = Builder.CreateBitCast(outerBegin, element->getType()); + CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType, + elementAlign, + CGF.getDestroyer(dtorKind)); + cleanup = CGF.EHStack.stable_begin(); + } else { + dtorKind = QualType::DK_none; + } + + // Emit the actual filler expression. + { + // Temporaries created in an array initialization loop are destroyed + // at the end of each iteration. + CodeGenFunction::RunCleanupsScope CleanupsScope(CGF); + CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index); + LValue elementLV = + CGF.MakeAddrLValue(Address(element, elementAlign), elementType); + + if (InnerLoop) { + // If the subexpression is an ArrayInitLoopExpr, share its cleanup. + auto elementSlot = AggValueSlot::forLValue( + elementLV, AggValueSlot::IsDestructed, + AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased); + AggExprEmitter(CGF, elementSlot, false) + .VisitArrayInitLoopExpr(InnerLoop, outerBegin); + } else + EmitInitializationToLValue(E->getSubExpr(), elementLV); + } + + // Move on to the next element. + llvm::Value *nextIndex = Builder.CreateNUWAdd( + index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next"); + index->addIncoming(nextIndex, Builder.GetInsertBlock()); + + // Leave the loop if we're done. + llvm::Value *done = Builder.CreateICmpEQ( + nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements), + "arrayinit.done"); + llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end"); + Builder.CreateCondBr(done, endBB, bodyBB); + + CGF.EmitBlock(endBB); + + // Leave the partial-array cleanup if we entered one. + if (dtorKind) + CGF.DeactivateCleanupBlock(cleanup, index); +} + void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { AggValueSlot Dest = EnsureSlot(E->getType()); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp index eec2ace..71c8fb8 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp @@ -28,25 +28,18 @@ static RequiredArgs commonEmitCXXMemberOrOperatorCall(CodeGenFunction &CGF, const CXXMethodDecl *MD, llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *CE, - CallArgList &Args) { + CallArgList &Args, CallArgList *RtlArgs) { assert(CE == nullptr || isa<CXXMemberCallExpr>(CE) || isa<CXXOperatorCallExpr>(CE)); assert(MD->isInstance() && "Trying to emit a member or operator call expr on a static method!"); - - // C++11 [class.mfct.non-static]p2: - // If a non-static member function of a class X is called for an object that - // is not of type X, or of a type derived from X, the behavior is undefined. - SourceLocation CallLoc; - if (CE) - CallLoc = CE->getExprLoc(); - CGF.EmitTypeCheck( - isa<CXXConstructorDecl>(MD) ? CodeGenFunction::TCK_ConstructorCall - : CodeGenFunction::TCK_MemberCall, - CallLoc, This, CGF.getContext().getRecordType(MD->getParent())); + ASTContext &C = CGF.getContext(); // Push the this ptr. - Args.add(RValue::get(This), MD->getThisType(CGF.getContext())); + const CXXRecordDecl *RD = + CGF.CGM.getCXXABI().getThisArgumentTypeForMethod(MD); + Args.add(RValue::get(This), + RD ? C.getPointerType(C.getTypeDeclType(RD)) : C.VoidPtrTy); // If there is an implicit parameter (e.g. VTT), emit it. if (ImplicitParam) { @@ -57,7 +50,12 @@ commonEmitCXXMemberOrOperatorCall(CodeGenFunction &CGF, const CXXMethodDecl *MD, RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, Args.size(), MD); // And the rest of the call args. - if (CE) { + if (RtlArgs) { + // Special case: if the caller emitted the arguments right-to-left already + // (prior to emitting the *this argument), we're done. This happens for + // assignment operators. + Args.addFrom(*RtlArgs); + } else if (CE) { // Special case: skip first argument of CXXOperatorCall (it is "this"). unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0; CGF.EmitCallArgs(Args, FPT, drop_begin(CE->arguments(), ArgsToSkip), @@ -71,26 +69,78 @@ commonEmitCXXMemberOrOperatorCall(CodeGenFunction &CGF, const CXXMethodDecl *MD, } RValue CodeGenFunction::EmitCXXMemberOrOperatorCall( - const CXXMethodDecl *MD, llvm::Value *Callee, ReturnValueSlot ReturnValue, + const CXXMethodDecl *MD, const CGCallee &Callee, + ReturnValueSlot ReturnValue, llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy, - const CallExpr *CE) { + const CallExpr *CE, CallArgList *RtlArgs) { const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>(); CallArgList Args; RequiredArgs required = commonEmitCXXMemberOrOperatorCall( - *this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args); - return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required), - Callee, ReturnValue, Args, MD); + *this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args, RtlArgs); + auto &FnInfo = CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required); + return EmitCall(FnInfo, Callee, ReturnValue, Args); } RValue CodeGenFunction::EmitCXXDestructorCall( - const CXXDestructorDecl *DD, llvm::Value *Callee, llvm::Value *This, + const CXXDestructorDecl *DD, const CGCallee &Callee, llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *CE, StructorType Type) { CallArgList Args; commonEmitCXXMemberOrOperatorCall(*this, DD, This, ImplicitParam, - ImplicitParamTy, CE, Args); + ImplicitParamTy, CE, Args, nullptr); return EmitCall(CGM.getTypes().arrangeCXXStructorDeclaration(DD, Type), - Callee, ReturnValueSlot(), Args, DD); + Callee, ReturnValueSlot(), Args); +} + +RValue CodeGenFunction::EmitCXXPseudoDestructorExpr( + const CXXPseudoDestructorExpr *E) { + QualType DestroyedType = E->getDestroyedType(); + if (DestroyedType.hasStrongOrWeakObjCLifetime()) { + // Automatic Reference Counting: + // If the pseudo-expression names a retainable object with weak or + // strong lifetime, the object shall be released. + Expr *BaseExpr = E->getBase(); + Address BaseValue = Address::invalid(); + Qualifiers BaseQuals; + + // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar. + if (E->isArrow()) { + BaseValue = EmitPointerWithAlignment(BaseExpr); + const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>(); + BaseQuals = PTy->getPointeeType().getQualifiers(); + } else { + LValue BaseLV = EmitLValue(BaseExpr); + BaseValue = BaseLV.getAddress(); + QualType BaseTy = BaseExpr->getType(); + BaseQuals = BaseTy.getQualifiers(); + } + + switch (DestroyedType.getObjCLifetime()) { + case Qualifiers::OCL_None: + case Qualifiers::OCL_ExplicitNone: + case Qualifiers::OCL_Autoreleasing: + break; + + case Qualifiers::OCL_Strong: + EmitARCRelease(Builder.CreateLoad(BaseValue, + DestroyedType.isVolatileQualified()), + ARCPreciseLifetime); + break; + + case Qualifiers::OCL_Weak: + EmitARCDestroyWeak(BaseValue); + break; + } + } else { + // C++ [expr.pseudo]p1: + // The result shall only be used as the operand for the function call + // operator (), and the result of such a call has type void. The only + // effect is the evaluation of the postfix-expression before the dot or + // arrow. + EmitIgnoredExpr(E->getBase()); + } + + return RValue::get(nullptr); } static CXXRecordDecl *getCXXRecord(const Expr *E) { @@ -115,8 +165,8 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE, if (MD->isStatic()) { // The method is static, emit it as we would a regular call. - llvm::Value *Callee = CGM.GetAddrOfFunction(MD); - return EmitCall(getContext().getPointerType(MD->getType()), Callee, CE, + CGCallee callee = CGCallee::forDirect(CGM.GetAddrOfFunction(MD), MD); + return EmitCall(getContext().getPointerType(MD->getType()), callee, CE, ReturnValue); } @@ -166,6 +216,19 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( } } + // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment + // operator before the LHS. + CallArgList RtlArgStorage; + CallArgList *RtlArgs = nullptr; + if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(CE)) { + if (OCE->isAssignmentOp()) { + RtlArgs = &RtlArgStorage; + EmitCallArgs(*RtlArgs, MD->getType()->castAs<FunctionProtoType>(), + drop_begin(CE->arguments(), 1), CE->getDirectCallee(), + /*ParamsToSkip*/0, EvaluationOrder::ForceRightToLeft); + } + } + Address This = Address::invalid(); if (IsArrow) This = EmitPointerWithAlignment(Base); @@ -183,10 +246,12 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) { // We don't like to generate the trivial copy/move assignment operator // when it isn't necessary; just produce the proper effect here. - // Special case: skip first argument of CXXOperatorCall (it is "this"). - unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0; - Address RHS = EmitLValue(*(CE->arg_begin() + ArgsToSkip)).getAddress(); - EmitAggregateAssign(This, RHS, CE->getType()); + LValue RHS = isa<CXXOperatorCallExpr>(CE) + ? MakeNaturalAlignAddrLValue( + (*RtlArgs)[0].RV.getScalarVal(), + (*(CE->arg_begin() + 1))->getType()) + : EmitLValue(*CE->arg_begin()); + EmitAggregateAssign(This, RHS.getAddress(), CE->getType()); return RValue::get(This.getPointer()); } @@ -217,6 +282,22 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( llvm::FunctionType *Ty = CGM.getTypes().GetFunctionType(*FInfo); + // C++11 [class.mfct.non-static]p2: + // If a non-static member function of a class X is called for an object that + // is not of type X, or of a type derived from X, the behavior is undefined. + SourceLocation CallLoc; + ASTContext &C = getContext(); + if (CE) + CallLoc = CE->getExprLoc(); + + EmitTypeCheck(isa<CXXConstructorDecl>(CalleeDecl) + ? CodeGenFunction::TCK_ConstructorCall + : CodeGenFunction::TCK_MemberCall, + CallLoc, This.getPointer(), C.getRecordType(CalleeDecl->getParent())); + + // FIXME: Uses of 'MD' past this point need to be audited. We may need to use + // 'CalleeDecl' instead. + // C++ [class.virtual]p12: // Explicit qualification with the scope operator (5.1) suppresses the // virtual call mechanism. @@ -224,8 +305,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( // We also don't emit a virtual call if the base expression has a record type // because then we know what the type is. bool UseVirtualCall = CanUseVirtualCall && !DevirtualizedMethod; - llvm::Value *Callee; - + if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(MD)) { assert(CE->arg_begin() == CE->arg_end() && "Destructor shouldn't have explicit parameters"); @@ -234,24 +314,32 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( CGM.getCXXABI().EmitVirtualDestructorCall( *this, Dtor, Dtor_Complete, This, cast<CXXMemberCallExpr>(CE)); } else { + CGCallee Callee; if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier) Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty); else if (!DevirtualizedMethod) - Callee = - CGM.getAddrOfCXXStructor(Dtor, StructorType::Complete, FInfo, Ty); + Callee = CGCallee::forDirect( + CGM.getAddrOfCXXStructor(Dtor, StructorType::Complete, FInfo, Ty), + Dtor); else { const CXXDestructorDecl *DDtor = cast<CXXDestructorDecl>(DevirtualizedMethod); - Callee = CGM.GetAddrOfFunction(GlobalDecl(DDtor, Dtor_Complete), Ty); + Callee = CGCallee::forDirect( + CGM.GetAddrOfFunction(GlobalDecl(DDtor, Dtor_Complete), Ty), + DDtor); } - EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This.getPointer(), - /*ImplicitParam=*/nullptr, QualType(), CE); + EmitCXXMemberOrOperatorCall( + CalleeDecl, Callee, ReturnValue, This.getPointer(), + /*ImplicitParam=*/nullptr, QualType(), CE, nullptr); } return RValue::get(nullptr); } + CGCallee Callee; if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) { - Callee = CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty); + Callee = CGCallee::forDirect( + CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty), + Ctor); } else if (UseVirtualCall) { Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty, CE->getLocStart()); @@ -266,9 +354,11 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier) Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty); else if (!DevirtualizedMethod) - Callee = CGM.GetAddrOfFunction(MD, Ty); + Callee = CGCallee::forDirect(CGM.GetAddrOfFunction(MD, Ty), MD); else { - Callee = CGM.GetAddrOfFunction(DevirtualizedMethod, Ty); + Callee = CGCallee::forDirect( + CGM.GetAddrOfFunction(DevirtualizedMethod, Ty), + DevirtualizedMethod); } } @@ -277,8 +367,9 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( *this, CalleeDecl, This, UseVirtualCall); } - return EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This.getPointer(), - /*ImplicitParam=*/nullptr, QualType(), CE); + return EmitCXXMemberOrOperatorCall( + CalleeDecl, Callee, ReturnValue, This.getPointer(), + /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs); } RValue @@ -297,9 +388,6 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, const CXXRecordDecl *RD = cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl()); - // Get the member function pointer. - llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr); - // Emit the 'this' pointer. Address This = Address::invalid(); if (BO->getOpcode() == BO_PtrMemI) @@ -310,9 +398,12 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), QualType(MPT->getClass(), 0)); + // Get the member function pointer. + llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr); + // Ask the ABI to load the callee. Note that This is modified. llvm::Value *ThisPtrForCall = nullptr; - llvm::Value *Callee = + CGCallee Callee = CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This, ThisPtrForCall, MemFnPtr, MPT); @@ -851,8 +942,68 @@ void CodeGenFunction::EmitNewArrayInitializer( CharUnits ElementAlign = BeginPtr.getAlignment().alignmentOfArrayElement(ElementSize); + // Attempt to perform zero-initialization using memset. + auto TryMemsetInitialization = [&]() -> bool { + // FIXME: If the type is a pointer-to-data-member under the Itanium ABI, + // we can initialize with a memset to -1. + if (!CGM.getTypes().isZeroInitializable(ElementType)) + return false; + + // Optimization: since zero initialization will just set the memory + // to all zeroes, generate a single memset to do it in one shot. + + // Subtract out the size of any elements we've already initialized. + auto *RemainingSize = AllocSizeWithoutCookie; + if (InitListElements) { + // We know this can't overflow; we check this when doing the allocation. + auto *InitializedSize = llvm::ConstantInt::get( + RemainingSize->getType(), + getContext().getTypeSizeInChars(ElementType).getQuantity() * + InitListElements); + RemainingSize = Builder.CreateSub(RemainingSize, InitializedSize); + } + + // Create the memset. + Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, false); + return true; + }; + // If the initializer is an initializer list, first do the explicit elements. if (const InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) { + // Initializing from a (braced) string literal is a special case; the init + // list element does not initialize a (single) array element. + if (ILE->isStringLiteralInit()) { + // Initialize the initial portion of length equal to that of the string + // literal. The allocation must be for at least this much; we emitted a + // check for that earlier. + AggValueSlot Slot = + AggValueSlot::forAddr(CurPtr, ElementType.getQualifiers(), + AggValueSlot::IsDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased); + EmitAggExpr(ILE->getInit(0), Slot); + + // Move past these elements. + InitListElements = + cast<ConstantArrayType>(ILE->getType()->getAsArrayTypeUnsafe()) + ->getSize().getZExtValue(); + CurPtr = + Address(Builder.CreateInBoundsGEP(CurPtr.getPointer(), + Builder.getSize(InitListElements), + "string.init.end"), + CurPtr.getAlignment().alignmentAtOffset(InitListElements * + ElementSize)); + + // Zero out the rest, if any remain. + llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements); + if (!ConstNum || !ConstNum->equalsInt(InitListElements)) { + bool OK = TryMemsetInitialization(); + (void)OK; + assert(OK && "couldn't memset character type?"); + } + return; + } + InitListElements = ILE->getNumInits(); // If this is a multi-dimensional array new, we will initialize multiple @@ -919,32 +1070,6 @@ void CodeGenFunction::EmitNewArrayInitializer( CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr.getType()); } - // Attempt to perform zero-initialization using memset. - auto TryMemsetInitialization = [&]() -> bool { - // FIXME: If the type is a pointer-to-data-member under the Itanium ABI, - // we can initialize with a memset to -1. - if (!CGM.getTypes().isZeroInitializable(ElementType)) - return false; - - // Optimization: since zero initialization will just set the memory - // to all zeroes, generate a single memset to do it in one shot. - - // Subtract out the size of any elements we've already initialized. - auto *RemainingSize = AllocSizeWithoutCookie; - if (InitListElements) { - // We know this can't overflow; we check this when doing the allocation. - auto *InitializedSize = llvm::ConstantInt::get( - RemainingSize->getType(), - getContext().getTypeSizeInChars(ElementType).getQuantity() * - InitListElements); - RemainingSize = Builder.CreateSub(RemainingSize, InitializedSize); - } - - // Create the memset. - Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, false); - return true; - }; - // If all elements have already been initialized, skip any further // initialization. llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements); @@ -1110,23 +1235,24 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E, /// Emit a call to an operator new or operator delete function, as implicitly /// created by new-expressions and delete-expressions. static RValue EmitNewDeleteCall(CodeGenFunction &CGF, - const FunctionDecl *Callee, + const FunctionDecl *CalleeDecl, const FunctionProtoType *CalleeType, const CallArgList &Args) { llvm::Instruction *CallOrInvoke; - llvm::Value *CalleeAddr = CGF.CGM.GetAddrOfFunction(Callee); + llvm::Constant *CalleePtr = CGF.CGM.GetAddrOfFunction(CalleeDecl); + CGCallee Callee = CGCallee::forDirect(CalleePtr, CalleeDecl); RValue RV = CGF.EmitCall(CGF.CGM.getTypes().arrangeFreeFunctionCall( Args, CalleeType, /*chainCall=*/false), - CalleeAddr, ReturnValueSlot(), Args, Callee, &CallOrInvoke); + Callee, ReturnValueSlot(), Args, &CallOrInvoke); /// C++1y [expr.new]p10: /// [In a new-expression,] an implementation is allowed to omit a call /// to a replaceable global allocation function. /// /// We model such elidable calls with the 'builtin' attribute. - llvm::Function *Fn = dyn_cast<llvm::Function>(CalleeAddr); - if (Callee->isReplaceableGlobalAllocationFunction() && + llvm::Function *Fn = dyn_cast<llvm::Function>(CalleePtr); + if (CalleeDecl->isReplaceableGlobalAllocationFunction() && Fn && Fn->hasFnAttribute(llvm::Attribute::NoBuiltin)) { // FIXME: Add addAttribute to CallSite. if (llvm::CallInst *CI = dyn_cast<llvm::CallInst>(CallOrInvoke)) @@ -1159,111 +1285,116 @@ RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, llvm_unreachable("predeclared global operator new/delete is missing"); } -namespace { - /// A cleanup to call the given 'operator delete' function upon - /// abnormal exit from a new expression. - class CallDeleteDuringNew final : public EHScopeStack::Cleanup { - size_t NumPlacementArgs; - const FunctionDecl *OperatorDelete; - llvm::Value *Ptr; - llvm::Value *AllocSize; - - RValue *getPlacementArgs() { return reinterpret_cast<RValue*>(this+1); } +static std::pair<bool, bool> +shouldPassSizeAndAlignToUsualDelete(const FunctionProtoType *FPT) { + auto AI = FPT->param_type_begin(), AE = FPT->param_type_end(); - public: - static size_t getExtraSize(size_t NumPlacementArgs) { - return NumPlacementArgs * sizeof(RValue); - } + // The first argument is always a void*. + ++AI; - CallDeleteDuringNew(size_t NumPlacementArgs, - const FunctionDecl *OperatorDelete, - llvm::Value *Ptr, - llvm::Value *AllocSize) - : NumPlacementArgs(NumPlacementArgs), OperatorDelete(OperatorDelete), - Ptr(Ptr), AllocSize(AllocSize) {} - - void setPlacementArg(unsigned I, RValue Arg) { - assert(I < NumPlacementArgs && "index out of range"); - getPlacementArgs()[I] = Arg; - } - - void Emit(CodeGenFunction &CGF, Flags flags) override { - const FunctionProtoType *FPT - = OperatorDelete->getType()->getAs<FunctionProtoType>(); - assert(FPT->getNumParams() == NumPlacementArgs + 1 || - (FPT->getNumParams() == 2 && NumPlacementArgs == 0)); + // Figure out what other parameters we should be implicitly passing. + bool PassSize = false; + bool PassAlignment = false; - CallArgList DeleteArgs; - - // The first argument is always a void*. - FunctionProtoType::param_type_iterator AI = FPT->param_type_begin(); - DeleteArgs.add(RValue::get(Ptr), *AI++); - - // A member 'operator delete' can take an extra 'size_t' argument. - if (FPT->getNumParams() == NumPlacementArgs + 2) - DeleteArgs.add(RValue::get(AllocSize), *AI++); + if (AI != AE && (*AI)->isIntegerType()) { + PassSize = true; + ++AI; + } - // Pass the rest of the arguments, which must match exactly. - for (unsigned I = 0; I != NumPlacementArgs; ++I) - DeleteArgs.add(getPlacementArgs()[I], *AI++); + if (AI != AE && (*AI)->isAlignValT()) { + PassAlignment = true; + ++AI; + } - // Call 'operator delete'. - EmitNewDeleteCall(CGF, OperatorDelete, FPT, DeleteArgs); - } - }; + assert(AI == AE && "unexpected usual deallocation function parameter"); + return {PassSize, PassAlignment}; +} - /// A cleanup to call the given 'operator delete' function upon - /// abnormal exit from a new expression when the new expression is - /// conditional. - class CallDeleteDuringConditionalNew final : public EHScopeStack::Cleanup { - size_t NumPlacementArgs; +namespace { + /// A cleanup to call the given 'operator delete' function upon abnormal + /// exit from a new expression. Templated on a traits type that deals with + /// ensuring that the arguments dominate the cleanup if necessary. + template<typename Traits> + class CallDeleteDuringNew final : public EHScopeStack::Cleanup { + /// Type used to hold llvm::Value*s. + typedef typename Traits::ValueTy ValueTy; + /// Type used to hold RValues. + typedef typename Traits::RValueTy RValueTy; + struct PlacementArg { + RValueTy ArgValue; + QualType ArgType; + }; + + unsigned NumPlacementArgs : 31; + unsigned PassAlignmentToPlacementDelete : 1; const FunctionDecl *OperatorDelete; - DominatingValue<RValue>::saved_type Ptr; - DominatingValue<RValue>::saved_type AllocSize; + ValueTy Ptr; + ValueTy AllocSize; + CharUnits AllocAlign; - DominatingValue<RValue>::saved_type *getPlacementArgs() { - return reinterpret_cast<DominatingValue<RValue>::saved_type*>(this+1); + PlacementArg *getPlacementArgs() { + return reinterpret_cast<PlacementArg *>(this + 1); } public: static size_t getExtraSize(size_t NumPlacementArgs) { - return NumPlacementArgs * sizeof(DominatingValue<RValue>::saved_type); + return NumPlacementArgs * sizeof(PlacementArg); } - CallDeleteDuringConditionalNew(size_t NumPlacementArgs, - const FunctionDecl *OperatorDelete, - DominatingValue<RValue>::saved_type Ptr, - DominatingValue<RValue>::saved_type AllocSize) - : NumPlacementArgs(NumPlacementArgs), OperatorDelete(OperatorDelete), - Ptr(Ptr), AllocSize(AllocSize) {} - - void setPlacementArg(unsigned I, DominatingValue<RValue>::saved_type Arg) { + CallDeleteDuringNew(size_t NumPlacementArgs, + const FunctionDecl *OperatorDelete, ValueTy Ptr, + ValueTy AllocSize, bool PassAlignmentToPlacementDelete, + CharUnits AllocAlign) + : NumPlacementArgs(NumPlacementArgs), + PassAlignmentToPlacementDelete(PassAlignmentToPlacementDelete), + OperatorDelete(OperatorDelete), Ptr(Ptr), AllocSize(AllocSize), + AllocAlign(AllocAlign) {} + + void setPlacementArg(unsigned I, RValueTy Arg, QualType Type) { assert(I < NumPlacementArgs && "index out of range"); - getPlacementArgs()[I] = Arg; + getPlacementArgs()[I] = {Arg, Type}; } void Emit(CodeGenFunction &CGF, Flags flags) override { - const FunctionProtoType *FPT - = OperatorDelete->getType()->getAs<FunctionProtoType>(); - assert(FPT->getNumParams() == NumPlacementArgs + 1 || - (FPT->getNumParams() == 2 && NumPlacementArgs == 0)); - + const FunctionProtoType *FPT = + OperatorDelete->getType()->getAs<FunctionProtoType>(); CallArgList DeleteArgs; // The first argument is always a void*. - FunctionProtoType::param_type_iterator AI = FPT->param_type_begin(); - DeleteArgs.add(Ptr.restore(CGF), *AI++); - - // A member 'operator delete' can take an extra 'size_t' argument. - if (FPT->getNumParams() == NumPlacementArgs + 2) { - RValue RV = AllocSize.restore(CGF); - DeleteArgs.add(RV, *AI++); + DeleteArgs.add(Traits::get(CGF, Ptr), FPT->getParamType(0)); + + // Figure out what other parameters we should be implicitly passing. + bool PassSize = false; + bool PassAlignment = false; + if (NumPlacementArgs) { + // A placement deallocation function is implicitly passed an alignment + // if the placement allocation function was, but is never passed a size. + PassAlignment = PassAlignmentToPlacementDelete; + } else { + // For a non-placement new-expression, 'operator delete' can take a + // size and/or an alignment if it has the right parameters. + std::tie(PassSize, PassAlignment) = + shouldPassSizeAndAlignToUsualDelete(FPT); } + // The second argument can be a std::size_t (for non-placement delete). + if (PassSize) + DeleteArgs.add(Traits::get(CGF, AllocSize), + CGF.getContext().getSizeType()); + + // The next (second or third) argument can be a std::align_val_t, which + // is an enum whose underlying type is std::size_t. + // FIXME: Use the right type as the parameter type. Note that in a call + // to operator delete(size_t, ...), we may not have it available. + if (PassAlignment) + DeleteArgs.add(RValue::get(llvm::ConstantInt::get( + CGF.SizeTy, AllocAlign.getQuantity())), + CGF.getContext().getSizeType()); + // Pass the rest of the arguments, which must match exactly. for (unsigned I = 0; I != NumPlacementArgs; ++I) { - RValue RV = getPlacementArgs()[I].restore(CGF); - DeleteArgs.add(RV, *AI++); + auto Arg = getPlacementArgs()[I]; + DeleteArgs.add(Traits::get(CGF, Arg.ArgValue), Arg.ArgType); } // Call 'operator delete'. @@ -1278,18 +1409,34 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, const CXXNewExpr *E, Address NewPtr, llvm::Value *AllocSize, + CharUnits AllocAlign, const CallArgList &NewArgs) { + unsigned NumNonPlacementArgs = E->passAlignment() ? 2 : 1; + // If we're not inside a conditional branch, then the cleanup will // dominate and we can do the easier (and more efficient) thing. if (!CGF.isInConditionalBranch()) { - CallDeleteDuringNew *Cleanup = CGF.EHStack - .pushCleanupWithExtra<CallDeleteDuringNew>(EHCleanup, - E->getNumPlacementArgs(), - E->getOperatorDelete(), - NewPtr.getPointer(), - AllocSize); - for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) - Cleanup->setPlacementArg(I, NewArgs[I+1].RV); + struct DirectCleanupTraits { + typedef llvm::Value *ValueTy; + typedef RValue RValueTy; + static RValue get(CodeGenFunction &, ValueTy V) { return RValue::get(V); } + static RValue get(CodeGenFunction &, RValueTy V) { return V; } + }; + + typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup; + + DirectCleanup *Cleanup = CGF.EHStack + .pushCleanupWithExtra<DirectCleanup>(EHCleanup, + E->getNumPlacementArgs(), + E->getOperatorDelete(), + NewPtr.getPointer(), + AllocSize, + E->passAlignment(), + AllocAlign); + for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) { + auto &Arg = NewArgs[I + NumNonPlacementArgs]; + Cleanup->setPlacementArg(I, Arg.RV, Arg.Ty); + } return; } @@ -1300,15 +1447,28 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, DominatingValue<RValue>::saved_type SavedAllocSize = DominatingValue<RValue>::save(CGF, RValue::get(AllocSize)); - CallDeleteDuringConditionalNew *Cleanup = CGF.EHStack - .pushCleanupWithExtra<CallDeleteDuringConditionalNew>(EHCleanup, - E->getNumPlacementArgs(), - E->getOperatorDelete(), - SavedNewPtr, - SavedAllocSize); - for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) - Cleanup->setPlacementArg(I, - DominatingValue<RValue>::save(CGF, NewArgs[I+1].RV)); + struct ConditionalCleanupTraits { + typedef DominatingValue<RValue>::saved_type ValueTy; + typedef DominatingValue<RValue>::saved_type RValueTy; + static RValue get(CodeGenFunction &CGF, ValueTy V) { + return V.restore(CGF); + } + }; + typedef CallDeleteDuringNew<ConditionalCleanupTraits> ConditionalCleanup; + + ConditionalCleanup *Cleanup = CGF.EHStack + .pushCleanupWithExtra<ConditionalCleanup>(EHCleanup, + E->getNumPlacementArgs(), + E->getOperatorDelete(), + SavedNewPtr, + SavedAllocSize, + E->passAlignment(), + AllocAlign); + for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) { + auto &Arg = NewArgs[I + NumNonPlacementArgs]; + Cleanup->setPlacementArg(I, DominatingValue<RValue>::save(CGF, Arg.RV), + Arg.Ty); + } CGF.initFullExprCleanup(); } @@ -1323,7 +1483,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // If there is a brace-initializer, cannot allocate fewer elements than inits. unsigned minElements = 0; if (E->isArray() && E->hasInitializer()) { - if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E->getInitializer())) + const InitListExpr *ILE = dyn_cast<InitListExpr>(E->getInitializer()); + if (ILE && ILE->isStringLiteralInit()) + minElements = + cast<ConstantArrayType>(ILE->getType()->getAsArrayTypeUnsafe()) + ->getSize().getZExtValue(); + else if (ILE) minElements = ILE->getNumInits(); } @@ -1332,6 +1497,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::Value *allocSize = EmitCXXNewAllocSize(*this, E, minElements, numElements, allocSizeWithoutCookie); + CharUnits allocAlign = getContext().getTypeAlignInChars(allocType); // Emit the allocation call. If the allocator is a global placement // operator, just "inline" it directly. @@ -1347,10 +1513,8 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // The pointer expression will, in many cases, be an opaque void*. // In these cases, discard the computed alignment and use the // formal alignment of the allocated type. - if (alignSource != AlignmentSource::Decl) { - allocation = Address(allocation.getPointer(), - getContext().getTypeAlignInChars(allocType)); - } + if (alignSource != AlignmentSource::Decl) + allocation = Address(allocation.getPointer(), allocAlign); // Set up allocatorArgs for the call to operator delete if it's not // the reserved global operator. @@ -1363,28 +1527,55 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { } else { const FunctionProtoType *allocatorType = allocator->getType()->castAs<FunctionProtoType>(); + unsigned ParamsToSkip = 0; // The allocation size is the first argument. QualType sizeType = getContext().getSizeType(); allocatorArgs.add(RValue::get(allocSize), sizeType); + ++ParamsToSkip; + + if (allocSize != allocSizeWithoutCookie) { + CharUnits cookieAlign = getSizeAlign(); // FIXME: Ask the ABI. + allocAlign = std::max(allocAlign, cookieAlign); + } + + // The allocation alignment may be passed as the second argument. + if (E->passAlignment()) { + QualType AlignValT = sizeType; + if (allocatorType->getNumParams() > 1) { + AlignValT = allocatorType->getParamType(1); + assert(getContext().hasSameUnqualifiedType( + AlignValT->castAs<EnumType>()->getDecl()->getIntegerType(), + sizeType) && + "wrong type for alignment parameter"); + ++ParamsToSkip; + } else { + // Corner case, passing alignment to 'operator new(size_t, ...)'. + assert(allocator->isVariadic() && "can't pass alignment to allocator"); + } + allocatorArgs.add( + RValue::get(llvm::ConstantInt::get(SizeTy, allocAlign.getQuantity())), + AlignValT); + } - // We start at 1 here because the first argument (the allocation size) - // has already been emitted. + // FIXME: Why do we not pass a CalleeDecl here? EmitCallArgs(allocatorArgs, allocatorType, E->placement_arguments(), - /* CalleeDecl */ nullptr, - /*ParamsToSkip*/ 1); + /*CalleeDecl*/nullptr, /*ParamsToSkip*/ParamsToSkip); RValue RV = EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs); - // For now, only assume that the allocation function returns - // something satisfactorily aligned for the element type, plus - // the cookie if we have one. - CharUnits allocationAlign = - getContext().getTypeAlignInChars(allocType); - if (allocSize != allocSizeWithoutCookie) { - CharUnits cookieAlign = getSizeAlign(); // FIXME? - allocationAlign = std::max(allocationAlign, cookieAlign); + // If this was a call to a global replaceable allocation function that does + // not take an alignment argument, the allocator is known to produce + // storage that's suitably aligned for any object that fits, up to a known + // threshold. Otherwise assume it's suitably aligned for the allocated type. + CharUnits allocationAlign = allocAlign; + if (!E->passAlignment() && + allocator->isReplaceableGlobalAllocationFunction()) { + unsigned AllocatorAlign = llvm::PowerOf2Floor(std::min<uint64_t>( + Target.getNewAlign(), getContext().getTypeSize(allocType))); + allocationAlign = std::max( + allocationAlign, getContext().toCharUnitsFromBits(AllocatorAlign)); } allocation = Address(RV.getScalarVal(), allocationAlign); @@ -1423,7 +1614,8 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::Instruction *cleanupDominator = nullptr; if (E->getOperatorDelete() && !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) { - EnterNewDeleteCleanup(*this, E, allocation, allocSize, allocatorArgs); + EnterNewDeleteCleanup(*this, E, allocation, allocSize, allocAlign, + allocatorArgs); operatorDeleteCleanup = EHStack.stable_begin(); cleanupDominator = Builder.CreateUnreachable(); } @@ -1485,31 +1677,58 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { } void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD, - llvm::Value *Ptr, - QualType DeleteTy) { - assert(DeleteFD->getOverloadedOperator() == OO_Delete); + llvm::Value *Ptr, QualType DeleteTy, + llvm::Value *NumElements, + CharUnits CookieSize) { + assert((!NumElements && CookieSize.isZero()) || + DeleteFD->getOverloadedOperator() == OO_Array_Delete); const FunctionProtoType *DeleteFTy = DeleteFD->getType()->getAs<FunctionProtoType>(); CallArgList DeleteArgs; - // Check if we need to pass the size to the delete operator. - llvm::Value *Size = nullptr; - QualType SizeTy; - if (DeleteFTy->getNumParams() == 2) { - SizeTy = DeleteFTy->getParamType(1); - CharUnits DeleteTypeSize = getContext().getTypeSizeInChars(DeleteTy); - Size = llvm::ConstantInt::get(ConvertType(SizeTy), - DeleteTypeSize.getQuantity()); - } + std::pair<bool, bool> PassSizeAndAlign = + shouldPassSizeAndAlignToUsualDelete(DeleteFTy); + + auto ParamTypeIt = DeleteFTy->param_type_begin(); - QualType ArgTy = DeleteFTy->getParamType(0); + // Pass the pointer itself. + QualType ArgTy = *ParamTypeIt++; llvm::Value *DeletePtr = Builder.CreateBitCast(Ptr, ConvertType(ArgTy)); DeleteArgs.add(RValue::get(DeletePtr), ArgTy); - if (Size) - DeleteArgs.add(RValue::get(Size), SizeTy); + // Pass the size if the delete function has a size_t parameter. + if (PassSizeAndAlign.first) { + QualType SizeType = *ParamTypeIt++; + CharUnits DeleteTypeSize = getContext().getTypeSizeInChars(DeleteTy); + llvm::Value *Size = llvm::ConstantInt::get(ConvertType(SizeType), + DeleteTypeSize.getQuantity()); + + // For array new, multiply by the number of elements. + if (NumElements) + Size = Builder.CreateMul(Size, NumElements); + + // If there is a cookie, add the cookie size. + if (!CookieSize.isZero()) + Size = Builder.CreateAdd( + Size, llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity())); + + DeleteArgs.add(RValue::get(Size), SizeType); + } + + // Pass the alignment if the delete function has an align_val_t parameter. + if (PassSizeAndAlign.second) { + QualType AlignValType = *ParamTypeIt++; + CharUnits DeleteTypeAlign = getContext().toCharUnitsFromBits( + getContext().getTypeAlignIfKnown(DeleteTy)); + llvm::Value *Align = llvm::ConstantInt::get(ConvertType(AlignValType), + DeleteTypeAlign.getQuantity()); + DeleteArgs.add(RValue::get(Align), AlignValType); + } + + assert(ParamTypeIt == DeleteFTy->param_type_end() && + "unknown parameter to usual delete function"); // Emit the call to delete. EmitNewDeleteCall(*this, DeleteFD, DeleteFTy, DeleteArgs); @@ -1546,6 +1765,15 @@ static void EmitObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, Address Ptr, QualType ElementType) { + // C++11 [expr.delete]p3: + // If the static type of the object to be deleted is different from its + // dynamic type, the static type shall be a base class of the dynamic type + // of the object to be deleted and the static type shall have a virtual + // destructor or the behavior is undefined. + CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, + DE->getExprLoc(), Ptr.getPointer(), + ElementType); + // Find the destructor for the type, if applicable. If the // destructor is virtual, we'll just emit the vcall and return. const CXXDestructorDecl *Dtor = nullptr; @@ -1613,45 +1841,8 @@ namespace { ElementType(ElementType), CookieSize(CookieSize) {} void Emit(CodeGenFunction &CGF, Flags flags) override { - const FunctionProtoType *DeleteFTy = - OperatorDelete->getType()->getAs<FunctionProtoType>(); - assert(DeleteFTy->getNumParams() == 1 || DeleteFTy->getNumParams() == 2); - - CallArgList Args; - - // Pass the pointer as the first argument. - QualType VoidPtrTy = DeleteFTy->getParamType(0); - llvm::Value *DeletePtr - = CGF.Builder.CreateBitCast(Ptr, CGF.ConvertType(VoidPtrTy)); - Args.add(RValue::get(DeletePtr), VoidPtrTy); - - // Pass the original requested size as the second argument. - if (DeleteFTy->getNumParams() == 2) { - QualType size_t = DeleteFTy->getParamType(1); - llvm::IntegerType *SizeTy - = cast<llvm::IntegerType>(CGF.ConvertType(size_t)); - - CharUnits ElementTypeSize = - CGF.CGM.getContext().getTypeSizeInChars(ElementType); - - // The size of an element, multiplied by the number of elements. - llvm::Value *Size - = llvm::ConstantInt::get(SizeTy, ElementTypeSize.getQuantity()); - if (NumElements) - Size = CGF.Builder.CreateMul(Size, NumElements); - - // Plus the size of the cookie if applicable. - if (!CookieSize.isZero()) { - llvm::Value *CookieSizeV - = llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity()); - Size = CGF.Builder.CreateAdd(Size, CookieSizeV); - } - - Args.add(RValue::get(Size), size_t); - } - - // Emit the call to delete. - EmitNewDeleteCall(CGF, OperatorDelete, DeleteFTy, Args); + CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType, NumElements, + CookieSize); } }; } @@ -1949,10 +2140,7 @@ void CodeGenFunction::EmitLambdaExpr(const LambdaExpr *E, AggValueSlot Slot) { auto VAT = CurField->getCapturedVLAType(); EmitStoreThroughLValue(RValue::get(VLASizeMap[VAT->getSizeExpr()]), LV); } else { - ArrayRef<VarDecl *> ArrayIndexes; - if (CurField->getType()->isArrayType()) - ArrayIndexes = E->getCaptureInitIndexVars(i); - EmitInitializerForField(*CurField, LV, *i, ArrayIndexes); + EmitInitializerForField(*CurField, LV, *i); } } } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp index 22910d9..59bc9cd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp @@ -13,12 +13,9 @@ #include "CodeGenFunction.h" #include "CodeGenModule.h" -#include "clang/AST/ASTContext.h" #include "clang/AST/StmtVisitor.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallString.h" #include "llvm/IR/Constants.h" -#include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" @@ -483,7 +480,9 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, case CK_CopyAndAutoreleaseBlockObject: case CK_BuiltinFnToFnPtr: case CK_ZeroToOCLEvent: + case CK_ZeroToOCLQueue: case CK_AddressSpaceConversion: + case CK_IntToOCLSampler: llvm_unreachable("invalid cast kind for complex value"); case CK_FloatingRealToComplex: @@ -600,10 +599,10 @@ ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName, llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo); llvm::Constant *Func = CGF.CGM.CreateBuiltinFunction(FTy, LibCallName); - llvm::Instruction *Call; + CGCallee Callee = CGCallee::forDirect(Func, FQTy->getAs<FunctionProtoType>()); - RValue Res = CGF.EmitCall(FuncInfo, Func, ReturnValueSlot(), Args, - FQTy->getAs<FunctionProtoType>(), &Call); + llvm::Instruction *Call; + RValue Res = CGF.EmitCall(FuncInfo, Callee, ReturnValueSlot(), Args, &Call); cast<llvm::CallInst>(Call)->setCallingConv(CGF.CGM.getBuiltinCC()); return Res.getComplexVal(); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp index 803b399..3db15c6 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp @@ -16,6 +16,7 @@ #include "CGObjCRuntime.h" #include "CGRecordLayout.h" #include "CodeGenModule.h" +#include "TargetInfo.h" #include "clang/AST/APValue.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RecordLayout.h" @@ -690,6 +691,9 @@ public: case CK_ConstructorConversion: return C; + case CK_IntToOCLSampler: + llvm_unreachable("global sampler variables are not generated"); + case CK_Dependent: llvm_unreachable("saw dependent cast!"); case CK_BuiltinFnToFnPtr: @@ -749,6 +753,7 @@ public: case CK_FloatingToBoolean: case CK_FloatingCast: case CK_ZeroToOCLEvent: + case CK_ZeroToOCLQueue: return nullptr; } llvm_unreachable("Invalid CastKind"); @@ -775,9 +780,6 @@ public: } llvm::Constant *EmitArrayInitialization(InitListExpr *ILE) { - if (ILE->isStringLiteralInit()) - return Visit(ILE->getInit(0)); - llvm::ArrayType *AType = cast<llvm::ArrayType>(ConvertType(ILE->getType())); llvm::Type *ElemTy = AType->getElementType(); @@ -842,6 +844,9 @@ public: } llvm::Constant *VisitInitListExpr(InitListExpr *ILE) { + if (ILE->isTransparent()) + return Visit(ILE->getInit(0)); + if (ILE->getType()->isArrayType()) return EmitArrayInitialization(ILE); @@ -1018,16 +1023,17 @@ public: switch (E->getStmtClass()) { default: break; case Expr::CompoundLiteralExprClass: { - // Note that due to the nature of compound literals, this is guaranteed - // to be the only use of the variable, so we just generate it here. CompoundLiteralExpr *CLE = cast<CompoundLiteralExpr>(E); + CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType()); + if (llvm::GlobalVariable *Addr = + CGM.getAddrOfConstantCompoundLiteralIfEmitted(CLE)) + return ConstantAddress(Addr, Align); + llvm::Constant* C = CGM.EmitConstantExpr(CLE->getInitializer(), CLE->getType(), CGF); // FIXME: "Leaked" on failure. if (!C) return ConstantAddress::invalid(); - CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType()); - auto GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(), E->getType().isConstant(CGM.getContext()), llvm::GlobalValue::InternalLinkage, @@ -1035,6 +1041,7 @@ public: llvm::GlobalVariable::NotThreadLocal, CGM.getContext().getTargetAddressSpace(E->getType())); GV->setAlignment(Align.getQuantity()); + CGM.setAddrOfConstantCompoundLiteral(CLE, GV); return ConstantAddress(GV, Align); } case Expr::StringLiteralClass: @@ -1083,7 +1090,7 @@ public: return CGM.GetAddrOfConstantCFString(Literal); } case Expr::BlockExprClass: { - std::string FunctionName; + StringRef FunctionName; if (CGF) FunctionName = CGF->CurFn->getName(); else @@ -1091,7 +1098,7 @@ public: // This is not really an l-value. llvm::Constant *Ptr = - CGM.GetAddrOfGlobalBlock(cast<BlockExpr>(E), FunctionName.c_str()); + CGM.GetAddrOfGlobalBlock(cast<BlockExpr>(E), FunctionName); return ConstantAddress(Ptr, CGM.getPointerAlign()); } case Expr::CXXTypeidExprClass: { @@ -1259,6 +1266,10 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E, return C; } +llvm::Constant *CodeGenModule::getNullPointer(llvm::PointerType *T, QualType QT) { + return getTargetCodeGenInfo().getNullPointer(*this, T, QT); +} + llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, QualType DestType, CodeGenFunction *CGF) { @@ -1290,6 +1301,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, llvm::ConstantInt::get(Int64Ty, Value.getLValueOffset().getQuantity()); llvm::Constant *C = nullptr; + if (APValue::LValueBase LVBase = Value.getLValueBase()) { // An array can be represented as an lvalue referring to the base. if (isa<llvm::ArrayType>(DestTy)) { @@ -1320,7 +1332,9 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, // Convert to the appropriate type; this could be an lvalue for // an integer. - if (isa<llvm::PointerType>(DestTy)) { + if (auto PT = dyn_cast<llvm::PointerType>(DestTy)) { + if (Value.isNullPointer()) + return getNullPointer(PT, DestType); // Convert the integer to a pointer-sized integer before converting it // to a pointer. C = llvm::ConstantExpr::getIntegerCast( @@ -1354,7 +1368,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, } case APValue::Float: { const llvm::APFloat &Init = Value.getFloat(); - if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf && + if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf() && !Context.getLangOpts().NativeHalfType && !Context.getLangOpts().HalfArgsAndReturns) return llvm::ConstantInt::get(VMContext, Init.bitcastToAPInt()); @@ -1480,6 +1494,18 @@ CodeGenModule::EmitConstantValueForMemory(const APValue &Value, return C; } +llvm::GlobalVariable *CodeGenModule::getAddrOfConstantCompoundLiteralIfEmitted( + const CompoundLiteralExpr *E) { + return EmittedCompoundLiterals.lookup(E); +} + +void CodeGenModule::setAddrOfConstantCompoundLiteral( + const CompoundLiteralExpr *CLE, llvm::GlobalVariable *GV) { + bool Ok = EmittedCompoundLiterals.insert(std::make_pair(CLE, GV)).second; + (void)Ok; + assert(Ok && "CLE has already been emitted!"); +} + ConstantAddress CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) { assert(E->isFileScope() && "not a file-scope compound literal expr"); @@ -1507,7 +1533,7 @@ static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM, const CXXRecordDecl *base); static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, - const CXXRecordDecl *record, + const RecordDecl *record, bool asCompleteObject) { const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record); llvm::StructType *structure = @@ -1517,24 +1543,29 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, unsigned numElements = structure->getNumElements(); std::vector<llvm::Constant *> elements(numElements); + auto CXXR = dyn_cast<CXXRecordDecl>(record); // Fill in all the bases. - for (const auto &I : record->bases()) { - if (I.isVirtual()) { - // Ignore virtual bases; if we're laying out for a complete - // object, we'll lay these out later. - continue; - } + if (CXXR) { + for (const auto &I : CXXR->bases()) { + if (I.isVirtual()) { + // Ignore virtual bases; if we're laying out for a complete + // object, we'll lay these out later. + continue; + } - const CXXRecordDecl *base = - cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); + const CXXRecordDecl *base = + cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); - // Ignore empty bases. - if (base->isEmpty()) - continue; - - unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(base); - llvm::Type *baseType = structure->getElementType(fieldIndex); - elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base); + // Ignore empty bases. + if (base->isEmpty() || + CGM.getContext().getASTRecordLayout(base).getNonVirtualSize() + .isZero()) + continue; + + unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(base); + llvm::Type *baseType = structure->getElementType(fieldIndex); + elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base); + } } // Fill in all the fields. @@ -1558,8 +1589,8 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, } // Fill in the virtual bases, if we're working with the complete object. - if (asCompleteObject) { - for (const auto &I : record->vbases()) { + if (CXXR && asCompleteObject) { + for (const auto &I : CXXR->vbases()) { const CXXRecordDecl *base = cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); @@ -1601,6 +1632,10 @@ static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM, } llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { + if (T->getAs<PointerType>()) + return getNullPointer( + cast<llvm::PointerType>(getTypes().ConvertTypeForMem(T)), T); + if (getTypes().isZeroInitializable(T)) return llvm::Constant::getNullValue(getTypes().ConvertTypeForMem(T)); @@ -1616,10 +1651,8 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { return llvm::ConstantArray::get(ATy, Array); } - if (const RecordType *RT = T->getAs<RecordType>()) { - const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl()); - return ::EmitNullConstant(*this, RD, /*complete object*/ true); - } + if (const RecordType *RT = T->getAs<RecordType>()) + return ::EmitNullConstant(*this, RT->getDecl(), /*complete object*/ true); assert(T->isMemberDataPointerType() && "Should only see pointers to data members here!"); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp index 120dacf..1b85c45 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp @@ -19,6 +19,7 @@ #include "TargetInfo.h" #include "clang/AST/ASTContext.h" #include "clang/AST/DeclObjC.h" +#include "clang/AST/Expr.h" #include "clang/AST/RecordLayout.h" #include "clang/AST/StmtVisitor.h" #include "clang/Basic/TargetInfo.h" @@ -171,9 +172,9 @@ public: } /// EmitPointerToBoolConversion - Perform a pointer to boolean conversion. - Value *EmitPointerToBoolConversion(Value *V) { - Value *Zero = llvm::ConstantPointerNull::get( - cast<llvm::PointerType>(V->getType())); + Value *EmitPointerToBoolConversion(Value *V, QualType QT) { + Value *Zero = CGF.CGM.getNullPointer(cast<llvm::PointerType>(V->getType()), QT); + return Builder.CreateICmpNE(V, Zero, "tobool"); } @@ -310,6 +311,12 @@ public: Value *VisitInitListExpr(InitListExpr *E); + Value *VisitArrayInitIndexExpr(ArrayInitIndexExpr *E) { + assert(CGF.getArrayInitIndex() && + "ArrayInitIndexExpr not inside an ArrayInitLoopExpr?"); + return CGF.getArrayInitIndex(); + } + Value *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) { return EmitNullValue(E->getType()); } @@ -591,7 +598,7 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) { return EmitIntToBoolConversion(Src); assert(isa<llvm::PointerType>(Src->getType())); - return EmitPointerToBoolConversion(Src); + return EmitPointerToBoolConversion(Src, SrcType); } void ScalarExprEmitter::EmitFloatConversionCheck( @@ -724,7 +731,7 @@ void ScalarExprEmitter::EmitFloatConversionCheck( CGF.EmitCheckTypeDescriptor(OrigSrcType), CGF.EmitCheckTypeDescriptor(DstType)}; CGF.EmitCheck(std::make_pair(Check, SanitizerKind::FloatCastOverflow), - "float_cast_overflow", StaticArgs, OrigSrc); + SanitizerHandler::FloatCastOverflow, StaticArgs, OrigSrc); } /// Emit a conversion from the specified type to the specified destination type, @@ -787,7 +794,7 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, // Handle pointer conversions next: pointers can only be converted to/from // other pointers and integers. Check for pointer types in terms of LLVM, as // some native types (like Obj-C id) may map to a pointer type. - if (isa<llvm::PointerType>(DstTy)) { + if (auto DstPT = dyn_cast<llvm::PointerType>(DstTy)) { // The source value may be an integer, or a pointer. if (isa<llvm::PointerType>(SrcTy)) return Builder.CreateBitCast(Src, DstTy, "conv"); @@ -795,7 +802,7 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, assert(SrcType->isIntegerType() && "Not ptr->ptr or int->ptr conversion?"); // First, convert to the correct width so that we control the kind of // extension. - llvm::Type *MiddleTy = CGF.IntPtrTy; + llvm::Type *MiddleTy = CGF.CGM.getDataLayout().getIntPtrType(DstPT); bool InputSigned = SrcType->isSignedIntegerOrEnumerationType(); llvm::Value* IntResult = Builder.CreateIntCast(Src, MiddleTy, InputSigned, "conv"); @@ -927,7 +934,7 @@ Value *ScalarExprEmitter::EmitNullValue(QualType Ty) { void ScalarExprEmitter::EmitBinOpCheck( ArrayRef<std::pair<Value *, SanitizerMask>> Checks, const BinOpInfo &Info) { assert(CGF.IsSanitizerScope); - StringRef CheckName; + SanitizerHandler Check; SmallVector<llvm::Constant *, 4> StaticData; SmallVector<llvm::Value *, 2> DynamicData; @@ -938,13 +945,13 @@ void ScalarExprEmitter::EmitBinOpCheck( StaticData.push_back(CGF.EmitCheckSourceLocation(Info.E->getExprLoc())); const UnaryOperator *UO = dyn_cast<UnaryOperator>(Info.E); if (UO && UO->getOpcode() == UO_Minus) { - CheckName = "negate_overflow"; + Check = SanitizerHandler::NegateOverflow; StaticData.push_back(CGF.EmitCheckTypeDescriptor(UO->getType())); DynamicData.push_back(Info.RHS); } else { if (BinaryOperator::isShiftOp(Opcode)) { // Shift LHS negative or too large, or RHS out of bounds. - CheckName = "shift_out_of_bounds"; + Check = SanitizerHandler::ShiftOutOfBounds; const BinaryOperator *BO = cast<BinaryOperator>(Info.E); StaticData.push_back( CGF.EmitCheckTypeDescriptor(BO->getLHS()->getType())); @@ -952,14 +959,14 @@ void ScalarExprEmitter::EmitBinOpCheck( CGF.EmitCheckTypeDescriptor(BO->getRHS()->getType())); } else if (Opcode == BO_Div || Opcode == BO_Rem) { // Divide or modulo by zero, or signed overflow (eg INT_MAX / -1). - CheckName = "divrem_overflow"; + Check = SanitizerHandler::DivremOverflow; StaticData.push_back(CGF.EmitCheckTypeDescriptor(Info.Ty)); } else { // Arithmetic overflow (+, -, *). switch (Opcode) { - case BO_Add: CheckName = "add_overflow"; break; - case BO_Sub: CheckName = "sub_overflow"; break; - case BO_Mul: CheckName = "mul_overflow"; break; + case BO_Add: Check = SanitizerHandler::AddOverflow; break; + case BO_Sub: Check = SanitizerHandler::SubOverflow; break; + case BO_Mul: Check = SanitizerHandler::MulOverflow; break; default: llvm_unreachable("unexpected opcode for bin op check"); } StaticData.push_back(CGF.EmitCheckTypeDescriptor(Info.Ty)); @@ -968,7 +975,7 @@ void ScalarExprEmitter::EmitBinOpCheck( DynamicData.push_back(Info.RHS); } - CGF.EmitCheck(Checks, CheckName, StaticData, DynamicData); + CGF.EmitCheck(Checks, Check, StaticData, DynamicData); } //===----------------------------------------------------------------------===// @@ -1394,11 +1401,23 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { return Builder.CreateBitCast(Src, DstTy); } case CK_AddressSpaceConversion: { - Value *Src = Visit(const_cast<Expr*>(E)); + Expr::EvalResult Result; + if (E->EvaluateAsRValue(Result, CGF.getContext()) && + Result.Val.isNullPointer()) { + // If E has side effect, it is emitted even if its final result is a + // null pointer. In that case, a DCE pass should be able to + // eliminate the useless instructions emitted during translating E. + if (Result.HasSideEffects) + Visit(E); + return CGF.CGM.getNullPointer(cast<llvm::PointerType>( + ConvertType(DestTy)), DestTy); + } // Since target may map different address spaces in AST to the same address // space, an address space conversion may end up as a bitcast. - return Builder.CreatePointerBitCastOrAddrSpaceCast(Src, - ConvertType(DestTy)); + auto *Src = Visit(E); + return CGF.CGM.getTargetCodeGenInfo().performAddrSpaceCast(CGF, Src, + E->getType(), + DestTy); } case CK_AtomicToNonAtomic: case CK_NonAtomicToAtomic: @@ -1453,8 +1472,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { if (MustVisitNullValue(E)) (void) Visit(E); - return llvm::ConstantPointerNull::get( - cast<llvm::PointerType>(ConvertType(DestTy))); + return CGF.CGM.getNullPointer(cast<llvm::PointerType>(ConvertType(DestTy)), + DestTy); case CK_NullToMemberPointer: { if (MustVisitNullValue(E)) @@ -1510,12 +1529,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // First, convert to the correct width so that we control the kind of // extension. - llvm::Type *MiddleTy = CGF.IntPtrTy; + auto DestLLVMTy = ConvertType(DestTy); + llvm::Type *MiddleTy = CGF.CGM.getDataLayout().getIntPtrType(DestLLVMTy); bool InputSigned = E->getType()->isSignedIntegerOrEnumerationType(); llvm::Value* IntResult = Builder.CreateIntCast(Src, MiddleTy, InputSigned, "conv"); - return Builder.CreateIntToPtr(IntResult, ConvertType(DestTy)); + return Builder.CreateIntToPtr(IntResult, DestLLVMTy); } case CK_PointerToIntegral: assert(!DestTy->isBooleanType() && "bool should use PointerToBool"); @@ -1546,7 +1566,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_IntegralToBoolean: return EmitIntToBoolConversion(Visit(E)); case CK_PointerToBoolean: - return EmitPointerToBoolConversion(Visit(E)); + return EmitPointerToBoolConversion(Visit(E), E->getType()); case CK_FloatingToBoolean: return EmitFloatToBoolConversion(Visit(E)); case CK_MemberPointerToBoolean: { @@ -1573,8 +1593,16 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { return llvm::Constant::getNullValue(ConvertType(DestTy)); } + case CK_ZeroToOCLQueue: { + assert(DestTy->isQueueT() && "CK_ZeroToOCLQueue cast on non queue_t type"); + return llvm::Constant::getNullValue(ConvertType(DestTy)); } + case CK_IntToOCLSampler: + return CGF.CGM.createOpenCLIntToSamplerConversion(E, CGF);
+ + } // end of switch + llvm_unreachable("unknown scalar cast"); } @@ -2273,8 +2301,13 @@ Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) { if (Ops.LHS->getType()->isFPOrFPVectorTy()) { llvm::Value *Val = Builder.CreateFDiv(Ops.LHS, Ops.RHS, "div"); - if (CGF.getLangOpts().OpenCL) { - // OpenCL 1.1 7.4: minimum accuracy of single precision / is 2.5ulp + if (CGF.getLangOpts().OpenCL && + !CGF.CGM.getCodeGenOpts().CorrectlyRoundedDivSqrt) { + // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5ulp + // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt + // build option allows an application to specify that single precision + // floating-point divide (x/y and 1/x) and sqrt used in the program + // source are correctly rounded. llvm::Type *ValTy = Val->getType(); if (ValTy->isFloatTy() || (isa<llvm::VectorType>(ValTy) && @@ -2363,9 +2396,8 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) { // Branch in case of overflow. llvm::BasicBlock *initialBB = Builder.GetInsertBlock(); - llvm::Function::iterator insertPt = initialBB->getIterator(); - llvm::BasicBlock *continueBB = CGF.createBasicBlock("nooverflow", CGF.CurFn, - &*std::next(insertPt)); + llvm::BasicBlock *continueBB = + CGF.createBasicBlock("nooverflow", CGF.CurFn, initialBB->getNextNode()); llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn); Builder.CreateCondBr(overflow, overflowBB, continueBB); @@ -2429,11 +2461,13 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF, } unsigned width = cast<llvm::IntegerType>(index->getType())->getBitWidth(); - if (width != CGF.PointerWidthInBits) { + auto &DL = CGF.CGM.getDataLayout(); + auto PtrTy = cast<llvm::PointerType>(pointer->getType()); + if (width != DL.getTypeSizeInBits(PtrTy)) { // Zero-extend or sign-extend the pointer value according to // whether the index is signed or not. bool isSigned = indexOperand->getType()->isSignedIntegerOrEnumerationType(); - index = CGF.Builder.CreateIntCast(index, CGF.PtrDiffTy, isSigned, + index = CGF.Builder.CreateIntCast(index, DL.getIntPtrType(PtrTy), isSigned, "idx.ext"); } @@ -3397,6 +3431,52 @@ static Value *ConvertVec3AndVec4(CGBuilderTy &Builder, CodeGenFunction &CGF, return Builder.CreateShuffleVector(Src, UnV, Mask); } +// Create cast instructions for converting LLVM value \p Src to LLVM type \p +// DstTy. \p Src has the same size as \p DstTy. Both are single value types +// but could be scalar or vectors of different lengths, and either can be +// pointer. +// There are 4 cases: +// 1. non-pointer -> non-pointer : needs 1 bitcast +// 2. pointer -> pointer : needs 1 bitcast or addrspacecast +// 3. pointer -> non-pointer +// a) pointer -> intptr_t : needs 1 ptrtoint +// b) pointer -> non-intptr_t : needs 1 ptrtoint then 1 bitcast +// 4. non-pointer -> pointer +// a) intptr_t -> pointer : needs 1 inttoptr +// b) non-intptr_t -> pointer : needs 1 bitcast then 1 inttoptr +// Note: for cases 3b and 4b two casts are required since LLVM casts do not +// allow casting directly between pointer types and non-integer non-pointer +// types. +static Value *createCastsForTypeOfSameSize(CGBuilderTy &Builder, + const llvm::DataLayout &DL, + Value *Src, llvm::Type *DstTy, + StringRef Name = "") { + auto SrcTy = Src->getType(); + + // Case 1. + if (!SrcTy->isPointerTy() && !DstTy->isPointerTy()) + return Builder.CreateBitCast(Src, DstTy, Name); + + // Case 2. + if (SrcTy->isPointerTy() && DstTy->isPointerTy()) + return Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy, Name); + + // Case 3. + if (SrcTy->isPointerTy() && !DstTy->isPointerTy()) { + // Case 3b. + if (!DstTy->isIntegerTy()) + Src = Builder.CreatePtrToInt(Src, DL.getIntPtrType(SrcTy)); + // Cases 3a and 3b. + return Builder.CreateBitOrPointerCast(Src, DstTy, Name); + } + + // Case 4b. + if (!SrcTy->isIntegerTy()) + Src = Builder.CreateBitCast(Src, DL.getIntPtrType(DstTy)); + // Cases 4a and 4b. + return Builder.CreateIntToPtr(Src, DstTy, Name); +} + Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) { Value *Src = CGF.EmitScalarExpr(E->getSrcExpr()); llvm::Type *DstTy = ConvertType(E->getType()); @@ -3411,7 +3491,8 @@ Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) { // vector to get a vec4, then a bitcast if the target type is different. if (NumElementsSrc == 3 && NumElementsDst != 3) { Src = ConvertVec3AndVec4(Builder, CGF, Src, 4); - Src = Builder.CreateBitCast(Src, DstTy); + Src = createCastsForTypeOfSameSize(Builder, CGF.CGM.getDataLayout(), Src, + DstTy); Src->setName("astype"); return Src; } @@ -3421,13 +3502,15 @@ Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) { // get a vec3. if (NumElementsSrc != 3 && NumElementsDst == 3) { auto Vec4Ty = llvm::VectorType::get(DstTy->getVectorElementType(), 4); - Src = Builder.CreateBitCast(Src, Vec4Ty); + Src = createCastsForTypeOfSameSize(Builder, CGF.CGM.getDataLayout(), Src, + Vec4Ty); Src = ConvertVec3AndVec4(Builder, CGF, Src, 3); Src->setName("astype"); return Src; } - return Builder.CreateBitCast(Src, DstTy, "astype"); + return Src = createCastsForTypeOfSameSize(Builder, CGF.CGM.getDataLayout(), + Src, DstTy, "astype"); } Value *ScalarExprEmitter::VisitAtomicExpr(AtomicExpr *E) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp index 51474f1..28998ce 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp @@ -20,14 +20,15 @@ using namespace clang::CodeGen; using namespace llvm; static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, - llvm::DebugLoc Location) { + const llvm::DebugLoc &StartLoc, + const llvm::DebugLoc &EndLoc) { if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.VectorizeEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.DistributeEnable == LoopAttributes::Unspecified && - !Location) + !StartLoc && !EndLoc) return nullptr; SmallVector<Metadata *, 4> Args; @@ -35,9 +36,14 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, auto TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); - // If we have a valid debug location for the loop, add it. - if (Location) - Args.push_back(Location.getAsMDNode()); + // If we have a valid start debug location for the loop, add it. + if (StartLoc) { + Args.push_back(StartLoc.getAsMDNode()); + + // If we also have a valid end debug location for the loop, add it. + if (EndLoc) + Args.push_back(EndLoc.getAsMDNode()); + } // Setting vectorize.width if (Attrs.VectorizeWidth > 0) { @@ -112,23 +118,26 @@ void LoopAttributes::clear() { UnrollCount = 0; VectorizeEnable = LoopAttributes::Unspecified; UnrollEnable = LoopAttributes::Unspecified; + DistributeEnable = LoopAttributes::Unspecified; } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, - llvm::DebugLoc Location) + const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc) : LoopID(nullptr), Header(Header), Attrs(Attrs) { - LoopID = createMetadata(Header->getContext(), Attrs, Location); + LoopID = createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc); } -void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) { - Active.push_back(LoopInfo(Header, StagedAttrs, Location)); +void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, + const llvm::DebugLoc &EndLoc) { + Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc)); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); } void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, ArrayRef<const clang::Attr *> Attrs, - llvm::DebugLoc Location) { + const llvm::DebugLoc &StartLoc, + const llvm::DebugLoc &EndLoc) { // Identify loop hint attributes from Attrs. for (const auto *Attr : Attrs) { @@ -266,7 +275,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, } /// Stage the attributes. - push(Header, Location); + push(Header, StartLoc, EndLoc); } void LoopInfoStack::pop() { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h index a0111ed..15608c1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h @@ -16,7 +16,6 @@ #define LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Value.h" @@ -68,7 +67,7 @@ class LoopInfo { public: /// \brief Construct a new LoopInfo for the loop with entry Header. LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, - llvm::DebugLoc Location); + const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc); /// \brief Get the loop id metadata for this loop. llvm::MDNode *getLoopID() const { return LoopID; } @@ -100,14 +99,14 @@ public: /// \brief Begin a new structured loop. The set of staged attributes will be /// applied to the loop and then cleared. - void push(llvm::BasicBlock *Header, - llvm::DebugLoc Location = llvm::DebugLoc()); + void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, + const llvm::DebugLoc &EndLoc); /// \brief Begin a new structured loop. Stage attributes from the Attrs list. /// The staged attributes are applied to the loop and then cleared. void push(llvm::BasicBlock *Header, clang::ASTContext &Ctx, - llvm::ArrayRef<const Attr *> Attrs, - llvm::DebugLoc Location = llvm::DebugLoc()); + llvm::ArrayRef<const Attr *> Attrs, const llvm::DebugLoc &StartLoc, + const llvm::DebugLoc &EndLoc); /// \brief End the current loop. void pop(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp index db894ce..932b8a1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp @@ -589,9 +589,10 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, args.add(RValue::get(CGF.Builder.getInt1(isAtomic)), Context.BoolTy); args.add(RValue::get(CGF.Builder.getInt1(hasStrong)), Context.BoolTy); - llvm::Value *fn = CGF.CGM.getObjCRuntime().GetGetStructFunction(); + llvm::Constant *fn = CGF.CGM.getObjCRuntime().GetGetStructFunction(); + CGCallee callee = CGCallee::forDirect(fn); CGF.EmitCall(CGF.getTypes().arrangeBuiltinFunctionCall(Context.VoidTy, args), - fn, ReturnValueSlot(), args); + callee, ReturnValueSlot(), args); } /// Determine whether the given architecture supports unaligned atomic @@ -852,11 +853,12 @@ static void emitCPPObjectAtomicGetterCall(CodeGenFunction &CGF, // Third argument is the helper function. args.add(RValue::get(AtomicHelperFn), CGF.getContext().VoidPtrTy); - llvm::Value *copyCppAtomicObjectFn = + llvm::Constant *copyCppAtomicObjectFn = CGF.CGM.getObjCRuntime().GetCppAtomicObjectGetFunction(); + CGCallee callee = CGCallee::forDirect(copyCppAtomicObjectFn); CGF.EmitCall( CGF.getTypes().arrangeBuiltinFunctionCall(CGF.getContext().VoidTy, args), - copyCppAtomicObjectFn, ReturnValueSlot(), args); + callee, ReturnValueSlot(), args); } void @@ -927,12 +929,13 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, } case PropertyImplStrategy::GetSetProperty: { - llvm::Value *getPropertyFn = + llvm::Constant *getPropertyFn = CGM.getObjCRuntime().GetPropertyGetFunction(); if (!getPropertyFn) { CGM.ErrorUnsupported(propImpl, "Obj-C getter requiring atomic copy"); return; } + CGCallee callee = CGCallee::forDirect(getPropertyFn); // Return (ivar-type) objc_getProperty((id) self, _cmd, offset, true). // FIXME: Can't this be simpler? This might even be worse than the @@ -955,8 +958,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, llvm::Instruction *CallInstruction; RValue RV = EmitCall( getTypes().arrangeBuiltinFunctionCall(propType, args), - getPropertyFn, ReturnValueSlot(), args, CGCalleeInfo(), - &CallInstruction); + callee, ReturnValueSlot(), args, &CallInstruction); if (llvm::CallInst *call = dyn_cast<llvm::CallInst>(CallInstruction)) call->setTailCall(); @@ -1068,10 +1070,11 @@ static void emitStructSetterCall(CodeGenFunction &CGF, ObjCMethodDecl *OMD, // FIXME: should this really always be false? args.add(RValue::get(CGF.Builder.getFalse()), CGF.getContext().BoolTy); - llvm::Value *copyStructFn = CGF.CGM.getObjCRuntime().GetSetStructFunction(); + llvm::Constant *fn = CGF.CGM.getObjCRuntime().GetSetStructFunction(); + CGCallee callee = CGCallee::forDirect(fn); CGF.EmitCall( CGF.getTypes().arrangeBuiltinFunctionCall(CGF.getContext().VoidTy, args), - copyStructFn, ReturnValueSlot(), args); + callee, ReturnValueSlot(), args); } /// emitCPPObjectAtomicSetterCall - Call the runtime function to store @@ -1103,11 +1106,12 @@ static void emitCPPObjectAtomicSetterCall(CodeGenFunction &CGF, // Third argument is the helper function. args.add(RValue::get(AtomicHelperFn), CGF.getContext().VoidPtrTy); - llvm::Value *copyCppAtomicObjectFn = + llvm::Constant *fn = CGF.CGM.getObjCRuntime().GetCppAtomicObjectSetFunction(); + CGCallee callee = CGCallee::forDirect(fn); CGF.EmitCall( CGF.getTypes().arrangeBuiltinFunctionCall(CGF.getContext().VoidTy, args), - copyCppAtomicObjectFn, ReturnValueSlot(), args); + callee, ReturnValueSlot(), args); } @@ -1197,8 +1201,8 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl, case PropertyImplStrategy::GetSetProperty: case PropertyImplStrategy::SetPropertyAndExpressionGet: { - llvm::Value *setOptimizedPropertyFn = nullptr; - llvm::Value *setPropertyFn = nullptr; + llvm::Constant *setOptimizedPropertyFn = nullptr; + llvm::Constant *setPropertyFn = nullptr; if (UseOptimizedSetter(CGM)) { // 10.8 and iOS 6.0 code and GC is off setOptimizedPropertyFn = @@ -1236,8 +1240,9 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl, if (setOptimizedPropertyFn) { args.add(RValue::get(arg), getContext().getObjCIdType()); args.add(RValue::get(ivarOffset), getContext().getPointerDiffType()); + CGCallee callee = CGCallee::forDirect(setOptimizedPropertyFn); EmitCall(getTypes().arrangeBuiltinFunctionCall(getContext().VoidTy, args), - setOptimizedPropertyFn, ReturnValueSlot(), args); + callee, ReturnValueSlot(), args); } else { args.add(RValue::get(ivarOffset), getContext().getPointerDiffType()); args.add(RValue::get(arg), getContext().getObjCIdType()); @@ -1247,8 +1252,9 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl, getContext().BoolTy); // FIXME: We shouldn't need to get the function info here, the runtime // already should have computed it to build the function. + CGCallee callee = CGCallee::forDirect(setPropertyFn); EmitCall(getTypes().arrangeBuiltinFunctionCall(getContext().VoidTy, args), - setPropertyFn, ReturnValueSlot(), args); + callee, ReturnValueSlot(), args); } return; @@ -1450,13 +1456,14 @@ QualType CodeGenFunction::TypeOfSelfObject() { } void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ - llvm::Constant *EnumerationMutationFn = + llvm::Constant *EnumerationMutationFnPtr = CGM.getObjCRuntime().EnumerationMutationFunction(); - - if (!EnumerationMutationFn) { + if (!EnumerationMutationFnPtr) { CGM.ErrorUnsupported(&S, "Obj-C fast enumeration for this runtime"); return; } + CGCallee EnumerationMutationFn = + CGCallee::forDirect(EnumerationMutationFnPtr); CGDebugInfo *DI = getDebugInfo(); if (DI) @@ -1662,7 +1669,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ elementLValue = EmitLValue(cast<Expr>(S.getElement())); EmitStoreThroughLValue(RValue::get(CurrentItem), elementLValue); } else { - EmitScalarInit(CurrentItem, elementLValue); + EmitStoreThroughLValue(RValue::get(CurrentItem), elementLValue, + /*isInit*/ true); } // If we do have an element variable, this assignment is the end of @@ -1803,7 +1811,8 @@ static llvm::Constant *createARCRuntimeFunction(CodeGenModule &CGM, // If the target runtime doesn't naturally support ARC, emit weak // references to the runtime support library. We don't really // permit this to fail, but we need a particular relocation style. - if (!CGM.getLangOpts().ObjCRuntime.hasNativeARC()) { + if (!CGM.getLangOpts().ObjCRuntime.hasNativeARC() && + !CGM.getTriple().isOSBinFormatCOFF()) { f->setLinkage(llvm::Function::ExternalWeakLinkage); } else if (fnName == "objc_retain" || fnName == "objc_release") { // If we have Native ARC, set nonlazybind attribute for these APIs for diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp index caafef8..fa2b3d8 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp @@ -18,6 +18,7 @@ #include "CGCleanup.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantBuilder.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclObjC.h" @@ -163,9 +164,8 @@ protected: /// Helper function that generates a constant string and returns a pointer to /// the start of the string. The result of this function can be used anywhere /// where the C code specifies const char*. - llvm::Constant *MakeConstantString(const std::string &Str, - const std::string &Name="") { - ConstantAddress Array = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + llvm::Constant *MakeConstantString(StringRef Str, const char *Name = "") { + ConstantAddress Array = CGM.GetAddrOfConstantCString(Str, Name); return llvm::ConstantExpr::getGetElementPtr(Array.getElementType(), Array.getPointer(), Zeros); } @@ -174,14 +174,14 @@ protected: /// string value. This allows the linker to combine the strings between /// different modules. Used for EH typeinfo names, selector strings, and a /// few other things. - llvm::Constant *ExportUniqueString(const std::string &Str, - const std::string prefix) { - std::string name = prefix + Str; - auto *ConstStr = TheModule.getGlobalVariable(name); + llvm::Constant *ExportUniqueString(const std::string &Str, StringRef Prefix) { + std::string Name = Prefix.str() + Str; + auto *ConstStr = TheModule.getGlobalVariable(Name); if (!ConstStr) { llvm::Constant *value = llvm::ConstantDataArray::getString(VMContext,Str); ConstStr = new llvm::GlobalVariable(TheModule, value->getType(), true, - llvm::GlobalValue::LinkOnceODRLinkage, value, prefix + Str); + llvm::GlobalValue::LinkOnceODRLinkage, + value, Name); } return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), ConstStr, Zeros); @@ -190,47 +190,17 @@ protected: /// Generates a global structure, initialized by the elements in the vector. /// The element types must match the types of the structure elements in the /// first argument. - llvm::GlobalVariable *MakeGlobal(llvm::StructType *Ty, - ArrayRef<llvm::Constant *> V, + llvm::GlobalVariable *MakeGlobal(llvm::Constant *C, CharUnits Align, StringRef Name="", llvm::GlobalValue::LinkageTypes linkage =llvm::GlobalValue::InternalLinkage) { - llvm::Constant *C = llvm::ConstantStruct::get(Ty, V); - auto GV = new llvm::GlobalVariable(TheModule, Ty, false, + auto GV = new llvm::GlobalVariable(TheModule, C->getType(), false, linkage, C, Name); GV->setAlignment(Align.getQuantity()); return GV; } - /// Generates a global array. The vector must contain the same number of - /// elements that the array type declares, of the type specified as the array - /// element type. - llvm::GlobalVariable *MakeGlobal(llvm::ArrayType *Ty, - ArrayRef<llvm::Constant *> V, - CharUnits Align, - StringRef Name="", - llvm::GlobalValue::LinkageTypes linkage - =llvm::GlobalValue::InternalLinkage) { - llvm::Constant *C = llvm::ConstantArray::get(Ty, V); - auto GV = new llvm::GlobalVariable(TheModule, Ty, false, - linkage, C, Name); - GV->setAlignment(Align.getQuantity()); - return GV; - } - - /// Generates a global array, inferring the array type from the specified - /// element type and the size of the initialiser. - llvm::GlobalVariable *MakeGlobalArray(llvm::Type *Ty, - ArrayRef<llvm::Constant *> V, - CharUnits Align, - StringRef Name="", - llvm::GlobalValue::LinkageTypes linkage - =llvm::GlobalValue::InternalLinkage) { - llvm::ArrayType *ArrayTy = llvm::ArrayType::get(Ty, V.size()); - return MakeGlobal(ArrayTy, V, Align, Name, linkage); - } - /// Returns a property name and encoding string. llvm::Constant *MakePropertyEncodingString(const ObjCPropertyDecl *PD, const Decl *Container) { @@ -238,8 +208,8 @@ protected: if ((R.getKind() == ObjCRuntime::GNUstep) && (R.getVersion() >= VersionTuple(1, 6))) { std::string NameAndAttributes; - std::string TypeStr; - CGM.getContext().getObjCEncodingForPropertyDecl(PD, Container, TypeStr); + std::string TypeStr = + CGM.getContext().getObjCEncodingForPropertyDecl(PD, Container); NameAndAttributes += '\0'; NameAndAttributes += TypeStr.length() + 3; NameAndAttributes += TypeStr; @@ -251,7 +221,7 @@ protected: } /// Push the property attributes into two structure fields. - void PushPropertyAttributes(std::vector<llvm::Constant*> &Fields, + void PushPropertyAttributes(ConstantStructBuilder &Fields, ObjCPropertyDecl *property, bool isSynthesized=true, bool isDynamic=true) { int attrs = property->getPropertyAttributes(); @@ -263,7 +233,7 @@ protected: attrs &= ~ObjCPropertyDecl::OBJC_PR_strong; } // The first flags field has the same attribute values as clang uses internally - Fields.push_back(llvm::ConstantInt::get(Int8Ty, attrs & 0xff)); + Fields.addInt(Int8Ty, attrs & 0xff); attrs >>= 8; attrs <<= 2; // For protocol properties, synthesized and dynamic have no meaning, so we @@ -273,10 +243,10 @@ protected: attrs |= isDynamic ? (1<<1) : 0; // The second field is the next four fields left shifted by two, with the // low bit set to indicate whether the field is synthesized or dynamic. - Fields.push_back(llvm::ConstantInt::get(Int8Ty, attrs & 0xff)); + Fields.addInt(Int8Ty, attrs & 0xff); // Two padding fields - Fields.push_back(llvm::ConstantInt::get(Int8Ty, 0)); - Fields.push_back(llvm::ConstantInt::get(Int8Ty, 0)); + Fields.addInt(Int8Ty, 0); + Fields.addInt(Int8Ty, 0); } /// Ensures that the value has the required type, by inserting a bitcast if @@ -590,11 +560,6 @@ public: llvm::Constant *BuildByrefLayout(CodeGenModule &CGM, QualType T) override { return NULLPtr; } - - llvm::GlobalVariable *GetClassGlobal(StringRef Name, - bool Weak = false) override { - return nullptr; - } }; /// Class representing the legacy GCC Objective-C ABI. This is the default when @@ -1152,8 +1117,7 @@ llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel) { llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, const ObjCMethodDecl *Method) { - std::string SelTypes; - CGM.getContext().getObjCEncodingForMethodDecl(Method, SelTypes); + std::string SelTypes = CGM.getContext().getObjCEncodingForMethodDecl(Method); return GetSelector(CGF, Method->getSelector(), SelTypes); } @@ -1233,14 +1197,15 @@ llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) { llvm::Constant *typeName = ExportUniqueString(className, "__objc_eh_typename_"); - std::vector<llvm::Constant*> fields; - fields.push_back(BVtable); - fields.push_back(typeName); - llvm::Constant *TI = - MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, nullptr), - fields, CGM.getPointerAlign(), - "__objc_eh_typeinfo_" + className, - llvm::GlobalValue::LinkOnceODRLinkage); + ConstantInitBuilder builder(CGM); + auto fields = builder.beginStruct(); + fields.add(BVtable); + fields.add(typeName); + llvm::Constant *TI = + fields.finishAndCreateGlobal("__objc_eh_typeinfo_" + className, + CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::LinkOnceODRLinkage); return llvm::ConstantExpr::getBitCast(TI, PtrToInt8Ty); } @@ -1270,13 +1235,13 @@ ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) { else if (isa->getType() != PtrToIdTy) isa = llvm::ConstantExpr::getBitCast(isa, PtrToIdTy); - std::vector<llvm::Constant*> Ivars; - Ivars.push_back(isa); - Ivars.push_back(MakeConstantString(Str)); - Ivars.push_back(llvm::ConstantInt::get(IntTy, Str.size())); - llvm::Constant *ObjCStr = MakeGlobal( - llvm::StructType::get(PtrToIdTy, PtrToInt8Ty, IntTy, nullptr), - Ivars, Align, ".objc_str"); + ConstantInitBuilder Builder(CGM); + auto Fields = Builder.beginStruct(); + Fields.add(isa); + Fields.add(MakeConstantString(Str)); + Fields.addInt(IntTy, Str.size()); + llvm::Constant *ObjCStr = + Fields.finishAndCreateGlobal(".objc_str", Align); ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty); ObjCStrings[Str] = ObjCStr; ConstantStrings.push_back(ObjCStr); @@ -1386,9 +1351,10 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, llvm::Type::getInt1Ty(VMContext), IsClassMessage))}; llvm::MDNode *node = llvm::MDNode::get(VMContext, impMD); + CGCallee callee(CGCalleeInfo(), imp); + llvm::Instruction *call; - RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, - CGCalleeInfo(), &call); + RValue msgRet = CGF.EmitCall(MSI.CallInfo, callee, Return, ActualArgs, &call); call->setMetadata(msgSendMDKind, node); return msgRet; } @@ -1500,8 +1466,8 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF, imp = EnforceType(Builder, imp, MSI.MessengerType); llvm::Instruction *call; - RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, - CGCalleeInfo(), &call); + CGCallee callee(CGCalleeInfo(), imp); + RValue msgRet = CGF.EmitCall(MSI.CallInfo, callee, Return, ActualArgs, &call); call->setMetadata(msgSendMDKind, node); @@ -1550,50 +1516,38 @@ GenerateMethodList(StringRef ClassName, bool isClassMethodList) { if (MethodSels.empty()) return NULLPtr; + + ConstantInitBuilder Builder(CGM); + + auto MethodList = Builder.beginStruct(); + MethodList.addNullPointer(CGM.Int8PtrTy); + MethodList.addInt(Int32Ty, MethodTypes.size()); + // Get the method structure type. - llvm::StructType *ObjCMethodTy = llvm::StructType::get( - PtrToInt8Ty, // Really a selector, but the runtime creates it us. - PtrToInt8Ty, // Method types - IMPTy, //Method pointer - nullptr); - std::vector<llvm::Constant*> Methods; + llvm::StructType *ObjCMethodTy = + llvm::StructType::get(CGM.getLLVMContext(), { + PtrToInt8Ty, // Really a selector, but the runtime creates it us. + PtrToInt8Ty, // Method types + IMPTy // Method pointer + }); + auto Methods = MethodList.beginArray(); for (unsigned int i = 0, e = MethodTypes.size(); i < e; ++i) { - llvm::Constant *Method = + llvm::Constant *FnPtr = TheModule.getFunction(SymbolNameForMethod(ClassName, CategoryName, MethodSels[i], isClassMethodList)); - assert(Method && "Can't generate metadata for method that doesn't exist"); - llvm::Constant *C = MakeConstantString(MethodSels[i].getAsString()); - Method = llvm::ConstantExpr::getBitCast(Method, - IMPTy); - Methods.push_back( - llvm::ConstantStruct::get(ObjCMethodTy, {C, MethodTypes[i], Method})); - } - - // Array of method structures - llvm::ArrayType *ObjCMethodArrayTy = llvm::ArrayType::get(ObjCMethodTy, - Methods.size()); - llvm::Constant *MethodArray = llvm::ConstantArray::get(ObjCMethodArrayTy, - Methods); - - // Structure containing list pointer, array and array count - llvm::StructType *ObjCMethodListTy = llvm::StructType::create(VMContext); - llvm::Type *NextPtrTy = llvm::PointerType::getUnqual(ObjCMethodListTy); - ObjCMethodListTy->setBody( - NextPtrTy, - IntTy, - ObjCMethodArrayTy, - nullptr); - - Methods.clear(); - Methods.push_back(llvm::ConstantPointerNull::get( - llvm::PointerType::getUnqual(ObjCMethodListTy))); - Methods.push_back(llvm::ConstantInt::get(Int32Ty, MethodTypes.size())); - Methods.push_back(MethodArray); + assert(FnPtr && "Can't generate metadata for method that doesn't exist"); + auto Method = Methods.beginStruct(ObjCMethodTy); + Method.add(MakeConstantString(MethodSels[i].getAsString())); + Method.add(MethodTypes[i]); + Method.addBitCast(FnPtr, IMPTy); + Method.finishAndAddTo(Methods); + } + Methods.finishAndAddTo(MethodList); // Create an instance of the structure - return MakeGlobal(ObjCMethodListTy, Methods, CGM.getPointerAlign(), - ".objc_method_list"); + return MethodList.finishAndCreateGlobal(".objc_method_list", + CGM.getPointerAlign()); } /// Generates an IvarList. Used in construction of a objc_class. @@ -1601,35 +1555,36 @@ llvm::Constant *CGObjCGNU:: GenerateIvarList(ArrayRef<llvm::Constant *> IvarNames, ArrayRef<llvm::Constant *> IvarTypes, ArrayRef<llvm::Constant *> IvarOffsets) { - if (IvarNames.size() == 0) + if (IvarNames.empty()) return NULLPtr; - // Get the method structure type. + + ConstantInitBuilder Builder(CGM); + + // Structure containing array count followed by array. + auto IvarList = Builder.beginStruct(); + IvarList.addInt(IntTy, (int)IvarNames.size()); + + // Get the ivar structure type. llvm::StructType *ObjCIvarTy = llvm::StructType::get( PtrToInt8Ty, PtrToInt8Ty, IntTy, nullptr); - std::vector<llvm::Constant*> Ivars; + + // Array of ivar structures. + auto Ivars = IvarList.beginArray(ObjCIvarTy); for (unsigned int i = 0, e = IvarNames.size() ; i < e ; i++) { - Ivars.push_back(llvm::ConstantStruct::get( - ObjCIvarTy, {IvarNames[i], IvarTypes[i], IvarOffsets[i]})); + auto Ivar = Ivars.beginStruct(ObjCIvarTy); + Ivar.add(IvarNames[i]); + Ivar.add(IvarTypes[i]); + Ivar.add(IvarOffsets[i]); + Ivar.finishAndAddTo(Ivars); } - - // Array of method structures - llvm::ArrayType *ObjCIvarArrayTy = llvm::ArrayType::get(ObjCIvarTy, - IvarNames.size()); - - llvm::Constant *Elements[] = { - llvm::ConstantInt::get(IntTy, (int)IvarNames.size()), - llvm::ConstantArray::get(ObjCIvarArrayTy, Ivars)}; - // Structure containing array and array count - llvm::StructType *ObjCIvarListTy = llvm::StructType::get(IntTy, - ObjCIvarArrayTy, - nullptr); + Ivars.finishAndAddTo(IvarList); // Create an instance of the structure - return MakeGlobal(ObjCIvarListTy, Elements, CGM.getPointerAlign(), - ".objc_ivar_list"); + return IvarList.finishAndCreateGlobal(".objc_ivar_list", + CGM.getPointerAlign()); } /// Generate a class structure @@ -1677,34 +1632,55 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure( IntPtrTy, // strong_pointers IntPtrTy, // weak_pointers nullptr); - llvm::Constant *Zero = llvm::ConstantInt::get(LongTy, 0); + + ConstantInitBuilder Builder(CGM); + auto Elements = Builder.beginStruct(ClassTy); + // Fill in the structure - std::vector<llvm::Constant*> Elements; - Elements.push_back(llvm::ConstantExpr::getBitCast(MetaClass, PtrToInt8Ty)); - Elements.push_back(SuperClass); - Elements.push_back(MakeConstantString(Name, ".class_name")); - Elements.push_back(Zero); - Elements.push_back(llvm::ConstantInt::get(LongTy, info)); + + // isa + Elements.addBitCast(MetaClass, PtrToInt8Ty); + // super_class + Elements.add(SuperClass); + // name + Elements.add(MakeConstantString(Name, ".class_name")); + // version + Elements.addInt(LongTy, 0); + // info + Elements.addInt(LongTy, info); + // instance_size if (isMeta) { llvm::DataLayout td(&TheModule); - Elements.push_back( - llvm::ConstantInt::get(LongTy, - td.getTypeSizeInBits(ClassTy) / - CGM.getContext().getCharWidth())); + Elements.addInt(LongTy, + td.getTypeSizeInBits(ClassTy) / + CGM.getContext().getCharWidth()); } else - Elements.push_back(InstanceSize); - Elements.push_back(IVars); - Elements.push_back(Methods); - Elements.push_back(NULLPtr); - Elements.push_back(NULLPtr); - Elements.push_back(NULLPtr); - Elements.push_back(llvm::ConstantExpr::getBitCast(Protocols, PtrTy)); - Elements.push_back(NULLPtr); - Elements.push_back(llvm::ConstantInt::get(LongTy, 1)); - Elements.push_back(IvarOffsets); - Elements.push_back(Properties); - Elements.push_back(StrongIvarBitmap); - Elements.push_back(WeakIvarBitmap); + Elements.add(InstanceSize); + // ivars + Elements.add(IVars); + // methods + Elements.add(Methods); + // These are all filled in by the runtime, so we pretend + // dtable + Elements.add(NULLPtr); + // subclass_list + Elements.add(NULLPtr); + // sibling_class + Elements.add(NULLPtr); + // protocols + Elements.addBitCast(Protocols, PtrTy); + // gc_object_type + Elements.add(NULLPtr); + // abi_version + Elements.addInt(LongTy, 1); + // ivar_offsets + Elements.add(IvarOffsets); + // properties + Elements.add(Properties); + // strong_pointers + Elements.add(StrongIvarBitmap); + // weak_pointers + Elements.add(WeakIvarBitmap); // Create an instance of the structure // This is now an externally visible symbol, so that we can speed up class // messages in the next ABI. We may already have some weak references to @@ -1713,13 +1689,13 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure( std::string(Name)); llvm::GlobalVariable *ClassRef = TheModule.getNamedGlobal(ClassSym); llvm::Constant *Class = - MakeGlobal(ClassTy, Elements, CGM.getPointerAlign(), ClassSym, - llvm::GlobalValue::ExternalLinkage); + Elements.finishAndCreateGlobal(ClassSym, CGM.getPointerAlign(), false, + llvm::GlobalValue::ExternalLinkage); if (ClassRef) { - ClassRef->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(Class, + ClassRef->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(Class, ClassRef->getType())); - ClassRef->removeFromParent(); - Class->setName(ClassSym); + ClassRef->removeFromParent(); + Class->setName(ClassSym); } return Class; } @@ -1728,38 +1704,33 @@ llvm::Constant *CGObjCGNU:: GenerateProtocolMethodList(ArrayRef<llvm::Constant *> MethodNames, ArrayRef<llvm::Constant *> MethodTypes) { // Get the method structure type. - llvm::StructType *ObjCMethodDescTy = llvm::StructType::get( - PtrToInt8Ty, // Really a selector, but the runtime does the casting for us. - PtrToInt8Ty, - nullptr); - std::vector<llvm::Constant*> Methods; + llvm::StructType *ObjCMethodDescTy = + llvm::StructType::get(CGM.getLLVMContext(), { PtrToInt8Ty, PtrToInt8Ty }); + ConstantInitBuilder Builder(CGM); + auto MethodList = Builder.beginStruct(); + MethodList.addInt(IntTy, MethodNames.size()); + auto Methods = MethodList.beginArray(ObjCMethodDescTy); for (unsigned int i = 0, e = MethodTypes.size() ; i < e ; i++) { - Methods.push_back(llvm::ConstantStruct::get( - ObjCMethodDescTy, {MethodNames[i], MethodTypes[i]})); - } - llvm::ArrayType *ObjCMethodArrayTy = llvm::ArrayType::get(ObjCMethodDescTy, - MethodNames.size()); - llvm::Constant *Array = llvm::ConstantArray::get(ObjCMethodArrayTy, - Methods); - llvm::StructType *ObjCMethodDescListTy = llvm::StructType::get( - IntTy, ObjCMethodArrayTy, nullptr); - Methods.clear(); - Methods.push_back(llvm::ConstantInt::get(IntTy, MethodNames.size())); - Methods.push_back(Array); - return MakeGlobal(ObjCMethodDescListTy, Methods, CGM.getPointerAlign(), - ".objc_method_list"); + auto Method = Methods.beginStruct(ObjCMethodDescTy); + Method.add(MethodNames[i]); + Method.add(MethodTypes[i]); + Method.finishAndAddTo(Methods); + } + Methods.finishAndAddTo(MethodList); + return MethodList.finishAndCreateGlobal(".objc_method_list", + CGM.getPointerAlign()); } // Create the protocol list structure used in classes, categories and so on -llvm::Constant *CGObjCGNU::GenerateProtocolList(ArrayRef<std::string>Protocols){ - llvm::ArrayType *ProtocolArrayTy = llvm::ArrayType::get(PtrToInt8Ty, - Protocols.size()); - llvm::StructType *ProtocolListTy = llvm::StructType::get( - PtrTy, //Should be a recurisve pointer, but it's always NULL here. - SizeTy, - ProtocolArrayTy, - nullptr); - std::vector<llvm::Constant*> Elements; +llvm::Constant * +CGObjCGNU::GenerateProtocolList(ArrayRef<std::string> Protocols) { + + ConstantInitBuilder Builder(CGM); + auto ProtocolList = Builder.beginStruct(); + ProtocolList.add(NULLPtr); + ProtocolList.addInt(LongTy, Protocols.size()); + + auto Elements = ProtocolList.beginArray(PtrToInt8Ty); for (const std::string *iter = Protocols.begin(), *endIter = Protocols.end(); iter != endIter ; iter++) { llvm::Constant *protocol = nullptr; @@ -1770,18 +1741,11 @@ llvm::Constant *CGObjCGNU::GenerateProtocolList(ArrayRef<std::string>Protocols){ } else { protocol = value->getValue(); } - llvm::Constant *Ptr = llvm::ConstantExpr::getBitCast(protocol, - PtrToInt8Ty); - Elements.push_back(Ptr); - } - llvm::Constant * ProtocolArray = llvm::ConstantArray::get(ProtocolArrayTy, - Elements); - Elements.clear(); - Elements.push_back(NULLPtr); - Elements.push_back(llvm::ConstantInt::get(LongTy, Protocols.size())); - Elements.push_back(ProtocolArray); - return MakeGlobal(ProtocolListTy, Elements, CGM.getPointerAlign(), - ".objc_protocol_list"); + Elements.addBitCast(protocol, PtrToInt8Ty); + } + Elements.finishAndAddTo(ProtocolList); + return ProtocolList.finishAndCreateGlobal(".objc_protocol_list", + CGM.getPointerAlign()); } llvm::Value *CGObjCGNU::GenerateProtocolRef(CodeGenFunction &CGF, @@ -1792,33 +1756,28 @@ llvm::Value *CGObjCGNU::GenerateProtocolRef(CodeGenFunction &CGF, return CGF.Builder.CreateBitCast(protocol, llvm::PointerType::getUnqual(T)); } -llvm::Constant *CGObjCGNU::GenerateEmptyProtocol( - const std::string &ProtocolName) { - SmallVector<std::string, 0> EmptyStringVector; - SmallVector<llvm::Constant*, 0> EmptyConstantVector; - - llvm::Constant *ProtocolList = GenerateProtocolList(EmptyStringVector); - llvm::Constant *MethodList = - GenerateProtocolMethodList(EmptyConstantVector, EmptyConstantVector); +llvm::Constant * +CGObjCGNU::GenerateEmptyProtocol(const std::string &ProtocolName) { + llvm::Constant *ProtocolList = GenerateProtocolList({}); + llvm::Constant *MethodList = GenerateProtocolMethodList({}, {}); // Protocols are objects containing lists of the methods implemented and // protocols adopted. - llvm::StructType *ProtocolTy = llvm::StructType::get(IdTy, - PtrToInt8Ty, - ProtocolList->getType(), - MethodList->getType(), - MethodList->getType(), - MethodList->getType(), - MethodList->getType(), - nullptr); + ConstantInitBuilder Builder(CGM); + auto Elements = Builder.beginStruct(); + // The isa pointer must be set to a magic number so the runtime knows it's // the correct layout. - llvm::Constant *Elements[] = { - llvm::ConstantExpr::getIntToPtr( - llvm::ConstantInt::get(Int32Ty, ProtocolVersion), IdTy), - MakeConstantString(ProtocolName, ".objc_protocol_name"), ProtocolList, - MethodList, MethodList, MethodList, MethodList}; - return MakeGlobal(ProtocolTy, Elements, CGM.getPointerAlign(), - ".objc_protocol"); + Elements.add(llvm::ConstantExpr::getIntToPtr( + llvm::ConstantInt::get(Int32Ty, ProtocolVersion), IdTy)); + + Elements.add(MakeConstantString(ProtocolName, ".objc_protocol_name")); + Elements.add(ProtocolList); + Elements.add(MethodList); + Elements.add(MethodList); + Elements.add(MethodList); + Elements.add(MethodList); + return Elements.finishAndCreateGlobal(".objc_protocol", + CGM.getPointerAlign()); } void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) { @@ -1837,8 +1796,7 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) { SmallVector<llvm::Constant*, 16> OptionalInstanceMethodNames; SmallVector<llvm::Constant*, 16> OptionalInstanceMethodTypes; for (const auto *I : PD->instance_methods()) { - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(I, TypeStr); + std::string TypeStr = Context.getObjCEncodingForMethodDecl(I); if (I->getImplementationControl() == ObjCMethodDecl::Optional) { OptionalInstanceMethodNames.push_back( MakeConstantString(I->getSelector().getAsString())); @@ -1855,8 +1813,7 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) { SmallVector<llvm::Constant*, 16> OptionalClassMethodNames; SmallVector<llvm::Constant*, 16> OptionalClassMethodTypes; for (const auto *I : PD->class_methods()) { - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(I,TypeStr); + std::string TypeStr = Context.getObjCEncodingForMethodDecl(I); if (I->getImplementationControl() == ObjCMethodDecl::Optional) { OptionalClassMethodNames.push_back( MakeConstantString(I->getSelector().getAsString())); @@ -1885,142 +1842,139 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) { // The isSynthesized value is always set to 0 in a protocol. It exists to // simplify the runtime library by allowing it to use the same data // structures for protocol metadata everywhere. - llvm::StructType *PropertyMetadataTy = llvm::StructType::get( - PtrToInt8Ty, Int8Ty, Int8Ty, Int8Ty, Int8Ty, PtrToInt8Ty, - PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty, nullptr); - std::vector<llvm::Constant*> Properties; - std::vector<llvm::Constant*> OptionalProperties; - // Add all of the property methods need adding to the method list and to the - // property metadata list. - for (auto *property : PD->instance_properties()) { - std::vector<llvm::Constant*> Fields; + llvm::Constant *PropertyList; + llvm::Constant *OptionalPropertyList; + { + llvm::StructType *propertyMetadataTy = + llvm::StructType::get(CGM.getLLVMContext(), + { PtrToInt8Ty, Int8Ty, Int8Ty, Int8Ty, Int8Ty, PtrToInt8Ty, + PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty }); + + unsigned numReqProperties = 0, numOptProperties = 0; + for (auto property : PD->instance_properties()) { + if (property->isOptional()) + numOptProperties++; + else + numReqProperties++; + } - Fields.push_back(MakePropertyEncodingString(property, nullptr)); - PushPropertyAttributes(Fields, property); + ConstantInitBuilder reqPropertyListBuilder(CGM); + auto reqPropertiesList = reqPropertyListBuilder.beginStruct(); + reqPropertiesList.addInt(IntTy, numReqProperties); + reqPropertiesList.add(NULLPtr); + auto reqPropertiesArray = reqPropertiesList.beginArray(propertyMetadataTy); + + ConstantInitBuilder optPropertyListBuilder(CGM); + auto optPropertiesList = optPropertyListBuilder.beginStruct(); + optPropertiesList.addInt(IntTy, numOptProperties); + optPropertiesList.add(NULLPtr); + auto optPropertiesArray = optPropertiesList.beginArray(propertyMetadataTy); + + // Add all of the property methods need adding to the method list and to the + // property metadata list. + for (auto *property : PD->instance_properties()) { + auto &propertiesArray = + (property->isOptional() ? optPropertiesArray : reqPropertiesArray); + auto fields = propertiesArray.beginStruct(propertyMetadataTy); + + fields.add(MakePropertyEncodingString(property, nullptr)); + PushPropertyAttributes(fields, property); + + if (ObjCMethodDecl *getter = property->getGetterMethodDecl()) { + std::string typeStr = Context.getObjCEncodingForMethodDecl(getter); + llvm::Constant *typeEncoding = MakeConstantString(typeStr); + InstanceMethodTypes.push_back(typeEncoding); + fields.add(MakeConstantString(getter->getSelector().getAsString())); + fields.add(typeEncoding); + } else { + fields.add(NULLPtr); + fields.add(NULLPtr); + } + if (ObjCMethodDecl *setter = property->getSetterMethodDecl()) { + std::string typeStr = Context.getObjCEncodingForMethodDecl(setter); + llvm::Constant *typeEncoding = MakeConstantString(typeStr); + InstanceMethodTypes.push_back(typeEncoding); + fields.add(MakeConstantString(setter->getSelector().getAsString())); + fields.add(typeEncoding); + } else { + fields.add(NULLPtr); + fields.add(NULLPtr); + } - if (ObjCMethodDecl *getter = property->getGetterMethodDecl()) { - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(getter,TypeStr); - llvm::Constant *TypeEncoding = MakeConstantString(TypeStr); - InstanceMethodTypes.push_back(TypeEncoding); - Fields.push_back(MakeConstantString(getter->getSelector().getAsString())); - Fields.push_back(TypeEncoding); - } else { - Fields.push_back(NULLPtr); - Fields.push_back(NULLPtr); - } - if (ObjCMethodDecl *setter = property->getSetterMethodDecl()) { - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(setter,TypeStr); - llvm::Constant *TypeEncoding = MakeConstantString(TypeStr); - InstanceMethodTypes.push_back(TypeEncoding); - Fields.push_back(MakeConstantString(setter->getSelector().getAsString())); - Fields.push_back(TypeEncoding); - } else { - Fields.push_back(NULLPtr); - Fields.push_back(NULLPtr); - } - if (property->getPropertyImplementation() == ObjCPropertyDecl::Optional) { - OptionalProperties.push_back(llvm::ConstantStruct::get(PropertyMetadataTy, Fields)); - } else { - Properties.push_back(llvm::ConstantStruct::get(PropertyMetadataTy, Fields)); + fields.finishAndAddTo(propertiesArray); } + + reqPropertiesArray.finishAndAddTo(reqPropertiesList); + PropertyList = + reqPropertiesList.finishAndCreateGlobal(".objc_property_list", + CGM.getPointerAlign()); + + optPropertiesArray.finishAndAddTo(optPropertiesList); + OptionalPropertyList = + optPropertiesList.finishAndCreateGlobal(".objc_property_list", + CGM.getPointerAlign()); } - llvm::Constant *PropertyArray = llvm::ConstantArray::get( - llvm::ArrayType::get(PropertyMetadataTy, Properties.size()), Properties); - llvm::Constant* PropertyListInitFields[] = - {llvm::ConstantInt::get(IntTy, Properties.size()), NULLPtr, PropertyArray}; - - llvm::Constant *PropertyListInit = - llvm::ConstantStruct::getAnon(PropertyListInitFields); - llvm::Constant *PropertyList = new llvm::GlobalVariable(TheModule, - PropertyListInit->getType(), false, llvm::GlobalValue::InternalLinkage, - PropertyListInit, ".objc_property_list"); - - llvm::Constant *OptionalPropertyArray = - llvm::ConstantArray::get(llvm::ArrayType::get(PropertyMetadataTy, - OptionalProperties.size()) , OptionalProperties); - llvm::Constant* OptionalPropertyListInitFields[] = { - llvm::ConstantInt::get(IntTy, OptionalProperties.size()), NULLPtr, - OptionalPropertyArray }; - - llvm::Constant *OptionalPropertyListInit = - llvm::ConstantStruct::getAnon(OptionalPropertyListInitFields); - llvm::Constant *OptionalPropertyList = new llvm::GlobalVariable(TheModule, - OptionalPropertyListInit->getType(), false, - llvm::GlobalValue::InternalLinkage, OptionalPropertyListInit, - ".objc_property_list"); // Protocols are objects containing lists of the methods implemented and // protocols adopted. - llvm::StructType *ProtocolTy = llvm::StructType::get(IdTy, - PtrToInt8Ty, - ProtocolList->getType(), - InstanceMethodList->getType(), - ClassMethodList->getType(), - OptionalInstanceMethodList->getType(), - OptionalClassMethodList->getType(), - PropertyList->getType(), - OptionalPropertyList->getType(), - nullptr); // The isa pointer must be set to a magic number so the runtime knows it's // the correct layout. - llvm::Constant *Elements[] = { + ConstantInitBuilder Builder(CGM); + auto Elements = Builder.beginStruct(); + Elements.add( llvm::ConstantExpr::getIntToPtr( - llvm::ConstantInt::get(Int32Ty, ProtocolVersion), IdTy), - MakeConstantString(ProtocolName, ".objc_protocol_name"), ProtocolList, - InstanceMethodList, ClassMethodList, OptionalInstanceMethodList, - OptionalClassMethodList, PropertyList, OptionalPropertyList}; + llvm::ConstantInt::get(Int32Ty, ProtocolVersion), IdTy)); + Elements.add( + MakeConstantString(ProtocolName, ".objc_protocol_name")); + Elements.add(ProtocolList); + Elements.add(InstanceMethodList); + Elements.add(ClassMethodList); + Elements.add(OptionalInstanceMethodList); + Elements.add(OptionalClassMethodList); + Elements.add(PropertyList); + Elements.add(OptionalPropertyList); ExistingProtocols[ProtocolName] = - llvm::ConstantExpr::getBitCast(MakeGlobal(ProtocolTy, Elements, - CGM.getPointerAlign(), ".objc_protocol"), IdTy); + llvm::ConstantExpr::getBitCast( + Elements.finishAndCreateGlobal(".objc_protocol", CGM.getPointerAlign()), + IdTy); } void CGObjCGNU::GenerateProtocolHolderCategory() { // Collect information about instance methods SmallVector<Selector, 1> MethodSels; SmallVector<llvm::Constant*, 1> MethodTypes; - std::vector<llvm::Constant*> Elements; + ConstantInitBuilder Builder(CGM); + auto Elements = Builder.beginStruct(); + const std::string ClassName = "__ObjC_Protocol_Holder_Ugly_Hack"; const std::string CategoryName = "AnotherHack"; - Elements.push_back(MakeConstantString(CategoryName)); - Elements.push_back(MakeConstantString(ClassName)); + Elements.add(MakeConstantString(CategoryName)); + Elements.add(MakeConstantString(ClassName)); // Instance method list - Elements.push_back(llvm::ConstantExpr::getBitCast(GenerateMethodList( - ClassName, CategoryName, MethodSels, MethodTypes, false), PtrTy)); + Elements.addBitCast(GenerateMethodList( + ClassName, CategoryName, MethodSels, MethodTypes, false), PtrTy); // Class method list - Elements.push_back(llvm::ConstantExpr::getBitCast(GenerateMethodList( - ClassName, CategoryName, MethodSels, MethodTypes, true), PtrTy)); + Elements.addBitCast(GenerateMethodList( + ClassName, CategoryName, MethodSels, MethodTypes, true), PtrTy); + // Protocol list - llvm::ArrayType *ProtocolArrayTy = llvm::ArrayType::get(PtrTy, - ExistingProtocols.size()); - llvm::StructType *ProtocolListTy = llvm::StructType::get( - PtrTy, //Should be a recurisve pointer, but it's always NULL here. - SizeTy, - ProtocolArrayTy, - nullptr); - std::vector<llvm::Constant*> ProtocolElements; - for (llvm::StringMapIterator<llvm::Constant*> iter = - ExistingProtocols.begin(), endIter = ExistingProtocols.end(); + ConstantInitBuilder ProtocolListBuilder(CGM); + auto ProtocolList = ProtocolListBuilder.beginStruct(); + ProtocolList.add(NULLPtr); + ProtocolList.addInt(LongTy, ExistingProtocols.size()); + auto ProtocolElements = ProtocolList.beginArray(PtrTy); + for (auto iter = ExistingProtocols.begin(), endIter = ExistingProtocols.end(); iter != endIter ; iter++) { - llvm::Constant *Ptr = llvm::ConstantExpr::getBitCast(iter->getValue(), - PtrTy); - ProtocolElements.push_back(Ptr); - } - llvm::Constant * ProtocolArray = llvm::ConstantArray::get(ProtocolArrayTy, - ProtocolElements); - ProtocolElements.clear(); - ProtocolElements.push_back(NULLPtr); - ProtocolElements.push_back(llvm::ConstantInt::get(LongTy, - ExistingProtocols.size())); - ProtocolElements.push_back(ProtocolArray); - Elements.push_back(llvm::ConstantExpr::getBitCast(MakeGlobal(ProtocolListTy, - ProtocolElements, CGM.getPointerAlign(), - ".objc_protocol_list"), PtrTy)); + ProtocolElements.addBitCast(iter->getValue(), PtrTy); + } + ProtocolElements.finishAndAddTo(ProtocolList); + Elements.addBitCast( + ProtocolList.finishAndCreateGlobal(".objc_protocol_list", + CGM.getPointerAlign()), + PtrTy); Categories.push_back(llvm::ConstantExpr::getBitCast( - MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, - PtrTy, PtrTy, PtrTy, nullptr), Elements, CGM.getPointerAlign()), + Elements.finishAndCreateGlobal("", CGM.getPointerAlign()), PtrTy)); } @@ -2055,13 +2009,16 @@ llvm::Constant *CGObjCGNU::MakeBitField(ArrayRef<bool> bits) { } values.push_back(llvm::ConstantInt::get(Int32Ty, word)); } - llvm::ArrayType *arrayTy = llvm::ArrayType::get(Int32Ty, values.size()); - llvm::Constant *array = llvm::ConstantArray::get(arrayTy, values); - llvm::Constant *fields[2] = { - llvm::ConstantInt::get(Int32Ty, values.size()), - array }; - llvm::Constant *GS = MakeGlobal(llvm::StructType::get(Int32Ty, arrayTy, - nullptr), fields, CharUnits::fromQuantity(4)); + + ConstantInitBuilder builder(CGM); + auto fields = builder.beginStruct(); + fields.addInt(Int32Ty, values.size()); + auto array = fields.beginArray(); + for (auto v : values) array.add(v); + array.finishAndAddTo(fields); + + llvm::Constant *GS = + fields.finishAndCreateGlobal("", CharUnits::fromQuantity(4)); llvm::Constant *ptr = llvm::ConstantExpr::getPtrToInt(GS, IntPtrTy); return ptr; } @@ -2074,8 +2031,7 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) { SmallVector<llvm::Constant*, 16> InstanceMethodTypes; for (const auto *I : OCD->instance_methods()) { InstanceMethodSels.push_back(I->getSelector()); - std::string TypeStr; - CGM.getContext().getObjCEncodingForMethodDecl(I,TypeStr); + std::string TypeStr = CGM.getContext().getObjCEncodingForMethodDecl(I); InstanceMethodTypes.push_back(MakeConstantString(TypeStr)); } @@ -2084,8 +2040,7 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) { SmallVector<llvm::Constant*, 16> ClassMethodTypes; for (const auto *I : OCD->class_methods()) { ClassMethodSels.push_back(I->getSelector()); - std::string TypeStr; - CGM.getContext().getObjCEncodingForMethodDecl(I,TypeStr); + std::string TypeStr = CGM.getContext().getObjCEncodingForMethodDecl(I); ClassMethodTypes.push_back(MakeConstantString(TypeStr)); } @@ -2097,23 +2052,24 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) { E = Protos.end(); I != E; ++I) Protocols.push_back((*I)->getNameAsString()); - llvm::Constant *Elements[] = { - MakeConstantString(CategoryName), MakeConstantString(ClassName), - // Instance method list - llvm::ConstantExpr::getBitCast( + ConstantInitBuilder Builder(CGM); + auto Elements = Builder.beginStruct(); + Elements.add(MakeConstantString(CategoryName)); + Elements.add(MakeConstantString(ClassName)); + // Instance method list + Elements.addBitCast( GenerateMethodList(ClassName, CategoryName, InstanceMethodSels, InstanceMethodTypes, false), - PtrTy), - // Class method list - llvm::ConstantExpr::getBitCast(GenerateMethodList(ClassName, CategoryName, - ClassMethodSels, - ClassMethodTypes, true), - PtrTy), - // Protocol list - llvm::ConstantExpr::getBitCast(GenerateProtocolList(Protocols), PtrTy)}; + PtrTy); + // Class method list + Elements.addBitCast( + GenerateMethodList(ClassName, CategoryName, ClassMethodSels, + ClassMethodTypes, true), + PtrTy); + // Protocol list + Elements.addBitCast(GenerateProtocolList(Protocols), PtrTy); Categories.push_back(llvm::ConstantExpr::getBitCast( - MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, - PtrTy, PtrTy, PtrTy, nullptr), Elements, CGM.getPointerAlign()), + Elements.finishAndCreateGlobal("", CGM.getPointerAlign()), PtrTy)); } @@ -2123,65 +2079,67 @@ llvm::Constant *CGObjCGNU::GeneratePropertyList(const ObjCImplementationDecl *OI ASTContext &Context = CGM.getContext(); // Property metadata: name, attributes, attributes2, padding1, padding2, // setter name, setter types, getter name, getter types. - llvm::StructType *PropertyMetadataTy = llvm::StructType::get( - PtrToInt8Ty, Int8Ty, Int8Ty, Int8Ty, Int8Ty, PtrToInt8Ty, - PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty, nullptr); - std::vector<llvm::Constant*> Properties; + llvm::StructType *propertyMetadataTy = + llvm::StructType::get(CGM.getLLVMContext(), + { PtrToInt8Ty, Int8Ty, Int8Ty, Int8Ty, Int8Ty, PtrToInt8Ty, + PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty }); + + unsigned numProperties = 0; + for (auto *propertyImpl : OID->property_impls()) { + (void) propertyImpl; + numProperties++; + } + + ConstantInitBuilder builder(CGM); + auto propertyList = builder.beginStruct(); + propertyList.addInt(IntTy, numProperties); + propertyList.add(NULLPtr); + auto properties = propertyList.beginArray(propertyMetadataTy); // Add all of the property methods need adding to the method list and to the // property metadata list. for (auto *propertyImpl : OID->property_impls()) { - std::vector<llvm::Constant*> Fields; + auto fields = properties.beginStruct(propertyMetadataTy); ObjCPropertyDecl *property = propertyImpl->getPropertyDecl(); bool isSynthesized = (propertyImpl->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize); bool isDynamic = (propertyImpl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic); - Fields.push_back(MakePropertyEncodingString(property, OID)); - PushPropertyAttributes(Fields, property, isSynthesized, isDynamic); + fields.add(MakePropertyEncodingString(property, OID)); + PushPropertyAttributes(fields, property, isSynthesized, isDynamic); if (ObjCMethodDecl *getter = property->getGetterMethodDecl()) { - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(getter,TypeStr); + std::string TypeStr = Context.getObjCEncodingForMethodDecl(getter); llvm::Constant *TypeEncoding = MakeConstantString(TypeStr); if (isSynthesized) { InstanceMethodTypes.push_back(TypeEncoding); InstanceMethodSels.push_back(getter->getSelector()); } - Fields.push_back(MakeConstantString(getter->getSelector().getAsString())); - Fields.push_back(TypeEncoding); + fields.add(MakeConstantString(getter->getSelector().getAsString())); + fields.add(TypeEncoding); } else { - Fields.push_back(NULLPtr); - Fields.push_back(NULLPtr); + fields.add(NULLPtr); + fields.add(NULLPtr); } if (ObjCMethodDecl *setter = property->getSetterMethodDecl()) { - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(setter,TypeStr); + std::string TypeStr = Context.getObjCEncodingForMethodDecl(setter); llvm::Constant *TypeEncoding = MakeConstantString(TypeStr); if (isSynthesized) { InstanceMethodTypes.push_back(TypeEncoding); InstanceMethodSels.push_back(setter->getSelector()); } - Fields.push_back(MakeConstantString(setter->getSelector().getAsString())); - Fields.push_back(TypeEncoding); + fields.add(MakeConstantString(setter->getSelector().getAsString())); + fields.add(TypeEncoding); } else { - Fields.push_back(NULLPtr); - Fields.push_back(NULLPtr); + fields.add(NULLPtr); + fields.add(NULLPtr); } - Properties.push_back(llvm::ConstantStruct::get(PropertyMetadataTy, Fields)); - } - llvm::ArrayType *PropertyArrayTy = - llvm::ArrayType::get(PropertyMetadataTy, Properties.size()); - llvm::Constant *PropertyArray = llvm::ConstantArray::get(PropertyArrayTy, - Properties); - llvm::Constant* PropertyListInitFields[] = - {llvm::ConstantInt::get(IntTy, Properties.size()), NULLPtr, PropertyArray}; - - llvm::Constant *PropertyListInit = - llvm::ConstantStruct::getAnon(PropertyListInitFields); - return new llvm::GlobalVariable(TheModule, PropertyListInit->getType(), false, - llvm::GlobalValue::InternalLinkage, PropertyListInit, - ".objc_property_list"); + fields.finishAndAddTo(properties); + } + properties.finishAndAddTo(propertyList); + + return propertyList.finishAndCreateGlobal(".objc_property_list", + CGM.getPointerAlign()); } void CGObjCGNU::RegisterAlias(const ObjCCompatibleAliasDecl *OAD) { @@ -2230,7 +2188,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { SmallVector<llvm::Constant*, 16> IvarTypes; SmallVector<llvm::Constant*, 16> IvarOffsets; - std::vector<llvm::Constant*> IvarOffsetValues; + ConstantInitBuilder IvarOffsetBuilder(CGM); + auto IvarOffsetValues = IvarOffsetBuilder.beginArray(PtrToIntTy); SmallVector<bool, 16> WeakIvars; SmallVector<bool, 16> StrongIvars; @@ -2274,7 +2233,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { "__objc_ivar_offset_value_" + ClassName +"." + IVD->getNameAsString()); IvarOffsets.push_back(OffsetValue); - IvarOffsetValues.push_back(OffsetVar); + IvarOffsetValues.add(OffsetVar); Qualifiers::ObjCLifetime lt = IVD->getType().getQualifiers().getObjCLifetime(); switch (lt) { case Qualifiers::OCL_Strong: @@ -2293,16 +2252,15 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { llvm::Constant *StrongIvarBitmap = MakeBitField(StrongIvars); llvm::Constant *WeakIvarBitmap = MakeBitField(WeakIvars); llvm::GlobalVariable *IvarOffsetArray = - MakeGlobalArray(PtrToIntTy, IvarOffsetValues, CGM.getPointerAlign(), - ".ivar.offsets"); + IvarOffsetValues.finishAndCreateGlobal(".ivar.offsets", + CGM.getPointerAlign()); // Collect information about instance methods SmallVector<Selector, 16> InstanceMethodSels; SmallVector<llvm::Constant*, 16> InstanceMethodTypes; for (const auto *I : OID->instance_methods()) { InstanceMethodSels.push_back(I->getSelector()); - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(I,TypeStr); + std::string TypeStr = Context.getObjCEncodingForMethodDecl(I); InstanceMethodTypes.push_back(MakeConstantString(TypeStr)); } @@ -2314,8 +2272,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { SmallVector<llvm::Constant*, 16> ClassMethodTypes; for (const auto *I : OID->class_methods()) { ClassMethodSels.push_back(I->getSelector()); - std::string TypeStr; - Context.getObjCEncodingForMethodDecl(I,TypeStr); + std::string TypeStr = Context.getObjCEncodingForMethodDecl(I); ClassMethodTypes.push_back(MakeConstantString(TypeStr)); } // Collect the names of referenced protocols @@ -2439,170 +2396,180 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { // Add all referenced protocols to a category. GenerateProtocolHolderCategory(); - llvm::StructType *SelStructTy = dyn_cast<llvm::StructType>( - SelectorTy->getElementType()); - llvm::Type *SelStructPtrTy = SelectorTy; - if (!SelStructTy) { - SelStructTy = llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, nullptr); - SelStructPtrTy = llvm::PointerType::getUnqual(SelStructTy); + llvm::StructType *selStructTy = + dyn_cast<llvm::StructType>(SelectorTy->getElementType()); + llvm::Type *selStructPtrTy = SelectorTy; + if (!selStructTy) { + selStructTy = llvm::StructType::get(CGM.getLLVMContext(), + { PtrToInt8Ty, PtrToInt8Ty }); + selStructPtrTy = llvm::PointerType::getUnqual(selStructTy); } - std::vector<llvm::Constant*> Elements; - llvm::Constant *Statics = NULLPtr; // Generate statics list: + llvm::Constant *statics = NULLPtr; if (!ConstantStrings.empty()) { - llvm::ArrayType *StaticsArrayTy = llvm::ArrayType::get(PtrToInt8Ty, - ConstantStrings.size() + 1); - ConstantStrings.push_back(NULLPtr); - - StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass; - - if (StringClass.empty()) StringClass = "NXConstantString"; - - Elements.push_back(MakeConstantString(StringClass, - ".objc_static_class_name")); - Elements.push_back(llvm::ConstantArray::get(StaticsArrayTy, - ConstantStrings)); - llvm::StructType *StaticsListTy = - llvm::StructType::get(PtrToInt8Ty, StaticsArrayTy, nullptr); - llvm::Type *StaticsListPtrTy = - llvm::PointerType::getUnqual(StaticsListTy); - Statics = MakeGlobal(StaticsListTy, Elements, CGM.getPointerAlign(), - ".objc_statics"); - llvm::ArrayType *StaticsListArrayTy = - llvm::ArrayType::get(StaticsListPtrTy, 2); - Elements.clear(); - Elements.push_back(Statics); - Elements.push_back(llvm::Constant::getNullValue(StaticsListPtrTy)); - Statics = MakeGlobal(StaticsListArrayTy, Elements, - CGM.getPointerAlign(), ".objc_statics_ptr"); - Statics = llvm::ConstantExpr::getBitCast(Statics, PtrTy); - } - // Array of classes, categories, and constant objects - llvm::ArrayType *ClassListTy = llvm::ArrayType::get(PtrToInt8Ty, - Classes.size() + Categories.size() + 2); - llvm::StructType *SymTabTy = llvm::StructType::get(LongTy, SelStructPtrTy, - llvm::Type::getInt16Ty(VMContext), - llvm::Type::getInt16Ty(VMContext), - ClassListTy, nullptr); - - Elements.clear(); - // Pointer to an array of selectors used in this module. - std::vector<llvm::Constant*> Selectors; - std::vector<llvm::GlobalAlias*> SelectorAliases; - for (SelectorMap::iterator iter = SelectorTable.begin(), - iterEnd = SelectorTable.end(); iter != iterEnd ; ++iter) { + llvm::GlobalVariable *fileStatics = [&] { + ConstantInitBuilder builder(CGM); + auto staticsStruct = builder.beginStruct(); - std::string SelNameStr = iter->first.getAsString(); - llvm::Constant *SelName = ExportUniqueString(SelNameStr, ".objc_sel_name"); + StringRef stringClass = CGM.getLangOpts().ObjCConstantStringClass; + if (stringClass.empty()) stringClass = "NXConstantString"; + staticsStruct.add(MakeConstantString(stringClass, + ".objc_static_class_name")); - SmallVectorImpl<TypedSelector> &Types = iter->second; - for (SmallVectorImpl<TypedSelector>::iterator i = Types.begin(), - e = Types.end() ; i!=e ; i++) { + auto array = staticsStruct.beginArray(); + array.addAll(ConstantStrings); + array.add(NULLPtr); + array.finishAndAddTo(staticsStruct); - llvm::Constant *SelectorTypeEncoding = NULLPtr; - if (!i->first.empty()) - SelectorTypeEncoding = MakeConstantString(i->first, ".objc_sel_types"); + return staticsStruct.finishAndCreateGlobal(".objc_statics", + CGM.getPointerAlign()); + }(); - Elements.push_back(SelName); - Elements.push_back(SelectorTypeEncoding); - Selectors.push_back(llvm::ConstantStruct::get(SelStructTy, Elements)); - Elements.clear(); + ConstantInitBuilder builder(CGM); + auto allStaticsArray = builder.beginArray(fileStatics->getType()); + allStaticsArray.add(fileStatics); + allStaticsArray.addNullPointer(fileStatics->getType()); - // Store the selector alias for later replacement - SelectorAliases.push_back(i->second); - } + statics = allStaticsArray.finishAndCreateGlobal(".objc_statics_ptr", + CGM.getPointerAlign()); + statics = llvm::ConstantExpr::getBitCast(statics, PtrTy); } - unsigned SelectorCount = Selectors.size(); - // NULL-terminate the selector list. This should not actually be required, - // because the selector list has a length field. Unfortunately, the GCC - // runtime decides to ignore the length field and expects a NULL terminator, - // and GCC cooperates with this by always setting the length to 0. - Elements.push_back(NULLPtr); - Elements.push_back(NULLPtr); - Selectors.push_back(llvm::ConstantStruct::get(SelStructTy, Elements)); - Elements.clear(); - - // Number of static selectors - Elements.push_back(llvm::ConstantInt::get(LongTy, SelectorCount)); - llvm::GlobalVariable *SelectorList = - MakeGlobalArray(SelStructTy, Selectors, CGM.getPointerAlign(), - ".objc_selector_list"); - Elements.push_back(llvm::ConstantExpr::getBitCast(SelectorList, - SelStructPtrTy)); - // Now that all of the static selectors exist, create pointers to them. - for (unsigned int i=0 ; i<SelectorCount ; i++) { + // Array of classes, categories, and constant objects. + + SmallVector<llvm::GlobalAlias*, 16> selectorAliases; + unsigned selectorCount; + + // Pointer to an array of selectors used in this module. + llvm::GlobalVariable *selectorList = [&] { + ConstantInitBuilder builder(CGM); + auto selectors = builder.beginArray(selStructTy); + auto &table = SelectorTable; // MSVC workaround + for (auto &entry : table) { + + std::string selNameStr = entry.first.getAsString(); + llvm::Constant *selName = ExportUniqueString(selNameStr, ".objc_sel_name"); + + for (TypedSelector &sel : entry.second) { + llvm::Constant *selectorTypeEncoding = NULLPtr; + if (!sel.first.empty()) + selectorTypeEncoding = + MakeConstantString(sel.first, ".objc_sel_types"); + + auto selStruct = selectors.beginStruct(selStructTy); + selStruct.add(selName); + selStruct.add(selectorTypeEncoding); + selStruct.finishAndAddTo(selectors); + + // Store the selector alias for later replacement + selectorAliases.push_back(sel.second); + } + } + + // Remember the number of entries in the selector table. + selectorCount = selectors.size(); - llvm::Constant *Idxs[] = {Zeros[0], - llvm::ConstantInt::get(Int32Ty, i), Zeros[0]}; + // NULL-terminate the selector list. This should not actually be required, + // because the selector list has a length field. Unfortunately, the GCC + // runtime decides to ignore the length field and expects a NULL terminator, + // and GCC cooperates with this by always setting the length to 0. + auto selStruct = selectors.beginStruct(selStructTy); + selStruct.add(NULLPtr); + selStruct.add(NULLPtr); + selStruct.finishAndAddTo(selectors); + + return selectors.finishAndCreateGlobal(".objc_selector_list", + CGM.getPointerAlign()); + }(); + + // Now that all of the static selectors exist, create pointers to them. + for (unsigned i = 0; i < selectorCount; ++i) { + llvm::Constant *idxs[] = { + Zeros[0], + llvm::ConstantInt::get(Int32Ty, i) + }; // FIXME: We're generating redundant loads and stores here! - llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr( - SelectorList->getValueType(), SelectorList, makeArrayRef(Idxs, 2)); + llvm::Constant *selPtr = llvm::ConstantExpr::getGetElementPtr( + selectorList->getValueType(), selectorList, idxs); // If selectors are defined as an opaque type, cast the pointer to this // type. - SelPtr = llvm::ConstantExpr::getBitCast(SelPtr, SelectorTy); - SelectorAliases[i]->replaceAllUsesWith(SelPtr); - SelectorAliases[i]->eraseFromParent(); - } - - // Number of classes defined. - Elements.push_back(llvm::ConstantInt::get(llvm::Type::getInt16Ty(VMContext), - Classes.size())); - // Number of categories defined - Elements.push_back(llvm::ConstantInt::get(llvm::Type::getInt16Ty(VMContext), - Categories.size())); - // Create an array of classes, then categories, then static object instances - Classes.insert(Classes.end(), Categories.begin(), Categories.end()); - // NULL-terminated list of static object instances (mainly constant strings) - Classes.push_back(Statics); - Classes.push_back(NULLPtr); - llvm::Constant *ClassList = llvm::ConstantArray::get(ClassListTy, Classes); - Elements.push_back(ClassList); - // Construct the symbol table - llvm::Constant *SymTab = - MakeGlobal(SymTabTy, Elements, CGM.getPointerAlign()); + selPtr = llvm::ConstantExpr::getBitCast(selPtr, SelectorTy); + selectorAliases[i]->replaceAllUsesWith(selPtr); + selectorAliases[i]->eraseFromParent(); + } + + llvm::GlobalVariable *symtab = [&] { + ConstantInitBuilder builder(CGM); + auto symtab = builder.beginStruct(); + + // Number of static selectors + symtab.addInt(LongTy, selectorCount); + + symtab.addBitCast(selectorList, selStructPtrTy); + + // Number of classes defined. + symtab.addInt(CGM.Int16Ty, Classes.size()); + // Number of categories defined + symtab.addInt(CGM.Int16Ty, Categories.size()); + + // Create an array of classes, then categories, then static object instances + auto classList = symtab.beginArray(PtrToInt8Ty); + classList.addAll(Classes); + classList.addAll(Categories); + // NULL-terminated list of static object instances (mainly constant strings) + classList.add(statics); + classList.add(NULLPtr); + classList.finishAndAddTo(symtab); + + // Construct the symbol table. + return symtab.finishAndCreateGlobal("", CGM.getPointerAlign()); + }(); // The symbol table is contained in a module which has some version-checking // constants - llvm::StructType * ModuleTy = llvm::StructType::get(LongTy, LongTy, - PtrToInt8Ty, llvm::PointerType::getUnqual(SymTabTy), - (RuntimeVersion >= 10) ? IntTy : nullptr, nullptr); - Elements.clear(); - // Runtime version, used for ABI compatibility checking. - Elements.push_back(llvm::ConstantInt::get(LongTy, RuntimeVersion)); - // sizeof(ModuleTy) - llvm::DataLayout td(&TheModule); - Elements.push_back( - llvm::ConstantInt::get(LongTy, - td.getTypeSizeInBits(ModuleTy) / - CGM.getContext().getCharWidth())); - - // The path to the source file where this module was declared - SourceManager &SM = CGM.getContext().getSourceManager(); - const FileEntry *mainFile = SM.getFileEntryForID(SM.getMainFileID()); - std::string path = - std::string(mainFile->getDir()->getName()) + '/' + mainFile->getName(); - Elements.push_back(MakeConstantString(path, ".objc_source_file_name")); - Elements.push_back(SymTab); - - if (RuntimeVersion >= 10) - switch (CGM.getLangOpts().getGC()) { + llvm::Constant *module = [&] { + llvm::Type *moduleEltTys[] = { + LongTy, LongTy, PtrToInt8Ty, symtab->getType(), IntTy + }; + llvm::StructType *moduleTy = + llvm::StructType::get(CGM.getLLVMContext(), + makeArrayRef(moduleEltTys).drop_back(unsigned(RuntimeVersion < 10))); + + ConstantInitBuilder builder(CGM); + auto module = builder.beginStruct(moduleTy); + // Runtime version, used for ABI compatibility checking. + module.addInt(LongTy, RuntimeVersion); + // sizeof(ModuleTy) + module.addInt(LongTy, CGM.getDataLayout().getTypeStoreSize(moduleTy)); + + // The path to the source file where this module was declared + SourceManager &SM = CGM.getContext().getSourceManager(); + const FileEntry *mainFile = SM.getFileEntryForID(SM.getMainFileID()); + std::string path = + (Twine(mainFile->getDir()->getName()) + "/" + mainFile->getName()).str(); + module.add(MakeConstantString(path, ".objc_source_file_name")); + module.add(symtab); + + if (RuntimeVersion >= 10) { + switch (CGM.getLangOpts().getGC()) { case LangOptions::GCOnly: - Elements.push_back(llvm::ConstantInt::get(IntTy, 2)); + module.addInt(IntTy, 2); break; case LangOptions::NonGC: if (CGM.getLangOpts().ObjCAutoRefCount) - Elements.push_back(llvm::ConstantInt::get(IntTy, 1)); + module.addInt(IntTy, 1); else - Elements.push_back(llvm::ConstantInt::get(IntTy, 0)); + module.addInt(IntTy, 0); break; case LangOptions::HybridGC: - Elements.push_back(llvm::ConstantInt::get(IntTy, 1)); + module.addInt(IntTy, 1); break; + } } - llvm::Value *Module = MakeGlobal(ModuleTy, Elements, CGM.getPointerAlign()); + return module.finishAndCreateGlobal("", CGM.getPointerAlign()); + }(); // Create the load function calling the runtime entry point with the module // structure @@ -2616,10 +2583,9 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { Builder.SetInsertPoint(EntryBB); llvm::FunctionType *FT = - llvm::FunctionType::get(Builder.getVoidTy(), - llvm::PointerType::getUnqual(ModuleTy), true); + llvm::FunctionType::get(Builder.getVoidTy(), module->getType(), true); llvm::Value *Register = CGM.CreateRuntimeFunction(FT, "__objc_exec_class"); - Builder.CreateCall(Register, Module); + Builder.CreateCall(Register, module); if (!ClassAliases.empty()) { llvm::Type *ArgTypes[2] = {PtrTy, PtrToInt8Ty}; @@ -2646,8 +2612,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { for (std::vector<ClassAliasPair>::iterator iter = ClassAliases.begin(); iter != ClassAliases.end(); ++iter) { llvm::Constant *TheClass = - TheModule.getGlobalVariable(("_OBJC_CLASS_" + iter->first).c_str(), - true); + TheModule.getGlobalVariable("_OBJC_CLASS_" + iter->first, true); if (TheClass) { TheClass = llvm::ConstantExpr::getBitCast(TheClass, PtrTy); Builder.CreateCall(RegisterAlias, @@ -2910,9 +2875,11 @@ llvm::Value *CGObjCGNU::EmitIvarOffset(CodeGenFunction &CGF, if (RuntimeVersion < 10 || CGF.CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) return CGF.Builder.CreateZExtOrBitCast( - CGF.Builder.CreateDefaultAlignedLoad(CGF.Builder.CreateAlignedLoad( - ObjCIvarOffsetVariable(Interface, Ivar), - CGF.getPointerAlign(), "ivar")), + CGF.Builder.CreateAlignedLoad( + Int32Ty, CGF.Builder.CreateAlignedLoad( + ObjCIvarOffsetVariable(Interface, Ivar), + CGF.getPointerAlign(), "ivar"), + CharUnits::fromQuantity(4)), PtrDiffTy); std::string name = "__objc_ivar_offset_value_" + Interface->getNameAsString() +"." + Ivar->getNameAsString(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp index 5ab9fc4..7219592 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp @@ -11,12 +11,13 @@ // //===----------------------------------------------------------------------===// -#include "CGObjCRuntime.h" #include "CGBlocks.h" #include "CGCleanup.h" +#include "CGObjCRuntime.h" #include "CGRecordLayout.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantBuilder.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclObjC.h" @@ -25,6 +26,7 @@ #include "clang/Basic/LangOptions.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" +#include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" @@ -172,18 +174,18 @@ protected: CodeGen::CodeGenModule &CGM; public: - llvm::Type *ShortTy, *IntTy, *LongTy, *LongLongTy; - llvm::Type *Int8PtrTy, *Int8PtrPtrTy; + llvm::IntegerType *ShortTy, *IntTy, *LongTy; + llvm::PointerType *Int8PtrTy, *Int8PtrPtrTy; llvm::Type *IvarOffsetVarTy; /// ObjectPtrTy - LLVM type for object handles (typeof(id)) - llvm::Type *ObjectPtrTy; + llvm::PointerType *ObjectPtrTy; /// PtrObjectPtrTy - LLVM type for id * - llvm::Type *PtrObjectPtrTy; + llvm::PointerType *PtrObjectPtrTy; /// SelectorPtrTy - LLVM type for selector handles (typeof(SEL)) - llvm::Type *SelectorPtrTy; + llvm::PointerType *SelectorPtrTy; private: /// ProtocolPtrTy - LLVM type for external protocol handles @@ -212,7 +214,7 @@ public: /// SuperTy - LLVM type for struct objc_super. llvm::StructType *SuperTy; /// SuperPtrTy - LLVM type for struct objc_super *. - llvm::Type *SuperPtrTy; + llvm::PointerType *SuperPtrTy; /// PropertyTy - LLVM type for struct objc_property (struct _prop_t /// in GCC parlance). @@ -222,7 +224,7 @@ public: /// (_prop_list_t in GCC parlance). llvm::StructType *PropertyListTy; /// PropertyListPtrTy - LLVM type for struct objc_property_list*. - llvm::Type *PropertyListPtrTy; + llvm::PointerType *PropertyListPtrTy; // MethodTy - LLVM type for struct objc_method. llvm::StructType *MethodTy; @@ -230,7 +232,7 @@ public: /// CacheTy - LLVM type for struct objc_cache. llvm::Type *CacheTy; /// CachePtrTy - LLVM type for struct objc_cache *. - llvm::Type *CachePtrTy; + llvm::PointerType *CachePtrTy; llvm::Constant *getGetPropertyFn() { CodeGen::CodeGenTypes &Types = CGM.getTypes(); @@ -500,20 +502,20 @@ public: /// SymtabTy - LLVM type for struct objc_symtab. llvm::StructType *SymtabTy; /// SymtabPtrTy - LLVM type for struct objc_symtab *. - llvm::Type *SymtabPtrTy; + llvm::PointerType *SymtabPtrTy; /// ModuleTy - LLVM type for struct objc_module. llvm::StructType *ModuleTy; /// ProtocolTy - LLVM type for struct objc_protocol. llvm::StructType *ProtocolTy; /// ProtocolPtrTy - LLVM type for struct objc_protocol *. - llvm::Type *ProtocolPtrTy; + llvm::PointerType *ProtocolPtrTy; /// ProtocolExtensionTy - LLVM type for struct /// objc_protocol_extension. llvm::StructType *ProtocolExtensionTy; /// ProtocolExtensionTy - LLVM type for struct /// objc_protocol_extension *. - llvm::Type *ProtocolExtensionPtrTy; + llvm::PointerType *ProtocolExtensionPtrTy; /// MethodDescriptionTy - LLVM type for struct /// objc_method_description. llvm::StructType *MethodDescriptionTy; @@ -522,34 +524,34 @@ public: llvm::StructType *MethodDescriptionListTy; /// MethodDescriptionListPtrTy - LLVM type for struct /// objc_method_description_list *. - llvm::Type *MethodDescriptionListPtrTy; + llvm::PointerType *MethodDescriptionListPtrTy; /// ProtocolListTy - LLVM type for struct objc_property_list. llvm::StructType *ProtocolListTy; /// ProtocolListPtrTy - LLVM type for struct objc_property_list*. - llvm::Type *ProtocolListPtrTy; + llvm::PointerType *ProtocolListPtrTy; /// CategoryTy - LLVM type for struct objc_category. llvm::StructType *CategoryTy; /// ClassTy - LLVM type for struct objc_class. llvm::StructType *ClassTy; /// ClassPtrTy - LLVM type for struct objc_class *. - llvm::Type *ClassPtrTy; + llvm::PointerType *ClassPtrTy; /// ClassExtensionTy - LLVM type for struct objc_class_ext. llvm::StructType *ClassExtensionTy; /// ClassExtensionPtrTy - LLVM type for struct objc_class_ext *. - llvm::Type *ClassExtensionPtrTy; + llvm::PointerType *ClassExtensionPtrTy; // IvarTy - LLVM type for struct objc_ivar. llvm::StructType *IvarTy; /// IvarListTy - LLVM type for struct objc_ivar_list. - llvm::Type *IvarListTy; + llvm::StructType *IvarListTy; /// IvarListPtrTy - LLVM type for struct objc_ivar_list *. - llvm::Type *IvarListPtrTy; + llvm::PointerType *IvarListPtrTy; /// MethodListTy - LLVM type for struct objc_method_list. - llvm::Type *MethodListTy; + llvm::StructType *MethodListTy; /// MethodListPtrTy - LLVM type for struct objc_method_list *. - llvm::Type *MethodListPtrTy; + llvm::PointerType *MethodListPtrTy; /// ExceptionDataTy - LLVM type for struct _objc_exception_data. - llvm::Type *ExceptionDataTy; + llvm::StructType *ExceptionDataTy; /// ExceptionTryEnterFn - LLVM objc_exception_try_enter function. llvm::Constant *getExceptionTryEnterFn() { @@ -608,25 +610,25 @@ public: llvm::StructType *MethodListnfABITy; // MethodListnfABIPtrTy - LLVM for struct _method_list_t* - llvm::Type *MethodListnfABIPtrTy; + llvm::PointerType *MethodListnfABIPtrTy; // ProtocolnfABITy = LLVM for struct _protocol_t llvm::StructType *ProtocolnfABITy; // ProtocolnfABIPtrTy = LLVM for struct _protocol_t* - llvm::Type *ProtocolnfABIPtrTy; + llvm::PointerType *ProtocolnfABIPtrTy; // ProtocolListnfABITy - LLVM for struct _objc_protocol_list llvm::StructType *ProtocolListnfABITy; // ProtocolListnfABIPtrTy - LLVM for struct _objc_protocol_list* - llvm::Type *ProtocolListnfABIPtrTy; + llvm::PointerType *ProtocolListnfABIPtrTy; // ClassnfABITy - LLVM for struct _class_t llvm::StructType *ClassnfABITy; // ClassnfABIPtrTy - LLVM for struct _class_t* - llvm::Type *ClassnfABIPtrTy; + llvm::PointerType *ClassnfABIPtrTy; // IvarnfABITy - LLVM for struct _ivar_t llvm::StructType *IvarnfABITy; @@ -635,13 +637,13 @@ public: llvm::StructType *IvarListnfABITy; // IvarListnfABIPtrTy = LLVM for struct _ivar_list_t* - llvm::Type *IvarListnfABIPtrTy; + llvm::PointerType *IvarListnfABIPtrTy; // ClassRonfABITy - LLVM for struct _class_ro_t llvm::StructType *ClassRonfABITy; // ImpnfABITy - LLVM for id (*)(id, SEL, ...) - llvm::Type *ImpnfABITy; + llvm::PointerType *ImpnfABITy; // CategorynfABITy - LLVM for struct _category_t llvm::StructType *CategorynfABITy; @@ -670,7 +672,7 @@ public: llvm::StructType *SuperMessageRefTy; // SuperMessageRefPtrTy - LLVM for struct _super_message_ref_t* - llvm::Type *SuperMessageRefPtrTy; + llvm::PointerType *SuperMessageRefPtrTy; llvm::Constant *getMessageSendFixupFn() { // id objc_msgSend_fixup(id, struct message_ref_t*, ...) @@ -733,6 +735,13 @@ public: ObjCNonFragileABITypesHelper(CodeGen::CodeGenModule &cgm); }; +enum class ObjCLabelType { + ClassName, + MethodVarName, + MethodVarType, + PropertyName, +}; + class CGObjCCommonMac : public CodeGen::CGObjCRuntime { public: class SKIP_SCAN { @@ -836,7 +845,7 @@ protected: llvm::DenseMap<Selector, llvm::GlobalVariable*> MethodVarNames; /// DefinedCategoryNames - list of category names in form Class_Category. - llvm::SmallSetVector<std::string, 16> DefinedCategoryNames; + llvm::SmallSetVector<llvm::CachedHashString, 16> DefinedCategoryNames; /// MethodVarTypes - uniqued method type signatures. We have to use /// a StringMap here because have no other unique reference. @@ -879,6 +888,15 @@ protected: /// DefinedNonLazyCategories - List of defined "non-lazy" categories. SmallVector<llvm::GlobalValue*, 16> DefinedNonLazyCategories; + /// Cached reference to the class for constant strings. This value has type + /// int * but is actually an Obj-C class pointer. + llvm::WeakVH ConstantStringClassRef; + + /// \brief The LLVM type corresponding to NSConstantString. + llvm::StructType *NSConstantStringType = nullptr; + + llvm::StringMap<llvm::GlobalVariable *> NSConstantStringMap; + /// GetNameForMethod - Return a name for the given method. /// \param[out] NameOut - The return value. void GetNameForMethod(const ObjCMethodDecl *OMD, @@ -979,15 +997,6 @@ protected: ArrayRef<llvm::Constant*> MethodTypes, const ObjCCommonTypesHelper &ObjCTypes); - /// PushProtocolProperties - Push protocol's property on the input stack. - void PushProtocolProperties( - llvm::SmallPtrSet<const IdentifierInfo*, 16> &PropertySet, - SmallVectorImpl<llvm::Constant*> &Properties, - const Decl *Container, - const ObjCProtocolDecl *Proto, - const ObjCCommonTypesHelper &ObjCTypes, - bool IsClassProperty); - /// GetProtocolRef - Return a reference to the internal protocol /// description, creating an empty one if it has not been /// defined. The return value has type ProtocolPtrTy. @@ -1009,15 +1018,25 @@ public: /// /// \param Name - The variable name. /// \param Init - The variable initializer; this is also used to - /// define the type of the variable. + /// define the type of the variable. /// \param Section - The section the variable should go into, or empty. /// \param Align - The alignment for the variable, or 0. /// \param AddToUsed - Whether the variable should be added to - /// "llvm.used". - llvm::GlobalVariable *CreateMetadataVar(Twine Name, llvm::Constant *Init, + /// "llvm.used". + llvm::GlobalVariable *CreateMetadataVar(Twine Name, + ConstantStructBuilder &Init, + StringRef Section, CharUnits Align, + bool AddToUsed); + llvm::GlobalVariable *CreateMetadataVar(Twine Name, + llvm::Constant *Init, StringRef Section, CharUnits Align, bool AddToUsed); + llvm::GlobalVariable *CreateCStringLiteral(StringRef Name, + ObjCLabelType LabelType, + bool ForceNonFragileABI = false, + bool NullTerminate = true); + protected: CodeGen::RValue EmitMessageSend(CodeGen::CodeGenFunction &CGF, ReturnValueSlot Return, @@ -1044,6 +1063,7 @@ public: } ConstantAddress GenerateConstantString(const StringLiteral *SL) override; + ConstantAddress GenerateConstantNSString(const StringLiteral *SL); llvm::Function *GenerateMethod(const ObjCMethodDecl *OMD, const ObjCContainerDecl *CD=nullptr) override; @@ -1060,6 +1080,9 @@ public: /// forward references will be filled in with empty bodies if no /// definition is seen. The return value has type ProtocolPtrTy. virtual llvm::Constant *GetOrEmitProtocolRef(const ObjCProtocolDecl *PD)=0; + + virtual llvm::Constant *getNSConstantStringClassRef() = 0; + llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM, const CGBlockInfo &blockInfo) override; llvm::Constant *BuildRCBlockLayout(CodeGen::CodeGenModule &CGM, @@ -1069,8 +1092,95 @@ public: QualType T) override; }; +namespace { + +enum class MethodListType { + CategoryInstanceMethods, + CategoryClassMethods, + InstanceMethods, + ClassMethods, + ProtocolInstanceMethods, + ProtocolClassMethods, + OptionalProtocolInstanceMethods, + OptionalProtocolClassMethods, +}; + +/// A convenience class for splitting the methods of a protocol into +/// the four interesting groups. +class ProtocolMethodLists { +public: + enum Kind { + RequiredInstanceMethods, + RequiredClassMethods, + OptionalInstanceMethods, + OptionalClassMethods + }; + enum { + NumProtocolMethodLists = 4 + }; + + static MethodListType getMethodListKind(Kind kind) { + switch (kind) { + case RequiredInstanceMethods: + return MethodListType::ProtocolInstanceMethods; + case RequiredClassMethods: + return MethodListType::ProtocolClassMethods; + case OptionalInstanceMethods: + return MethodListType::OptionalProtocolInstanceMethods; + case OptionalClassMethods: + return MethodListType::OptionalProtocolClassMethods; + } + llvm_unreachable("bad kind"); + } + + SmallVector<const ObjCMethodDecl *, 4> Methods[NumProtocolMethodLists]; + + static ProtocolMethodLists get(const ObjCProtocolDecl *PD) { + ProtocolMethodLists result; + + for (auto MD : PD->methods()) { + size_t index = (2 * size_t(MD->isOptional())) + + (size_t(MD->isClassMethod())); + result.Methods[index].push_back(MD); + } + + return result; + } + + template <class Self> + SmallVector<llvm::Constant*, 8> emitExtendedTypesArray(Self *self) const { + // In both ABIs, the method types list is parallel with the + // concatenation of the methods arrays in the following order: + // instance methods + // class methods + // optional instance methods + // optional class methods + SmallVector<llvm::Constant*, 8> result; + + // Methods is already in the correct order for both ABIs. + for (auto &list : Methods) { + for (auto MD : list) { + result.push_back(self->GetMethodVarType(MD, true)); + } + } + + return result; + } + + template <class Self> + llvm::Constant *emitMethodList(Self *self, const ObjCProtocolDecl *PD, + Kind kind) const { + return self->emitMethodList(PD->getObjCRuntimeNameAsString(), + getMethodListKind(kind), Methods[kind]); + } +}; + +} // end anonymous namespace + class CGObjCMac : public CGObjCCommonMac { private: + friend ProtocolMethodLists; + ObjCTypesHelper ObjCTypes; /// EmitModuleInfo - Another marker encoding module level @@ -1091,7 +1201,7 @@ private: llvm::Constant *EmitClassExtension(const ObjCImplementationDecl *ID, CharUnits instanceSize, bool hasMRCWeakIvars, - bool isClassProperty); + bool isMetaclass); /// EmitClassRef - Return a Value*, of type ObjCTypes.ClassPtrTy, /// for the given class. @@ -1123,30 +1233,18 @@ private: /// given implementation. The return value has type ClassPtrTy. llvm::Constant *EmitMetaClass(const ObjCImplementationDecl *ID, llvm::Constant *Protocols, - ArrayRef<llvm::Constant*> Methods); + ArrayRef<const ObjCMethodDecl *> Methods); - llvm::Constant *GetMethodConstant(const ObjCMethodDecl *MD); + void emitMethodConstant(ConstantArrayBuilder &builder, + const ObjCMethodDecl *MD); - llvm::Constant *GetMethodDescriptionConstant(const ObjCMethodDecl *MD); + void emitMethodDescriptionConstant(ConstantArrayBuilder &builder, + const ObjCMethodDecl *MD); /// EmitMethodList - Emit the method list for the given /// implementation. The return value has type MethodListPtrTy. - llvm::Constant *EmitMethodList(Twine Name, StringRef Section, - ArrayRef<llvm::Constant *> Methods); - - /// EmitMethodDescList - Emit a method description list for a list of - /// method declarations. - /// - TypeName: The name for the type containing the methods. - /// - IsProtocol: True iff these methods are for a protocol. - /// - ClassMethds: True iff these are class methods. - /// - Required: When true, only "required" methods are - /// listed. Similarly, when false only "optional" methods are - /// listed. For classes this should always be true. - /// - begin, end: The method list to output. - /// - /// The return value has type MethodDescriptionListPtrTy. - llvm::Constant *EmitMethodDescList(Twine Name, StringRef Section, - ArrayRef<llvm::Constant *> Methods); + llvm::Constant *emitMethodList(Twine Name, MethodListType MLT, + ArrayRef<const ObjCMethodDecl *> Methods); /// GetOrEmitProtocol - Get the protocol object for the given /// declaration, emitting it if necessary. The return value has type @@ -1165,9 +1263,7 @@ private: /// ProtocolExtensionPtrTy. llvm::Constant * EmitProtocolExtension(const ObjCProtocolDecl *PD, - ArrayRef<llvm::Constant*> OptInstanceMethods, - ArrayRef<llvm::Constant*> OptClassMethods, - ArrayRef<llvm::Constant*> MethodTypesExt); + const ProtocolMethodLists &methodLists); /// EmitProtocolList - Generate the list of referenced /// protocols. The return value has type ProtocolListPtrTy. @@ -1183,6 +1279,8 @@ private: public: CGObjCMac(CodeGen::CodeGenModule &cgm); + llvm::Constant *getNSConstantStringClassRef() override; + llvm::Function *ModuleInitFunction() override; CodeGen::RValue GenerateMessageSend(CodeGen::CodeGenFunction &CGF, @@ -1262,20 +1360,14 @@ public: llvm::Value *EmitIvarOffset(CodeGen::CodeGenFunction &CGF, const ObjCInterfaceDecl *Interface, const ObjCIvarDecl *Ivar) override; - - /// GetClassGlobal - Return the global variable for the Objective-C - /// class of the given name. - llvm::GlobalVariable *GetClassGlobal(StringRef Name, - bool Weak = false) override { - llvm_unreachable("CGObjCMac::GetClassGlobal"); - } }; class CGObjCNonFragileABIMac : public CGObjCCommonMac { private: + friend ProtocolMethodLists; ObjCNonFragileABITypesHelper ObjCTypes; llvm::GlobalVariable* ObjCEmptyCacheVar; - llvm::GlobalVariable* ObjCEmptyVtableVar; + llvm::Constant* ObjCEmptyVtableVar; /// SuperClassReferences - uniqued super class references. llvm::DenseMap<IdentifierInfo*, llvm::GlobalVariable*> SuperClassReferences; @@ -1310,21 +1402,22 @@ private: unsigned InstanceStart, unsigned InstanceSize, const ObjCImplementationDecl *ID); - llvm::GlobalVariable * BuildClassMetaData(const std::string &ClassName, - llvm::Constant *IsAGV, - llvm::Constant *SuperClassGV, - llvm::Constant *ClassRoGV, - bool HiddenVisibility, - bool Weak); - - llvm::Constant *GetMethodConstant(const ObjCMethodDecl *MD); + llvm::GlobalVariable *BuildClassObject(const ObjCInterfaceDecl *CI, + bool isMetaclass, + llvm::Constant *IsAGV, + llvm::Constant *SuperClassGV, + llvm::Constant *ClassRoGV, + bool HiddenVisibility); + + void emitMethodConstant(ConstantArrayBuilder &builder, + const ObjCMethodDecl *MD, + bool forProtocol); + + /// Emit the method list for the given implementation. The return value + /// has type MethodListnfABITy. + llvm::Constant *emitMethodList(Twine Name, MethodListType MLT, + ArrayRef<const ObjCMethodDecl *> Methods); - llvm::Constant *GetMethodDescriptionConstant(const ObjCMethodDecl *MD); - - /// EmitMethodList - Emit the method list for the given - /// implementation. The return value has type MethodListnfABITy. - llvm::Constant *EmitMethodList(Twine Name, StringRef Section, - ArrayRef<llvm::Constant *> Methods); /// EmitIvarList - Emit the ivar list for the given /// implementation. If ForClass is true the list of class ivars /// (i.e. metaclass ivars) is emitted, otherwise the list of @@ -1365,8 +1458,12 @@ private: /// GetClassGlobal - Return the global variable for the Objective-C /// class of the given name. - llvm::GlobalVariable *GetClassGlobal(StringRef Name, - bool Weak = false) override; + llvm::Constant *GetClassGlobal(StringRef Name, + ForDefinition_t IsForDefinition, + bool Weak = false, bool DLLImport = false); + llvm::Constant *GetClassGlobal(const ObjCInterfaceDecl *ID, + bool isMetaclass, + ForDefinition_t isForDefinition); /// EmitClassRef - Return a Value*, of type ObjCTypes.ClassPtrTy, /// for the given class reference. @@ -1374,7 +1471,7 @@ private: const ObjCInterfaceDecl *ID); llvm::Value *EmitClassRefFromId(CodeGenFunction &CGF, - IdentifierInfo *II, bool Weak, + IdentifierInfo *II, const ObjCInterfaceDecl *ID); llvm::Value *EmitNSAutoreleasePoolClassRef(CodeGenFunction &CGF) override; @@ -1404,7 +1501,7 @@ private: /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C /// interface. The return value has type EHTypePtrTy. llvm::Constant *GetInterfaceEHType(const ObjCInterfaceDecl *ID, - bool ForDefinition); + ForDefinition_t IsForDefinition); StringRef getMetaclassSymbolPrefix() const { return "OBJC_METACLASS_$_"; } @@ -1451,7 +1548,9 @@ private: public: CGObjCNonFragileABIMac(CodeGen::CodeGenModule &cgm); - // FIXME. All stubs for now! + + llvm::Constant *getNSConstantStringClassRef() override; + llvm::Function *ModuleInitFunction() override; CodeGen::RValue GenerateMessageSend(CodeGen::CodeGenFunction &CGF, @@ -1761,11 +1860,115 @@ llvm::Constant *CGObjCMac::GetEHType(QualType T) { }; */ -ConstantAddress CGObjCCommonMac::GenerateConstantString( - const StringLiteral *SL) { - return (CGM.getLangOpts().NoConstantCFStrings == 0 ? - CGM.GetAddrOfConstantCFString(SL) : - CGM.GetAddrOfConstantString(SL)); +ConstantAddress +CGObjCCommonMac::GenerateConstantString(const StringLiteral *SL) { + return (!CGM.getLangOpts().NoConstantCFStrings + ? CGM.GetAddrOfConstantCFString(SL) + : GenerateConstantNSString(SL)); +} + +static llvm::StringMapEntry<llvm::GlobalVariable *> & +GetConstantStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, + const StringLiteral *Literal, unsigned &StringLength) { + StringRef String = Literal->getString(); + StringLength = String.size(); + return *Map.insert(std::make_pair(String, nullptr)).first; +} + +llvm::Constant *CGObjCMac::getNSConstantStringClassRef() { + if (llvm::Value *V = ConstantStringClassRef) + return cast<llvm::Constant>(V); + + auto &StringClass = CGM.getLangOpts().ObjCConstantStringClass; + std::string str = + StringClass.empty() ? "_NSConstantStringClassReference" + : "_" + StringClass + "ClassReference"; + + llvm::Type *PTy = llvm::ArrayType::get(CGM.IntTy, 0); + auto GV = CGM.CreateRuntimeVariable(PTy, str); + auto V = llvm::ConstantExpr::getBitCast(GV, CGM.IntTy->getPointerTo()); + ConstantStringClassRef = V; + return V; +} + +llvm::Constant *CGObjCNonFragileABIMac::getNSConstantStringClassRef() { + if (llvm::Value *V = ConstantStringClassRef) + return cast<llvm::Constant>(V); + + auto &StringClass = CGM.getLangOpts().ObjCConstantStringClass; + std::string str = + StringClass.empty() ? "OBJC_CLASS_$_NSConstantString" + : "OBJC_CLASS_$_" + StringClass; + auto GV = GetClassGlobal(str, NotForDefinition); + + // Make sure the result is of the correct type. + auto V = llvm::ConstantExpr::getBitCast(GV, CGM.IntTy->getPointerTo()); + + ConstantStringClassRef = V; + return V; +} + +ConstantAddress +CGObjCCommonMac::GenerateConstantNSString(const StringLiteral *Literal) { + unsigned StringLength = 0; + llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = + GetConstantStringEntry(NSConstantStringMap, Literal, StringLength); + + if (auto *C = Entry.second) + return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); + + // If we don't already have it, get _NSConstantStringClassReference. + llvm::Constant *Class = getNSConstantStringClassRef(); + + // If we don't already have it, construct the type for a constant NSString. + if (!NSConstantStringType) { + NSConstantStringType = + llvm::StructType::create({ + CGM.Int32Ty->getPointerTo(), + CGM.Int8PtrTy, + CGM.IntTy + }, "struct.__builtin_NSString"); + } + + ConstantInitBuilder Builder(CGM); + auto Fields = Builder.beginStruct(NSConstantStringType); + + // Class pointer. + Fields.add(Class); + + // String pointer. + llvm::Constant *C = + llvm::ConstantDataArray::getString(VMContext, Entry.first()); + + llvm::GlobalValue::LinkageTypes Linkage = llvm::GlobalValue::PrivateLinkage; + bool isConstant = !CGM.getLangOpts().WritableStrings; + + auto *GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(), isConstant, + Linkage, C, ".str"); + GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); + // Don't enforce the target's minimum global alignment, since the only use + // of the string is via this class initializer. + GV->setAlignment(1); + Fields.addBitCast(GV, CGM.Int8PtrTy); + + // String length. + Fields.addInt(CGM.IntTy, StringLength); + + // The struct. + CharUnits Alignment = CGM.getPointerAlign(); + GV = Fields.finishAndCreateGlobal("_unnamed_nsstring_", Alignment, + /*constant*/ true, + llvm::GlobalVariable::PrivateLinkage); + const char *NSStringSection = "__OBJC,__cstring_object,regular,no_dead_strip"; + const char *NSStringNonFragileABISection = + "__DATA,__objc_stringobj,regular,no_dead_strip"; + // FIXME. Fix section. + GV->setSection(CGM.getLangOpts().ObjCRuntime.isNonFragile() + ? NSStringNonFragileABISection + : NSStringSection); + Entry.second = GV; + + return ConstantAddress(GV, Alignment); } enum { @@ -1953,8 +2156,9 @@ CGObjCCommonMac::EmitMessageSend(CodeGen::CodeGenFunction &CGF, llvm::Instruction *CallSite; Fn = llvm::ConstantExpr::getBitCast(Fn, MSI.MessengerType); - RValue rvalue = CGF.EmitCall(MSI.CallInfo, Fn, Return, ActualArgs, - CGCalleeInfo(), &CallSite); + CGCallee Callee = CGCallee::forDirect(Fn); + RValue rvalue = CGF.EmitCall(MSI.CallInfo, Callee, Return, ActualArgs, + &CallSite); // Mark the call as noreturn if the method is marked noreturn and the // receiver cannot be null. @@ -2576,10 +2780,9 @@ llvm::Constant *CGObjCCommonMac::getBitmapBlockLayout(bool ComputeByrefLayout) { } } - llvm::GlobalVariable *Entry = CreateMetadataVar( - "OBJC_CLASS_NAME_", - llvm::ConstantDataArray::getString(VMContext, BitMap, false), - "__TEXT,__objc_classname,cstring_literals", CharUnits::One(), true); + auto *Entry = CreateCStringLiteral(BitMap, ObjCLabelType::ClassName, + /*ForceNonFragileABI=*/true, + /*NullTerminate=*/false); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -2730,66 +2933,29 @@ llvm::Constant *CGObjCMac::GetOrEmitProtocol(const ObjCProtocolDecl *PD) { LazySymbols.insert(&CGM.getContext().Idents.get("Protocol")); // Construct method lists. - std::vector<llvm::Constant*> InstanceMethods, ClassMethods; - std::vector<llvm::Constant*> OptInstanceMethods, OptClassMethods; - std::vector<llvm::Constant*> MethodTypesExt, OptMethodTypesExt; - for (const auto *MD : PD->instance_methods()) { - llvm::Constant *C = GetMethodDescriptionConstant(MD); - if (!C) - return GetOrEmitProtocolRef(PD); - - if (MD->getImplementationControl() == ObjCMethodDecl::Optional) { - OptInstanceMethods.push_back(C); - OptMethodTypesExt.push_back(GetMethodVarType(MD, true)); - } else { - InstanceMethods.push_back(C); - MethodTypesExt.push_back(GetMethodVarType(MD, true)); - } - } - - for (const auto *MD : PD->class_methods()) { - llvm::Constant *C = GetMethodDescriptionConstant(MD); - if (!C) - return GetOrEmitProtocolRef(PD); - - if (MD->getImplementationControl() == ObjCMethodDecl::Optional) { - OptClassMethods.push_back(C); - OptMethodTypesExt.push_back(GetMethodVarType(MD, true)); - } else { - ClassMethods.push_back(C); - MethodTypesExt.push_back(GetMethodVarType(MD, true)); - } - } - - MethodTypesExt.insert(MethodTypesExt.end(), - OptMethodTypesExt.begin(), OptMethodTypesExt.end()); - - llvm::Constant *Values[] = { - EmitProtocolExtension(PD, OptInstanceMethods, OptClassMethods, - MethodTypesExt), - GetClassName(PD->getObjCRuntimeNameAsString()), - EmitProtocolList("OBJC_PROTOCOL_REFS_" + PD->getName(), - PD->protocol_begin(), PD->protocol_end()), - EmitMethodDescList("OBJC_PROTOCOL_INSTANCE_METHODS_" + PD->getName(), - "__OBJC,__cat_inst_meth,regular,no_dead_strip", - InstanceMethods), - EmitMethodDescList("OBJC_PROTOCOL_CLASS_METHODS_" + PD->getName(), - "__OBJC,__cat_cls_meth,regular,no_dead_strip", - ClassMethods)}; - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ProtocolTy, - Values); + auto methodLists = ProtocolMethodLists::get(PD); + + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ProtocolTy); + values.add(EmitProtocolExtension(PD, methodLists)); + values.add(GetClassName(PD->getObjCRuntimeNameAsString())); + values.add(EmitProtocolList("OBJC_PROTOCOL_REFS_" + PD->getName(), + PD->protocol_begin(), PD->protocol_end())); + values.add(methodLists.emitMethodList(this, PD, + ProtocolMethodLists::RequiredInstanceMethods)); + values.add(methodLists.emitMethodList(this, PD, + ProtocolMethodLists::RequiredClassMethods)); if (Entry) { // Already created, update the initializer. assert(Entry->hasPrivateLinkage()); - Entry->setInitializer(Init); + values.finishAndSetAsInitializer(Entry); } else { - Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ProtocolTy, - false, llvm::GlobalValue::PrivateLinkage, - Init, "OBJC_PROTOCOL_" + PD->getName()); + Entry = values.finishAndCreateGlobal("OBJC_PROTOCOL_" + PD->getName(), + CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); Entry->setSection("__OBJC,__protocol,regular,no_dead_strip"); - // FIXME: Is this necessary? Why only for protocol? - Entry->setAlignment(4); Protocols[PD->getIdentifier()] = Entry; } @@ -2828,37 +2994,49 @@ llvm::Constant *CGObjCMac::GetOrEmitProtocolRef(const ObjCProtocolDecl *PD) { */ llvm::Constant * CGObjCMac::EmitProtocolExtension(const ObjCProtocolDecl *PD, - ArrayRef<llvm::Constant*> OptInstanceMethods, - ArrayRef<llvm::Constant*> OptClassMethods, - ArrayRef<llvm::Constant*> MethodTypesExt) { - uint64_t Size = - CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ProtocolExtensionTy); - llvm::Constant *Values[] = { - llvm::ConstantInt::get(ObjCTypes.IntTy, Size), - EmitMethodDescList("OBJC_PROTOCOL_INSTANCE_METHODS_OPT_" + PD->getName(), - "__OBJC,__cat_inst_meth,regular,no_dead_strip", - OptInstanceMethods), - EmitMethodDescList("OBJC_PROTOCOL_CLASS_METHODS_OPT_" + PD->getName(), - "__OBJC,__cat_cls_meth,regular,no_dead_strip", - OptClassMethods), - EmitPropertyList("OBJC_$_PROP_PROTO_LIST_" + PD->getName(), nullptr, PD, - ObjCTypes, false), - EmitProtocolMethodTypes("OBJC_PROTOCOL_METHOD_TYPES_" + PD->getName(), - MethodTypesExt, ObjCTypes), - EmitPropertyList("OBJC_$_CLASS_PROP_PROTO_LIST_" + PD->getName(), nullptr, - PD, ObjCTypes, true)}; + const ProtocolMethodLists &methodLists) { + auto optInstanceMethods = + methodLists.emitMethodList(this, PD, + ProtocolMethodLists::OptionalInstanceMethods); + auto optClassMethods = + methodLists.emitMethodList(this, PD, + ProtocolMethodLists::OptionalClassMethods); + + auto extendedMethodTypes = + EmitProtocolMethodTypes("OBJC_PROTOCOL_METHOD_TYPES_" + PD->getName(), + methodLists.emitExtendedTypesArray(this), + ObjCTypes); + + auto instanceProperties = + EmitPropertyList("OBJC_$_PROP_PROTO_LIST_" + PD->getName(), nullptr, PD, + ObjCTypes, false); + auto classProperties = + EmitPropertyList("OBJC_$_CLASS_PROP_PROTO_LIST_" + PD->getName(), nullptr, + PD, ObjCTypes, true); // Return null if no extension bits are used. - if (Values[1]->isNullValue() && Values[2]->isNullValue() && - Values[3]->isNullValue() && Values[4]->isNullValue() && - Values[5]->isNullValue()) + if (optInstanceMethods->isNullValue() && + optClassMethods->isNullValue() && + extendedMethodTypes->isNullValue() && + instanceProperties->isNullValue() && + classProperties->isNullValue()) { return llvm::Constant::getNullValue(ObjCTypes.ProtocolExtensionPtrTy); + } - llvm::Constant *Init = - llvm::ConstantStruct::get(ObjCTypes.ProtocolExtensionTy, Values); + uint64_t size = + CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ProtocolExtensionTy); + + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ProtocolExtensionTy); + values.addInt(ObjCTypes.IntTy, size); + values.add(optInstanceMethods); + values.add(optClassMethods); + values.add(instanceProperties); + values.add(extendedMethodTypes); + values.add(classProperties); // No special section, but goes in llvm.used - return CreateMetadataVar("\01l_OBJC_PROTOCOLEXT_" + PD->getName(), Init, + return CreateMetadataVar("\01l_OBJC_PROTOCOLEXT_" + PD->getName(), values, StringRef(), CGM.getPointerAlign(), true); } @@ -2870,59 +3048,57 @@ CGObjCMac::EmitProtocolExtension(const ObjCProtocolDecl *PD, }; */ llvm::Constant * -CGObjCMac::EmitProtocolList(Twine Name, +CGObjCMac::EmitProtocolList(Twine name, ObjCProtocolDecl::protocol_iterator begin, ObjCProtocolDecl::protocol_iterator end) { - SmallVector<llvm::Constant *, 16> ProtocolRefs; - - for (; begin != end; ++begin) - ProtocolRefs.push_back(GetProtocolRef(*begin)); - // Just return null for empty protocol lists - if (ProtocolRefs.empty()) + if (begin == end) return llvm::Constant::getNullValue(ObjCTypes.ProtocolListPtrTy); - // This list is null terminated. - ProtocolRefs.push_back(llvm::Constant::getNullValue(ObjCTypes.ProtocolPtrTy)); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); - llvm::Constant *Values[3]; // This field is only used by the runtime. - Values[0] = llvm::Constant::getNullValue(ObjCTypes.ProtocolListPtrTy); - Values[1] = llvm::ConstantInt::get(ObjCTypes.LongTy, - ProtocolRefs.size() - 1); - Values[2] = - llvm::ConstantArray::get(llvm::ArrayType::get(ObjCTypes.ProtocolPtrTy, - ProtocolRefs.size()), - ProtocolRefs); - - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + values.addNullPointer(ObjCTypes.ProtocolListPtrTy); + + // Reserve a slot for the count. + auto countSlot = values.addPlaceholder(); + + auto refsArray = values.beginArray(ObjCTypes.ProtocolPtrTy); + for (; begin != end; ++begin) { + refsArray.add(GetProtocolRef(*begin)); + } + auto count = refsArray.size(); + + // This list is null terminated. + refsArray.addNullPointer(ObjCTypes.ProtocolPtrTy); + + refsArray.finishAndAddTo(values); + values.fillPlaceholderWithInt(countSlot, ObjCTypes.LongTy, count); + + StringRef section; + if (CGM.getTriple().isOSBinFormatMachO()) + section = "__OBJC,__cat_cls_meth,regular,no_dead_strip"; + llvm::GlobalVariable *GV = - CreateMetadataVar(Name, Init, "__OBJC,__cat_cls_meth,regular,no_dead_strip", - CGM.getPointerAlign(), false); + CreateMetadataVar(name, values, section, CGM.getPointerAlign(), false); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.ProtocolListPtrTy); } -void CGObjCCommonMac:: +static void PushProtocolProperties(llvm::SmallPtrSet<const IdentifierInfo*,16> &PropertySet, - SmallVectorImpl<llvm::Constant *> &Properties, - const Decl *Container, + SmallVectorImpl<const ObjCPropertyDecl *> &Properties, const ObjCProtocolDecl *Proto, - const ObjCCommonTypesHelper &ObjCTypes, bool IsClassProperty) { for (const auto *P : Proto->protocols()) - PushProtocolProperties(PropertySet, Properties, Container, P, ObjCTypes, - IsClassProperty); + PushProtocolProperties(PropertySet, Properties, P, IsClassProperty); for (const auto *PD : Proto->properties()) { if (IsClassProperty != PD->isClassProperty()) continue; if (!PropertySet.insert(PD->getIdentifier()).second) continue; - llvm::Constant *Prop[] = { - GetPropertyName(PD->getIdentifier()), - GetPropertyTypeString(PD, Container) - }; - Properties.push_back(llvm::ConstantStruct::get(ObjCTypes.PropertyTy, Prop)); + Properties.push_back(PD); } } @@ -2952,21 +3128,16 @@ llvm::Constant *CGObjCCommonMac::EmitPropertyList(Twine Name, return llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); } - SmallVector<llvm::Constant *, 16> Properties; + SmallVector<const ObjCPropertyDecl *, 16> Properties; llvm::SmallPtrSet<const IdentifierInfo*, 16> PropertySet; - auto AddProperty = [&](const ObjCPropertyDecl *PD) { - llvm::Constant *Prop[] = {GetPropertyName(PD->getIdentifier()), - GetPropertyTypeString(PD, Container)}; - Properties.push_back(llvm::ConstantStruct::get(ObjCTypes.PropertyTy, Prop)); - }; if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD)) for (const ObjCCategoryDecl *ClassExt : OID->known_extensions()) for (auto *PD : ClassExt->properties()) { if (IsClassProperty != PD->isClassProperty()) continue; PropertySet.insert(PD->getIdentifier()); - AddProperty(PD); + Properties.push_back(PD); } for (const auto *PD : OCD->properties()) { @@ -2976,40 +3147,45 @@ llvm::Constant *CGObjCCommonMac::EmitPropertyList(Twine Name, // class extension. if (!PropertySet.insert(PD->getIdentifier()).second) continue; - AddProperty(PD); + Properties.push_back(PD); } if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD)) { for (const auto *P : OID->all_referenced_protocols()) - PushProtocolProperties(PropertySet, Properties, Container, P, ObjCTypes, - IsClassProperty); + PushProtocolProperties(PropertySet, Properties, P, IsClassProperty); } else if (const ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(OCD)) { for (const auto *P : CD->protocols()) - PushProtocolProperties(PropertySet, Properties, Container, P, ObjCTypes, - IsClassProperty); + PushProtocolProperties(PropertySet, Properties, P, IsClassProperty); } // Return null for empty list. if (Properties.empty()) return llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); - unsigned PropertySize = + unsigned propertySize = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.PropertyTy); - llvm::Constant *Values[3]; - Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, PropertySize); - Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Properties.size()); - llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.PropertyTy, - Properties.size()); - Values[2] = llvm::ConstantArray::get(AT, Properties); - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + values.addInt(ObjCTypes.IntTy, propertySize); + values.addInt(ObjCTypes.IntTy, Properties.size()); + auto propertiesArray = values.beginArray(ObjCTypes.PropertyTy); + for (auto PD : Properties) { + auto property = propertiesArray.beginStruct(ObjCTypes.PropertyTy); + property.add(GetPropertyName(PD->getIdentifier())); + property.add(GetPropertyTypeString(PD, Container)); + property.finishAndAddTo(propertiesArray); + } + propertiesArray.finishAndAddTo(values); + + StringRef Section; + if (CGM.getTriple().isOSBinFormatMachO()) + Section = (ObjCABI == 2) ? "__DATA, __objc_const" + : "__OBJC,__property,regular,no_dead_strip"; llvm::GlobalVariable *GV = - CreateMetadataVar(Name, Init, - (ObjCABI == 2) ? "__DATA, __objc_const" : - "__OBJC,__property,regular,no_dead_strip", - CGM.getPointerAlign(), - true); + CreateMetadataVar(Name, values, Section, CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.PropertyListPtrTy); } @@ -3025,50 +3201,13 @@ CGObjCCommonMac::EmitProtocolMethodTypes(Twine Name, MethodTypes.size()); llvm::Constant *Init = llvm::ConstantArray::get(AT, MethodTypes); - llvm::GlobalVariable *GV = CreateMetadataVar( - Name, Init, (ObjCABI == 2) ? "__DATA, __objc_const" : StringRef(), - CGM.getPointerAlign(), true); - return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.Int8PtrPtrTy); -} - -/* - struct objc_method_description_list { - int count; - struct objc_method_description list[]; - }; -*/ -llvm::Constant * -CGObjCMac::GetMethodDescriptionConstant(const ObjCMethodDecl *MD) { - llvm::Constant *Desc[] = { - llvm::ConstantExpr::getBitCast(GetMethodVarName(MD->getSelector()), - ObjCTypes.SelectorPtrTy), - GetMethodVarType(MD) - }; - if (!Desc[1]) - return nullptr; - - return llvm::ConstantStruct::get(ObjCTypes.MethodDescriptionTy, - Desc); -} - -llvm::Constant * -CGObjCMac::EmitMethodDescList(Twine Name, StringRef Section, - ArrayRef<llvm::Constant *> Methods) { - // Return null for empty list. - if (Methods.empty()) - return llvm::Constant::getNullValue(ObjCTypes.MethodDescriptionListPtrTy); - - llvm::Constant *Values[2]; - Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Methods.size()); - llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.MethodDescriptionTy, - Methods.size()); - Values[1] = llvm::ConstantArray::get(AT, Methods); - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + StringRef Section; + if (CGM.getTriple().isOSBinFormatMachO() && ObjCABI == 2) + Section = "__DATA, __objc_const"; llvm::GlobalVariable *GV = - CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); - return llvm::ConstantExpr::getBitCast(GV, - ObjCTypes.MethodDescriptionListPtrTy); + CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); + return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.Int8PtrPtrTy); } /* @@ -3098,54 +3237,53 @@ void CGObjCMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) { llvm::raw_svector_ostream(ExtName) << Interface->getName() << '_' << OCD->getName(); - SmallVector<llvm::Constant *, 16> InstanceMethods, ClassMethods; - for (const auto *I : OCD->instance_methods()) - // Instance methods should always be defined. - InstanceMethods.push_back(GetMethodConstant(I)); + ConstantInitBuilder Builder(CGM); + auto Values = Builder.beginStruct(ObjCTypes.CategoryTy); - for (const auto *I : OCD->class_methods()) - // Class methods should always be defined. - ClassMethods.push_back(GetMethodConstant(I)); + enum { + InstanceMethods, + ClassMethods, + NumMethodLists + }; + SmallVector<const ObjCMethodDecl *, 16> Methods[NumMethodLists]; + for (const auto *MD : OCD->methods()) { + Methods[unsigned(MD->isClassMethod())].push_back(MD); + } - llvm::Constant *Values[8]; - Values[0] = GetClassName(OCD->getName()); - Values[1] = GetClassName(Interface->getObjCRuntimeNameAsString()); + Values.add(GetClassName(OCD->getName())); + Values.add(GetClassName(Interface->getObjCRuntimeNameAsString())); LazySymbols.insert(Interface->getIdentifier()); - Values[2] = EmitMethodList("OBJC_CATEGORY_INSTANCE_METHODS_" + ExtName.str(), - "__OBJC,__cat_inst_meth,regular,no_dead_strip", - InstanceMethods); - Values[3] = EmitMethodList("OBJC_CATEGORY_CLASS_METHODS_" + ExtName.str(), - "__OBJC,__cat_cls_meth,regular,no_dead_strip", - ClassMethods); + + Values.add(emitMethodList(ExtName, MethodListType::CategoryInstanceMethods, + Methods[InstanceMethods])); + Values.add(emitMethodList(ExtName, MethodListType::CategoryClassMethods, + Methods[ClassMethods])); if (Category) { - Values[4] = + Values.add( EmitProtocolList("OBJC_CATEGORY_PROTOCOLS_" + ExtName.str(), - Category->protocol_begin(), Category->protocol_end()); + Category->protocol_begin(), Category->protocol_end())); } else { - Values[4] = llvm::Constant::getNullValue(ObjCTypes.ProtocolListPtrTy); + Values.addNullPointer(ObjCTypes.ProtocolListPtrTy); } - Values[5] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); + Values.addInt(ObjCTypes.IntTy, Size); // If there is no category @interface then there can be no properties. if (Category) { - Values[6] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ExtName.str(), - OCD, Category, ObjCTypes, false); - Values[7] = EmitPropertyList("\01l_OBJC_$_CLASS_PROP_LIST_" + ExtName.str(), - OCD, Category, ObjCTypes, true); + Values.add(EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ExtName.str(), + OCD, Category, ObjCTypes, false)); + Values.add(EmitPropertyList("\01l_OBJC_$_CLASS_PROP_LIST_" + ExtName.str(), + OCD, Category, ObjCTypes, true)); } else { - Values[6] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); - Values[7] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); + Values.addNullPointer(ObjCTypes.PropertyListPtrTy); + Values.addNullPointer(ObjCTypes.PropertyListPtrTy); } - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.CategoryTy, - Values); - llvm::GlobalVariable *GV = - CreateMetadataVar("OBJC_CATEGORY_" + ExtName.str(), Init, + CreateMetadataVar("OBJC_CATEGORY_" + ExtName.str(), Values, "__OBJC,__category,regular,no_dead_strip", CGM.getPointerAlign(), true); DefinedCategories.push_back(GV); - DefinedCategoryNames.insert(ExtName.str()); + DefinedCategoryNames.insert(llvm::CachedHashString(ExtName)); // method definition entries must be clear for next implementation. MethodDefinitions.clear(); } @@ -3282,57 +3420,56 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { if (ID->getClassInterface()->getVisibility() == HiddenVisibility) Flags |= FragileABI_Class_Hidden; - SmallVector<llvm::Constant *, 16> InstanceMethods, ClassMethods; - for (const auto *I : ID->instance_methods()) - // Instance methods should always be defined. - InstanceMethods.push_back(GetMethodConstant(I)); - - for (const auto *I : ID->class_methods()) - // Class methods should always be defined. - ClassMethods.push_back(GetMethodConstant(I)); + enum { + InstanceMethods, + ClassMethods, + NumMethodLists + }; + SmallVector<const ObjCMethodDecl *, 16> Methods[NumMethodLists]; + for (const auto *MD : ID->methods()) { + Methods[unsigned(MD->isClassMethod())].push_back(MD); + } for (const auto *PID : ID->property_impls()) { if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize) { ObjCPropertyDecl *PD = PID->getPropertyDecl(); if (ObjCMethodDecl *MD = PD->getGetterMethodDecl()) - if (llvm::Constant *C = GetMethodConstant(MD)) - InstanceMethods.push_back(C); + if (GetMethodDefinition(MD)) + Methods[InstanceMethods].push_back(MD); if (ObjCMethodDecl *MD = PD->getSetterMethodDecl()) - if (llvm::Constant *C = GetMethodConstant(MD)) - InstanceMethods.push_back(C); + if (GetMethodDefinition(MD)) + Methods[InstanceMethods].push_back(MD); } } - llvm::Constant *Values[12]; - Values[ 0] = EmitMetaClass(ID, Protocols, ClassMethods); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ClassTy); + values.add(EmitMetaClass(ID, Protocols, Methods[ClassMethods])); if (ObjCInterfaceDecl *Super = Interface->getSuperClass()) { // Record a reference to the super class. LazySymbols.insert(Super->getIdentifier()); - Values[ 1] = - llvm::ConstantExpr::getBitCast(GetClassName(Super->getObjCRuntimeNameAsString()), - ObjCTypes.ClassPtrTy); + values.addBitCast(GetClassName(Super->getObjCRuntimeNameAsString()), + ObjCTypes.ClassPtrTy); } else { - Values[ 1] = llvm::Constant::getNullValue(ObjCTypes.ClassPtrTy); + values.addNullPointer(ObjCTypes.ClassPtrTy); } - Values[ 2] = GetClassName(ID->getObjCRuntimeNameAsString()); + values.add(GetClassName(ID->getObjCRuntimeNameAsString())); // Version is always 0. - Values[ 3] = llvm::ConstantInt::get(ObjCTypes.LongTy, 0); - Values[ 4] = llvm::ConstantInt::get(ObjCTypes.LongTy, Flags); - Values[ 5] = llvm::ConstantInt::get(ObjCTypes.LongTy, Size.getQuantity()); - Values[ 6] = EmitIvarList(ID, false); - Values[7] = EmitMethodList("OBJC_INSTANCE_METHODS_" + ID->getName(), - "__OBJC,__inst_meth,regular,no_dead_strip", - InstanceMethods); + values.addInt(ObjCTypes.LongTy, 0); + values.addInt(ObjCTypes.LongTy, Flags); + values.addInt(ObjCTypes.LongTy, Size.getQuantity()); + values.add(EmitIvarList(ID, false)); + values.add(emitMethodList(ID->getName(), MethodListType::InstanceMethods, + Methods[InstanceMethods])); // cache is always NULL. - Values[ 8] = llvm::Constant::getNullValue(ObjCTypes.CachePtrTy); - Values[ 9] = Protocols; - Values[10] = BuildStrongIvarLayout(ID, CharUnits::Zero(), Size); - Values[11] = EmitClassExtension(ID, Size, hasMRCWeak, - false/*isClassProperty*/); - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassTy, - Values); + values.addNullPointer(ObjCTypes.CachePtrTy); + values.add(Protocols); + values.add(BuildStrongIvarLayout(ID, CharUnits::Zero(), Size)); + values.add(EmitClassExtension(ID, Size, hasMRCWeak, + /*isMetaclass*/ false)); + std::string Name("OBJC_CLASS_"); Name += ClassName; const char *Section = "__OBJC,__class,regular,no_dead_strip"; @@ -3341,12 +3478,12 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { if (GV) { assert(GV->getType()->getElementType() == ObjCTypes.ClassTy && "Forward metaclass reference has incorrect type."); - GV->setInitializer(Init); + values.finishAndSetAsInitializer(GV); GV->setSection(Section); GV->setAlignment(CGM.getPointerAlign().getQuantity()); CGM.addCompilerUsedGlobal(GV); } else - GV = CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); + GV = CreateMetadataVar(Name, values, Section, CGM.getPointerAlign(), true); DefinedClasses.push_back(GV); ImplementedClasses.push_back(Interface); // method definition entries must be clear for next implementation. @@ -3355,50 +3492,46 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { llvm::Constant *CGObjCMac::EmitMetaClass(const ObjCImplementationDecl *ID, llvm::Constant *Protocols, - ArrayRef<llvm::Constant*> Methods) { + ArrayRef<const ObjCMethodDecl*> Methods) { unsigned Flags = FragileABI_Class_Meta; unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassTy); if (ID->getClassInterface()->getVisibility() == HiddenVisibility) Flags |= FragileABI_Class_Hidden; - llvm::Constant *Values[12]; + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ClassTy); // The isa for the metaclass is the root of the hierarchy. const ObjCInterfaceDecl *Root = ID->getClassInterface(); while (const ObjCInterfaceDecl *Super = Root->getSuperClass()) Root = Super; - Values[ 0] = - llvm::ConstantExpr::getBitCast(GetClassName(Root->getObjCRuntimeNameAsString()), - ObjCTypes.ClassPtrTy); + values.addBitCast(GetClassName(Root->getObjCRuntimeNameAsString()), + ObjCTypes.ClassPtrTy); // The super class for the metaclass is emitted as the name of the // super class. The runtime fixes this up to point to the // *metaclass* for the super class. if (ObjCInterfaceDecl *Super = ID->getClassInterface()->getSuperClass()) { - Values[ 1] = - llvm::ConstantExpr::getBitCast(GetClassName(Super->getObjCRuntimeNameAsString()), - ObjCTypes.ClassPtrTy); + values.addBitCast(GetClassName(Super->getObjCRuntimeNameAsString()), + ObjCTypes.ClassPtrTy); } else { - Values[ 1] = llvm::Constant::getNullValue(ObjCTypes.ClassPtrTy); + values.addNullPointer(ObjCTypes.ClassPtrTy); } - Values[ 2] = GetClassName(ID->getObjCRuntimeNameAsString()); + values.add(GetClassName(ID->getObjCRuntimeNameAsString())); // Version is always 0. - Values[ 3] = llvm::ConstantInt::get(ObjCTypes.LongTy, 0); - Values[ 4] = llvm::ConstantInt::get(ObjCTypes.LongTy, Flags); - Values[ 5] = llvm::ConstantInt::get(ObjCTypes.LongTy, Size); - Values[ 6] = EmitIvarList(ID, true); - Values[7] = - EmitMethodList("OBJC_CLASS_METHODS_" + ID->getNameAsString(), - "__OBJC,__cls_meth,regular,no_dead_strip", Methods); + values.addInt(ObjCTypes.LongTy, 0); + values.addInt(ObjCTypes.LongTy, Flags); + values.addInt(ObjCTypes.LongTy, Size); + values.add(EmitIvarList(ID, true)); + values.add(emitMethodList(ID->getName(), MethodListType::ClassMethods, + Methods)); // cache is always NULL. - Values[ 8] = llvm::Constant::getNullValue(ObjCTypes.CachePtrTy); - Values[ 9] = Protocols; + values.addNullPointer(ObjCTypes.CachePtrTy); + values.add(Protocols); // ivar_layout for metaclass is always NULL. - Values[10] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); + values.addNullPointer(ObjCTypes.Int8PtrTy); // The class extension is used to store class properties for metaclasses. - Values[11] = EmitClassExtension(ID, CharUnits::Zero(), false/*hasMRCWeak*/, - true/*isClassProperty*/); - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassTy, - Values); + values.add(EmitClassExtension(ID, CharUnits::Zero(), false/*hasMRCWeak*/, + /*isMetaclass*/true)); std::string Name("OBJC_METACLASS_"); Name += ID->getName(); @@ -3408,14 +3541,13 @@ llvm::Constant *CGObjCMac::EmitMetaClass(const ObjCImplementationDecl *ID, if (GV) { assert(GV->getType()->getElementType() == ObjCTypes.ClassTy && "Forward metaclass reference has incorrect type."); - GV->setInitializer(Init); + values.finishAndSetAsInitializer(GV); } else { - GV = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassTy, false, - llvm::GlobalValue::PrivateLinkage, - Init, Name); + GV = values.finishAndCreateGlobal(Name, CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); } GV->setSection("__OBJC,__meta_class,regular,no_dead_strip"); - GV->setAlignment(4); CGM.addCompilerUsedGlobal(GV); return GV; @@ -3471,32 +3603,38 @@ llvm::Value *CGObjCMac::EmitSuperClassRef(const ObjCInterfaceDecl *ID) { llvm::Constant * CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID, CharUnits InstanceSize, bool hasMRCWeakIvars, - bool isClassProperty) { - uint64_t Size = - CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassExtensionTy); + bool isMetaclass) { + // Weak ivar layout. + llvm::Constant *layout; + if (isMetaclass) { + layout = llvm::ConstantPointerNull::get(CGM.Int8PtrTy); + } else { + layout = BuildWeakIvarLayout(ID, CharUnits::Zero(), InstanceSize, + hasMRCWeakIvars); + } - llvm::Constant *Values[3]; - Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); - if (isClassProperty) { - llvm::Type *PtrTy = CGM.Int8PtrTy; - Values[1] = llvm::Constant::getNullValue(PtrTy); - } else - Values[1] = BuildWeakIvarLayout(ID, CharUnits::Zero(), InstanceSize, - hasMRCWeakIvars); - if (isClassProperty) - Values[2] = EmitPropertyList("\01l_OBJC_$_CLASS_PROP_LIST_" + ID->getName(), - ID, ID->getClassInterface(), ObjCTypes, true); - else - Values[2] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ID->getName(), - ID, ID->getClassInterface(), ObjCTypes, false); + // Properties. + llvm::Constant *propertyList = + EmitPropertyList((isMetaclass ? Twine("\01l_OBJC_$_CLASS_PROP_LIST_") + : Twine("\01l_OBJC_$_PROP_LIST_")) + + ID->getName(), + ID, ID->getClassInterface(), ObjCTypes, isMetaclass); // Return null if no extension bits are used. - if ((!Values[1] || Values[1]->isNullValue()) && Values[2]->isNullValue()) + if (layout->isNullValue() && propertyList->isNullValue()) { return llvm::Constant::getNullValue(ObjCTypes.ClassExtensionPtrTy); + } - llvm::Constant *Init = - llvm::ConstantStruct::get(ObjCTypes.ClassExtensionTy, Values); - return CreateMetadataVar("OBJC_CLASSEXT_" + ID->getName(), Init, + uint64_t size = + CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassExtensionTy); + + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ClassExtensionTy); + values.addInt(ObjCTypes.IntTy, size); + values.add(layout); + values.add(propertyList); + + return CreateMetadataVar("OBJC_CLASSEXT_" + ID->getName(), values, "__OBJC,__class_ext,regular,no_dead_strip", CGM.getPointerAlign(), true); } @@ -3515,8 +3653,6 @@ CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID, */ llvm::Constant *CGObjCMac::EmitIvarList(const ObjCImplementationDecl *ID, bool ForClass) { - std::vector<llvm::Constant*> Ivars; - // When emitting the root class GCC emits ivar entries for the // actual class structure. It is not clear if we need to follow this // behavior; for now lets try and get away with not doing it. If so, @@ -3527,91 +3663,181 @@ llvm::Constant *CGObjCMac::EmitIvarList(const ObjCImplementationDecl *ID, const ObjCInterfaceDecl *OID = ID->getClassInterface(); + ConstantInitBuilder builder(CGM); + auto ivarList = builder.beginStruct(); + auto countSlot = ivarList.addPlaceholder(); + auto ivars = ivarList.beginArray(ObjCTypes.IvarTy); + for (const ObjCIvarDecl *IVD = OID->all_declared_ivar_begin(); IVD; IVD = IVD->getNextIvar()) { // Ignore unnamed bit-fields. if (!IVD->getDeclName()) continue; - llvm::Constant *Ivar[] = { - GetMethodVarName(IVD->getIdentifier()), - GetMethodVarType(IVD), - llvm::ConstantInt::get(ObjCTypes.IntTy, - ComputeIvarBaseOffset(CGM, OID, IVD)) - }; - Ivars.push_back(llvm::ConstantStruct::get(ObjCTypes.IvarTy, Ivar)); + + auto ivar = ivars.beginStruct(ObjCTypes.IvarTy); + ivar.add(GetMethodVarName(IVD->getIdentifier())); + ivar.add(GetMethodVarType(IVD)); + ivar.addInt(ObjCTypes.IntTy, ComputeIvarBaseOffset(CGM, OID, IVD)); + ivar.finishAndAddTo(ivars); } // Return null for empty list. - if (Ivars.empty()) + auto count = ivars.size(); + if (count == 0) { + ivars.abandon(); + ivarList.abandon(); return llvm::Constant::getNullValue(ObjCTypes.IvarListPtrTy); + } - llvm::Constant *Values[2]; - Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Ivars.size()); - llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.IvarTy, - Ivars.size()); - Values[1] = llvm::ConstantArray::get(AT, Ivars); - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + ivars.finishAndAddTo(ivarList); + ivarList.fillPlaceholderWithInt(countSlot, ObjCTypes.IntTy, count); llvm::GlobalVariable *GV; if (ForClass) GV = - CreateMetadataVar("OBJC_CLASS_VARIABLES_" + ID->getName(), Init, + CreateMetadataVar("OBJC_CLASS_VARIABLES_" + ID->getName(), ivarList, "__OBJC,__class_vars,regular,no_dead_strip", CGM.getPointerAlign(), true); else - GV = CreateMetadataVar("OBJC_INSTANCE_VARIABLES_" + ID->getName(), Init, + GV = CreateMetadataVar("OBJC_INSTANCE_VARIABLES_" + ID->getName(), ivarList, "__OBJC,__instance_vars,regular,no_dead_strip", CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.IvarListPtrTy); } -/* - struct objc_method { - SEL method_name; - char *method_types; - void *method; - }; +/// Build a struct objc_method_description constant for the given method. +/// +/// struct objc_method_description { +/// SEL method_name; +/// char *method_types; +/// }; +void CGObjCMac::emitMethodDescriptionConstant(ConstantArrayBuilder &builder, + const ObjCMethodDecl *MD) { + auto description = builder.beginStruct(ObjCTypes.MethodDescriptionTy); + description.addBitCast(GetMethodVarName(MD->getSelector()), + ObjCTypes.SelectorPtrTy); + description.add(GetMethodVarType(MD)); + description.finishAndAddTo(builder); +} - struct objc_method_list { - struct objc_method_list *obsolete; - int count; - struct objc_method methods_list[count]; - }; -*/ +/// Build a struct objc_method constant for the given method. +/// +/// struct objc_method { +/// SEL method_name; +/// char *method_types; +/// void *method; +/// }; +void CGObjCMac::emitMethodConstant(ConstantArrayBuilder &builder, + const ObjCMethodDecl *MD) { + llvm::Function *fn = GetMethodDefinition(MD); + assert(fn && "no definition registered for method"); -/// GetMethodConstant - Return a struct objc_method constant for the -/// given method if it has been defined. The result is null if the -/// method has not been defined. The return value has type MethodPtrTy. -llvm::Constant *CGObjCMac::GetMethodConstant(const ObjCMethodDecl *MD) { - llvm::Function *Fn = GetMethodDefinition(MD); - if (!Fn) - return nullptr; - - llvm::Constant *Method[] = { - llvm::ConstantExpr::getBitCast(GetMethodVarName(MD->getSelector()), - ObjCTypes.SelectorPtrTy), - GetMethodVarType(MD), - llvm::ConstantExpr::getBitCast(Fn, ObjCTypes.Int8PtrTy) - }; - return llvm::ConstantStruct::get(ObjCTypes.MethodTy, Method); + auto method = builder.beginStruct(ObjCTypes.MethodTy); + method.addBitCast(GetMethodVarName(MD->getSelector()), + ObjCTypes.SelectorPtrTy); + method.add(GetMethodVarType(MD)); + method.addBitCast(fn, ObjCTypes.Int8PtrTy); + method.finishAndAddTo(builder); } -llvm::Constant *CGObjCMac::EmitMethodList(Twine Name, StringRef Section, - ArrayRef<llvm::Constant *> Methods) { +/// Build a struct objc_method_list or struct objc_method_description_list, +/// as appropriate. +/// +/// struct objc_method_list { +/// struct objc_method_list *obsolete; +/// int count; +/// struct objc_method methods_list[count]; +/// }; +/// +/// struct objc_method_description_list { +/// int count; +/// struct objc_method_description list[count]; +/// }; +llvm::Constant *CGObjCMac::emitMethodList(Twine name, MethodListType MLT, + ArrayRef<const ObjCMethodDecl *> methods) { + StringRef prefix; + StringRef section; + bool forProtocol = false; + switch (MLT) { + case MethodListType::CategoryInstanceMethods: + prefix = "OBJC_CATEGORY_INSTANCE_METHODS_"; + section = "__OBJC,__cat_inst_meth,regular,no_dead_strip"; + forProtocol = false; + break; + case MethodListType::CategoryClassMethods: + prefix = "OBJC_CATEGORY_CLASS_METHODS_"; + section = "__OBJC,__cat_cls_meth,regular,no_dead_strip"; + forProtocol = false; + break; + case MethodListType::InstanceMethods: + prefix = "OBJC_INSTANCE_METHODS_"; + section = "__OBJC,__inst_meth,regular,no_dead_strip"; + forProtocol = false; + break; + case MethodListType::ClassMethods: + prefix = "OBJC_CLASS_METHODS_"; + section = "__OBJC,__cls_meth,regular,no_dead_strip"; + forProtocol = false; + break; + case MethodListType::ProtocolInstanceMethods: + prefix = "OBJC_PROTOCOL_INSTANCE_METHODS_"; + section = "__OBJC,__cat_inst_meth,regular,no_dead_strip"; + forProtocol = true; + break; + case MethodListType::ProtocolClassMethods: + prefix = "OBJC_PROTOCOL_CLASS_METHODS_"; + section = "__OBJC,__cat_cls_meth,regular,no_dead_strip"; + forProtocol = true; + break; + case MethodListType::OptionalProtocolInstanceMethods: + prefix = "OBJC_PROTOCOL_INSTANCE_METHODS_OPT_"; + section = "__OBJC,__cat_inst_meth,regular,no_dead_strip"; + forProtocol = true; + break; + case MethodListType::OptionalProtocolClassMethods: + prefix = "OBJC_PROTOCOL_CLASS_METHODS_OPT_"; + section = "__OBJC,__cat_cls_meth,regular,no_dead_strip"; + forProtocol = true; + break; + } + // Return null for empty list. - if (Methods.empty()) - return llvm::Constant::getNullValue(ObjCTypes.MethodListPtrTy); + if (methods.empty()) + return llvm::Constant::getNullValue(forProtocol + ? ObjCTypes.MethodDescriptionListPtrTy + : ObjCTypes.MethodListPtrTy); + + // For protocols, this is an objc_method_description_list, which has + // a slightly different structure. + if (forProtocol) { + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + values.addInt(ObjCTypes.IntTy, methods.size()); + auto methodArray = values.beginArray(ObjCTypes.MethodDescriptionTy); + for (auto MD : methods) { + emitMethodDescriptionConstant(methodArray, MD); + } + methodArray.finishAndAddTo(values); + + llvm::GlobalVariable *GV = CreateMetadataVar(prefix + name, values, section, + CGM.getPointerAlign(), true); + return llvm::ConstantExpr::getBitCast(GV, + ObjCTypes.MethodDescriptionListPtrTy); + } - llvm::Constant *Values[3]; - Values[0] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); - Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Methods.size()); - llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.MethodTy, - Methods.size()); - Values[2] = llvm::ConstantArray::get(AT, Methods); - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + // Otherwise, it's an objc_method_list. + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + values.addNullPointer(ObjCTypes.Int8PtrTy); + values.addInt(ObjCTypes.IntTy, methods.size()); + auto methodArray = values.beginArray(ObjCTypes.MethodTy); + for (auto MD : methods) { + emitMethodConstant(methodArray, MD); + } + methodArray.finishAndAddTo(values); - llvm::GlobalVariable *GV = - CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); + llvm::GlobalVariable *GV = CreateMetadataVar(prefix + name, values, section, + CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.MethodListPtrTy); } @@ -3634,6 +3860,21 @@ llvm::Function *CGObjCCommonMac::GenerateMethod(const ObjCMethodDecl *OMD, } llvm::GlobalVariable *CGObjCCommonMac::CreateMetadataVar(Twine Name, + ConstantStructBuilder &Init, + StringRef Section, + CharUnits Align, + bool AddToUsed) { + llvm::GlobalVariable *GV = + Init.finishAndCreateGlobal(Name, Align, /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); + if (!Section.empty()) + GV->setSection(Section); + if (AddToUsed) + CGM.addCompilerUsedGlobal(GV); + return GV; +} + +llvm::GlobalVariable *CGObjCCommonMac::CreateMetadataVar(Twine Name, llvm::Constant *Init, StringRef Section, CharUnits Align, @@ -3650,6 +3891,54 @@ llvm::GlobalVariable *CGObjCCommonMac::CreateMetadataVar(Twine Name, return GV; } +llvm::GlobalVariable * +CGObjCCommonMac::CreateCStringLiteral(StringRef Name, ObjCLabelType Type, + bool ForceNonFragileABI, + bool NullTerminate) { + StringRef Label; + switch (Type) { + case ObjCLabelType::ClassName: Label = "OBJC_CLASS_NAME_"; break; + case ObjCLabelType::MethodVarName: Label = "OBJC_METH_VAR_NAME_"; break; + case ObjCLabelType::MethodVarType: Label = "OBJC_METH_VAR_TYPE_"; break; + case ObjCLabelType::PropertyName: Label = "OBJC_PROP_NAME_ATTR_"; break; + } + + bool NonFragile = ForceNonFragileABI || isNonFragileABI(); + + StringRef Section; + switch (Type) { + case ObjCLabelType::ClassName: + Section = NonFragile ? "__TEXT,__objc_classname,cstring_literals" + : "__TEXT,__cstring,cstring_literals"; + break; + case ObjCLabelType::MethodVarName: + Section = NonFragile ? "__TEXT,__objc_methname,cstring_literals" + : "__TEXT,__cstring,cstring_literals"; + break; + case ObjCLabelType::MethodVarType: + Section = NonFragile ? "__TEXT,__objc_methtype,cstring_literals" + : "__TEXT,__cstring,cstring_literals"; + break; + case ObjCLabelType::PropertyName: + Section = "__TEXT,__cstring,cstring_literals"; + break; + } + + llvm::Constant *Value = + llvm::ConstantDataArray::getString(VMContext, Name, NullTerminate); + llvm::GlobalVariable *GV = + new llvm::GlobalVariable(CGM.getModule(), Value->getType(), + /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, Value, Label); + if (CGM.getTriple().isOSBinFormatMachO()) + GV->setSection(Section); + GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); + GV->setAlignment(CharUnits::One().getQuantity()); + CGM.addCompilerUsedGlobal(GV); + + return GV; +} + llvm::Function *CGObjCMac::ModuleInitFunction() { // Abuse this interface function as a place to finalize. FinishModule(); @@ -4390,8 +4679,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); assert(Size <= 8 && "does not support size > 8"); - src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy) - : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy); + src = (Size == 4) ? CGF.Builder.CreateBitCast(src, CGM.Int32Ty) + : CGF.Builder.CreateBitCast(src, CGM.Int64Ty); src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); @@ -4411,8 +4700,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); assert(Size <= 8 && "does not support size > 8"); - src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy) - : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy); + src = (Size == 4) ? CGF.Builder.CreateBitCast(src, CGM.Int32Ty) + : CGF.Builder.CreateBitCast(src, CGM.Int64Ty); src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); @@ -4437,8 +4726,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); assert(Size <= 8 && "does not support size > 8"); - src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy) - : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy); + src = (Size == 4) ? CGF.Builder.CreateBitCast(src, CGM.Int32Ty) + : CGF.Builder.CreateBitCast(src, CGM.Int64Ty); src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); @@ -4456,8 +4745,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); assert(Size <= 8 && "does not support size > 8"); - src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy) - : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy); + src = (Size == 4) ? CGF.Builder.CreateBitCast(src, CGM.Int32Ty) + : CGF.Builder.CreateBitCast(src, CGM.Int64Ty); src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); @@ -4590,15 +4879,14 @@ static const int ModuleVersion = 7; void CGObjCMac::EmitModuleInfo() { uint64_t Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ModuleTy); - llvm::Constant *Values[] = { - llvm::ConstantInt::get(ObjCTypes.LongTy, ModuleVersion), - llvm::ConstantInt::get(ObjCTypes.LongTy, Size), - // This used to be the filename, now it is unused. <rdr://4327263> - GetClassName(StringRef("")), - EmitModuleSymbols() - }; - CreateMetadataVar("OBJC_MODULES", - llvm::ConstantStruct::get(ObjCTypes.ModuleTy, Values), + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ModuleTy); + values.addInt(ObjCTypes.LongTy, ModuleVersion); + values.addInt(ObjCTypes.LongTy, Size); + // This used to be the filename, now it is unused. <rdr://4327263> + values.add(GetClassName(StringRef(""))); + values.add(EmitModuleSymbols()); + CreateMetadataVar("OBJC_MODULES", values, "__OBJC,__module_info,regular,no_dead_strip", CGM.getPointerAlign(), true); } @@ -4611,15 +4899,16 @@ llvm::Constant *CGObjCMac::EmitModuleSymbols() { if (!NumClasses && !NumCategories) return llvm::Constant::getNullValue(ObjCTypes.SymtabPtrTy); - llvm::Constant *Values[5]; - Values[0] = llvm::ConstantInt::get(ObjCTypes.LongTy, 0); - Values[1] = llvm::Constant::getNullValue(ObjCTypes.SelectorPtrTy); - Values[2] = llvm::ConstantInt::get(ObjCTypes.ShortTy, NumClasses); - Values[3] = llvm::ConstantInt::get(ObjCTypes.ShortTy, NumCategories); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + values.addInt(ObjCTypes.LongTy, 0); + values.addNullPointer(ObjCTypes.SelectorPtrTy); + values.addInt(ObjCTypes.ShortTy, NumClasses); + values.addInt(ObjCTypes.ShortTy, NumCategories); // The runtime expects exactly the list of defined classes followed // by the list of defined categories, in a single array. - SmallVector<llvm::Constant*, 8> Symbols(NumClasses + NumCategories); + auto array = values.beginArray(ObjCTypes.Int8PtrTy); for (unsigned i=0; i<NumClasses; i++) { const ObjCInterfaceDecl *ID = ImplementedClasses[i]; assert(ID); @@ -4627,24 +4916,16 @@ llvm::Constant *CGObjCMac::EmitModuleSymbols() { // We are implementing a weak imported interface. Give it external linkage if (ID->isWeakImported() && !IMP->isWeakImported()) DefinedClasses[i]->setLinkage(llvm::GlobalVariable::ExternalLinkage); - - Symbols[i] = llvm::ConstantExpr::getBitCast(DefinedClasses[i], - ObjCTypes.Int8PtrTy); + + array.addBitCast(DefinedClasses[i], ObjCTypes.Int8PtrTy); } for (unsigned i=0; i<NumCategories; i++) - Symbols[NumClasses + i] = - llvm::ConstantExpr::getBitCast(DefinedCategories[i], - ObjCTypes.Int8PtrTy); + array.addBitCast(DefinedCategories[i], ObjCTypes.Int8PtrTy); - Values[4] = - llvm::ConstantArray::get(llvm::ArrayType::get(ObjCTypes.Int8PtrTy, - Symbols.size()), - Symbols); - - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + array.finishAndAddTo(values); llvm::GlobalVariable *GV = CreateMetadataVar( - "OBJC_SYMBOLS", Init, "__OBJC,__symbols,regular,no_dead_strip", + "OBJC_SYMBOLS", values, "__OBJC,__symbols,regular,no_dead_strip", CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.SymtabPtrTy); } @@ -4707,12 +4988,7 @@ Address CGObjCMac::EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel) { llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { llvm::GlobalVariable *&Entry = ClassNames[RuntimeName]; if (!Entry) - Entry = CreateMetadataVar( - "OBJC_CLASS_NAME_", - llvm::ConstantDataArray::getString(VMContext, RuntimeName), - ((ObjCABI == 2) ? "__TEXT,__objc_classname,cstring_literals" - : "__TEXT,__cstring,cstring_literals"), - CharUnits::One(), true); + Entry = CreateCStringLiteral(RuntimeName, ObjCLabelType::ClassName); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -4960,14 +5236,8 @@ llvm::Constant *IvarLayoutBuilder::buildBitmap(CGObjCCommonMac &CGObjC, // Null terminate the string. buffer.push_back(0); - bool isNonFragileABI = CGObjC.isNonFragileABI(); - - llvm::GlobalVariable *Entry = CGObjC.CreateMetadataVar( - "OBJC_CLASS_NAME_", - llvm::ConstantDataArray::get(CGM.getLLVMContext(), buffer), - (isNonFragileABI ? "__TEXT,__objc_classname,cstring_literals" - : "__TEXT,__cstring,cstring_literals"), - CharUnits::One(), true); + auto *Entry = CGObjC.CreateCStringLiteral( + reinterpret_cast<char *>(buffer.data()), ObjCLabelType::ClassName); return getConstantGEP(CGM.getLLVMContext(), Entry, 0, 0); } @@ -5062,16 +5332,9 @@ CGObjCCommonMac::BuildIvarLayout(const ObjCImplementationDecl *OMD, llvm::Constant *CGObjCCommonMac::GetMethodVarName(Selector Sel) { llvm::GlobalVariable *&Entry = MethodVarNames[Sel]; - // FIXME: Avoid std::string in "Sel.getAsString()" if (!Entry) - Entry = CreateMetadataVar( - "OBJC_METH_VAR_NAME_", - llvm::ConstantDataArray::getString(VMContext, Sel.getAsString()), - ((ObjCABI == 2) ? "__TEXT,__objc_methname,cstring_literals" - : "__TEXT,__cstring,cstring_literals"), - CharUnits::One(), true); - + Entry = CreateCStringLiteral(Sel.getAsString(), ObjCLabelType::MethodVarName); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -5085,47 +5348,27 @@ llvm::Constant *CGObjCCommonMac::GetMethodVarType(const FieldDecl *Field) { CGM.getContext().getObjCEncodingForType(Field->getType(), TypeStr, Field); llvm::GlobalVariable *&Entry = MethodVarTypes[TypeStr]; - if (!Entry) - Entry = CreateMetadataVar( - "OBJC_METH_VAR_TYPE_", - llvm::ConstantDataArray::getString(VMContext, TypeStr), - ((ObjCABI == 2) ? "__TEXT,__objc_methtype,cstring_literals" - : "__TEXT,__cstring,cstring_literals"), - CharUnits::One(), true); - + Entry = CreateCStringLiteral(TypeStr, ObjCLabelType::MethodVarType); return getConstantGEP(VMContext, Entry, 0, 0); } llvm::Constant *CGObjCCommonMac::GetMethodVarType(const ObjCMethodDecl *D, bool Extended) { - std::string TypeStr; - if (CGM.getContext().getObjCEncodingForMethodDecl(D, TypeStr, Extended)) - return nullptr; + std::string TypeStr = + CGM.getContext().getObjCEncodingForMethodDecl(D, Extended); llvm::GlobalVariable *&Entry = MethodVarTypes[TypeStr]; - if (!Entry) - Entry = CreateMetadataVar( - "OBJC_METH_VAR_TYPE_", - llvm::ConstantDataArray::getString(VMContext, TypeStr), - ((ObjCABI == 2) ? "__TEXT,__objc_methtype,cstring_literals" - : "__TEXT,__cstring,cstring_literals"), - CharUnits::One(), true); - + Entry = CreateCStringLiteral(TypeStr, ObjCLabelType::MethodVarType); return getConstantGEP(VMContext, Entry, 0, 0); } // FIXME: Merge into a single cstring creation function. llvm::Constant *CGObjCCommonMac::GetPropertyName(IdentifierInfo *Ident) { llvm::GlobalVariable *&Entry = PropertyNames[Ident]; - if (!Entry) - Entry = CreateMetadataVar( - "OBJC_PROP_NAME_ATTR_", - llvm::ConstantDataArray::getString(VMContext, Ident->getName()), - "__TEXT,__cstring,cstring_literals", CharUnits::One(), true); - + Entry = CreateCStringLiteral(Ident->getName(), ObjCLabelType::PropertyName); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -5134,8 +5377,8 @@ llvm::Constant *CGObjCCommonMac::GetPropertyName(IdentifierInfo *Ident) { llvm::Constant * CGObjCCommonMac::GetPropertyTypeString(const ObjCPropertyDecl *PD, const Decl *Container) { - std::string TypeStr; - CGM.getContext().getObjCEncodingForPropertyDecl(PD, Container, TypeStr); + std::string TypeStr = + CGM.getContext().getObjCEncodingForPropertyDecl(PD, Container); return GetPropertyName(&CGM.getContext().Idents.get(TypeStr)); } @@ -5157,20 +5400,20 @@ void CGObjCMac::FinishModule() { // Emit the dummy bodies for any protocols which were referenced but // never defined. - for (llvm::DenseMap<IdentifierInfo*, llvm::GlobalVariable*>::iterator - I = Protocols.begin(), e = Protocols.end(); I != e; ++I) { - if (I->second->hasInitializer()) + for (auto &entry : Protocols) { + llvm::GlobalVariable *global = entry.second; + if (global->hasInitializer()) continue; - llvm::Constant *Values[5]; - Values[0] = llvm::Constant::getNullValue(ObjCTypes.ProtocolExtensionPtrTy); - Values[1] = GetClassName(I->first->getName()); - Values[2] = llvm::Constant::getNullValue(ObjCTypes.ProtocolListPtrTy); - Values[3] = Values[4] = - llvm::Constant::getNullValue(ObjCTypes.MethodDescriptionListPtrTy); - I->second->setInitializer(llvm::ConstantStruct::get(ObjCTypes.ProtocolTy, - Values)); - CGM.addCompilerUsedGlobal(I->second); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ProtocolTy); + values.addNullPointer(ObjCTypes.ProtocolExtensionPtrTy); + values.add(GetClassName(entry.first->getName())); + values.addNullPointer(ObjCTypes.ProtocolListPtrTy); + values.addNullPointer(ObjCTypes.MethodDescriptionListPtrTy); + values.addNullPointer(ObjCTypes.MethodDescriptionListPtrTy); + values.finishAndSetAsInitializer(global); + CGM.addCompilerUsedGlobal(global); } // Add assembler directives to add lazy undefined symbol references @@ -5178,27 +5421,23 @@ void CGObjCMac::FinishModule() { // important for correct linker interaction. // // FIXME: It would be nice if we had an LLVM construct for this. - if (!LazySymbols.empty() || !DefinedSymbols.empty()) { + if ((!LazySymbols.empty() || !DefinedSymbols.empty()) && + CGM.getTriple().isOSBinFormatMachO()) { SmallString<256> Asm; Asm += CGM.getModule().getModuleInlineAsm(); if (!Asm.empty() && Asm.back() != '\n') Asm += '\n'; llvm::raw_svector_ostream OS(Asm); - for (llvm::SetVector<IdentifierInfo*>::iterator I = DefinedSymbols.begin(), - e = DefinedSymbols.end(); I != e; ++I) - OS << "\t.objc_class_name_" << (*I)->getName() << "=0\n" - << "\t.globl .objc_class_name_" << (*I)->getName() << "\n"; - for (llvm::SetVector<IdentifierInfo*>::iterator I = LazySymbols.begin(), - e = LazySymbols.end(); I != e; ++I) { - OS << "\t.lazy_reference .objc_class_name_" << (*I)->getName() << "\n"; - } + for (const auto *Sym : DefinedSymbols) + OS << "\t.objc_class_name_" << Sym->getName() << "=0\n" + << "\t.globl .objc_class_name_" << Sym->getName() << "\n"; + for (const auto *Sym : LazySymbols) + OS << "\t.lazy_reference .objc_class_name_" << Sym->getName() << "\n"; + for (const auto &Category : DefinedCategoryNames) + OS << "\t.objc_category_name_" << Category << "=0\n" + << "\t.globl .objc_category_name_" << Category << "\n"; - for (size_t i = 0, e = DefinedCategoryNames.size(); i < e; ++i) { - OS << "\t.objc_category_name_" << DefinedCategoryNames[i] << "=0\n" - << "\t.globl .objc_category_name_" << DefinedCategoryNames[i] << "\n"; - } - CGM.getModule().setModuleInlineAsm(OS.str()); } } @@ -5217,10 +5456,9 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm) CodeGen::CodeGenTypes &Types = CGM.getTypes(); ASTContext &Ctx = CGM.getContext(); - ShortTy = Types.ConvertType(Ctx.ShortTy); - IntTy = Types.ConvertType(Ctx.IntTy); - LongTy = Types.ConvertType(Ctx.LongTy); - LongLongTy = Types.ConvertType(Ctx.LongLongTy); + ShortTy = cast<llvm::IntegerType>(Types.ConvertType(Ctx.ShortTy)); + IntTy = CGM.IntTy; + LongTy = cast<llvm::IntegerType>(Types.ConvertType(Ctx.LongTy)); Int8PtrTy = CGM.Int8PtrTy; Int8PtrPtrTy = CGM.Int8PtrPtrTy; @@ -5231,9 +5469,12 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm) else IvarOffsetVarTy = LongTy; - ObjectPtrTy = Types.ConvertType(Ctx.getObjCIdType()); - PtrObjectPtrTy = llvm::PointerType::getUnqual(ObjectPtrTy); - SelectorPtrTy = Types.ConvertType(Ctx.getObjCSelType()); + ObjectPtrTy = + cast<llvm::PointerType>(Types.ConvertType(Ctx.getObjCIdType())); + PtrObjectPtrTy = + llvm::PointerType::getUnqual(ObjectPtrTy); + SelectorPtrTy = + cast<llvm::PointerType>(Types.ConvertType(Ctx.getObjCSelType())); // I'm not sure I like this. The implicit coordination is a bit // gross. We should solve this in a reasonable fashion because this @@ -5831,7 +6072,6 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer( unsigned InstanceSize, const ObjCImplementationDecl *ID) { std::string ClassName = ID->getObjCRuntimeNameAsString(); - llvm::Constant *Values[10]; // 11 for 64bit targets! CharUnits beginInstance = CharUnits::fromQuantity(InstanceStart); CharUnits endInstance = CharUnits::fromQuantity(InstanceSize); @@ -5842,85 +6082,84 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer( else if ((hasMRCWeak = hasMRCWeakIvars(CGM, ID))) flags |= NonFragileABI_Class_HasMRCWeakIvars; - Values[ 0] = llvm::ConstantInt::get(ObjCTypes.IntTy, flags); - Values[ 1] = llvm::ConstantInt::get(ObjCTypes.IntTy, InstanceStart); - Values[ 2] = llvm::ConstantInt::get(ObjCTypes.IntTy, InstanceSize); - // FIXME. For 64bit targets add 0 here. - Values[ 3] = (flags & NonFragileABI_Class_Meta) - ? GetIvarLayoutName(nullptr, ObjCTypes) - : BuildStrongIvarLayout(ID, beginInstance, endInstance); - Values[ 4] = GetClassName(ID->getObjCRuntimeNameAsString()); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ClassRonfABITy); + + values.addInt(ObjCTypes.IntTy, flags); + values.addInt(ObjCTypes.IntTy, InstanceStart); + values.addInt(ObjCTypes.IntTy, InstanceSize); + values.add((flags & NonFragileABI_Class_Meta) + ? GetIvarLayoutName(nullptr, ObjCTypes) + : BuildStrongIvarLayout(ID, beginInstance, endInstance)); + values.add(GetClassName(ID->getObjCRuntimeNameAsString())); + // const struct _method_list_t * const baseMethods; - std::vector<llvm::Constant*> Methods; - std::string MethodListName("\01l_OBJC_$_"); + SmallVector<const ObjCMethodDecl*, 16> methods; if (flags & NonFragileABI_Class_Meta) { - MethodListName += "CLASS_METHODS_"; - MethodListName += ID->getObjCRuntimeNameAsString(); - for (const auto *I : ID->class_methods()) - // Class methods should always be defined. - Methods.push_back(GetMethodConstant(I)); + for (const auto *MD : ID->class_methods()) + methods.push_back(MD); } else { - MethodListName += "INSTANCE_METHODS_"; - MethodListName += ID->getObjCRuntimeNameAsString(); - for (const auto *I : ID->instance_methods()) - // Instance methods should always be defined. - Methods.push_back(GetMethodConstant(I)); + for (const auto *MD : ID->instance_methods()) + methods.push_back(MD); for (const auto *PID : ID->property_impls()) { if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize){ ObjCPropertyDecl *PD = PID->getPropertyDecl(); - if (ObjCMethodDecl *MD = PD->getGetterMethodDecl()) - if (llvm::Constant *C = GetMethodConstant(MD)) - Methods.push_back(C); - if (ObjCMethodDecl *MD = PD->getSetterMethodDecl()) - if (llvm::Constant *C = GetMethodConstant(MD)) - Methods.push_back(C); + if (auto MD = PD->getGetterMethodDecl()) + if (GetMethodDefinition(MD)) + methods.push_back(MD); + if (auto MD = PD->getSetterMethodDecl()) + if (GetMethodDefinition(MD)) + methods.push_back(MD); } } } - Values[ 5] = EmitMethodList(MethodListName, - "__DATA, __objc_const", Methods); + + values.add(emitMethodList(ID->getObjCRuntimeNameAsString(), + (flags & NonFragileABI_Class_Meta) + ? MethodListType::ClassMethods + : MethodListType::InstanceMethods, + methods)); const ObjCInterfaceDecl *OID = ID->getClassInterface(); assert(OID && "CGObjCNonFragileABIMac::BuildClassRoTInitializer"); - Values[ 6] = EmitProtocolList("\01l_OBJC_CLASS_PROTOCOLS_$_" + values.add(EmitProtocolList("\01l_OBJC_CLASS_PROTOCOLS_$_" + OID->getObjCRuntimeNameAsString(), - OID->all_referenced_protocol_begin(), - OID->all_referenced_protocol_end()); + OID->all_referenced_protocol_begin(), + OID->all_referenced_protocol_end())); if (flags & NonFragileABI_Class_Meta) { - Values[ 7] = llvm::Constant::getNullValue(ObjCTypes.IvarListnfABIPtrTy); - Values[ 8] = GetIvarLayoutName(nullptr, ObjCTypes); - Values[ 9] = EmitPropertyList( + values.addNullPointer(ObjCTypes.IvarListnfABIPtrTy); + values.add(GetIvarLayoutName(nullptr, ObjCTypes)); + values.add(EmitPropertyList( "\01l_OBJC_$_CLASS_PROP_LIST_" + ID->getObjCRuntimeNameAsString(), - ID, ID->getClassInterface(), ObjCTypes, true); + ID, ID->getClassInterface(), ObjCTypes, true)); } else { - Values[ 7] = EmitIvarList(ID); - Values[ 8] = BuildWeakIvarLayout(ID, beginInstance, endInstance, - hasMRCWeak); - Values[ 9] = EmitPropertyList( + values.add(EmitIvarList(ID)); + values.add(BuildWeakIvarLayout(ID, beginInstance, endInstance, hasMRCWeak)); + values.add(EmitPropertyList( "\01l_OBJC_$_PROP_LIST_" + ID->getObjCRuntimeNameAsString(), - ID, ID->getClassInterface(), ObjCTypes, false); + ID, ID->getClassInterface(), ObjCTypes, false)); } - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassRonfABITy, - Values); + + llvm::SmallString<64> roLabel; + llvm::raw_svector_ostream(roLabel) + << ((flags & NonFragileABI_Class_Meta) ? "\01l_OBJC_METACLASS_RO_$_" + : "\01l_OBJC_CLASS_RO_$_") + << ClassName; + llvm::GlobalVariable *CLASS_RO_GV = - new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassRonfABITy, false, - llvm::GlobalValue::PrivateLinkage, - Init, - (flags & NonFragileABI_Class_Meta) ? - std::string("\01l_OBJC_METACLASS_RO_$_")+ClassName : - std::string("\01l_OBJC_CLASS_RO_$_")+ClassName); - CLASS_RO_GV->setAlignment( - CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassRonfABITy)); - CLASS_RO_GV->setSection("__DATA, __objc_const"); + values.finishAndCreateGlobal(roLabel, CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); + if (CGM.getTriple().isOSBinFormatMachO()) + CLASS_RO_GV->setSection("__DATA, __objc_const"); return CLASS_RO_GV; - } -/// BuildClassMetaData - This routine defines that to-level meta-data -/// for the given ClassName for: +/// Build the metaclass object for a class. +/// /// struct _class_t { /// struct _class_t *isa; /// struct _class_t * const superclass; @@ -5929,28 +6168,33 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer( /// struct class_ro_t *ro; /// } /// -llvm::GlobalVariable *CGObjCNonFragileABIMac::BuildClassMetaData( - const std::string &ClassName, llvm::Constant *IsAGV, llvm::Constant *SuperClassGV, - llvm::Constant *ClassRoGV, bool HiddenVisibility, bool Weak) { - llvm::Constant *Values[] = { - IsAGV, - SuperClassGV, - ObjCEmptyCacheVar, // &ObjCEmptyCacheVar - ObjCEmptyVtableVar, // &ObjCEmptyVtableVar - ClassRoGV // &CLASS_RO_GV - }; - if (!Values[1]) - Values[1] = llvm::Constant::getNullValue(ObjCTypes.ClassnfABIPtrTy); - if (!Values[3]) - Values[3] = llvm::Constant::getNullValue( - llvm::PointerType::getUnqual(ObjCTypes.ImpnfABITy)); - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassnfABITy, - Values); - llvm::GlobalVariable *GV = GetClassGlobal(ClassName, Weak); - GV->setInitializer(Init); - GV->setSection("__DATA, __objc_data"); +llvm::GlobalVariable * +CGObjCNonFragileABIMac::BuildClassObject(const ObjCInterfaceDecl *CI, + bool isMetaclass, + llvm::Constant *IsAGV, + llvm::Constant *SuperClassGV, + llvm::Constant *ClassRoGV, + bool HiddenVisibility) { + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ClassnfABITy); + values.add(IsAGV); + if (SuperClassGV) { + values.add(SuperClassGV); + } else { + values.addNullPointer(ObjCTypes.ClassnfABIPtrTy); + } + values.add(ObjCEmptyCacheVar); + values.add(ObjCEmptyVtableVar); + values.add(ClassRoGV); + + llvm::GlobalVariable *GV = + cast<llvm::GlobalVariable>(GetClassGlobal(CI, isMetaclass, ForDefinition)); + values.finishAndSetAsInitializer(GV); + + if (CGM.getTriple().isOSBinFormatMachO()) + GV->setSection("__DATA, __objc_data"); GV->setAlignment( - CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassnfABITy)); + CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassnfABITy)); if (!CGM.getTriple().isOSBinFormatCOFF()) if (HiddenVisibility) GV->setVisibility(llvm::GlobalValue::HiddenVisibility); @@ -6014,6 +6258,9 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ImpnfABITy, false, llvm::GlobalValue::ExternalLinkage, nullptr, "_objc_empty_vtable"); + else + ObjCEmptyVtableVar = + llvm::ConstantPointerNull::get(ObjCTypes.ImpnfABITy->getPointerTo()); } // FIXME: Is this correct (that meta class size is never computed)? @@ -6022,9 +6269,8 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { uint32_t InstanceSize = InstanceStart; uint32_t flags = NonFragileABI_Class_Meta; - llvm::GlobalVariable *SuperClassGV, *IsAGV; + llvm::Constant *SuperClassGV, *IsAGV; - StringRef ClassName = ID->getObjCRuntimeNameAsString(); const auto *CI = ID->getClassInterface(); assert(CI && "CGObjCNonFragileABIMac::GenerateClass - class is 0"); @@ -6047,17 +6293,8 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { // class is root flags |= NonFragileABI_Class_Root; - SuperClassGV = GetClassGlobal((getClassSymbolPrefix() + ClassName).str(), - CI->isWeakImported()); - if (CGM.getTriple().isOSBinFormatCOFF()) - if (CI->hasAttr<DLLImportAttr>()) - SuperClassGV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); - - IsAGV = GetClassGlobal((getMetaclassSymbolPrefix() + ClassName).str(), - CI->isWeakImported()); - if (CGM.getTriple().isOSBinFormatCOFF()) - if (CI->hasAttr<DLLImportAttr>()) - IsAGV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + SuperClassGV = GetClassGlobal(CI, /*metaclass*/ false, NotForDefinition); + IsAGV = GetClassGlobal(CI, /*metaclass*/ true, NotForDefinition); } else { // Has a root. Current class is not a root. const ObjCInterfaceDecl *Root = ID->getClassInterface(); @@ -6065,31 +6302,16 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { Root = Super; const auto *Super = CI->getSuperClass(); - StringRef RootClassName = Root->getObjCRuntimeNameAsString(); - StringRef SuperClassName = Super->getObjCRuntimeNameAsString(); - - IsAGV = GetClassGlobal((getMetaclassSymbolPrefix() + RootClassName).str(), - Root->isWeakImported()); - if (CGM.getTriple().isOSBinFormatCOFF()) - if (Root->hasAttr<DLLImportAttr>()) - IsAGV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); - - // work on super class metadata symbol. - SuperClassGV = - GetClassGlobal((getMetaclassSymbolPrefix() + SuperClassName).str(), - Super->isWeakImported()); - if (CGM.getTriple().isOSBinFormatCOFF()) - if (Super->hasAttr<DLLImportAttr>()) - SuperClassGV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + IsAGV = GetClassGlobal(Root, /*metaclass*/ true, NotForDefinition); + SuperClassGV = GetClassGlobal(Super, /*metaclass*/ true, NotForDefinition); } llvm::GlobalVariable *CLASS_RO_GV = BuildClassRoTInitializer(flags, InstanceStart, InstanceSize, ID); llvm::GlobalVariable *MetaTClass = - BuildClassMetaData((getMetaclassSymbolPrefix() + ClassName).str(), IsAGV, - SuperClassGV, CLASS_RO_GV, classIsHidden, - CI->isWeakImported()); + BuildClassObject(CI, /*metaclass*/ true, + IsAGV, SuperClassGV, CLASS_RO_GV, classIsHidden); if (CGM.getTriple().isOSBinFormatCOFF()) if (CI->hasAttr<DLLExportAttr>()) MetaTClass->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); @@ -6122,14 +6344,7 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { } else { // Has a root. Current class is not a root. const auto *Super = CI->getSuperClass(); - StringRef SuperClassName = Super->getObjCRuntimeNameAsString(); - - SuperClassGV = - GetClassGlobal((getClassSymbolPrefix() + SuperClassName).str(), - Super->isWeakImported()); - if (CGM.getTriple().isOSBinFormatCOFF()) - if (Super->hasAttr<DLLImportAttr>()) - SuperClassGV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + SuperClassGV = GetClassGlobal(Super, /*metaclass*/ false, NotForDefinition); } GetClassSizeInfo(ID, InstanceStart, InstanceSize); @@ -6137,9 +6352,8 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { BuildClassRoTInitializer(flags, InstanceStart, InstanceSize, ID); llvm::GlobalVariable *ClassMD = - BuildClassMetaData((getClassSymbolPrefix() + ClassName).str(), MetaTClass, - SuperClassGV, CLASS_RO_GV, classIsHidden, - CI->isWeakImported()); + BuildClassObject(CI, /*metaclass*/ false, + MetaTClass, SuperClassGV, CLASS_RO_GV, classIsHidden); if (CGM.getTriple().isOSBinFormatCOFF()) if (CI->hasAttr<DLLExportAttr>()) ClassMD->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); @@ -6152,7 +6366,7 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { // Force the definition of the EHType if necessary. if (flags & NonFragileABI_Class_Exception) - GetInterfaceEHType(CI, true); + (void) GetInterfaceEHType(CI, ForDefinition); // Make sure method definition entries are all clear for next implementation. MethodDefinitions.clear(); } @@ -6217,82 +6431,59 @@ void CGObjCNonFragileABIMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) { ExtCatName += "_$_"; ExtCatName += OCD->getNameAsString(); - llvm::SmallString<64> ExtClassName(getClassSymbolPrefix()); - ExtClassName += Interface->getObjCRuntimeNameAsString(); - - llvm::Constant *Values[8]; - Values[0] = GetClassName(OCD->getIdentifier()->getName()); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.CategorynfABITy); + values.add(GetClassName(OCD->getIdentifier()->getName())); // meta-class entry symbol - llvm::GlobalVariable *ClassGV = - GetClassGlobal(ExtClassName.str(), Interface->isWeakImported()); + values.add(GetClassGlobal(Interface, /*metaclass*/ false, NotForDefinition)); + std::string listName = + (Interface->getObjCRuntimeNameAsString() + "_$_" + OCD->getName()).str(); + + SmallVector<const ObjCMethodDecl *, 16> instanceMethods; + SmallVector<const ObjCMethodDecl *, 8> classMethods; + for (const auto *MD : OCD->methods()) { + if (MD->isInstanceMethod()) { + instanceMethods.push_back(MD); + } else { + classMethods.push_back(MD); + } + } + + values.add(emitMethodList(listName, MethodListType::CategoryInstanceMethods, + instanceMethods)); + values.add(emitMethodList(listName, MethodListType::CategoryClassMethods, + classMethods)); - Values[1] = ClassGV; - std::vector<llvm::Constant*> Methods; - llvm::SmallString<64> MethodListName(Prefix); - - MethodListName += "INSTANCE_METHODS_"; - MethodListName += Interface->getObjCRuntimeNameAsString(); - MethodListName += "_$_"; - MethodListName += OCD->getName(); - - for (const auto *I : OCD->instance_methods()) - // Instance methods should always be defined. - Methods.push_back(GetMethodConstant(I)); - - Values[2] = EmitMethodList(MethodListName.str(), - "__DATA, __objc_const", - Methods); - - MethodListName = Prefix; - MethodListName += "CLASS_METHODS_"; - MethodListName += Interface->getObjCRuntimeNameAsString(); - MethodListName += "_$_"; - MethodListName += OCD->getNameAsString(); - - Methods.clear(); - for (const auto *I : OCD->class_methods()) - // Class methods should always be defined. - Methods.push_back(GetMethodConstant(I)); - - Values[3] = EmitMethodList(MethodListName.str(), - "__DATA, __objc_const", - Methods); const ObjCCategoryDecl *Category = Interface->FindCategoryDeclaration(OCD->getIdentifier()); if (Category) { SmallString<256> ExtName; llvm::raw_svector_ostream(ExtName) << Interface->getObjCRuntimeNameAsString() << "_$_" << OCD->getName(); - Values[4] = EmitProtocolList("\01l_OBJC_CATEGORY_PROTOCOLS_$_" + values.add(EmitProtocolList("\01l_OBJC_CATEGORY_PROTOCOLS_$_" + Interface->getObjCRuntimeNameAsString() + "_$_" + Category->getName(), - Category->protocol_begin(), - Category->protocol_end()); - Values[5] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ExtName.str(), - OCD, Category, ObjCTypes, false); - Values[6] = EmitPropertyList("\01l_OBJC_$_CLASS_PROP_LIST_" + ExtName.str(), - OCD, Category, ObjCTypes, true); + Category->protocol_begin(), + Category->protocol_end())); + values.add(EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ExtName.str(), + OCD, Category, ObjCTypes, false)); + values.add(EmitPropertyList("\01l_OBJC_$_CLASS_PROP_LIST_" + ExtName.str(), + OCD, Category, ObjCTypes, true)); } else { - Values[4] = llvm::Constant::getNullValue(ObjCTypes.ProtocolListnfABIPtrTy); - Values[5] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); - Values[6] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); + values.addNullPointer(ObjCTypes.ProtocolListnfABIPtrTy); + values.addNullPointer(ObjCTypes.PropertyListPtrTy); + values.addNullPointer(ObjCTypes.PropertyListPtrTy); } unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.CategorynfABITy); - Values[7] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); - - llvm::Constant *Init = - llvm::ConstantStruct::get(ObjCTypes.CategorynfABITy, - Values); - llvm::GlobalVariable *GCATV - = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.CategorynfABITy, - false, - llvm::GlobalValue::PrivateLinkage, - Init, - ExtCatName.str()); - GCATV->setAlignment( - CGM.getDataLayout().getABITypeAlignment(ObjCTypes.CategorynfABITy)); - GCATV->setSection("__DATA, __objc_const"); + values.addInt(ObjCTypes.IntTy, Size); + + llvm::GlobalVariable *GCATV = + values.finishAndCreateGlobal(ExtCatName.str(), CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); + if (CGM.getTriple().isOSBinFormatMachO()) + GCATV->setSection("__DATA, __objc_const"); CGM.addCompilerUsedGlobal(GCATV); DefinedCategories.push_back(GCATV); @@ -6303,25 +6494,37 @@ void CGObjCNonFragileABIMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) { MethodDefinitions.clear(); } -/// GetMethodConstant - Return a struct objc_method constant for the -/// given method if it has been defined. The result is null if the -/// method has not been defined. The return value has type MethodPtrTy. -llvm::Constant *CGObjCNonFragileABIMac::GetMethodConstant( - const ObjCMethodDecl *MD) { - llvm::Function *Fn = GetMethodDefinition(MD); - if (!Fn) - return nullptr; - - llvm::Constant *Method[] = { - llvm::ConstantExpr::getBitCast(GetMethodVarName(MD->getSelector()), - ObjCTypes.SelectorPtrTy), - GetMethodVarType(MD), - llvm::ConstantExpr::getBitCast(Fn, ObjCTypes.Int8PtrTy) - }; - return llvm::ConstantStruct::get(ObjCTypes.MethodTy, Method); +/// emitMethodConstant - Return a struct objc_method constant. If +/// forProtocol is true, the implementation will be null; otherwise, +/// the method must have a definition registered with the runtime. +/// +/// struct _objc_method { +/// SEL _cmd; +/// char *method_type; +/// char *_imp; +/// } +void CGObjCNonFragileABIMac::emitMethodConstant(ConstantArrayBuilder &builder, + const ObjCMethodDecl *MD, + bool forProtocol) { + auto method = builder.beginStruct(ObjCTypes.MethodTy); + method.addBitCast(GetMethodVarName(MD->getSelector()), + ObjCTypes.SelectorPtrTy); + method.add(GetMethodVarType(MD)); + + if (forProtocol) { + // Protocol methods have no implementation. So, this entry is always NULL. + method.addNullPointer(ObjCTypes.Int8PtrTy); + } else { + llvm::Function *fn = GetMethodDefinition(MD); + assert(fn && "no definition for method?"); + method.addBitCast(fn, ObjCTypes.Int8PtrTy); + } + + method.finishAndAddTo(builder); } -/// EmitMethodList - Build meta-data for method declarations +/// Build meta-data for method declarations. +/// /// struct _method_list_t { /// uint32_t entsize; // sizeof(struct _objc_method) /// uint32_t method_count; @@ -6329,28 +6532,69 @@ llvm::Constant *CGObjCNonFragileABIMac::GetMethodConstant( /// } /// llvm::Constant * -CGObjCNonFragileABIMac::EmitMethodList(Twine Name, StringRef Section, - ArrayRef<llvm::Constant *> Methods) { +CGObjCNonFragileABIMac::emitMethodList(Twine name, MethodListType kind, + ArrayRef<const ObjCMethodDecl *> methods) { // Return null for empty list. - if (Methods.empty()) + if (methods.empty()) return llvm::Constant::getNullValue(ObjCTypes.MethodListnfABIPtrTy); - llvm::Constant *Values[3]; + StringRef prefix; + bool forProtocol; + switch (kind) { + case MethodListType::CategoryInstanceMethods: + prefix = "\01l_OBJC_$_CATEGORY_INSTANCE_METHODS_"; + forProtocol = false; + break; + case MethodListType::CategoryClassMethods: + prefix = "\01l_OBJC_$_CATEGORY_CLASS_METHODS_"; + forProtocol = false; + break; + case MethodListType::InstanceMethods: + prefix = "\01l_OBJC_$_INSTANCE_METHODS_"; + forProtocol = false; + break; + case MethodListType::ClassMethods: + prefix = "\01l_OBJC_$_CLASS_METHODS_"; + forProtocol = false; + break; + + case MethodListType::ProtocolInstanceMethods: + prefix = "\01l_OBJC_$_PROTOCOL_INSTANCE_METHODS_"; + forProtocol = true; + break; + case MethodListType::ProtocolClassMethods: + prefix = "\01l_OBJC_$_PROTOCOL_CLASS_METHODS_"; + forProtocol = true; + break; + case MethodListType::OptionalProtocolInstanceMethods: + prefix = "\01l_OBJC_$_PROTOCOL_INSTANCE_METHODS_OPT_"; + forProtocol = true; + break; + case MethodListType::OptionalProtocolClassMethods: + prefix = "\01l_OBJC_$_PROTOCOL_CLASS_METHODS_OPT_"; + forProtocol = true; + break; + } + + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + // sizeof(struct _objc_method) unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.MethodTy); - Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); + values.addInt(ObjCTypes.IntTy, Size); // method_count - Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Methods.size()); - llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.MethodTy, - Methods.size()); - Values[2] = llvm::ConstantArray::get(AT, Methods); - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); + values.addInt(ObjCTypes.IntTy, methods.size()); + auto methodArray = values.beginArray(ObjCTypes.MethodTy); + for (auto MD : methods) { + emitMethodConstant(methodArray, MD, forProtocol); + } + methodArray.finishAndAddTo(values); - llvm::GlobalVariable *GV = - new llvm::GlobalVariable(CGM.getModule(), Init->getType(), false, - llvm::GlobalValue::PrivateLinkage, Init, Name); - GV->setAlignment(CGM.getDataLayout().getABITypeAlignment(Init->getType())); - GV->setSection(Section); + auto *GV = values.finishAndCreateGlobal(prefix + name, CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); + if (CGM.getTriple().isOSBinFormatMachO()) + GV->setSection("__DATA, __objc_const"); CGM.addCompilerUsedGlobal(GV); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.MethodListnfABIPtrTy); } @@ -6406,7 +6650,8 @@ CGObjCNonFragileABIMac::EmitIvarOffsetVar(const ObjCInterfaceDecl *ID, IvarOffsetGV->setVisibility(llvm::GlobalValue::DefaultVisibility); } - IvarOffsetGV->setSection("__DATA, __objc_ivar"); + if (CGM.getTriple().isOSBinFormatMachO()) + IvarOffsetGV->setSection("__DATA, __objc_ivar"); return IvarOffsetGV; } @@ -6430,7 +6675,12 @@ CGObjCNonFragileABIMac::EmitIvarOffsetVar(const ObjCInterfaceDecl *ID, llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList( const ObjCImplementationDecl *ID) { - std::vector<llvm::Constant*> Ivars; + ConstantInitBuilder builder(CGM); + auto ivarList = builder.beginStruct(); + ivarList.addInt(ObjCTypes.IntTy, + CGM.getDataLayout().getTypeAllocSize(ObjCTypes.IvarnfABITy)); + auto ivarCountSlot = ivarList.addPlaceholder(); + auto ivars = ivarList.beginArray(ObjCTypes.IvarnfABITy); const ObjCInterfaceDecl *OID = ID->getClassInterface(); assert(OID && "CGObjCNonFragileABIMac::EmitIvarList - null interface"); @@ -6442,48 +6692,45 @@ llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList( // Ignore unnamed bit-fields. if (!IVD->getDeclName()) continue; - llvm::Constant *Ivar[5]; - Ivar[0] = EmitIvarOffsetVar(ID->getClassInterface(), IVD, - ComputeIvarBaseOffset(CGM, ID, IVD)); - Ivar[1] = GetMethodVarName(IVD->getIdentifier()); - Ivar[2] = GetMethodVarType(IVD); + + auto ivar = ivars.beginStruct(ObjCTypes.IvarnfABITy); + ivar.add(EmitIvarOffsetVar(ID->getClassInterface(), IVD, + ComputeIvarBaseOffset(CGM, ID, IVD))); + ivar.add(GetMethodVarName(IVD->getIdentifier())); + ivar.add(GetMethodVarType(IVD)); llvm::Type *FieldTy = CGM.getTypes().ConvertTypeForMem(IVD->getType()); unsigned Size = CGM.getDataLayout().getTypeAllocSize(FieldTy); unsigned Align = CGM.getContext().getPreferredTypeAlign( IVD->getType().getTypePtr()) >> 3; Align = llvm::Log2_32(Align); - Ivar[3] = llvm::ConstantInt::get(ObjCTypes.IntTy, Align); + ivar.addInt(ObjCTypes.IntTy, Align); // NOTE. Size of a bitfield does not match gcc's, because of the // way bitfields are treated special in each. But I am told that // 'size' for bitfield ivars is ignored by the runtime so it does // not matter. If it matters, there is enough info to get the // bitfield right! - Ivar[4] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); - Ivars.push_back(llvm::ConstantStruct::get(ObjCTypes.IvarnfABITy, Ivar)); + ivar.addInt(ObjCTypes.IntTy, Size); + ivar.finishAndAddTo(ivars); } // Return null for empty list. - if (Ivars.empty()) + if (ivars.empty()) { + ivars.abandon(); + ivarList.abandon(); return llvm::Constant::getNullValue(ObjCTypes.IvarListnfABIPtrTy); + } + + auto ivarCount = ivars.size(); + ivars.finishAndAddTo(ivarList); + ivarList.fillPlaceholderWithInt(ivarCountSlot, ObjCTypes.IntTy, ivarCount); - llvm::Constant *Values[3]; - unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.IvarnfABITy); - Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); - Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Ivars.size()); - llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.IvarnfABITy, - Ivars.size()); - Values[2] = llvm::ConstantArray::get(AT, Ivars); - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); const char *Prefix = "\01l_OBJC_$_INSTANCE_VARIABLES_"; llvm::GlobalVariable *GV = - new llvm::GlobalVariable(CGM.getModule(), Init->getType(), false, - llvm::GlobalValue::PrivateLinkage, - Init, - Prefix + OID->getObjCRuntimeNameAsString()); - GV->setAlignment( - CGM.getDataLayout().getABITypeAlignment(Init->getType())); - GV->setSection("__DATA, __objc_const"); - + ivarList.finishAndCreateGlobal(Prefix + OID->getObjCRuntimeNameAsString(), + CGM.getPointerAlign(), /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); + if (CGM.getTriple().isOSBinFormatMachO()) + GV->setSection("__DATA, __objc_const"); CGM.addCompilerUsedGlobal(GV); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.IvarListnfABIPtrTy); } @@ -6492,15 +6739,20 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocolRef( const ObjCProtocolDecl *PD) { llvm::GlobalVariable *&Entry = Protocols[PD->getIdentifier()]; - if (!Entry) + if (!Entry) { // We use the initializer as a marker of whether this is a forward // reference or not. At module finalization we add the empty // contents for protocols which were referenced but never defined. - Entry = - new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ProtocolnfABITy, - false, llvm::GlobalValue::ExternalLinkage, - nullptr, - "\01l_OBJC_PROTOCOL_$_" + PD->getObjCRuntimeNameAsString()); + llvm::SmallString<64> Protocol; + llvm::raw_svector_ostream(Protocol) << "\01l_OBJC_PROTOCOL_$_" + << PD->getObjCRuntimeNameAsString(); + + Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ProtocolnfABITy, + false, llvm::GlobalValue::ExternalLinkage, + nullptr, Protocol); + if (!CGM.getTriple().isOSBinFormatMachO()) + Entry->setComdat(CGM.getModule().getOrInsertComdat(Protocol)); + } return Entry; } @@ -6537,96 +6789,59 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol( if (const ObjCProtocolDecl *Def = PD->getDefinition()) PD = Def; - // Construct method lists. - std::vector<llvm::Constant*> InstanceMethods, ClassMethods; - std::vector<llvm::Constant*> OptInstanceMethods, OptClassMethods; - std::vector<llvm::Constant*> MethodTypesExt, OptMethodTypesExt; - for (const auto *MD : PD->instance_methods()) { - llvm::Constant *C = GetMethodDescriptionConstant(MD); - if (!C) - return GetOrEmitProtocolRef(PD); - - if (MD->getImplementationControl() == ObjCMethodDecl::Optional) { - OptInstanceMethods.push_back(C); - OptMethodTypesExt.push_back(GetMethodVarType(MD, true)); - } else { - InstanceMethods.push_back(C); - MethodTypesExt.push_back(GetMethodVarType(MD, true)); - } - } - - for (const auto *MD : PD->class_methods()) { - llvm::Constant *C = GetMethodDescriptionConstant(MD); - if (!C) - return GetOrEmitProtocolRef(PD); - - if (MD->getImplementationControl() == ObjCMethodDecl::Optional) { - OptClassMethods.push_back(C); - OptMethodTypesExt.push_back(GetMethodVarType(MD, true)); - } else { - ClassMethods.push_back(C); - MethodTypesExt.push_back(GetMethodVarType(MD, true)); - } - } + auto methodLists = ProtocolMethodLists::get(PD); - MethodTypesExt.insert(MethodTypesExt.end(), - OptMethodTypesExt.begin(), OptMethodTypesExt.end()); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.ProtocolnfABITy); - llvm::Constant *Values[13]; // isa is NULL - Values[0] = llvm::Constant::getNullValue(ObjCTypes.ObjectPtrTy); - Values[1] = GetClassName(PD->getObjCRuntimeNameAsString()); - Values[2] = EmitProtocolList("\01l_OBJC_$_PROTOCOL_REFS_" + PD->getObjCRuntimeNameAsString(), + values.addNullPointer(ObjCTypes.ObjectPtrTy); + values.add(GetClassName(PD->getObjCRuntimeNameAsString())); + values.add(EmitProtocolList("\01l_OBJC_$_PROTOCOL_REFS_" + + PD->getObjCRuntimeNameAsString(), PD->protocol_begin(), - PD->protocol_end()); - - Values[3] = EmitMethodList("\01l_OBJC_$_PROTOCOL_INSTANCE_METHODS_" - + PD->getObjCRuntimeNameAsString(), - "__DATA, __objc_const", - InstanceMethods); - Values[4] = EmitMethodList("\01l_OBJC_$_PROTOCOL_CLASS_METHODS_" - + PD->getObjCRuntimeNameAsString(), - "__DATA, __objc_const", - ClassMethods); - Values[5] = EmitMethodList("\01l_OBJC_$_PROTOCOL_INSTANCE_METHODS_OPT_" - + PD->getObjCRuntimeNameAsString(), - "__DATA, __objc_const", - OptInstanceMethods); - Values[6] = EmitMethodList("\01l_OBJC_$_PROTOCOL_CLASS_METHODS_OPT_" - + PD->getObjCRuntimeNameAsString(), - "__DATA, __objc_const", - OptClassMethods); - Values[7] = EmitPropertyList( - "\01l_OBJC_$_PROP_LIST_" + PD->getObjCRuntimeNameAsString(), - nullptr, PD, ObjCTypes, false); + PD->protocol_end())); + values.add(methodLists.emitMethodList(this, PD, + ProtocolMethodLists::RequiredInstanceMethods)); + values.add(methodLists.emitMethodList(this, PD, + ProtocolMethodLists::RequiredClassMethods)); + values.add(methodLists.emitMethodList(this, PD, + ProtocolMethodLists::OptionalInstanceMethods)); + values.add(methodLists.emitMethodList(this, PD, + ProtocolMethodLists::OptionalClassMethods)); + values.add(EmitPropertyList( + "\01l_OBJC_$_PROP_LIST_" + PD->getObjCRuntimeNameAsString(), + nullptr, PD, ObjCTypes, false)); uint32_t Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ProtocolnfABITy); - Values[8] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); - Values[9] = llvm::Constant::getNullValue(ObjCTypes.IntTy); - Values[10] = EmitProtocolMethodTypes("\01l_OBJC_$_PROTOCOL_METHOD_TYPES_" + values.addInt(ObjCTypes.IntTy, Size); + values.addInt(ObjCTypes.IntTy, 0); + values.add(EmitProtocolMethodTypes("\01l_OBJC_$_PROTOCOL_METHOD_TYPES_" + PD->getObjCRuntimeNameAsString(), - MethodTypesExt, ObjCTypes); + methodLists.emitExtendedTypesArray(this), + ObjCTypes)); + // const char *demangledName; - Values[11] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); + values.addNullPointer(ObjCTypes.Int8PtrTy); - Values[12] = EmitPropertyList( + values.add(EmitPropertyList( "\01l_OBJC_$_CLASS_PROP_LIST_" + PD->getObjCRuntimeNameAsString(), - nullptr, PD, ObjCTypes, true); + nullptr, PD, ObjCTypes, true)); - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ProtocolnfABITy, - Values); - if (Entry) { // Already created, fix the linkage and update the initializer. Entry->setLinkage(llvm::GlobalValue::WeakAnyLinkage); - Entry->setInitializer(Init); + values.finishAndSetAsInitializer(Entry); } else { - Entry = - new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ProtocolnfABITy, - false, llvm::GlobalValue::WeakAnyLinkage, Init, - "\01l_OBJC_PROTOCOL_$_" + PD->getObjCRuntimeNameAsString()); - Entry->setAlignment( - CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ProtocolnfABITy)); + llvm::SmallString<64> symbolName; + llvm::raw_svector_ostream(symbolName) + << "\01l_OBJC_PROTOCOL_$_" << PD->getObjCRuntimeNameAsString(); + + Entry = values.finishAndCreateGlobal(symbolName, CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::WeakAnyLinkage); + if (!CGM.getTriple().isOSBinFormatMachO()) + Entry->setComdat(CGM.getModule().getOrInsertComdat(symbolName)); Protocols[PD->getIdentifier()] = Entry; } @@ -6635,13 +6850,20 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol( // Use this protocol meta-data to build protocol list table in section // __DATA, __objc_protolist + llvm::SmallString<64> ProtocolRef; + llvm::raw_svector_ostream(ProtocolRef) << "\01l_OBJC_LABEL_PROTOCOL_$_" + << PD->getObjCRuntimeNameAsString(); + llvm::GlobalVariable *PTGV = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ProtocolnfABIPtrTy, false, llvm::GlobalValue::WeakAnyLinkage, Entry, - "\01l_OBJC_LABEL_PROTOCOL_$_" + PD->getObjCRuntimeNameAsString()); + ProtocolRef); + if (!CGM.getTriple().isOSBinFormatMachO()) + PTGV->setComdat(CGM.getModule().getOrInsertComdat(ProtocolRef)); PTGV->setAlignment( CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ProtocolnfABIPtrTy)); - PTGV->setSection("__DATA, __objc_protolist, coalesced, no_dead_strip"); + if (CGM.getTriple().isOSBinFormatMachO()) + PTGV->setSection("__DATA, __objc_protolist, coalesced, no_dead_strip"); PTGV->setVisibility(llvm::GlobalValue::HiddenVisibility); CGM.addCompilerUsedGlobal(PTGV); return Entry; @@ -6673,55 +6895,30 @@ CGObjCNonFragileABIMac::EmitProtocolList(Twine Name, if (GV) return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.ProtocolListnfABIPtrTy); - for (; begin != end; ++begin) - ProtocolRefs.push_back(GetProtocolRef(*begin)); // Implemented??? + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + auto countSlot = values.addPlaceholder(); - // This list is null terminated. - ProtocolRefs.push_back(llvm::Constant::getNullValue( - ObjCTypes.ProtocolnfABIPtrTy)); - - llvm::Constant *Values[2]; - Values[0] = - llvm::ConstantInt::get(ObjCTypes.LongTy, ProtocolRefs.size() - 1); - Values[1] = - llvm::ConstantArray::get(llvm::ArrayType::get(ObjCTypes.ProtocolnfABIPtrTy, - ProtocolRefs.size()), - ProtocolRefs); - - llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); - GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(), false, - llvm::GlobalValue::PrivateLinkage, - Init, Name); - GV->setSection("__DATA, __objc_const"); - GV->setAlignment( - CGM.getDataLayout().getABITypeAlignment(Init->getType())); + // A null-terminated array of protocols. + auto array = values.beginArray(ObjCTypes.ProtocolnfABIPtrTy); + for (; begin != end; ++begin) + array.add(GetProtocolRef(*begin)); // Implemented??? + auto count = array.size(); + array.addNullPointer(ObjCTypes.ProtocolnfABIPtrTy); + + array.finishAndAddTo(values); + values.fillPlaceholderWithInt(countSlot, ObjCTypes.LongTy, count); + + GV = values.finishAndCreateGlobal(Name, CGM.getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::PrivateLinkage); + if (CGM.getTriple().isOSBinFormatMachO()) + GV->setSection("__DATA, __objc_const"); CGM.addCompilerUsedGlobal(GV); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.ProtocolListnfABIPtrTy); } -/// GetMethodDescriptionConstant - This routine build following meta-data: -/// struct _objc_method { -/// SEL _cmd; -/// char *method_type; -/// char *_imp; -/// } - -llvm::Constant * -CGObjCNonFragileABIMac::GetMethodDescriptionConstant(const ObjCMethodDecl *MD) { - llvm::Constant *Desc[3]; - Desc[0] = - llvm::ConstantExpr::getBitCast(GetMethodVarName(MD->getSelector()), - ObjCTypes.SelectorPtrTy); - Desc[1] = GetMethodVarType(MD); - if (!Desc[1]) - return nullptr; - - // Protocol methods have no implementation. So, this entry is always NULL. - Desc[2] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); - return llvm::ConstantStruct::get(ObjCTypes.MethodTy, Desc); -} - /// EmitObjCValueForIvar - Code Gen for nonfragile ivar reference. /// This code gen. amounts to generating code for: /// @code @@ -6853,16 +7050,15 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, = CGM.getModule().getGlobalVariable(messageRefName); if (!messageRef) { // Build the message ref structure. - llvm::Constant *values[] = { fn, GetMethodVarName(selector) }; - llvm::Constant *init = llvm::ConstantStruct::getAnon(values); - messageRef = new llvm::GlobalVariable(CGM.getModule(), - init->getType(), - /*constant*/ false, - llvm::GlobalValue::WeakAnyLinkage, - init, - messageRefName); + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(); + values.add(fn); + values.add(GetMethodVarName(selector)); + messageRef = values.finishAndCreateGlobal(messageRefName, + CharUnits::fromQuantity(16), + /*constant*/ false, + llvm::GlobalValue::WeakAnyLinkage); messageRef->setVisibility(llvm::GlobalValue::HiddenVisibility); - messageRef->setAlignment(16); messageRef->setSection("__DATA, __objc_msgrefs, coalesced"); } @@ -6887,9 +7083,10 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, // Load the function to call from the message ref table. Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0, CharUnits::Zero()); - llvm::Value *callee = CGF.Builder.CreateLoad(calleeAddr, "msgSend_fn"); + llvm::Value *calleePtr = CGF.Builder.CreateLoad(calleeAddr, "msgSend_fn"); - callee = CGF.Builder.CreateBitCast(callee, MSI.MessengerType); + calleePtr = CGF.Builder.CreateBitCast(calleePtr, MSI.MessengerType); + CGCallee callee(CGCalleeInfo(), calleePtr); RValue result = CGF.EmitCall(MSI.CallInfo, callee, returnSlot, args); return nullReturn.complete(CGF, result, resultType, formalArgs, @@ -6916,33 +7113,59 @@ CGObjCNonFragileABIMac::GenerateMessageSend(CodeGen::CodeGenFunction &CGF, false, CallArgs, Method, Class, ObjCTypes); } -llvm::GlobalVariable * -CGObjCNonFragileABIMac::GetClassGlobal(StringRef Name, bool Weak) { +llvm::Constant * +CGObjCNonFragileABIMac::GetClassGlobal(const ObjCInterfaceDecl *ID, + bool metaclass, + ForDefinition_t isForDefinition) { + auto prefix = + (metaclass ? getMetaclassSymbolPrefix() : getClassSymbolPrefix()); + return GetClassGlobal((prefix + ID->getObjCRuntimeNameAsString()).str(), + isForDefinition, + ID->isWeakImported(), + !isForDefinition + && CGM.getTriple().isOSBinFormatCOFF() + && ID->hasAttr<DLLImportAttr>()); +} + +llvm::Constant * +CGObjCNonFragileABIMac::GetClassGlobal(StringRef Name, + ForDefinition_t IsForDefinition, + bool Weak, bool DLLImport) { llvm::GlobalValue::LinkageTypes L = Weak ? llvm::GlobalValue::ExternalWeakLinkage : llvm::GlobalValue::ExternalLinkage; - llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name); - if (!GV) + + llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name); + if (!GV) { GV = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABITy, false, L, nullptr, Name); + if (DLLImport) + GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + } + assert(GV->getLinkage() == L); return GV; } -llvm::Value *CGObjCNonFragileABIMac::EmitClassRefFromId(CodeGenFunction &CGF, - IdentifierInfo *II, - bool Weak, - const ObjCInterfaceDecl *ID) { +llvm::Value * +CGObjCNonFragileABIMac::EmitClassRefFromId(CodeGenFunction &CGF, + IdentifierInfo *II, + const ObjCInterfaceDecl *ID) { CharUnits Align = CGF.getPointerAlign(); llvm::GlobalVariable *&Entry = ClassReferences[II]; if (!Entry) { - StringRef Name = ID ? ID->getObjCRuntimeNameAsString() : II->getName(); - std::string ClassName = (getClassSymbolPrefix() + Name).str(); - llvm::GlobalVariable *ClassGV = GetClassGlobal(ClassName, Weak); + llvm::Constant *ClassGV; + if (ID) { + ClassGV = GetClassGlobal(ID, /*metaclass*/ false, NotForDefinition); + } else { + ClassGV = GetClassGlobal((getClassSymbolPrefix() + II->getName()).str(), + NotForDefinition); + } + Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy, false, llvm::GlobalValue::PrivateLinkage, ClassGV, "OBJC_CLASSLIST_REFERENCES_$_"); @@ -6960,13 +7183,13 @@ llvm::Value *CGObjCNonFragileABIMac::EmitClassRef(CodeGenFunction &CGF, if (ID->hasAttr<ObjCRuntimeVisibleAttr>()) return EmitClassRefViaRuntime(CGF, ID, ObjCTypes); - return EmitClassRefFromId(CGF, ID->getIdentifier(), ID->isWeakImported(), ID); + return EmitClassRefFromId(CGF, ID->getIdentifier(), ID); } llvm::Value *CGObjCNonFragileABIMac::EmitNSAutoreleasePoolClassRef( CodeGenFunction &CGF) { IdentifierInfo *II = &CGM.getContext().Idents.get("NSAutoreleasePool"); - return EmitClassRefFromId(CGF, II, false, nullptr); + return EmitClassRefFromId(CGF, II, nullptr); } llvm::Value * @@ -6976,10 +7199,7 @@ CGObjCNonFragileABIMac::EmitSuperClassRef(CodeGenFunction &CGF, llvm::GlobalVariable *&Entry = SuperClassReferences[ID->getIdentifier()]; if (!Entry) { - llvm::SmallString<64> ClassName(getClassSymbolPrefix()); - ClassName += ID->getObjCRuntimeNameAsString(); - llvm::GlobalVariable *ClassGV = GetClassGlobal(ClassName.str(), - ID->isWeakImported()); + auto ClassGV = GetClassGlobal(ID, /*metaclass*/ false, NotForDefinition); Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy, false, llvm::GlobalValue::PrivateLinkage, ClassGV, "OBJC_CLASSLIST_SUP_REFS_$_"); @@ -6999,10 +7219,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CodeGenFunction &CGF, CharUnits Align = CGF.getPointerAlign(); llvm::GlobalVariable * &Entry = MetaClassReferences[ID->getIdentifier()]; if (!Entry) { - llvm::SmallString<64> MetaClassName(getMetaclassSymbolPrefix()); - MetaClassName += ID->getObjCRuntimeNameAsString(); - llvm::GlobalVariable *MetaClassGV = - GetClassGlobal(MetaClassName.str(), Weak); + auto MetaClassGV = GetClassGlobal(ID, /*metaclass*/ true, NotForDefinition); Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy, false, llvm::GlobalValue::PrivateLinkage, @@ -7021,11 +7238,10 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CodeGenFunction &CGF, llvm::Value *CGObjCNonFragileABIMac::GetClass(CodeGenFunction &CGF, const ObjCInterfaceDecl *ID) { if (ID->isWeakImported()) { - llvm::SmallString<64> ClassName(getClassSymbolPrefix()); - ClassName += ID->getObjCRuntimeNameAsString(); - llvm::GlobalVariable *ClassGV = GetClassGlobal(ClassName.str(), true); + auto ClassGV = GetClassGlobal(ID, /*metaclass*/ false, NotForDefinition); (void)ClassGV; - assert(ClassGV->hasExternalWeakLinkage()); + assert(!isa<llvm::GlobalVariable>(ClassGV) || + cast<llvm::GlobalVariable>(ClassGV)->hasExternalWeakLinkage()); } return EmitClassRef(CGF, ID); @@ -7258,7 +7474,7 @@ CGObjCNonFragileABIMac::GetEHType(QualType T) { const ObjCInterfaceType *IT = PT->getInterfaceType(); assert(IT && "Invalid @catch type."); - return GetInterfaceEHType(IT->getDecl(), false); + return GetInterfaceEHType(IT->getDecl(), NotForDefinition); } void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF, @@ -7290,13 +7506,13 @@ void CGObjCNonFragileABIMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF, llvm::Constant * CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID, - bool ForDefinition) { + ForDefinition_t IsForDefinition) { llvm::GlobalVariable * &Entry = EHTypeReferences[ID->getIdentifier()]; StringRef ClassName = ID->getObjCRuntimeNameAsString(); // If we don't need a definition, return the entry if found or check // if we use an external reference. - if (!ForDefinition) { + if (!IsForDefinition) { if (Entry) return Entry; @@ -7332,23 +7548,24 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID, } llvm::Value *VTableIdx = llvm::ConstantInt::get(CGM.Int32Ty, 2); - llvm::Constant *Values[] = { - llvm::ConstantExpr::getGetElementPtr(VTableGV->getValueType(), VTableGV, - VTableIdx), - GetClassName(ID->getObjCRuntimeNameAsString()), - GetClassGlobal((getClassSymbolPrefix() + ClassName).str()), - }; - llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.EHTypeTy, Values); - - llvm::GlobalValue::LinkageTypes L = ForDefinition + ConstantInitBuilder builder(CGM); + auto values = builder.beginStruct(ObjCTypes.EHTypeTy); + values.add(llvm::ConstantExpr::getGetElementPtr(VTableGV->getValueType(), + VTableGV, VTableIdx)); + values.add(GetClassName(ClassName)); + values.add(GetClassGlobal(ID, /*metaclass*/ false, NotForDefinition)); + + llvm::GlobalValue::LinkageTypes L = IsForDefinition ? llvm::GlobalValue::ExternalLinkage : llvm::GlobalValue::WeakAnyLinkage; if (Entry) { - Entry->setInitializer(Init); + values.finishAndSetAsInitializer(Entry); + Entry->setAlignment(CGM.getPointerAlign().getQuantity()); } else { - Entry = - new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.EHTypeTy, false, L, - Init, ("OBJC_EHTYPE_$_" + ClassName).str()); + Entry = values.finishAndCreateGlobal("OBJC_EHTYPE_$_" + ClassName, + CGM.getPointerAlign(), + /*constant*/ false, + L); if (CGM.getTriple().isOSBinFormatCOFF()) if (hasObjCExceptionAttribute(CGM.getContext(), ID)) if (ID->hasAttr<DLLExportAttr>()) @@ -7360,11 +7577,9 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID, if (ID->getVisibility() == HiddenVisibility) Entry->setVisibility(llvm::GlobalValue::HiddenVisibility); - const auto &DL = CGM.getDataLayout(); - Entry->setAlignment(DL.getABITypeAlignment(ObjCTypes.EHTypeTy)); - - if (ForDefinition) - Entry->setSection("__DATA,__objc_const"); + if (IsForDefinition) + if (CGM.getTriple().isOSBinFormatMachO()) + Entry->setSection("__DATA,__objc_const"); return Entry; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp index 0caf6d9..3da7ed2 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -90,7 +90,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, unsigned CVRQualifiers, llvm::Value *Offset) { // Compute (type*) ( (char *) BaseValue + Offset) - QualType IvarTy = Ivar->getType(); + QualType IvarTy = Ivar->getType().withCVRQualifiers(CVRQualifiers); llvm::Type *LTy = CGF.CGM.getTypes().ConvertTypeForMem(IvarTy); llvm::Value *V = CGF.Builder.CreateBitCast(BaseValue, CGF.Int8PtrTy); V = CGF.Builder.CreateInBoundsGEP(V, Offset, "add.ptr"); @@ -98,7 +98,6 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, if (!Ivar->isBitField()) { V = CGF.Builder.CreateBitCast(V, llvm::PointerType::getUnqual(LTy)); LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy); - LV.getQuals().addCVRQualifiers(CVRQualifiers); return LV; } @@ -139,9 +138,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, Addr = CGF.Builder.CreateElementBitCast(Addr, llvm::Type::getIntNTy(CGF.getLLVMContext(), Info->StorageSize)); - return LValue::MakeBitfield(Addr, *Info, - IvarTy.withCVRQualifiers(CVRQualifiers), - AlignmentSource::Decl); + return LValue::MakeBitfield(Addr, *Info, IvarTy, AlignmentSource::Decl); } namespace { @@ -153,18 +150,16 @@ namespace { }; struct CallObjCEndCatch final : EHScopeStack::Cleanup { - CallObjCEndCatch(bool MightThrow, llvm::Value *Fn) : - MightThrow(MightThrow), Fn(Fn) {} + CallObjCEndCatch(bool MightThrow, llvm::Value *Fn) + : MightThrow(MightThrow), Fn(Fn) {} bool MightThrow; llvm::Value *Fn; void Emit(CodeGenFunction &CGF, Flags flags) override { - if (!MightThrow) { - CGF.Builder.CreateCall(Fn)->setDoesNotThrow(); - return; - } - - CGF.EmitRuntimeCallOrInvoke(Fn); + if (MightThrow) + CGF.EmitRuntimeCallOrInvoke(Fn); + else + CGF.EmitNounwindRuntimeCall(Fn); } }; } @@ -233,10 +228,8 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, // Enter the catch. llvm::Value *Exn = RawExn; - if (beginCatchFn) { - Exn = CGF.Builder.CreateCall(beginCatchFn, RawExn, "exn.adjusted"); - cast<llvm::CallInst>(Exn)->setDoesNotThrow(); - } + if (beginCatchFn) + Exn = CGF.EmitNounwindRuntimeCall(beginCatchFn, RawExn, "exn.adjusted"); CodeGenFunction::LexicalScope cleanups(CGF, Handler.Body->getSourceRange()); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h index 6c33059..a14b44a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h @@ -280,9 +280,6 @@ public: virtual llvm::Constant *BuildByrefLayout(CodeGen::CodeGenModule &CGM, QualType T) = 0; - virtual llvm::GlobalVariable *GetClassGlobal(StringRef Name, - bool Weak = false) = 0; - struct MessageSendInfo { const CGFunctionInfo &CallInfo; llvm::PointerType *MessengerType; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp index 38aebea..9062936 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp @@ -15,6 +15,7 @@ #include "CGOpenCLRuntime.h" #include "CodeGenFunction.h" +#include "TargetInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalValue.h" #include <assert.h> @@ -34,10 +35,10 @@ llvm::Type *CGOpenCLRuntime::convertOpenCLSpecificType(const Type *T) { "Not an OpenCL specific type!"); llvm::LLVMContext& Ctx = CGM.getLLVMContext(); - uint32_t ImgAddrSpc = - CGM.getContext().getTargetAddressSpace(LangAS::opencl_global); + uint32_t ImgAddrSpc = CGM.getContext().getTargetAddressSpace( + CGM.getTarget().getOpenCLImageAddrSpace()); switch (cast<BuiltinType>(T)->getKind()) { - default: + default: llvm_unreachable("Unexpected opencl builtin type!"); return nullptr; #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \ @@ -47,7 +48,7 @@ llvm::Type *CGOpenCLRuntime::convertOpenCLSpecificType(const Type *T) { ImgAddrSpc); #include "clang/Basic/OpenCLImageTypes.def" case BuiltinType::OCLSampler: - return llvm::IntegerType::get(Ctx, 32); + return getSamplerType(); case BuiltinType::OCLEvent: return llvm::PointerType::get(llvm::StructType::create( Ctx, "opencl.event_t"), 0); @@ -76,3 +77,32 @@ llvm::Type *CGOpenCLRuntime::getPipeType() { return PipeTy; } + +llvm::PointerType *CGOpenCLRuntime::getSamplerType() { + if (!SamplerTy) + SamplerTy = llvm::PointerType::get(llvm::StructType::create( + CGM.getLLVMContext(), "opencl.sampler_t"), + CGM.getContext().getTargetAddressSpace( + LangAS::opencl_constant)); + return SamplerTy; +} + +llvm::Value *CGOpenCLRuntime::getPipeElemSize(const Expr *PipeArg) { + const PipeType *PipeTy = PipeArg->getType()->getAs<PipeType>(); + // The type of the last (implicit) argument to be passed. + llvm::Type *Int32Ty = llvm::IntegerType::getInt32Ty(CGM.getLLVMContext()); + unsigned TypeSize = CGM.getContext() + .getTypeSizeInChars(PipeTy->getElementType()) + .getQuantity(); + return llvm::ConstantInt::get(Int32Ty, TypeSize, false); +} + +llvm::Value *CGOpenCLRuntime::getPipeElemAlign(const Expr *PipeArg) { + const PipeType *PipeTy = PipeArg->getType()->getAs<PipeType>(); + // The type of the last (implicit) argument to be passed. + llvm::Type *Int32Ty = llvm::IntegerType::getInt32Ty(CGM.getLLVMContext()); + unsigned TypeSize = CGM.getContext() + .getTypeAlignInChars(PipeTy->getElementType()) + .getQuantity(); + return llvm::ConstantInt::get(Int32Ty, TypeSize, false); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h index f1a7a31..ee3cb3d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.h @@ -33,9 +33,11 @@ class CGOpenCLRuntime { protected: CodeGenModule &CGM; llvm::Type *PipeTy; + llvm::PointerType *SamplerTy; public: - CGOpenCLRuntime(CodeGenModule &CGM) : CGM(CGM), PipeTy(nullptr) {} + CGOpenCLRuntime(CodeGenModule &CGM) : CGM(CGM), PipeTy(nullptr), + SamplerTy(nullptr) {} virtual ~CGOpenCLRuntime(); /// Emit the IR required for a work-group-local variable declaration, and add @@ -47,6 +49,16 @@ public: virtual llvm::Type *convertOpenCLSpecificType(const Type *T); virtual llvm::Type *getPipeType(); + + llvm::PointerType *getSamplerType(); + + // \brief Returnes a value which indicates the size in bytes of the pipe + // element. + virtual llvm::Value *getPipeElemSize(const Expr *PipeArg); + + // \brief Returnes a value which indicates the alignment in bytes of the pipe + // element. + virtual llvm::Value *getPipeElemAlign(const Expr *PipeArg); }; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 6a0edbe..4025217 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -15,10 +15,11 @@ #include "CGCleanup.h" #include "CGOpenMPRuntime.h" #include "CodeGenFunction.h" +#include "ConstantBuilder.h" #include "clang/AST/Decl.h" #include "clang/AST/StmtOpenMP.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalValue.h" @@ -98,10 +99,11 @@ class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo { public: CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, const RegionCodeGenTy &CodeGen, - OpenMPDirectiveKind Kind, bool HasCancel) + OpenMPDirectiveKind Kind, bool HasCancel, + StringRef HelperName) : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind, HasCancel), - ThreadIDVar(ThreadIDVar) { + ThreadIDVar(ThreadIDVar), HelperName(HelperName) { assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); } @@ -110,7 +112,7 @@ public: const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } /// \brief Get the name of the capture helper. - StringRef getHelperName() const override { return ".omp_outlined."; } + StringRef getHelperName() const override { return HelperName; } static bool classof(const CGCapturedStmtInfo *Info) { return CGOpenMPRegionInfo::classof(Info) && @@ -122,6 +124,7 @@ private: /// \brief A variable or parameter storing global thread id for OpenMP /// constructs. const VarDecl *ThreadIDVar; + StringRef HelperName; }; /// \brief API for captured statement code generation in OpenMP constructs. @@ -489,7 +492,7 @@ enum OpenMPSchedType { OMP_sch_runtime = 37, OMP_sch_auto = 38, /// static with chunk adjustment (e.g., simd) - OMP_sch_static_balanced_chunked = 45, + OMP_sch_static_balanced_chunked = 45, /// \brief Lower bound for 'ordered' versions. OMP_ord_lower = 64, OMP_ord_static_chunked = 65, @@ -756,6 +759,7 @@ emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, FnTy, llvm::GlobalValue::InternalLinkage, IsCombiner ? ".omp_combiner." : ".omp_initializer.", &CGM.getModule()); CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, FnInfo); + Fn->removeFnAttr(llvm::Attribute::NoInline); Fn->addFnAttr(llvm::Attribute::AlwaysInline); CodeGenFunction CGF(CGM); // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions. @@ -853,7 +857,7 @@ llvm::Value *CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction( else if (auto *OPFD = dyn_cast<OMPParallelForDirective>(&D)) HasCancel = OPFD->hasCancel(); CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind, - HasCancel); + HasCancel, getOutlinedHelperName()); CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); return CGF.GenerateOpenMPCapturedStmtFunction(*CS); } @@ -906,18 +910,19 @@ Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) { DefaultOpenMPPSource = llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy); } - auto DefaultOpenMPLocation = new llvm::GlobalVariable( - CGM.getModule(), IdentTy, /*isConstant*/ true, - llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr); + + ConstantInitBuilder builder(CGM); + auto fields = builder.beginStruct(IdentTy); + fields.addInt(CGM.Int32Ty, 0); + fields.addInt(CGM.Int32Ty, Flags); + fields.addInt(CGM.Int32Ty, 0); + fields.addInt(CGM.Int32Ty, 0); + fields.add(DefaultOpenMPPSource); + auto DefaultOpenMPLocation = + fields.finishAndCreateGlobal("", Align, /*isConstant*/ true, + llvm::GlobalValue::PrivateLinkage); DefaultOpenMPLocation->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - DefaultOpenMPLocation->setAlignment(Align.getQuantity()); - - llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true); - llvm::Constant *Values[] = {Zero, - llvm::ConstantInt::get(CGM.Int32Ty, Flags), - Zero, Zero, DefaultOpenMPPSource}; - llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values); - DefaultOpenMPLocation->setInitializer(Init); + OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation; } return Address(Entry, Align); @@ -1889,9 +1894,9 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( /// } else { /// ElseGen(); /// } -static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, - const RegionCodeGenTy &ThenGen, - const RegionCodeGenTy &ElseGen) { +void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, + const RegionCodeGenTy &ThenGen, + const RegionCodeGenTy &ElseGen) { CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); // If the condition constant folds and can be elided, try to avoid emitting @@ -2698,14 +2703,16 @@ void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: "only required for the device " "code generation."); OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = - OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr); + OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr, + /*Flags=*/0); ++OffloadingEntriesNum; } void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, - llvm::Constant *Addr, llvm::Constant *ID) { + llvm::Constant *Addr, llvm::Constant *ID, + int32_t Flags) { // If we are emitting code for a target, the entry is already initialized, // only has to be registered. if (CGM.getLangOpts().OpenMPIsDevice) { @@ -2716,9 +2723,10 @@ void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: assert(Entry.isValid() && "Entry not initialized!"); Entry.setAddress(Addr); Entry.setID(ID); + Entry.setFlags(Flags); return; } else { - OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID); + OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID, Flags); OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry; } } @@ -2767,7 +2775,6 @@ createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name, Args.push_back(&DummyPtr); CodeGenFunction CGF(CGM); - GlobalDecl(); auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); auto FTy = CGM.getTypes().GetFunctionType(FI); auto *Fn = @@ -2810,9 +2817,10 @@ CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() { ".omp_offloading.entries_end"); // Create all device images - llvm::SmallVector<llvm::Constant *, 4> DeviceImagesEntires; auto *DeviceImageTy = cast<llvm::StructType>( CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy())); + ConstantInitBuilder DeviceImagesBuilder(CGM); + auto DeviceImagesEntries = DeviceImagesBuilder.beginArray(DeviceImageTy); for (unsigned i = 0; i < Devices.size(); ++i) { StringRef T = Devices[i].getTriple(); @@ -2824,22 +2832,19 @@ CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() { M, CGM.Int8Ty, /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr, Twine(".omp_offloading.img_end.") + Twine(T)); - llvm::Constant *Dev = - llvm::ConstantStruct::get(DeviceImageTy, ImgBegin, ImgEnd, - HostEntriesBegin, HostEntriesEnd, nullptr); - DeviceImagesEntires.push_back(Dev); + auto Dev = DeviceImagesEntries.beginStruct(DeviceImageTy); + Dev.add(ImgBegin); + Dev.add(ImgEnd); + Dev.add(HostEntriesBegin); + Dev.add(HostEntriesEnd); + Dev.finishAndAddTo(DeviceImagesEntries); } // Create device images global array. - llvm::ArrayType *DeviceImagesInitTy = - llvm::ArrayType::get(DeviceImageTy, DeviceImagesEntires.size()); - llvm::Constant *DeviceImagesInit = - llvm::ConstantArray::get(DeviceImagesInitTy, DeviceImagesEntires); - - llvm::GlobalVariable *DeviceImages = new llvm::GlobalVariable( - M, DeviceImagesInitTy, /*isConstant=*/true, - llvm::GlobalValue::InternalLinkage, DeviceImagesInit, - ".omp_offloading.device_images"); + llvm::GlobalVariable *DeviceImages = + DeviceImagesEntries.finishAndCreateGlobal(".omp_offloading.device_images", + CGM.getPointerAlign(), + /*isConstant=*/true); DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); // This is a Zero array to be used in the creation of the constant expressions @@ -2849,16 +2854,18 @@ CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() { // Create the target region descriptor. auto *BinaryDescriptorTy = cast<llvm::StructType>( CGM.getTypes().ConvertTypeForMem(getTgtBinaryDescriptorQTy())); - llvm::Constant *TargetRegionsDescriptorInit = llvm::ConstantStruct::get( - BinaryDescriptorTy, llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()), - llvm::ConstantExpr::getGetElementPtr(DeviceImagesInitTy, DeviceImages, - Index), - HostEntriesBegin, HostEntriesEnd, nullptr); - - auto *Desc = new llvm::GlobalVariable( - M, BinaryDescriptorTy, /*isConstant=*/true, - llvm::GlobalValue::InternalLinkage, TargetRegionsDescriptorInit, - ".omp_offloading.descriptor"); + ConstantInitBuilder DescBuilder(CGM); + auto DescInit = DescBuilder.beginStruct(BinaryDescriptorTy); + DescInit.addInt(CGM.Int32Ty, Devices.size()); + DescInit.add(llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(), + DeviceImages, + Index)); + DescInit.add(HostEntriesBegin); + DescInit.add(HostEntriesEnd); + + auto *Desc = DescInit.finishAndCreateGlobal(".omp_offloading.descriptor", + CGM.getPointerAlign(), + /*isConstant=*/true); // Emit code to register or unregister the descriptor at execution // startup or closing, respectively. @@ -2886,7 +2893,8 @@ CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() { } void CGOpenMPRuntime::createOffloadEntry(llvm::Constant *ID, - llvm::Constant *Addr, uint64_t Size) { + llvm::Constant *Addr, uint64_t Size, + int32_t Flags) { StringRef Name = Addr->getName(); auto *TgtOffloadEntryType = cast<llvm::StructType>( CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy())); @@ -2906,25 +2914,32 @@ void CGOpenMPRuntime::createOffloadEntry(llvm::Constant *ID, Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy); + // We can't have any padding between symbols, so we need to have 1-byte + // alignment. + auto Align = CharUnits::fromQuantity(1); + // Create the entry struct. - llvm::Constant *EntryInit = llvm::ConstantStruct::get( - TgtOffloadEntryType, AddrPtr, StrPtr, - llvm::ConstantInt::get(CGM.SizeTy, Size), nullptr); - llvm::GlobalVariable *Entry = new llvm::GlobalVariable( - M, TgtOffloadEntryType, true, llvm::GlobalValue::ExternalLinkage, - EntryInit, ".omp_offloading.entry"); + ConstantInitBuilder EntryBuilder(CGM); + auto EntryInit = EntryBuilder.beginStruct(TgtOffloadEntryType); + EntryInit.add(AddrPtr); + EntryInit.add(StrPtr); + EntryInit.addInt(CGM.SizeTy, Size); + EntryInit.addInt(CGM.Int32Ty, Flags); + EntryInit.addInt(CGM.Int32Ty, 0); + llvm::GlobalVariable *Entry = + EntryInit.finishAndCreateGlobal(".omp_offloading.entry", + Align, + /*constant*/ true, + llvm::GlobalValue::ExternalLinkage); // The entry has to be created in the section the linker expects it to be. Entry->setSection(".omp_offloading.entries"); - // We can't have any padding between symbols, so we need to have 1-byte - // alignment. - Entry->setAlignment(1); } void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() { // Emit the offloading entries and metadata so that the device codegen side - // can - // easily figure out what to emit. The produced metadata looks like this: + // can easily figure out what to emit. The produced metadata looks like + // this: // // !omp_offload.info = !{!1, ...} // @@ -3012,7 +3027,8 @@ void CGOpenMPRuntime::loadOffloadInfoMetadata() { return; llvm::LLVMContext C; - auto ME = llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C); + auto ME = expectedToErrorOrAndEmitErrors( + C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C)); if (ME.getError()) return; @@ -3082,6 +3098,8 @@ QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() { // // (function or global) // char *name; // Name of the function or global. // size_t size; // Size of the entry info (0 if it a function). + // int32_t flags; // Flags associated with the entry, e.g. 'link'. + // int32_t reserved; // Reserved, to use by the runtime library. // }; if (TgtOffloadEntryQTy.isNull()) { ASTContext &C = CGM.getContext(); @@ -3090,6 +3108,10 @@ QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() { addFieldToRecordDecl(C, RD, C.VoidPtrTy); addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy)); addFieldToRecordDecl(C, RD, C.getSizeType()); + addFieldToRecordDecl( + C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true)); + addFieldToRecordDecl( + C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true)); RD->completeDefinition(); TgtOffloadEntryQTy = C.getRecordType(RD); } @@ -3465,6 +3487,7 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, ".omp_task_privates_map.", &CGM.getModule()); CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskPrivatesMap, TaskPrivatesMapFnInfo); + TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline); TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline); CodeGenFunction CGF(CGM); CGF.disableDebugInfo(); @@ -4436,9 +4459,8 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, auto *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); auto *ThreadId = getThreadID(CGF, Loc); auto *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); - auto *RL = - CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList.getPointer(), - CGF.VoidPtrTy); + auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + ReductionList.getPointer(), CGF.VoidPtrTy); llvm::Value *Args[] = { IdentTLoc, // ident_t *<loc> ThreadId, // i32 <gtid> @@ -4675,7 +4697,9 @@ void CGOpenMPRuntime::emitCancellationPointCall( // global_tid, kmp_int32 cncl_kind); if (auto *OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - if (OMPRegionInfo->hasCancel()) { + // For 'cancellation point taskgroup', the task region info may not have a + // cancel. This may instead happen in another adjacent task. + if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) { llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; @@ -4683,7 +4707,6 @@ void CGOpenMPRuntime::emitCancellationPointCall( auto *Result = CGF.EmitRuntimeCall( createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args); // if (__kmpc_cancellationpoint()) { - // __kmpc_cancel_barrier(); // exit from construct; // } auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); @@ -4691,8 +4714,6 @@ void CGOpenMPRuntime::emitCancellationPointCall( auto *Cmp = CGF.Builder.CreateIsNotNull(Result); CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); CGF.EmitBlock(ExitBB); - // __kmpc_cancel_barrier(); - emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); // exit from construct; auto CancelDest = CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); @@ -4721,7 +4742,6 @@ void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, auto *Result = CGF.EmitRuntimeCall( RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args); // if (__kmpc_cancel()) { - // __kmpc_cancel_barrier(); // exit from construct; // } auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); @@ -4729,8 +4749,6 @@ void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, auto *Cmp = CGF.Builder.CreateIsNotNull(Result); CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); CGF.EmitBlock(ExitBB); - // __kmpc_cancel_barrier(); - RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); // exit from construct; auto CancelDest = CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); @@ -4844,7 +4862,8 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper( // Register the information for the entry associated with this target region. OffloadEntriesInfoManager.registerTargetRegionEntryInfo( - DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID); + DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID, + /*Flags=*/0); } /// discard all CompoundStmts intervening between two constructs @@ -4981,6 +5000,9 @@ public: /// map/privatization results in multiple arguments passed to the runtime /// library. OMP_MAP_FIRST_REF = 0x20, + /// \brief Signal that the runtime library has to return the device pointer + /// in the current position for the data being mapped. + OMP_MAP_RETURN_PTR = 0x40, /// \brief This flag signals that the reference being passed is a pointer to /// private data. OMP_MAP_PRIVATE_PTR = 0x80, @@ -4988,12 +5010,30 @@ public: OMP_MAP_PRIVATE_VAL = 0x100, }; + /// Class that associates information with a base pointer to be passed to the + /// runtime library. + class BasePointerInfo { + /// The base pointer. + llvm::Value *Ptr = nullptr; + /// The base declaration that refers to this device pointer, or null if + /// there is none. + const ValueDecl *DevPtrDecl = nullptr; + + public: + BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr) + : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {} + llvm::Value *operator*() const { return Ptr; } + const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; } + void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; } + }; + + typedef SmallVector<BasePointerInfo, 16> MapBaseValuesArrayTy; typedef SmallVector<llvm::Value *, 16> MapValuesArrayTy; typedef SmallVector<unsigned, 16> MapFlagsArrayTy; private: /// \brief Directive from where the map clauses were extracted. - const OMPExecutableDirective &Directive; + const OMPExecutableDirective &CurDir; /// \brief Function the directive is being generated for. CodeGenFunction &CGF; @@ -5001,6 +5041,13 @@ private: /// \brief Set of all first private variables in the current directive. llvm::SmallPtrSet<const VarDecl *, 8> FirstPrivateDecls; + /// Map between device pointer declarations and their expression components. + /// The key value for declarations in 'this' is null. + llvm::DenseMap< + const ValueDecl *, + SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>> + DevPointersMap; + llvm::Value *getExprTypeSize(const Expr *E) const { auto ExprTy = E->getType().getCanonicalType(); @@ -5129,7 +5176,7 @@ private: void generateInfoForComponentList( OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapTypeModifier, OMPClauseMappableExprCommon::MappableExprComponentListRef Components, - MapValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, + MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types, bool IsFirstComponentList) const { @@ -5271,15 +5318,13 @@ private: // If the variable is a pointer and is being dereferenced (i.e. is not // the last component), the base has to be the pointer itself, not its - // reference. - if (I->getAssociatedDeclaration()->getType()->isAnyPointerType() && - std::next(I) != CE) { - auto PtrAddr = CGF.MakeNaturalAlignAddrLValue( - BP, I->getAssociatedDeclaration()->getType()); + // reference. References are ignored for mapping purposes. + QualType Ty = + I->getAssociatedDeclaration()->getType().getNonReferenceType(); + if (Ty->isAnyPointerType() && std::next(I) != CE) { + auto PtrAddr = CGF.MakeNaturalAlignAddrLValue(BP, Ty); BP = CGF.EmitLoadOfPointerLValue(PtrAddr.getAddress(), - I->getAssociatedDeclaration() - ->getType() - ->getAs<PointerType>()) + Ty->castAs<PointerType>()) .getPointer(); // We do not need to generate individual map information for the @@ -5322,14 +5367,34 @@ private: isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && "Unexpected expression"); - // Save the base we are currently using. - BasePointers.push_back(BP); - auto *LB = CGF.EmitLValue(I->getAssociatedExpression()).getPointer(); auto *Size = getExprTypeSize(I->getAssociatedExpression()); + // If we have a member expression and the current component is a + // reference, we have to map the reference too. Whenever we have a + // reference, the section that reference refers to is going to be a + // load instruction from the storage assigned to the reference. + if (isa<MemberExpr>(I->getAssociatedExpression()) && + I->getAssociatedDeclaration()->getType()->isReferenceType()) { + auto *LI = cast<llvm::LoadInst>(LB); + auto *RefAddr = LI->getPointerOperand(); + + BasePointers.push_back(BP); + Pointers.push_back(RefAddr); + Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy)); + Types.push_back(getMapTypeBits( + /*MapType*/ OMPC_MAP_alloc, /*MapTypeModifier=*/OMPC_MAP_unknown, + !IsExpressionFirstInfo, IsCaptureFirstInfo)); + IsExpressionFirstInfo = false; + IsCaptureFirstInfo = false; + // The reference will be the next base address. + BP = RefAddr; + } + + BasePointers.push_back(BP); Pointers.push_back(LB); Sizes.push_back(Size); + // We need to add a pointer flag for each map that comes from the // same expression except for the first one. We also need to signal // this map is the first one that relates with the current capture @@ -5373,17 +5438,23 @@ private: public: MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) - : Directive(Dir), CGF(CGF) { + : CurDir(Dir), CGF(CGF) { // Extract firstprivate clause information. for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>()) for (const auto *D : C->varlists()) FirstPrivateDecls.insert( cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl()); + // Extract device pointer clause information. + for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>()) + for (auto L : C->component_lists()) + DevPointersMap[L.first].push_back(L.second); } /// \brief Generate all the base pointers, section pointers, sizes and map - /// types for the extracted mappable expressions. - void generateAllInfo(MapValuesArrayTy &BasePointers, + /// types for the extracted mappable expressions. Also, for each item that + /// relates with a device pointer, a pair of the relevant declaration and + /// index where it occurs is appended to the device pointers info array. + void generateAllInfo(MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types) const { BasePointers.clear(); @@ -5392,9 +5463,32 @@ public: Types.clear(); struct MapInfo { + /// Kind that defines how a device pointer has to be returned. + enum ReturnPointerKind { + // Don't have to return any pointer. + RPK_None, + // Pointer is the base of the declaration. + RPK_Base, + // Pointer is a member of the base declaration - 'this' + RPK_Member, + // Pointer is a reference and a member of the base declaration - 'this' + RPK_MemberReference, + }; OMPClauseMappableExprCommon::MappableExprComponentListRef Components; OpenMPMapClauseKind MapType; OpenMPMapClauseKind MapTypeModifier; + ReturnPointerKind ReturnDevicePointer; + + MapInfo() + : MapType(OMPC_MAP_unknown), MapTypeModifier(OMPC_MAP_unknown), + ReturnDevicePointer(RPK_None) {} + MapInfo( + OMPClauseMappableExprCommon::MappableExprComponentListRef Components, + OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapTypeModifier, + ReturnPointerKind ReturnDevicePointer) + : Components(Components), MapType(MapType), + MapTypeModifier(MapTypeModifier), + ReturnDevicePointer(ReturnDevicePointer) {} }; // We have to process the component lists that relate with the same @@ -5404,24 +5498,77 @@ public: // Helper function to fill the information map for the different supported // clauses. - auto &&InfoGen = - [&Info](const ValueDecl *D, - OMPClauseMappableExprCommon::MappableExprComponentListRef L, - OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapModifier) { - const ValueDecl *VD = - D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr; - Info[VD].push_back({L, MapType, MapModifier}); - }; + auto &&InfoGen = [&Info]( + const ValueDecl *D, + OMPClauseMappableExprCommon::MappableExprComponentListRef L, + OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapModifier, + MapInfo::ReturnPointerKind ReturnDevicePointer) { + const ValueDecl *VD = + D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr; + Info[VD].push_back({L, MapType, MapModifier, ReturnDevicePointer}); + }; - for (auto *C : Directive.getClausesOfKind<OMPMapClause>()) + // FIXME: MSVC 2013 seems to require this-> to find member CurDir. + for (auto *C : this->CurDir.getClausesOfKind<OMPMapClause>()) for (auto L : C->component_lists()) - InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifier()); - for (auto *C : Directive.getClausesOfKind<OMPToClause>()) + InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifier(), + MapInfo::RPK_None); + for (auto *C : this->CurDir.getClausesOfKind<OMPToClause>()) for (auto L : C->component_lists()) - InfoGen(L.first, L.second, OMPC_MAP_to, OMPC_MAP_unknown); - for (auto *C : Directive.getClausesOfKind<OMPFromClause>()) + InfoGen(L.first, L.second, OMPC_MAP_to, OMPC_MAP_unknown, + MapInfo::RPK_None); + for (auto *C : this->CurDir.getClausesOfKind<OMPFromClause>()) for (auto L : C->component_lists()) - InfoGen(L.first, L.second, OMPC_MAP_from, OMPC_MAP_unknown); + InfoGen(L.first, L.second, OMPC_MAP_from, OMPC_MAP_unknown, + MapInfo::RPK_None); + + // Look at the use_device_ptr clause information and mark the existing map + // entries as such. If there is no map information for an entry in the + // use_device_ptr list, we create one with map type 'alloc' and zero size + // section. It is the user fault if that was not mapped before. + // FIXME: MSVC 2013 seems to require this-> to find member CurDir. + for (auto *C : this->CurDir.getClausesOfKind<OMPUseDevicePtrClause>()) + for (auto L : C->component_lists()) { + assert(!L.second.empty() && "Not expecting empty list of components!"); + const ValueDecl *VD = L.second.back().getAssociatedDeclaration(); + VD = cast<ValueDecl>(VD->getCanonicalDecl()); + auto *IE = L.second.back().getAssociatedExpression(); + // If the first component is a member expression, we have to look into + // 'this', which maps to null in the map of map information. Otherwise + // look directly for the information. + auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD); + + // We potentially have map information for this declaration already. + // Look for the first set of components that refer to it. + if (It != Info.end()) { + auto CI = std::find_if( + It->second.begin(), It->second.end(), [VD](const MapInfo &MI) { + return MI.Components.back().getAssociatedDeclaration() == VD; + }); + // If we found a map entry, signal that the pointer has to be returned + // and move on to the next declaration. + if (CI != It->second.end()) { + CI->ReturnDevicePointer = isa<MemberExpr>(IE) + ? (VD->getType()->isReferenceType() + ? MapInfo::RPK_MemberReference + : MapInfo::RPK_Member) + : MapInfo::RPK_Base; + continue; + } + } + + // We didn't find any match in our map information - generate a zero + // size array section. + // FIXME: MSVC 2013 seems to require this-> to find member CGF. + llvm::Value *Ptr = + this->CGF + .EmitLoadOfLValue(this->CGF.EmitLValue(IE), SourceLocation()) + .getScalarVal(); + BasePointers.push_back({Ptr, VD}); + Pointers.push_back(Ptr); + Sizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy)); + Types.push_back(OMP_MAP_RETURN_PTR | OMP_MAP_FIRST_REF); + } for (auto &M : Info) { // We need to know when we generate information for the first component @@ -5430,9 +5577,36 @@ public: for (MapInfo &L : M.second) { assert(!L.Components.empty() && "Not expecting declaration with no component lists."); - generateInfoForComponentList(L.MapType, L.MapTypeModifier, L.Components, - BasePointers, Pointers, Sizes, Types, - IsFirstComponentList); + + // Remember the current base pointer index. + unsigned CurrentBasePointersIdx = BasePointers.size(); + // FIXME: MSVC 2013 seems to require this-> to find the member method. + this->generateInfoForComponentList(L.MapType, L.MapTypeModifier, + L.Components, BasePointers, Pointers, + Sizes, Types, IsFirstComponentList); + + // If this entry relates with a device pointer, set the relevant + // declaration and add the 'return pointer' flag. + if (IsFirstComponentList && + L.ReturnDevicePointer != MapInfo::RPK_None) { + // If the pointer is not the base of the map, we need to skip the + // base. If it is a reference in a member field, we also need to skip + // the map of the reference. + if (L.ReturnDevicePointer != MapInfo::RPK_Base) { + ++CurrentBasePointersIdx; + if (L.ReturnDevicePointer == MapInfo::RPK_MemberReference) + ++CurrentBasePointersIdx; + } + assert(BasePointers.size() > CurrentBasePointersIdx && + "Unexpected number of mapped base pointers."); + + auto *RelevantVD = L.Components.back().getAssociatedDeclaration(); + assert(RelevantVD && + "No relevant declaration related with device pointer??"); + + BasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD); + Types[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PTR; + } IsFirstComponentList = false; } } @@ -5441,7 +5615,8 @@ public: /// \brief Generate the base pointers, section pointers, sizes and map types /// associated to a given capture. void generateInfoForCapture(const CapturedStmt::Capture *Cap, - MapValuesArrayTy &BasePointers, + llvm::Value *Arg, + MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types) const { @@ -5453,15 +5628,40 @@ public: Sizes.clear(); Types.clear(); + // We need to know when we generating information for the first component + // associated with a capture, because the mapping flags depend on it. + bool IsFirstComponentList = true; + const ValueDecl *VD = Cap->capturesThis() ? nullptr : cast<ValueDecl>(Cap->getCapturedVar()->getCanonicalDecl()); - // We need to know when we generating information for the first component - // associated with a capture, because the mapping flags depend on it. - bool IsFirstComponentList = true; - for (auto *C : Directive.getClausesOfKind<OMPMapClause>()) + // If this declaration appears in a is_device_ptr clause we just have to + // pass the pointer by value. If it is a reference to a declaration, we just + // pass its value, otherwise, if it is a member expression, we need to map + // 'to' the field. + if (!VD) { + auto It = DevPointersMap.find(VD); + if (It != DevPointersMap.end()) { + for (auto L : It->second) { + generateInfoForComponentList( + /*MapType=*/OMPC_MAP_to, /*MapTypeModifier=*/OMPC_MAP_unknown, L, + BasePointers, Pointers, Sizes, Types, IsFirstComponentList); + IsFirstComponentList = false; + } + return; + } + } else if (DevPointersMap.count(VD)) { + BasePointers.push_back({Arg, VD}); + Pointers.push_back(Arg); + Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy)); + Types.push_back(OMP_MAP_PRIVATE_VAL | OMP_MAP_FIRST_REF); + return; + } + + // FIXME: MSVC 2013 seems to require this-> to find member CurDir. + for (auto *C : this->CurDir.getClausesOfKind<OMPMapClause>()) for (auto L : C->decl_component_lists(VD)) { assert(L.first == VD && "We got information for the wrong declaration??"); @@ -5478,12 +5678,12 @@ public: /// \brief Generate the default map information for a given capture \a CI, /// record field declaration \a RI and captured value \a CV. - void generateDefaultMapInfo( - const CapturedStmt::Capture &CI, const FieldDecl &RI, llvm::Value *CV, - MappableExprsHandler::MapValuesArrayTy &CurBasePointers, - MappableExprsHandler::MapValuesArrayTy &CurPointers, - MappableExprsHandler::MapValuesArrayTy &CurSizes, - MappableExprsHandler::MapFlagsArrayTy &CurMapTypes) { + void generateDefaultMapInfo(const CapturedStmt::Capture &CI, + const FieldDecl &RI, llvm::Value *CV, + MapBaseValuesArrayTy &CurBasePointers, + MapValuesArrayTy &CurPointers, + MapValuesArrayTy &CurSizes, + MapFlagsArrayTy &CurMapTypes) { // Do the default mapping. if (CI.capturesThis()) { @@ -5492,15 +5692,14 @@ public: const PointerType *PtrTy = cast<PointerType>(RI.getType().getTypePtr()); CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType())); // Default map type. - CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_TO | - MappableExprsHandler::OMP_MAP_FROM); + CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM); } else if (CI.capturesVariableByCopy()) { CurBasePointers.push_back(CV); CurPointers.push_back(CV); if (!RI.getType()->isAnyPointerType()) { // We have to signal to the runtime captures passed by value that are // not pointers. - CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_PRIVATE_VAL); + CurMapTypes.push_back(OMP_MAP_PRIVATE_VAL); CurSizes.push_back(CGF.getTypeSize(RI.getType())); } else { // Pointers are implicitly mapped with a zero size and no flags @@ -5521,9 +5720,8 @@ public: // default the value doesn't have to be retrieved. For an aggregate // type, the default is 'tofrom'. CurMapTypes.push_back(ElementType->isAggregateType() - ? (MappableExprsHandler::OMP_MAP_TO | - MappableExprsHandler::OMP_MAP_FROM) - : MappableExprsHandler::OMP_MAP_TO); + ? (OMP_MAP_TO | OMP_MAP_FROM) + : OMP_MAP_TO); // If we have a capture by reference we may need to add the private // pointer flag if the base declaration shows in some first-private @@ -5533,7 +5731,7 @@ public: } // Every default map produces a single argument, so, it is always the // first one. - CurMapTypes.back() |= MappableExprsHandler::OMP_MAP_FIRST_REF; + CurMapTypes.back() |= OMP_MAP_FIRST_REF; } }; @@ -5548,19 +5746,20 @@ enum OpenMPOffloadingReservedDeviceIDs { /// offloading runtime library. If there is no map or capture information, /// return nullptr by reference. static void -emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, - llvm::Value *&PointersArray, llvm::Value *&SizesArray, - llvm::Value *&MapTypesArray, - MappableExprsHandler::MapValuesArrayTy &BasePointers, +emitOffloadingArrays(CodeGenFunction &CGF, + MappableExprsHandler::MapBaseValuesArrayTy &BasePointers, MappableExprsHandler::MapValuesArrayTy &Pointers, MappableExprsHandler::MapValuesArrayTy &Sizes, - MappableExprsHandler::MapFlagsArrayTy &MapTypes) { + MappableExprsHandler::MapFlagsArrayTy &MapTypes, + CGOpenMPRuntime::TargetDataInfo &Info) { auto &CGM = CGF.CGM; auto &Ctx = CGF.getContext(); - BasePointersArray = PointersArray = SizesArray = MapTypesArray = nullptr; + // Reset the array information. + Info.clearArrayInfo(); + Info.NumberOfPtrs = BasePointers.size(); - if (unsigned PointerNumVal = BasePointers.size()) { + if (Info.NumberOfPtrs) { // Detect if we have any capture size requiring runtime evaluation of the // size so that a constant array could be eventually used. bool hasRuntimeEvaluationCaptureSize = false; @@ -5570,14 +5769,14 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, break; } - llvm::APInt PointerNumAP(32, PointerNumVal, /*isSigned=*/true); + llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true); QualType PointerArrayType = Ctx.getConstantArrayType(Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal, /*IndexTypeQuals=*/0); - BasePointersArray = + Info.BasePointersArray = CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); - PointersArray = + Info.PointersArray = CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); // If we don't have any VLA types or other types that require runtime @@ -5587,7 +5786,7 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, QualType SizeArrayType = Ctx.getConstantArrayType( Ctx.getSizeType(), PointerNumAP, ArrayType::Normal, /*IndexTypeQuals=*/0); - SizesArray = + Info.SizesArray = CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); } else { // We expect all the sizes to be constant, so we collect them to create @@ -5603,7 +5802,7 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, SizesArrayInit, ".offload_sizes"); SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - SizesArray = SizesArrayGbl; + Info.SizesArray = SizesArrayGbl; } // The map types are always constant so we don't need to generate code to @@ -5615,10 +5814,10 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapTypesArrayInit, ".offload_maptypes"); MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - MapTypesArray = MapTypesArrayGbl; + Info.MapTypesArray = MapTypesArrayGbl; - for (unsigned i = 0; i < PointerNumVal; ++i) { - llvm::Value *BPVal = BasePointers[i]; + for (unsigned i = 0; i < Info.NumberOfPtrs; ++i) { + llvm::Value *BPVal = *BasePointers[i]; if (BPVal->getType()->isPointerTy()) BPVal = CGF.Builder.CreateBitCast(BPVal, CGM.VoidPtrTy); else { @@ -5627,11 +5826,15 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, BPVal = CGF.Builder.CreateIntToPtr(BPVal, CGM.VoidPtrTy); } llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), BasePointersArray, - 0, i); + llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), + Info.BasePointersArray, 0, i); Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy)); CGF.Builder.CreateStore(BPVal, BPAddr); + if (Info.requiresDevicePointerInfo()) + if (auto *DevVD = BasePointers[i].getDevicePtrDecl()) + Info.CaptureDeviceAddrMap.insert(std::make_pair(DevVD, BPAddr)); + llvm::Value *PVal = Pointers[i]; if (PVal->getType()->isPointerTy()) PVal = CGF.Builder.CreateBitCast(PVal, CGM.VoidPtrTy); @@ -5641,14 +5844,15 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, PVal = CGF.Builder.CreateIntToPtr(PVal, CGM.VoidPtrTy); } llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), PointersArray, 0, - i); + llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), + Info.PointersArray, 0, i); Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy)); CGF.Builder.CreateStore(PVal, PAddr); if (hasRuntimeEvaluationCaptureSize) { llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.SizeTy, PointerNumVal), SizesArray, + llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs), + Info.SizesArray, /*Idx0=*/0, /*Idx1=*/i); Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType())); @@ -5664,23 +5868,24 @@ emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, static void emitOffloadingArraysArgument( CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg, - llvm::Value *&MapTypesArrayArg, llvm::Value *BasePointersArray, - llvm::Value *PointersArray, llvm::Value *SizesArray, - llvm::Value *MapTypesArray, unsigned NumElems) { + llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) { auto &CGM = CGF.CGM; - if (NumElems) { + if (Info.NumberOfPtrs) { BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.VoidPtrTy, NumElems), BasePointersArray, + llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), + Info.BasePointersArray, /*Idx0=*/0, /*Idx1=*/0); PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.VoidPtrTy, NumElems), PointersArray, + llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), + Info.PointersArray, /*Idx0=*/0, /*Idx1=*/0); SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.SizeTy, NumElems), SizesArray, + llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs), Info.SizesArray, /*Idx0=*/0, /*Idx1=*/0); MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( - llvm::ArrayType::get(CGM.Int32Ty, NumElems), MapTypesArray, + llvm::ArrayType::get(CGM.Int32Ty, Info.NumberOfPtrs), + Info.MapTypesArray, /*Idx0=*/0, /*Idx1=*/0); } else { @@ -5707,12 +5912,12 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, // Fill up the arrays with all the captured variables. MappableExprsHandler::MapValuesArrayTy KernelArgs; - MappableExprsHandler::MapValuesArrayTy BasePointers; + MappableExprsHandler::MapBaseValuesArrayTy BasePointers; MappableExprsHandler::MapValuesArrayTy Pointers; MappableExprsHandler::MapValuesArrayTy Sizes; MappableExprsHandler::MapFlagsArrayTy MapTypes; - MappableExprsHandler::MapValuesArrayTy CurBasePointers; + MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers; MappableExprsHandler::MapValuesArrayTy CurPointers; MappableExprsHandler::MapValuesArrayTy CurSizes; MappableExprsHandler::MapFlagsArrayTy CurMapTypes; @@ -5746,7 +5951,7 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, } else { // If we have any information in the map clause, we use it, otherwise we // just do a default mapping. - MEHandler.generateInfoForCapture(CI, CurBasePointers, CurPointers, + MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers, CurSizes, CurMapTypes); if (CurBasePointers.empty()) MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers, @@ -5761,7 +5966,7 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, // The kernel args are always the first elements of the base pointers // associated with a capture. - KernelArgs.push_back(CurBasePointers.front()); + KernelArgs.push_back(*CurBasePointers.front()); // We need to append the results of this capture to what we already have. BasePointers.append(CurBasePointers.begin(), CurBasePointers.end()); Pointers.append(CurPointers.begin(), CurPointers.end()); @@ -5784,17 +5989,11 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, &D](CodeGenFunction &CGF, PrePostActionTy &) { auto &RT = CGF.CGM.getOpenMPRuntime(); // Emit the offloading arrays. - llvm::Value *BasePointersArray; - llvm::Value *PointersArray; - llvm::Value *SizesArray; - llvm::Value *MapTypesArray; - emitOffloadingArrays(CGF, BasePointersArray, PointersArray, SizesArray, - MapTypesArray, BasePointers, Pointers, Sizes, - MapTypes); - emitOffloadingArraysArgument(CGF, BasePointersArray, PointersArray, - SizesArray, MapTypesArray, BasePointersArray, - PointersArray, SizesArray, MapTypesArray, - BasePointers.size()); + TargetDataInfo Info; + emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); + emitOffloadingArraysArgument(CGF, Info.BasePointersArray, + Info.PointersArray, Info.SizesArray, + Info.MapTypesArray, Info); // On top of the arrays that were filled up, the target offloading call // takes as arguments the device id as well as the host pointer. The host @@ -5835,15 +6034,19 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, assert(ThreadLimit && "Thread limit expression should be available along " "with number of teams."); llvm::Value *OffloadingArgs[] = { - DeviceID, OutlinedFnID, PointerNum, - BasePointersArray, PointersArray, SizesArray, - MapTypesArray, NumTeams, ThreadLimit}; + DeviceID, OutlinedFnID, + PointerNum, Info.BasePointersArray, + Info.PointersArray, Info.SizesArray, + Info.MapTypesArray, NumTeams, + ThreadLimit}; Return = CGF.EmitRuntimeCall( RT.createRuntimeFunction(OMPRTL__tgt_target_teams), OffloadingArgs); } else { llvm::Value *OffloadingArgs[] = { - DeviceID, OutlinedFnID, PointerNum, BasePointersArray, - PointersArray, SizesArray, MapTypesArray}; + DeviceID, OutlinedFnID, + PointerNum, Info.BasePointersArray, + Info.PointersArray, Info.SizesArray, + Info.MapTypesArray}; Return = CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target), OffloadingArgs); } @@ -5951,7 +6154,7 @@ bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) { // Try to detect target regions in the function. scanForTargetRegionsFunctions(FD.getBody(), CGM.getMangledName(GD)); - // We should not emit any function othen that the ones created during the + // We should not emit any function other that the ones created during the // scanning. Therefore, we signal that this function is completely dealt // with. return true; @@ -6055,29 +6258,23 @@ void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, PushNumTeamsArgs); } -void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, - const OMPExecutableDirective &D, - const Expr *IfCond, - const Expr *Device, - const RegionCodeGenTy &CodeGen) { - +void CGOpenMPRuntime::emitTargetDataCalls( + CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, + const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) { if (!CGF.HaveInsertPoint()) return; - llvm::Value *BasePointersArray = nullptr; - llvm::Value *PointersArray = nullptr; - llvm::Value *SizesArray = nullptr; - llvm::Value *MapTypesArray = nullptr; - unsigned NumOfPtrs = 0; + // Action used to replace the default codegen action and turn privatization + // off. + PrePostActionTy NoPrivAction; // Generate the code for the opening of the data environment. Capture all the // arguments of the runtime call by reference because they are used in the // closing of the region. - auto &&BeginThenGen = [&D, &CGF, &BasePointersArray, &PointersArray, - &SizesArray, &MapTypesArray, Device, - &NumOfPtrs](CodeGenFunction &CGF, PrePostActionTy &) { + auto &&BeginThenGen = [&D, &CGF, Device, &Info, &CodeGen, &NoPrivAction]( + CodeGenFunction &CGF, PrePostActionTy &) { // Fill up the arrays with all the mapped variables. - MappableExprsHandler::MapValuesArrayTy BasePointers; + MappableExprsHandler::MapBaseValuesArrayTy BasePointers; MappableExprsHandler::MapValuesArrayTy Pointers; MappableExprsHandler::MapValuesArrayTy Sizes; MappableExprsHandler::MapFlagsArrayTy MapTypes; @@ -6085,21 +6282,16 @@ void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, // Get map clause information. MappableExprsHandler MCHandler(D, CGF); MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes); - NumOfPtrs = BasePointers.size(); // Fill up the arrays and create the arguments. - emitOffloadingArrays(CGF, BasePointersArray, PointersArray, SizesArray, - MapTypesArray, BasePointers, Pointers, Sizes, - MapTypes); + emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); llvm::Value *BasePointersArrayArg = nullptr; llvm::Value *PointersArrayArg = nullptr; llvm::Value *SizesArrayArg = nullptr; llvm::Value *MapTypesArrayArg = nullptr; emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, - SizesArrayArg, MapTypesArrayArg, - BasePointersArray, PointersArray, SizesArray, - MapTypesArray, NumOfPtrs); + SizesArrayArg, MapTypesArrayArg, Info); // Emit device ID if any. llvm::Value *DeviceID = nullptr; @@ -6110,7 +6302,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); // Emit the number of elements in the offloading arrays. - auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs); + auto *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); llvm::Value *OffloadingArgs[] = { DeviceID, PointerNum, BasePointersArrayArg, @@ -6118,23 +6310,24 @@ void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, auto &RT = CGF.CGM.getOpenMPRuntime(); CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_begin), OffloadingArgs); + + // If device pointer privatization is required, emit the body of the region + // here. It will have to be duplicated: with and without privatization. + if (!Info.CaptureDeviceAddrMap.empty()) + CodeGen(CGF); }; // Generate code for the closing of the data region. - auto &&EndThenGen = [&CGF, &BasePointersArray, &PointersArray, &SizesArray, - &MapTypesArray, Device, - &NumOfPtrs](CodeGenFunction &CGF, PrePostActionTy &) { - assert(BasePointersArray && PointersArray && SizesArray && MapTypesArray && - NumOfPtrs && "Invalid data environment closing arguments."); + auto &&EndThenGen = [&CGF, Device, &Info](CodeGenFunction &CGF, + PrePostActionTy &) { + assert(Info.isValid() && "Invalid data environment closing arguments."); llvm::Value *BasePointersArrayArg = nullptr; llvm::Value *PointersArrayArg = nullptr; llvm::Value *SizesArrayArg = nullptr; llvm::Value *MapTypesArrayArg = nullptr; emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, - SizesArrayArg, MapTypesArrayArg, - BasePointersArray, PointersArray, SizesArray, - MapTypesArray, NumOfPtrs); + SizesArrayArg, MapTypesArrayArg, Info); // Emit device ID if any. llvm::Value *DeviceID = nullptr; @@ -6145,7 +6338,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); // Emit the number of elements in the offloading arrays. - auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs); + auto *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); llvm::Value *OffloadingArgs[] = { DeviceID, PointerNum, BasePointersArrayArg, @@ -6155,24 +6348,40 @@ void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, OffloadingArgs); }; - // In the event we get an if clause, we don't have to take any action on the - // else side. - auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; + // If we need device pointer privatization, we need to emit the body of the + // region with no privatization in the 'else' branch of the conditional. + // Otherwise, we don't have to do anything. + auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, + PrePostActionTy &) { + if (!Info.CaptureDeviceAddrMap.empty()) { + CodeGen.setAction(NoPrivAction); + CodeGen(CGF); + } + }; + + // We don't have to do anything to close the region if the if clause evaluates + // to false. + auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; if (IfCond) { - emitOMPIfClause(CGF, IfCond, BeginThenGen, ElseGen); + emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); } else { - RegionCodeGenTy BeginThenRCG(BeginThenGen); - BeginThenRCG(CGF); + RegionCodeGenTy RCG(BeginThenGen); + RCG(CGF); } - CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, CodeGen); + // If we don't require privatization of device pointers, we emit the body in + // between the runtime calls. This avoids duplicating the body code. + if (Info.CaptureDeviceAddrMap.empty()) { + CodeGen.setAction(NoPrivAction); + CodeGen(CGF); + } if (IfCond) { - emitOMPIfClause(CGF, IfCond, EndThenGen, ElseGen); + emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen); } else { - RegionCodeGenTy EndThenRCG(EndThenGen); - EndThenRCG(CGF); + RegionCodeGenTy RCG(EndThenGen); + RCG(CGF); } } @@ -6190,7 +6399,7 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( // Generate the code for the opening of the data environment. auto &&ThenGen = [&D, &CGF, Device](CodeGenFunction &CGF, PrePostActionTy &) { // Fill up the arrays with all the mapped variables. - MappableExprsHandler::MapValuesArrayTy BasePointers; + MappableExprsHandler::MapBaseValuesArrayTy BasePointers; MappableExprsHandler::MapValuesArrayTy Pointers; MappableExprsHandler::MapValuesArrayTy Sizes; MappableExprsHandler::MapFlagsArrayTy MapTypes; @@ -6199,19 +6408,12 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( MappableExprsHandler MEHandler(D, CGF); MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes); - llvm::Value *BasePointersArrayArg = nullptr; - llvm::Value *PointersArrayArg = nullptr; - llvm::Value *SizesArrayArg = nullptr; - llvm::Value *MapTypesArrayArg = nullptr; - // Fill up the arrays and create the arguments. - emitOffloadingArrays(CGF, BasePointersArrayArg, PointersArrayArg, - SizesArrayArg, MapTypesArrayArg, BasePointers, - Pointers, Sizes, MapTypes); - emitOffloadingArraysArgument( - CGF, BasePointersArrayArg, PointersArrayArg, SizesArrayArg, - MapTypesArrayArg, BasePointersArrayArg, PointersArrayArg, SizesArrayArg, - MapTypesArrayArg, BasePointers.size()); + TargetDataInfo Info; + emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); + emitOffloadingArraysArgument(CGF, Info.BasePointersArray, + Info.PointersArray, Info.SizesArray, + Info.MapTypesArray, Info); // Emit device ID if any. llvm::Value *DeviceID = nullptr; @@ -6225,8 +6427,8 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( auto *PointerNum = CGF.Builder.getInt32(BasePointers.size()); llvm::Value *OffloadingArgs[] = { - DeviceID, PointerNum, BasePointersArrayArg, - PointersArrayArg, SizesArrayArg, MapTypesArrayArg}; + DeviceID, PointerNum, Info.BasePointersArray, + Info.PointersArray, Info.SizesArray, Info.MapTypesArray}; auto &RT = CGF.CGM.getOpenMPRuntime(); // Select the right runtime function call for each expected standalone @@ -6326,7 +6528,7 @@ static unsigned evaluateCDTSize(const FunctionDecl *FD, static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, - llvm::APSInt VLENVal, + const llvm::APSInt &VLENVal, ArrayRef<ParamAttrTy> ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State) { struct ISADataTy { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h index 270de8d..61ddc70 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -110,9 +110,9 @@ protected: CodeGenModule &CGM; /// \brief Creates offloading entry for the provided entry ID \a ID, - /// address \a Addr and size \a Size. + /// address \a Addr, size \a Size, and flags \a Flags. virtual void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, - uint64_t Size); + uint64_t Size, int32_t Flags = 0); /// \brief Helper to emit outlined function for 'target' directive. /// \param D Directive to emit. @@ -130,6 +130,35 @@ protected: bool IsOffloadEntry, const RegionCodeGenTy &CodeGen); + /// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen + /// function. Here is the logic: + /// if (Cond) { + /// ThenGen(); + /// } else { + /// ElseGen(); + /// } + void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, + const RegionCodeGenTy &ThenGen, + const RegionCodeGenTy &ElseGen); + + /// \brief Emits object of ident_t type with info for source location. + /// \param Flags Flags for OpenMP location. + /// + llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, + unsigned Flags = 0); + + /// \brief Returns pointer to ident_t type. + llvm::Type *getIdentTyPointerTy(); + + /// \brief Gets thread id value for the current thread. + /// + llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); + + /// \brief Get the function name of an outlined region. + // The name can be customized depending on the target. + // + virtual StringRef getOutlinedHelperName() const { return ".omp_outlined."; } + private: /// \brief Default const ident_t object used for initialization of all other /// ident_t objects. @@ -245,10 +274,10 @@ private: unsigned OffloadingEntriesNum; public: - /// \brief Base class of the entries info. + /// Base class of the entries info. class OffloadEntryInfo { public: - /// \brief Kind of a given entry. Currently, only target regions are + /// Kind of a given entry. Currently, only target regions are /// supported. enum OffloadingEntryInfoKinds : unsigned { // Entry is a target region. @@ -257,17 +286,24 @@ private: OFFLOAD_ENTRY_INFO_INVALID = ~0u }; - OffloadEntryInfo() : Order(~0u), Kind(OFFLOAD_ENTRY_INFO_INVALID) {} - explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind, unsigned Order) - : Order(Order), Kind(Kind) {} + OffloadEntryInfo() + : Flags(0), Order(~0u), Kind(OFFLOAD_ENTRY_INFO_INVALID) {} + explicit OffloadEntryInfo(OffloadingEntryInfoKinds Kind, unsigned Order, + int32_t Flags) + : Flags(Flags), Order(Order), Kind(Kind) {} bool isValid() const { return Order != ~0u; } unsigned getOrder() const { return Order; } OffloadingEntryInfoKinds getKind() const { return Kind; } + int32_t getFlags() const { return Flags; } + void setFlags(int32_t NewFlags) { Flags = NewFlags; } static bool classof(const OffloadEntryInfo *Info) { return true; } - protected: - // \brief Order this entry was emitted. + private: + /// Flags associated with the device global. + int32_t Flags; + + /// Order this entry was emitted. unsigned Order; OffloadingEntryInfoKinds Kind; @@ -292,12 +328,13 @@ private: public: OffloadEntryInfoTargetRegion() - : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_TARGET_REGION, ~0u), + : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_TARGET_REGION, ~0u, + /*Flags=*/0), Addr(nullptr), ID(nullptr) {} explicit OffloadEntryInfoTargetRegion(unsigned Order, llvm::Constant *Addr, - llvm::Constant *ID) - : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_TARGET_REGION, Order), + llvm::Constant *ID, int32_t Flags) + : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_TARGET_REGION, Order, Flags), Addr(Addr), ID(ID) {} llvm::Constant *getAddress() const { return Addr; } @@ -321,8 +358,8 @@ private: /// \brief Register target region entry. void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, - llvm::Constant *Addr, - llvm::Constant *ID); + llvm::Constant *Addr, llvm::Constant *ID, + int32_t Flags); /// \brief Return true if a target region entry with the provided /// information exists. bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, @@ -380,15 +417,6 @@ private: /// \brief Build type kmp_routine_entry_t (if not built yet). void emitKmpRoutineEntryT(QualType KmpInt32Ty); - /// \brief Emits object of ident_t type with info for source location. - /// \param Flags Flags for OpenMP location. - /// - llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, - unsigned Flags = 0); - - /// \brief Returns pointer to ident_t type. - llvm::Type *getIdentTyPointerTy(); - /// \brief Returns pointer to kmpc_micro type. llvm::Type *getKmpc_MicroPointerTy(); @@ -424,10 +452,6 @@ private: /// stored. virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc); - /// \brief Gets thread id value for the current thread. - /// - llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); - /// \brief Gets (if variable with the given name already exist) or creates /// internal global variable with the specified Name. The created variable has /// linkage CommonLinkage by default and is initialized by null value. @@ -997,17 +1021,59 @@ public: virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc); + /// Struct that keeps all the relevant information that should be kept + /// throughout a 'target data' region. + class TargetDataInfo { + /// Set to true if device pointer information have to be obtained. + bool RequiresDevicePointerInfo = false; + + public: + /// The array of base pointer passed to the runtime library. + llvm::Value *BasePointersArray = nullptr; + /// The array of section pointers passed to the runtime library. + llvm::Value *PointersArray = nullptr; + /// The array of sizes passed to the runtime library. + llvm::Value *SizesArray = nullptr; + /// The array of map types passed to the runtime library. + llvm::Value *MapTypesArray = nullptr; + /// The total number of pointers passed to the runtime library. + unsigned NumberOfPtrs = 0u; + /// Map between the a declaration of a capture and the corresponding base + /// pointer address where the runtime returns the device pointers. + llvm::DenseMap<const ValueDecl *, Address> CaptureDeviceAddrMap; + + explicit TargetDataInfo() {} + explicit TargetDataInfo(bool RequiresDevicePointerInfo) + : RequiresDevicePointerInfo(RequiresDevicePointerInfo) {} + /// Clear information about the data arrays. + void clearArrayInfo() { + BasePointersArray = nullptr; + PointersArray = nullptr; + SizesArray = nullptr; + MapTypesArray = nullptr; + NumberOfPtrs = 0u; + } + /// Return true if the current target data information has valid arrays. + bool isValid() { + return BasePointersArray && PointersArray && SizesArray && + MapTypesArray && NumberOfPtrs; + } + bool requiresDevicePointerInfo() { return RequiresDevicePointerInfo; } + }; + /// \brief Emit the target data mapping code associated with \a D. /// \param D Directive to emit. - /// \param IfCond Expression evaluated in if clause associated with the target - /// directive, or null if no if clause is used. + /// \param IfCond Expression evaluated in if clause associated with the + /// target directive, or null if no device clause is used. /// \param Device Expression evaluated in device clause associated with the /// target directive, or null if no device clause is used. - /// \param CodeGen Function that emits the enclosed region. + /// \param Info A record used to store information that needs to be preserved + /// until the region is closed. virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, - const RegionCodeGenTy &CodeGen); + const RegionCodeGenTy &CodeGen, + TargetDataInfo &Info); /// \brief Emit the data mapping/movement code associated with the directive /// \a D that should be of the form 'target [{enter|exit} data | update]'. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp index d64f6df..6a6d832 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp @@ -20,53 +20,118 @@ using namespace clang; using namespace CodeGen; -/// \brief Get the GPU warp size. -llvm::Value *CGOpenMPRuntimeNVPTX::getNVPTXWarpSize(CodeGenFunction &CGF) { +namespace { +enum OpenMPRTLFunctionNVPTX { + /// \brief Call to void __kmpc_kernel_init(kmp_int32 thread_limit); + OMPRTL_NVPTX__kmpc_kernel_init, + /// \brief Call to void __kmpc_kernel_deinit(); + OMPRTL_NVPTX__kmpc_kernel_deinit, + /// \brief Call to void __kmpc_kernel_prepare_parallel(void + /// *outlined_function); + OMPRTL_NVPTX__kmpc_kernel_prepare_parallel, + /// \brief Call to bool __kmpc_kernel_parallel(void **outlined_function); + OMPRTL_NVPTX__kmpc_kernel_parallel, + /// \brief Call to void __kmpc_kernel_end_parallel(); + OMPRTL_NVPTX__kmpc_kernel_end_parallel, + /// Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 + /// global_tid); + OMPRTL_NVPTX__kmpc_serialized_parallel, + /// Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 + /// global_tid); + OMPRTL_NVPTX__kmpc_end_serialized_parallel, +}; + +/// Pre(post)-action for different OpenMP constructs specialized for NVPTX. +class NVPTXActionTy final : public PrePostActionTy { + llvm::Value *EnterCallee; + ArrayRef<llvm::Value *> EnterArgs; + llvm::Value *ExitCallee; + ArrayRef<llvm::Value *> ExitArgs; + bool Conditional; + llvm::BasicBlock *ContBlock = nullptr; + +public: + NVPTXActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs, + llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs, + bool Conditional = false) + : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee), + ExitArgs(ExitArgs), Conditional(Conditional) {} + void Enter(CodeGenFunction &CGF) override { + llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); + if (Conditional) { + llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); + auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); + ContBlock = CGF.createBasicBlock("omp_if.end"); + // Generate the branch (If-stmt) + CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); + CGF.EmitBlock(ThenBlock); + } + } + void Done(CodeGenFunction &CGF) { + // Emit the rest of blocks/branches + CGF.EmitBranch(ContBlock); + CGF.EmitBlock(ContBlock, true); + } + void Exit(CodeGenFunction &CGF) override { + CGF.EmitRuntimeCall(ExitCallee, ExitArgs); + } +}; +} // anonymous namespace + +/// Get the GPU warp size. +static llvm::Value *getNVPTXWarpSize(CodeGenFunction &CGF) { CGBuilderTy &Bld = CGF.Builder; return Bld.CreateCall( llvm::Intrinsic::getDeclaration( - &CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_warpsize), + &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_warpsize), llvm::None, "nvptx_warp_size"); } -/// \brief Get the id of the current thread on the GPU. -llvm::Value *CGOpenMPRuntimeNVPTX::getNVPTXThreadID(CodeGenFunction &CGF) { +/// Get the id of the current thread on the GPU. +static llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF) { CGBuilderTy &Bld = CGF.Builder; return Bld.CreateCall( llvm::Intrinsic::getDeclaration( - &CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x), + &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x), llvm::None, "nvptx_tid"); } -// \brief Get the maximum number of threads in a block of the GPU. -llvm::Value *CGOpenMPRuntimeNVPTX::getNVPTXNumThreads(CodeGenFunction &CGF) { +/// Get the maximum number of threads in a block of the GPU. +static llvm::Value *getNVPTXNumThreads(CodeGenFunction &CGF) { CGBuilderTy &Bld = CGF.Builder; return Bld.CreateCall( llvm::Intrinsic::getDeclaration( - &CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x), + &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x), llvm::None, "nvptx_num_threads"); } -/// \brief Get barrier to synchronize all threads in a block. -void CGOpenMPRuntimeNVPTX::getNVPTXCTABarrier(CodeGenFunction &CGF) { +/// Get barrier to synchronize all threads in a block. +static void getNVPTXCTABarrier(CodeGenFunction &CGF) { CGBuilderTy &Bld = CGF.Builder; Bld.CreateCall(llvm::Intrinsic::getDeclaration( - &CGM.getModule(), llvm::Intrinsic::nvvm_barrier0)); + &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_barrier0)); } -// \brief Synchronize all GPU threads in a block. -void CGOpenMPRuntimeNVPTX::syncCTAThreads(CodeGenFunction &CGF) { - getNVPTXCTABarrier(CGF); +/// Synchronize all GPU threads in a block. +static void syncCTAThreads(CodeGenFunction &CGF) { getNVPTXCTABarrier(CGF); } + +/// Get the value of the thread_limit clause in the teams directive. +/// The runtime encodes thread_limit in the launch parameter, always starting +/// thread_limit+warpSize threads per team. +static llvm::Value *getThreadLimit(CodeGenFunction &CGF) { + CGBuilderTy &Bld = CGF.Builder; + return Bld.CreateSub(getNVPTXNumThreads(CGF), getNVPTXWarpSize(CGF), + "thread_limit"); } -/// \brief Get the thread id of the OMP master thread. +/// Get the thread id of the OMP master thread. /// The master thread id is the first thread (lane) of the last warp in the /// GPU block. Warp size is assumed to be some power of 2. /// Thread id is 0 indexed. /// E.g: If NumThreads is 33, master id is 32. /// If NumThreads is 64, master id is 32. /// If NumThreads is 1024, master id is 992. -llvm::Value *CGOpenMPRuntimeNVPTX::getMasterThreadID(CodeGenFunction &CGF) { +static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) { CGBuilderTy &Bld = CGF.Builder; llvm::Value *NumThreads = getNVPTXNumThreads(CGF); @@ -77,19 +142,6 @@ llvm::Value *CGOpenMPRuntimeNVPTX::getMasterThreadID(CodeGenFunction &CGF) { Bld.CreateNot(Mask), "master_tid"); } -namespace { -enum OpenMPRTLFunctionNVPTX { - /// \brief Call to void __kmpc_kernel_init(kmp_int32 omp_handle, - /// kmp_int32 thread_limit); - OMPRTL_NVPTX__kmpc_kernel_init, -}; - -// NVPTX Address space -enum ADDRESS_SPACE { - ADDRESS_SPACE_SHARED = 3, -}; -} // namespace - CGOpenMPRuntimeNVPTX::WorkerFunctionState::WorkerFunctionState( CodeGenModule &CGM) : WorkerFn(nullptr), CGFI(nullptr) { @@ -105,35 +157,106 @@ void CGOpenMPRuntimeNVPTX::WorkerFunctionState::createWorkerFunction( CGM.getTypes().GetFunctionType(*CGFI), llvm::GlobalValue::InternalLinkage, /* placeholder */ "_worker", &CGM.getModule()); CGM.SetInternalFunctionAttributes(/*D=*/nullptr, WorkerFn, *CGFI); - WorkerFn->setLinkage(llvm::GlobalValue::InternalLinkage); - WorkerFn->addFnAttr(llvm::Attribute::NoInline); } -void CGOpenMPRuntimeNVPTX::initializeEnvironment() { - // - // Initialize master-worker control state in shared memory. - // +void CGOpenMPRuntimeNVPTX::emitGenericKernel(const OMPExecutableDirective &D, + StringRef ParentName, + llvm::Function *&OutlinedFn, + llvm::Constant *&OutlinedFnID, + bool IsOffloadEntry, + const RegionCodeGenTy &CodeGen) { + EntryFunctionState EST; + WorkerFunctionState WST(CGM); + Work.clear(); + + // Emit target region as a standalone region. + class NVPTXPrePostActionTy : public PrePostActionTy { + CGOpenMPRuntimeNVPTX &RT; + CGOpenMPRuntimeNVPTX::EntryFunctionState &EST; + CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST; + + public: + NVPTXPrePostActionTy(CGOpenMPRuntimeNVPTX &RT, + CGOpenMPRuntimeNVPTX::EntryFunctionState &EST, + CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST) + : RT(RT), EST(EST), WST(WST) {} + void Enter(CodeGenFunction &CGF) override { + RT.emitGenericEntryHeader(CGF, EST, WST); + } + void Exit(CodeGenFunction &CGF) override { + RT.emitGenericEntryFooter(CGF, EST); + } + } Action(*this, EST, WST); + CodeGen.setAction(Action); + emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID, + IsOffloadEntry, CodeGen); + + // Create the worker function + emitWorkerFunction(WST); + + // Now change the name of the worker function to correspond to this target + // region's entry function. + WST.WorkerFn->setName(OutlinedFn->getName() + "_worker"); +} + +// Setup NVPTX threads for master-worker OpenMP scheme. +void CGOpenMPRuntimeNVPTX::emitGenericEntryHeader(CodeGenFunction &CGF, + EntryFunctionState &EST, + WorkerFunctionState &WST) { + CGBuilderTy &Bld = CGF.Builder; + + llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker"); + llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck"); + llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master"); + EST.ExitBB = CGF.createBasicBlock(".exit"); + + auto *IsWorker = + Bld.CreateICmpULT(getNVPTXThreadID(CGF), getThreadLimit(CGF)); + Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB); - auto DL = CGM.getDataLayout(); - ActiveWorkers = new llvm::GlobalVariable( - CGM.getModule(), CGM.Int32Ty, /*isConstant=*/false, - llvm::GlobalValue::CommonLinkage, - llvm::Constant::getNullValue(CGM.Int32Ty), "__omp_num_threads", 0, - llvm::GlobalVariable::NotThreadLocal, ADDRESS_SPACE_SHARED); - ActiveWorkers->setAlignment(DL.getPrefTypeAlignment(CGM.Int32Ty)); - - WorkID = new llvm::GlobalVariable( - CGM.getModule(), CGM.Int64Ty, /*isConstant=*/false, - llvm::GlobalValue::CommonLinkage, - llvm::Constant::getNullValue(CGM.Int64Ty), "__tgt_work_id", 0, - llvm::GlobalVariable::NotThreadLocal, ADDRESS_SPACE_SHARED); - WorkID->setAlignment(DL.getPrefTypeAlignment(CGM.Int64Ty)); + CGF.EmitBlock(WorkerBB); + CGF.EmitCallOrInvoke(WST.WorkerFn, llvm::None); + CGF.EmitBranch(EST.ExitBB); + + CGF.EmitBlock(MasterCheckBB); + auto *IsMaster = + Bld.CreateICmpEQ(getNVPTXThreadID(CGF), getMasterThreadID(CGF)); + Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB); + + CGF.EmitBlock(MasterBB); + // First action in sequential region: + // Initialize the state of the OpenMP runtime library on the GPU. + llvm::Value *Args[] = {getThreadLimit(CGF)}; + CGF.EmitRuntimeCall( + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_init), Args); +} + +void CGOpenMPRuntimeNVPTX::emitGenericEntryFooter(CodeGenFunction &CGF, + EntryFunctionState &EST) { + if (!EST.ExitBB) + EST.ExitBB = CGF.createBasicBlock(".exit"); + + llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier"); + CGF.EmitBranch(TerminateBB); + + CGF.EmitBlock(TerminateBB); + // Signal termination condition. + CGF.EmitRuntimeCall( + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_deinit), None); + // Barrier to terminate worker threads. + syncCTAThreads(CGF); + // Master thread jumps to exit point. + CGF.EmitBranch(EST.ExitBB); + + CGF.EmitBlock(EST.ExitBB); + EST.ExitBB = nullptr; } void CGOpenMPRuntimeNVPTX::emitWorkerFunction(WorkerFunctionState &WST) { auto &Ctx = CGM.getContext(); CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); + CGF.disableDebugInfo(); CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, *WST.CGFI, {}); emitWorkerLoop(CGF, WST); CGF.FinishFunction(); @@ -165,28 +288,68 @@ void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF, CGF.EmitBlock(AwaitBB); // Wait for parallel work syncCTAThreads(CGF); + + Address WorkFn = + CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn"); + Address ExecStatus = + CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status"); + CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0)); + CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy)); + + llvm::Value *Args[] = {WorkFn.getPointer()}; + llvm::Value *Ret = CGF.EmitRuntimeCall( + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_parallel), Args); + Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus); + // On termination condition (workid == 0), exit loop. - llvm::Value *ShouldTerminate = Bld.CreateICmpEQ( - Bld.CreateAlignedLoad(WorkID, WorkID->getAlignment()), - llvm::Constant::getNullValue(WorkID->getType()->getElementType()), - "should_terminate"); + llvm::Value *ShouldTerminate = + Bld.CreateIsNull(Bld.CreateLoad(WorkFn), "should_terminate"); Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB); // Activate requested workers. CGF.EmitBlock(SelectWorkersBB); - llvm::Value *ThreadID = getNVPTXThreadID(CGF); - llvm::Value *ActiveThread = Bld.CreateICmpSLT( - ThreadID, - Bld.CreateAlignedLoad(ActiveWorkers, ActiveWorkers->getAlignment()), - "active_thread"); - Bld.CreateCondBr(ActiveThread, ExecuteBB, BarrierBB); + llvm::Value *IsActive = + Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active"); + Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB); // Signal start of parallel region. CGF.EmitBlock(ExecuteBB); - // TODO: Add parallel work. + + // Process work items: outlined parallel functions. + for (auto *W : Work) { + // Try to match this outlined function. + auto *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy); + + llvm::Value *WorkFnMatch = + Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match"); + + llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn"); + llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next"); + Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB); + + // Execute this outlined function. + CGF.EmitBlock(ExecuteFNBB); + + // Insert call to work function. + // FIXME: Pass arguments to outlined function from master thread. + auto *Fn = cast<llvm::Function>(W); + Address ZeroAddr = + CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".zero.addr"); + CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C=*/0)); + llvm::Value *FnArgs[] = {ZeroAddr.getPointer(), ZeroAddr.getPointer()}; + CGF.EmitCallOrInvoke(Fn, FnArgs); + + // Go to end of parallel region. + CGF.EmitBranch(TerminateBB); + + CGF.EmitBlock(CheckNextBB); + } // Signal end of parallel region. CGF.EmitBlock(TerminateBB); + CGF.EmitRuntimeCall( + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_end_parallel), + llvm::None); CGF.EmitBranch(BarrierBB); // All active and inactive workers wait at a barrier after parallel region. @@ -199,68 +362,6 @@ void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF, CGF.EmitBlock(ExitBB); } -// Setup NVPTX threads for master-worker OpenMP scheme. -void CGOpenMPRuntimeNVPTX::emitEntryHeader(CodeGenFunction &CGF, - EntryFunctionState &EST, - WorkerFunctionState &WST) { - CGBuilderTy &Bld = CGF.Builder; - - // Get the master thread id. - llvm::Value *MasterID = getMasterThreadID(CGF); - // Current thread's identifier. - llvm::Value *ThreadID = getNVPTXThreadID(CGF); - - // Setup BBs in entry function. - llvm::BasicBlock *WorkerCheckBB = CGF.createBasicBlock(".check.for.worker"); - llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker"); - llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master"); - EST.ExitBB = CGF.createBasicBlock(".exit"); - - // The head (master thread) marches on while its body of companion threads in - // the warp go to sleep. - llvm::Value *ShouldDie = - Bld.CreateICmpUGT(ThreadID, MasterID, "excess_in_master_warp"); - Bld.CreateCondBr(ShouldDie, EST.ExitBB, WorkerCheckBB); - - // Select worker threads... - CGF.EmitBlock(WorkerCheckBB); - llvm::Value *IsWorker = Bld.CreateICmpULT(ThreadID, MasterID, "is_worker"); - Bld.CreateCondBr(IsWorker, WorkerBB, MasterBB); - - // ... and send to worker loop, awaiting parallel invocation. - CGF.EmitBlock(WorkerBB); - CGF.EmitCallOrInvoke(WST.WorkerFn, llvm::None); - CGF.EmitBranch(EST.ExitBB); - - // Only master thread executes subsequent serial code. - CGF.EmitBlock(MasterBB); - - // First action in sequential region: - // Initialize the state of the OpenMP runtime library on the GPU. - llvm::Value *Args[] = {Bld.getInt32(/*OmpHandle=*/0), getNVPTXThreadID(CGF)}; - CGF.EmitRuntimeCall(createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_init), - Args); -} - -void CGOpenMPRuntimeNVPTX::emitEntryFooter(CodeGenFunction &CGF, - EntryFunctionState &EST) { - CGBuilderTy &Bld = CGF.Builder; - llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier"); - CGF.EmitBranch(TerminateBB); - - CGF.EmitBlock(TerminateBB); - // Signal termination condition. - Bld.CreateAlignedStore( - llvm::Constant::getNullValue(WorkID->getType()->getElementType()), WorkID, - WorkID->getAlignment()); - // Barrier to terminate worker threads. - syncCTAThreads(CGF); - // Master thread jumps to exit point. - CGF.EmitBranch(EST.ExitBB); - - CGF.EmitBlock(EST.ExitBB); -} - /// \brief Returns specified OpenMP runtime function for the current OpenMP /// implementation. Specialized for the NVPTX device. /// \param Function OpenMP runtime function. @@ -270,21 +371,70 @@ CGOpenMPRuntimeNVPTX::createNVPTXRuntimeFunction(unsigned Function) { llvm::Constant *RTLFn = nullptr; switch (static_cast<OpenMPRTLFunctionNVPTX>(Function)) { case OMPRTL_NVPTX__kmpc_kernel_init: { - // Build void __kmpc_kernel_init(kmp_int32 omp_handle, - // kmp_int32 thread_limit); - llvm::Type *TypeParams[] = {CGM.Int32Ty, CGM.Int32Ty}; + // Build void __kmpc_kernel_init(kmp_int32 thread_limit); + llvm::Type *TypeParams[] = {CGM.Int32Ty}; llvm::FunctionType *FnTy = llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_init"); break; } + case OMPRTL_NVPTX__kmpc_kernel_deinit: { + // Build void __kmpc_kernel_deinit(); + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, llvm::None, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_deinit"); + break; + } + case OMPRTL_NVPTX__kmpc_kernel_prepare_parallel: { + /// Build void __kmpc_kernel_prepare_parallel( + /// void *outlined_function); + llvm::Type *TypeParams[] = {CGM.Int8PtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_prepare_parallel"); + break; + } + case OMPRTL_NVPTX__kmpc_kernel_parallel: { + /// Build bool __kmpc_kernel_parallel(void **outlined_function); + llvm::Type *TypeParams[] = {CGM.Int8PtrPtrTy}; + llvm::Type *RetTy = CGM.getTypes().ConvertType(CGM.getContext().BoolTy); + llvm::FunctionType *FnTy = + llvm::FunctionType::get(RetTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_parallel"); + break; + } + case OMPRTL_NVPTX__kmpc_kernel_end_parallel: { + /// Build void __kmpc_kernel_end_parallel(); + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, llvm::None, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_end_parallel"); + break; + } + case OMPRTL_NVPTX__kmpc_serialized_parallel: { + // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 + // global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel"); + break; + } + case OMPRTL_NVPTX__kmpc_end_serialized_parallel: { + // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 + // global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel"); + break; + } } return RTLFn; } void CGOpenMPRuntimeNVPTX::createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, - uint64_t Size) { + uint64_t Size, int32_t) { auto *F = dyn_cast<llvm::Function>(Addr); // TODO: Add support for global variables on the device after declare target // support. @@ -313,44 +463,14 @@ void CGOpenMPRuntimeNVPTX::emitTargetOutlinedFunction( assert(!ParentName.empty() && "Invalid target region parent name!"); - EntryFunctionState EST; - WorkerFunctionState WST(CGM); - - // Emit target region as a standalone region. - class NVPTXPrePostActionTy : public PrePostActionTy { - CGOpenMPRuntimeNVPTX &RT; - CGOpenMPRuntimeNVPTX::EntryFunctionState &EST; - CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST; - - public: - NVPTXPrePostActionTy(CGOpenMPRuntimeNVPTX &RT, - CGOpenMPRuntimeNVPTX::EntryFunctionState &EST, - CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST) - : RT(RT), EST(EST), WST(WST) {} - void Enter(CodeGenFunction &CGF) override { - RT.emitEntryHeader(CGF, EST, WST); - } - void Exit(CodeGenFunction &CGF) override { RT.emitEntryFooter(CGF, EST); } - } Action(*this, EST, WST); - CodeGen.setAction(Action); - emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID, - IsOffloadEntry, CodeGen); - - // Create the worker function - emitWorkerFunction(WST); - - // Now change the name of the worker function to correspond to this target - // region's entry function. - WST.WorkerFn->setName(OutlinedFn->getName() + "_worker"); + emitGenericKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry, + CodeGen); } CGOpenMPRuntimeNVPTX::CGOpenMPRuntimeNVPTX(CodeGenModule &CGM) - : CGOpenMPRuntime(CGM), ActiveWorkers(nullptr), WorkID(nullptr) { + : CGOpenMPRuntime(CGM) { if (!CGM.getLangOpts().OpenMPIsDevice) llvm_unreachable("OpenMP NVPTX can only handle device code."); - - // Called once per module during initialization. - initializeEnvironment(); } void CGOpenMPRuntimeNVPTX::emitNumTeamsClause(CodeGenFunction &CGF, @@ -368,10 +488,14 @@ llvm::Value *CGOpenMPRuntimeNVPTX::emitParallelOrTeamsOutlinedFunction( CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction( D, ThreadIDVar, InnermostKind, CodeGen); OutlinedFun = cast<llvm::Function>(OutlinedFunVal); + OutlinedFun->removeFnAttr(llvm::Attribute::NoInline); OutlinedFun->addFnAttr(llvm::Attribute::AlwaysInline); - } else - llvm_unreachable("parallel directive is not yet supported for nvptx " - "backend."); + } else { + llvm::Value *OutlinedFunVal = + CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction( + D, ThreadIDVar, InnermostKind, CodeGen); + OutlinedFun = cast<llvm::Function>(OutlinedFunVal); + } return OutlinedFun; } @@ -394,3 +518,81 @@ void CGOpenMPRuntimeNVPTX::emitTeamsCall(CodeGenFunction &CGF, OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); } + +void CGOpenMPRuntimeNVPTX::emitParallelCall( + CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, + ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) { + if (!CGF.HaveInsertPoint()) + return; + + emitGenericParallelCall(CGF, Loc, OutlinedFn, CapturedVars, IfCond); +} + +void CGOpenMPRuntimeNVPTX::emitGenericParallelCall( + CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, + ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) { + llvm::Function *Fn = cast<llvm::Function>(OutlinedFn); + + auto &&L0ParallelGen = [this, Fn, &CapturedVars](CodeGenFunction &CGF, + PrePostActionTy &) { + CGBuilderTy &Bld = CGF.Builder; + + // Prepare for parallel region. Indicate the outlined function. + llvm::Value *Args[] = {Bld.CreateBitOrPointerCast(Fn, CGM.Int8PtrTy)}; + CGF.EmitRuntimeCall( + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_prepare_parallel), + Args); + + // Activate workers. This barrier is used by the master to signal + // work for the workers. + syncCTAThreads(CGF); + + // OpenMP [2.5, Parallel Construct, p.49] + // There is an implied barrier at the end of a parallel region. After the + // end of a parallel region, only the master thread of the team resumes + // execution of the enclosing task region. + // + // The master waits at this barrier until all workers are done. + syncCTAThreads(CGF); + + // Remember for post-processing in worker loop. + Work.push_back(Fn); + }; + + auto *RTLoc = emitUpdateLocation(CGF, Loc); + auto *ThreadID = getThreadID(CGF, Loc); + llvm::Value *Args[] = {RTLoc, ThreadID}; + + auto &&SeqGen = [this, Fn, &CapturedVars, &Args](CodeGenFunction &CGF, + PrePostActionTy &) { + auto &&CodeGen = [this, Fn, &CapturedVars, &Args](CodeGenFunction &CGF, + PrePostActionTy &Action) { + Action.Enter(CGF); + + llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; + OutlinedFnArgs.push_back( + llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo())); + OutlinedFnArgs.push_back( + llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo())); + OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); + CGF.EmitCallOrInvoke(Fn, OutlinedFnArgs); + }; + + RegionCodeGenTy RCG(CodeGen); + NVPTXActionTy Action( + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_serialized_parallel), + Args, + createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_end_serialized_parallel), + Args); + RCG.setAction(Action); + RCG(CGF); + }; + + if (IfCond) + emitOMPIfClause(CGF, IfCond, L0ParallelGen, SeqGen); + else { + CodeGenFunction::RunCleanupsScope Scope(CGF); + RegionCodeGenTy ThenRCG(L0ParallelGen); + ThenRCG(CGF); + } +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h index a6c64b2..4010b46 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h @@ -24,12 +24,12 @@ namespace clang { namespace CodeGen { class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntime { -public: - class EntryFunctionState { - public: - llvm::BasicBlock *ExitBB; +private: + // Parallel outlined function work for workers to execute. + llvm::SmallVector<llvm::Function *, 16> Work; - EntryFunctionState() : ExitBB(nullptr){}; + struct EntryFunctionState { + llvm::BasicBlock *ExitBB = nullptr; }; class WorkerFunctionState { @@ -43,66 +43,21 @@ public: void createWorkerFunction(CodeGenModule &CGM); }; - /// \brief Helper for target entry function. Guide the master and worker - /// threads to their respective locations. - void emitEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST, - WorkerFunctionState &WST); - - /// \brief Signal termination of OMP execution. - void emitEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST); - -private: - // - // NVPTX calls. - // - - /// \brief Get the GPU warp size. - llvm::Value *getNVPTXWarpSize(CodeGenFunction &CGF); - - /// \brief Get the id of the current thread on the GPU. - llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF); - - // \brief Get the maximum number of threads in a block of the GPU. - llvm::Value *getNVPTXNumThreads(CodeGenFunction &CGF); - - /// \brief Get barrier to synchronize all threads in a block. - void getNVPTXCTABarrier(CodeGenFunction &CGF); - - // \brief Synchronize all GPU threads in a block. - void syncCTAThreads(CodeGenFunction &CGF); - - // - // OMP calls. - // - - /// \brief Get the thread id of the OMP master thread. - /// The master thread id is the first thread (lane) of the last warp in the - /// GPU block. Warp size is assumed to be some power of 2. - /// Thread id is 0 indexed. - /// E.g: If NumThreads is 33, master id is 32. - /// If NumThreads is 64, master id is 32. - /// If NumThreads is 1024, master id is 992. - llvm::Value *getMasterThreadID(CodeGenFunction &CGF); - - // - // Private state and methods. - // - - // Master-worker control state. - // Number of requested OMP threads in parallel region. - llvm::GlobalVariable *ActiveWorkers; - // Outlined function for the workers to execute. - llvm::GlobalVariable *WorkID; - - /// \brief Initialize master-worker control state. - void initializeEnvironment(); - /// \brief Emit the worker function for the current target region. void emitWorkerFunction(WorkerFunctionState &WST); /// \brief Helper for worker function. Emit body of worker loop. void emitWorkerLoop(CodeGenFunction &CGF, WorkerFunctionState &WST); + /// \brief Helper for generic target entry function. Guide the master and + /// worker threads to their respective locations. + void emitGenericEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST, + WorkerFunctionState &WST); + + /// \brief Signal termination of OMP execution for generic target entry + /// function. + void emitGenericEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST); + /// \brief Returns specified OpenMP runtime function for the current OpenMP /// implementation. Specialized for the NVPTX device. /// \param Function OpenMP runtime function. @@ -114,9 +69,23 @@ private: // /// \brief Creates offloading entry for the provided entry ID \a ID, - /// address \a Addr and size \a Size. + /// address \a Addr, size \a Size, and flags \a Flags. void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, - uint64_t Size) override; + uint64_t Size, int32_t Flags = 0) override; + + /// \brief Emit outlined function specialized for the Fork-Join + /// programming model for applicable target directives on the NVPTX device. + /// \param D Directive to emit. + /// \param ParentName Name of the function that encloses the target region. + /// \param OutlinedFn Outlined function value to be defined by this call. + /// \param OutlinedFnID Outlined function ID value to be defined by this call. + /// \param IsOffloadEntry True if the outlined function is an offload entry. + /// An outlined function may not be an entry if, e.g. the if clause always + /// evaluates to false. + void emitGenericKernel(const OMPExecutableDirective &D, StringRef ParentName, + llvm::Function *&OutlinedFn, + llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, + const RegionCodeGenTy &CodeGen); /// \brief Emit outlined function for 'target' directive on the NVPTX /// device. @@ -134,6 +103,29 @@ private: bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override; + /// \brief Emits code for parallel or serial call of the \a OutlinedFn with + /// variables captured in a record which address is stored in \a + /// CapturedStruct. + /// This call is for the Generic Execution Mode. + /// \param OutlinedFn Outlined function to be run in parallel threads. Type of + /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). + /// \param CapturedVars A pointer to the record with the references to + /// variables used in \a OutlinedFn function. + /// \param IfCond Condition in the associated 'if' clause, if it was + /// specified, nullptr otherwise. + void emitGenericParallelCall(CodeGenFunction &CGF, SourceLocation Loc, + llvm::Value *OutlinedFn, + ArrayRef<llvm::Value *> CapturedVars, + const Expr *IfCond); + +protected: + /// \brief Get the function name of an outlined region. + // The name can be customized depending on the target. + // + StringRef getOutlinedHelperName() const override { + return "__omp_outlined__"; + } + public: explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM); @@ -171,6 +163,20 @@ public: void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Value *OutlinedFn, ArrayRef<llvm::Value *> CapturedVars) override; + + /// \brief Emits code for parallel or serial call of the \a OutlinedFn with + /// variables captured in a record which address is stored in \a + /// CapturedStruct. + /// \param OutlinedFn Outlined function to be run in parallel threads. Type of + /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). + /// \param CapturedVars A pointer to the record with the references to + /// variables used in \a OutlinedFn function. + /// \param IfCond Condition in the associated 'if' clause, if it was + /// specified, nullptr otherwise. + void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, + llvm::Value *OutlinedFn, + ArrayRef<llvm::Value *> CapturedVars, + const Expr *IfCond) override; }; } // CodeGen namespace. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp index d815863..8370607 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp @@ -142,6 +142,8 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::GCCAsmStmtClass: // Intentional fall-through. case Stmt::MSAsmStmtClass: EmitAsmStmt(cast<AsmStmt>(*S)); break; case Stmt::CoroutineBodyStmtClass: + EmitCoroutineBody(cast<CoroutineBodyStmt>(*S)); + break; case Stmt::CoreturnStmtClass: CGM.ErrorUnsupported(S, "coroutine"); break; @@ -295,6 +297,43 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { EmitOMPTargetParallelForSimdDirective( cast<OMPTargetParallelForSimdDirective>(*S)); break; + case Stmt::OMPTargetSimdDirectiveClass: + EmitOMPTargetSimdDirective(cast<OMPTargetSimdDirective>(*S)); + break; + case Stmt::OMPTeamsDistributeDirectiveClass: + EmitOMPTeamsDistributeDirective(cast<OMPTeamsDistributeDirective>(*S)); + break; + case Stmt::OMPTeamsDistributeSimdDirectiveClass: + EmitOMPTeamsDistributeSimdDirective( + cast<OMPTeamsDistributeSimdDirective>(*S)); + break; + case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass: + EmitOMPTeamsDistributeParallelForSimdDirective( + cast<OMPTeamsDistributeParallelForSimdDirective>(*S)); + break; + case Stmt::OMPTeamsDistributeParallelForDirectiveClass: + EmitOMPTeamsDistributeParallelForDirective( + cast<OMPTeamsDistributeParallelForDirective>(*S)); + break; + case Stmt::OMPTargetTeamsDirectiveClass: + EmitOMPTargetTeamsDirective(cast<OMPTargetTeamsDirective>(*S)); + break; + case Stmt::OMPTargetTeamsDistributeDirectiveClass: + EmitOMPTargetTeamsDistributeDirective( + cast<OMPTargetTeamsDistributeDirective>(*S)); + break; + case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass: + EmitOMPTargetTeamsDistributeParallelForDirective( + cast<OMPTargetTeamsDistributeParallelForDirective>(*S)); + break; + case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass: + EmitOMPTargetTeamsDistributeParallelForSimdDirective( + cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S)); + break; + case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass: + EmitOMPTargetTeamsDistributeSimdDirective( + cast<OMPTargetTeamsDistributeSimdDirective>(*S)); + break; } } @@ -651,8 +690,10 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); EmitBlock(LoopHeader.getBlock()); + const SourceRange &R = S.getSourceRange(); LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), WhileAttrs, - Builder.getCurrentDebugLocation()); + SourceLocToDebugLoc(R.getBegin()), + SourceLocToDebugLoc(R.getEnd())); // Create an exit block for when the condition fails, which will // also become the break target. @@ -743,8 +784,10 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); + const SourceRange &R = S.getSourceRange(); LoopStack.push(LoopBody, CGM.getContext(), DoAttrs, - Builder.getCurrentDebugLocation()); + SourceLocToDebugLoc(R.getBegin()), + SourceLocToDebugLoc(R.getEnd())); EmitBlockWithFallThrough(LoopBody, &S); { @@ -796,8 +839,6 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, LexicalScope ForScope(*this, S.getSourceRange()); - llvm::DebugLoc DL = Builder.getCurrentDebugLocation(); - // Evaluate the first part before the loop. if (S.getInit()) EmitStmt(S.getInit()); @@ -809,7 +850,10 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, llvm::BasicBlock *CondBlock = Continue.getBlock(); EmitBlock(CondBlock); - LoopStack.push(CondBlock, CGM.getContext(), ForAttrs, DL); + const SourceRange &R = S.getSourceRange(); + LoopStack.push(CondBlock, CGM.getContext(), ForAttrs, + SourceLocToDebugLoc(R.getBegin()), + SourceLocToDebugLoc(R.getEnd())); // If the for loop doesn't have an increment we can just use the // condition as the continue block. Otherwise we'll need to create @@ -894,8 +938,6 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, LexicalScope ForScope(*this, S.getSourceRange()); - llvm::DebugLoc DL = Builder.getCurrentDebugLocation(); - // Evaluate the first pieces before the loop. EmitStmt(S.getRangeStmt()); EmitStmt(S.getBeginStmt()); @@ -907,7 +949,10 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, llvm::BasicBlock *CondBlock = createBasicBlock("for.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock, CGM.getContext(), ForAttrs, DL); + const SourceRange &R = S.getSourceRange(); + LoopStack.push(CondBlock, CGM.getContext(), ForAttrs, + SourceLocToDebugLoc(R.getBegin()), + SourceLocToDebugLoc(R.getEnd())); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. @@ -2085,15 +2130,6 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { Result->addAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::NoUnwind); - if (isa<MSAsmStmt>(&S)) { - // If the assembly contains any labels, mark the call noduplicate to prevent - // defining the same ASM label twice (PR23715). This is pretty hacky, but it - // works. - if (AsmString.find("__MSASMLABEL_") != std::string::npos) - Result->addAttribute(llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoDuplicate); - } - // Attach readnone and readonly attributes. if (!HasSideEffect) { if (ReadNone) @@ -2189,7 +2225,7 @@ LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) { auto VAT = CurField->getCapturedVLAType(); EmitStoreThroughLValue(RValue::get(VLASizeMap[VAT->getSizeExpr()]), LV); } else { - EmitInitializerForField(*CurField, LV, *I, None); + EmitInitializerForField(*CurField, LV, *I); } } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp index d214340..39e1cdf 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -188,7 +188,7 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, auto *RefVal = TmpAddr.getPointer(); TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref"); auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); - CGF.EmitScalarInit(RefVal, TmpLVal); + CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true); } return TmpAddr; @@ -271,7 +271,17 @@ CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) { // If we are capturing a pointer by copy we don't need to do anything, just // use the value that we get from the arguments. if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) { - setAddrOfLocalVar(I->getCapturedVar(), GetAddrOfLocalVar(Args[Cnt])); + const VarDecl *CurVD = I->getCapturedVar(); + Address LocalAddr = GetAddrOfLocalVar(Args[Cnt]); + // If the variable is a reference we need to materialize it here. + if (CurVD->getType()->isReferenceType()) { + Address RefAddr = CreateMemTemp(CurVD->getType(), getPointerAlign(), + ".materialized_ref"); + EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr, /*Volatile=*/false, + CurVD->getType()); + LocalAddr = RefAddr; + } + setAddrOfLocalVar(CurVD, LocalAddr); ++Cnt; ++I; continue; @@ -1294,7 +1304,9 @@ void CodeGenFunction::EmitOMPInnerLoop( // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.inner.for.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock, Builder.getCurrentDebugLocation()); + const SourceRange &R = S.getSourceRange(); + LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), + SourceLocToDebugLoc(R.getEnd())); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. @@ -1695,7 +1707,9 @@ void CodeGenFunction::EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic, // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.dispatch.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock, Builder.getCurrentDebugLocation()); + const SourceRange &R = S.getSourceRange(); + LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), + SourceLocToDebugLoc(R.getEnd())); llvm::Value *BoolCondVal = nullptr; if (!DynamicOrOrdered) { @@ -1930,6 +1944,114 @@ void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( }); } +void CodeGenFunction::EmitOMPTargetSimdDirective( + const OMPTargetSimdDirective &S) { + OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { + OMPLoopScope PreInitScope(CGF, S); + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTeamsDistributeDirective( + const OMPTeamsDistributeDirective &S) { + OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_teams_distribute, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + OMPLoopScope PreInitScope(CGF, S); + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( + const OMPTeamsDistributeSimdDirective &S) { + OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_teams_distribute_simd, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + OMPLoopScope PreInitScope(CGF, S); + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( + const OMPTeamsDistributeParallelForSimdDirective &S) { + OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_teams_distribute_parallel_for_simd, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + OMPLoopScope PreInitScope(CGF, S); + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( + const OMPTeamsDistributeParallelForDirective &S) { + OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_teams_distribute_parallel_for, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + OMPLoopScope PreInitScope(CGF, S); + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTargetTeamsDirective( + const OMPTargetTeamsDirective &S) { + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_teams, [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( + const OMPTargetTeamsDistributeDirective &S) { + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_teams_distribute, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( + const OMPTargetTeamsDistributeParallelForDirective &S) { + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_teams_distribute_parallel_for, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( + const OMPTargetTeamsDistributeParallelForSimdDirective &S) { + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_teams_distribute_parallel_for_simd, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + +void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( + const OMPTargetTeamsDistributeSimdDirective &S) { + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_teams_distribute_simd, + [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }); +} + /// \brief Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { @@ -2167,7 +2289,7 @@ static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, llvm::Value *Init = nullptr) { auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); if (Init) - CGF.EmitScalarInit(Init, LVal); + CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); return LVal; } @@ -2451,10 +2573,8 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, } // Check if the task has 'priority' clause. if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) { - // Runtime currently does not support codegen for priority clause argument. - // TODO: Add codegen for priority clause arg when runtime lib support it. auto *Prio = Clause->getPriority(); - Data.Priority.setInt(Prio); + Data.Priority.setInt(/*IntVal=*/true); Data.Priority.setPointer(EmitScalarConversion( EmitScalarExpr(Prio), Prio->getType(), getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), @@ -2660,6 +2780,7 @@ void CodeGenFunction::EmitOMPDistributeLoop(const OMPDistributeDirective &S) { auto &RT = CGM.getOpenMPRuntime(); + bool HasLastprivateClause = false; // Check pre-condition. { OMPLoopScope PreInitScope(*this, S); @@ -2693,6 +2814,16 @@ void CodeGenFunction::EmitOMPDistributeLoop(const OMPDistributeDirective &S) { EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); OMPPrivateScope LoopScope(*this); + if (EmitOMPFirstprivateClause(S, LoopScope)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables and post-update of + // lastprivate variables. + CGM.getOpenMPRuntime().emitBarrierCall( + *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, + /*ForceSimpleCall=*/true); + } + EmitOMPPrivateClause(S, LoopScope); + HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); EmitOMPPrivateLoopCounters(S, LoopScope); (void)LoopScope.Privatize(); @@ -2749,6 +2880,13 @@ void CodeGenFunction::EmitOMPDistributeLoop(const OMPDistributeDirective &S) { LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), Chunk); } + + // Emit final copy of the lastprivate variables if IsLastIter != 0. + if (HasLastprivateClause) + EmitOMPLastprivateClauseFinal( + S, /*NoFinals=*/false, + Builder.CreateIsNotNull( + EmitLoadOfScalar(IL, S.getLocStart()))); } // We're now done with the loop, so jump to the continuation block. @@ -3368,7 +3506,7 @@ static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, } void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) { - // Emit parallel region as a standalone region. + // Emit teams region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); @@ -3410,22 +3548,137 @@ CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) { return OMPCancelStack.getExitBlock(); } +void CodeGenFunction::EmitOMPUseDevicePtrClause( + const OMPClause &NC, OMPPrivateScope &PrivateScope, + const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) { + const auto &C = cast<OMPUseDevicePtrClause>(NC); + auto OrigVarIt = C.varlist_begin(); + auto InitIt = C.inits().begin(); + for (auto PvtVarIt : C.private_copies()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl()); + auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl()); + auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl()); + + // In order to identify the right initializer we need to match the + // declaration used by the mapping logic. In some cases we may get + // OMPCapturedExprDecl that refers to the original declaration. + const ValueDecl *MatchingVD = OrigVD; + if (auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) { + // OMPCapturedExprDecl are used to privative fields of the current + // structure. + auto *ME = cast<MemberExpr>(OED->getInit()); + assert(isa<CXXThisExpr>(ME->getBase()) && + "Base should be the current struct!"); + MatchingVD = ME->getMemberDecl(); + } + + // If we don't have information about the current list item, move on to + // the next one. + auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD); + if (InitAddrIt == CaptureDeviceAddrMap.end()) + continue; + + bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { + // Initialize the temporary initialization variable with the address we + // get from the runtime library. We have to cast the source address + // because it is always a void *. References are materialized in the + // privatization scope, so the initialization here disregards the fact + // the original variable is a reference. + QualType AddrQTy = + getContext().getPointerType(OrigVD->getType().getNonReferenceType()); + llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy); + Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy); + setAddrOfLocalVar(InitVD, InitAddr); + + // Emit private declaration, it will be initialized by the value we + // declaration we just added to the local declarations map. + EmitDecl(*PvtVD); + + // The initialization variables reached its purpose in the emission + // ofthe previous declaration, so we don't need it anymore. + LocalDeclMap.erase(InitVD); + + // Return the address of the private variable. + return GetAddrOfLocalVar(PvtVD); + }); + assert(IsRegistered && "firstprivate var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + + ++OrigVarIt; + ++InitIt; + } +} + // Generate the instructions for '#pragma omp target data' directive. void CodeGenFunction::EmitOMPTargetDataDirective( const OMPTargetDataDirective &S) { - // The target data enclosed region is implemented just by emitting the - // statement. - auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { - CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true); + + // Create a pre/post action to signal the privatization of the device pointer. + // This action can be replaced by the OpenMP runtime code generation to + // deactivate privatization. + bool PrivatizeDevicePointers = false; + class DevicePointerPrivActionTy : public PrePostActionTy { + bool &PrivatizeDevicePointers; + + public: + explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers) + : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {} + void Enter(CodeGenFunction &CGF) override { + PrivatizeDevicePointers = true; + } + }; + DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers); + + auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers]( + CodeGenFunction &CGF, PrePostActionTy &Action) { + auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + }; + + // Codegen that selects wheather to generate the privatization code or not. + auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers, + &InnermostCodeGen](CodeGenFunction &CGF, + PrePostActionTy &Action) { + RegionCodeGenTy RCG(InnermostCodeGen); + PrivatizeDevicePointers = false; + + // Call the pre-action to change the status of PrivatizeDevicePointers if + // needed. + Action.Enter(CGF); + + if (PrivatizeDevicePointers) { + OMPPrivateScope PrivateScope(CGF); + // Emit all instances of the use_device_ptr clause. + for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>()) + CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, + Info.CaptureDeviceAddrMap); + (void)PrivateScope.Privatize(); + RCG(CGF); + } else + RCG(CGF); + }; + + // Forward the provided action to the privatization codegen. + RegionCodeGenTy PrivRCG(PrivCodeGen); + PrivRCG.setAction(Action); + + // Notwithstanding the body of the region is emitted as inlined directive, + // we don't use an inline scope as changes in the references inside the + // region are expected to be visible outside, so we do not privative them. + OMPLexicalScope Scope(CGF, S); + CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, + PrivRCG); }; + RegionCodeGenTy RCG(CodeGen); + // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) { - OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); - - CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_data, - CodeGen); + RCG(*this); return; } @@ -3439,7 +3692,12 @@ void CodeGenFunction::EmitOMPTargetDataDirective( if (auto *C = S.getSingleClause<OMPDeviceClause>()) Device = C->getDevice(); - CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, CodeGen); + // Set the action to signal privatization of device pointers. + RCG.setAction(PrivAction); + + // Emit region code. + CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG, + Info); } void CodeGenFunction::EmitOMPTargetEnterDataDirective( diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp index 5b90ee6..92fd93b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp @@ -23,7 +23,7 @@ GetAddrOfVTTVTable(CodeGenVTables &CGVT, CodeGenModule &CGM, const CXXRecordDecl *MostDerivedClass, const VTTVTable &VTable, llvm::GlobalVariable::LinkageTypes Linkage, - llvm::DenseMap<BaseSubobject, uint64_t> &AddressPoints) { + VTableLayout::AddressPointsMapTy &AddressPoints) { if (VTable.getBase() == MostDerivedClass) { assert(VTable.getBaseOffset().isZero() && "Most derived class vtable must have a zero offset!"); @@ -62,25 +62,27 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, *e = Builder.getVTTComponents().end(); i != e; ++i) { const VTTVTable &VTTVT = Builder.getVTTVTables()[i->VTableIndex]; llvm::GlobalVariable *VTable = VTables[i->VTableIndex]; - uint64_t AddressPoint; + VTableLayout::AddressPointLocation AddressPoint; if (VTTVT.getBase() == RD) { // Just get the address point for the regular vtable. AddressPoint = getItaniumVTableContext().getVTableLayout(RD).getAddressPoint( i->VTableBase); - assert(AddressPoint != 0 && "Did not find vtable address point!"); } else { AddressPoint = VTableAddressPoints[i->VTableIndex].lookup(i->VTableBase); - assert(AddressPoint != 0 && "Did not find ctor vtable address point!"); + assert(AddressPoint.AddressPointIndex != 0 && + "Did not find ctor vtable address point!"); } llvm::Value *Idxs[] = { llvm::ConstantInt::get(Int32Ty, 0), - llvm::ConstantInt::get(Int32Ty, AddressPoint) + llvm::ConstantInt::get(Int32Ty, AddressPoint.VTableIndex), + llvm::ConstantInt::get(Int32Ty, AddressPoint.AddressPointIndex), }; - llvm::Constant *Init = llvm::ConstantExpr::getInBoundsGetElementPtr( - VTable->getValueType(), VTable, Idxs); + llvm::Constant *Init = llvm::ConstantExpr::getGetElementPtr( + VTable->getValueType(), VTable, Idxs, /*InBounds=*/true, + /*InRangeIndex=*/1); Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp index 9570550..1a09830 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp @@ -11,16 +11,14 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenFunction.h" #include "CGCXXABI.h" +#include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantBuilder.h" #include "clang/AST/CXXInheritance.h" #include "clang/AST/RecordLayout.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Format.h" #include "llvm/Transforms/Utils/Cloning.h" #include <algorithm> @@ -32,7 +30,7 @@ using namespace CodeGen; CodeGenVTables::CodeGenVTables(CodeGenModule &CGM) : CGM(CGM), VTContext(CGM.getContext().getVTableContext()) {} -llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD, +llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD, const ThunkInfo &Thunk) { const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); @@ -96,7 +94,7 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF, AdjustNull = CGF.createBasicBlock("adjust.null"); AdjustNotNull = CGF.createBasicBlock("adjust.notnull"); AdjustEnd = CGF.createBasicBlock("adjust.end"); - + llvm::Value *IsNull = CGF.Builder.CreateIsNull(ReturnValue); CGF.Builder.CreateCondBr(IsNull, AdjustNull, AdjustNotNull); CGF.EmitBlock(AdjustNotNull); @@ -113,14 +111,14 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF, CGF.EmitBlock(AdjustNull); CGF.Builder.CreateBr(AdjustEnd); CGF.EmitBlock(AdjustEnd); - + llvm::PHINode *PHI = CGF.Builder.CreatePHI(ReturnValue->getType(), 2); PHI->addIncoming(ReturnValue, AdjustNotNull); - PHI->addIncoming(llvm::Constant::getNullValue(ReturnValue->getType()), + PHI->addIncoming(llvm::Constant::getNullValue(ReturnValue->getType()), AdjustNull); ReturnValue = PHI; } - + return RValue::get(ReturnValue); } @@ -232,8 +230,11 @@ void CodeGenFunction::StartThunk(llvm::Function *Fn, GlobalDecl GD, CGM.getCXXABI().addImplicitStructorParams(*this, ResultType, FunctionArgs); // Start defining the function. + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(GlobalDecl(), ResultType, Fn, FnInfo, FunctionArgs, - MD->getLocation(), MD->getLocation()); + MD->getLocation()); + // Create a scope with an artificial location for the body of this function. + auto AL = ApplyDebugLocation::CreateArtificial(*this); // Since we didn't pass a GlobalDecl to StartFunction, do this ourselves. CGM.getCXXABI().EmitInstanceFunctionProlog(*this); @@ -251,7 +252,7 @@ void CodeGenFunction::FinishThunk() { FinishFunction(); } -void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, +void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Constant *CalleePtr, const ThunkInfo *Thunk) { assert(isa<CXXMethodDecl>(CurGD.getDecl()) && "Please use a new CGF for this thunk"); @@ -271,7 +272,7 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, CGM.ErrorUnsupported( MD, "non-trivial argument copy for return-adjusting thunk"); } - EmitMustTailThunk(MD, AdjustedThisPtr, Callee); + EmitMustTailThunk(MD, AdjustedThisPtr, CalleePtr); return; } @@ -285,7 +286,7 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, // Add the rest of the arguments. for (const ParmVarDecl *PD : MD->parameters()) - EmitDelegateCallArg(CallArgs, PD, PD->getLocStart()); + EmitDelegateCallArg(CallArgs, PD, SourceLocation()); const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>(); @@ -317,10 +318,11 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) Slot = ReturnValueSlot(ReturnValue, ResultType.isVolatileQualified()); - + // Now emit our call. llvm::Instruction *CallOrInvoke; - RValue RV = EmitCall(*CurFnInfo, Callee, Slot, CallArgs, MD, &CallOrInvoke); + CGCallee Callee = CGCallee::forDirect(CalleePtr, MD); + RValue RV = EmitCall(*CurFnInfo, Callee, Slot, CallArgs, &CallOrInvoke); // Consider return adjustment if we have ThunkInfo. if (Thunk && !Thunk->Return.isEmpty()) @@ -340,7 +342,7 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, void CodeGenFunction::EmitMustTailThunk(const CXXMethodDecl *MD, llvm::Value *AdjustedThisPtr, - llvm::Value *Callee) { + llvm::Value *CalleePtr) { // Emitting a musttail call thunk doesn't use any of the CGCall.cpp machinery // to translate AST arguments into LLVM IR arguments. For thunks, we know // that the caller prototype more or less matches the callee prototype with @@ -369,13 +371,14 @@ void CodeGenFunction::EmitMustTailThunk(const CXXMethodDecl *MD, // Emit the musttail call manually. Even if the prologue pushed cleanups, we // don't actually want to run them. - llvm::CallInst *Call = Builder.CreateCall(Callee, Args); + llvm::CallInst *Call = Builder.CreateCall(CalleePtr, Args); Call->setTailCallKind(llvm::CallInst::TCK_MustTail); // Apply the standard set of call attributes. unsigned CallingConv; CodeGen::AttributeListType AttributeList; - CGM.ConstructAttributeList(Callee->getName(), *CurFnInfo, MD, AttributeList, + CGM.ConstructAttributeList(CalleePtr->getName(), + *CurFnInfo, MD, AttributeList, CallingConv, /*AttrOnCallSite=*/true); llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(), AttributeList); @@ -397,11 +400,13 @@ void CodeGenFunction::generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo, GlobalDecl GD, const ThunkInfo &Thunk) { StartThunk(Fn, GD, FnInfo); + // Create a scope with an artificial location for the body of this function. + auto AL = ApplyDebugLocation::CreateArtificial(*this); // Get our callee. llvm::Type *Ty = CGM.getTypes().GetFunctionType(CGM.getTypes().arrangeGlobalDeclaration(GD)); - llvm::Value *Callee = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true); + llvm::Constant *Callee = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true); // Make the call and return the result. EmitCallAndReturnForThunk(Callee, &Thunk); @@ -436,14 +441,14 @@ void CodeGenVTables::emitThunk(GlobalDecl GD, const ThunkInfo &Thunk, // Remove the name from the old thunk function and get a new thunk. OldThunkFn->setName(StringRef()); Entry = cast<llvm::GlobalValue>(CGM.GetAddrOfThunk(GD, Thunk)); - + // If needed, replace the old thunk with a bitcast. if (!OldThunkFn->use_empty()) { llvm::Constant *NewPtrForOldDecl = llvm::ConstantExpr::getBitCast(Entry, OldThunkFn->getType()); OldThunkFn->replaceAllUsesWith(NewPtrForOldDecl); } - + // Remove the old thunk. OldThunkFn->eraseFromParent(); } @@ -503,7 +508,7 @@ void CodeGenVTables::maybeEmitThunkForVTable(GlobalDecl GD, void CodeGenVTables::EmitThunks(GlobalDecl GD) { - const CXXMethodDecl *MD = + const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl())->getCanonicalDecl(); // We don't need to generate thunks for the base destructor. @@ -520,146 +525,146 @@ void CodeGenVTables::EmitThunks(GlobalDecl GD) emitThunk(GD, Thunk, /*ForVTable=*/false); } -llvm::Constant *CodeGenVTables::CreateVTableInitializer( - const CXXRecordDecl *RD, const VTableComponent *Components, - unsigned NumComponents, const VTableLayout::VTableThunkTy *VTableThunks, - unsigned NumVTableThunks, llvm::Constant *RTTI) { - SmallVector<llvm::Constant *, 64> Inits; +void CodeGenVTables::addVTableComponent( + ConstantArrayBuilder &builder, const VTableLayout &layout, + unsigned idx, llvm::Constant *rtti, unsigned &nextVTableThunkIndex) { + auto &component = layout.vtable_components()[idx]; - llvm::Type *Int8PtrTy = CGM.Int8PtrTy; - - llvm::Type *PtrDiffTy = - CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()); + auto addOffsetConstant = [&](CharUnits offset) { + builder.add(llvm::ConstantExpr::getIntToPtr( + llvm::ConstantInt::get(CGM.PtrDiffTy, offset.getQuantity()), + CGM.Int8PtrTy)); + }; - unsigned NextVTableThunkIndex = 0; + switch (component.getKind()) { + case VTableComponent::CK_VCallOffset: + return addOffsetConstant(component.getVCallOffset()); - llvm::Constant *PureVirtualFn = nullptr, *DeletedVirtualFn = nullptr; + case VTableComponent::CK_VBaseOffset: + return addOffsetConstant(component.getVBaseOffset()); - for (unsigned I = 0; I != NumComponents; ++I) { - VTableComponent Component = Components[I]; + case VTableComponent::CK_OffsetToTop: + return addOffsetConstant(component.getOffsetToTop()); - llvm::Constant *Init = nullptr; + case VTableComponent::CK_RTTI: + return builder.add(llvm::ConstantExpr::getBitCast(rtti, CGM.Int8PtrTy)); - switch (Component.getKind()) { - case VTableComponent::CK_VCallOffset: - Init = llvm::ConstantInt::get(PtrDiffTy, - Component.getVCallOffset().getQuantity()); - Init = llvm::ConstantExpr::getIntToPtr(Init, Int8PtrTy); - break; - case VTableComponent::CK_VBaseOffset: - Init = llvm::ConstantInt::get(PtrDiffTy, - Component.getVBaseOffset().getQuantity()); - Init = llvm::ConstantExpr::getIntToPtr(Init, Int8PtrTy); + case VTableComponent::CK_FunctionPointer: + case VTableComponent::CK_CompleteDtorPointer: + case VTableComponent::CK_DeletingDtorPointer: { + GlobalDecl GD; + + // Get the right global decl. + switch (component.getKind()) { + default: + llvm_unreachable("Unexpected vtable component kind"); + case VTableComponent::CK_FunctionPointer: + GD = component.getFunctionDecl(); break; - case VTableComponent::CK_OffsetToTop: - Init = llvm::ConstantInt::get(PtrDiffTy, - Component.getOffsetToTop().getQuantity()); - Init = llvm::ConstantExpr::getIntToPtr(Init, Int8PtrTy); + case VTableComponent::CK_CompleteDtorPointer: + GD = GlobalDecl(component.getDestructorDecl(), Dtor_Complete); break; - case VTableComponent::CK_RTTI: - Init = llvm::ConstantExpr::getBitCast(RTTI, Int8PtrTy); + case VTableComponent::CK_DeletingDtorPointer: + GD = GlobalDecl(component.getDestructorDecl(), Dtor_Deleting); break; - case VTableComponent::CK_FunctionPointer: - case VTableComponent::CK_CompleteDtorPointer: - case VTableComponent::CK_DeletingDtorPointer: { - GlobalDecl GD; - - // Get the right global decl. - switch (Component.getKind()) { - default: - llvm_unreachable("Unexpected vtable component kind"); - case VTableComponent::CK_FunctionPointer: - GD = Component.getFunctionDecl(); - break; - case VTableComponent::CK_CompleteDtorPointer: - GD = GlobalDecl(Component.getDestructorDecl(), Dtor_Complete); - break; - case VTableComponent::CK_DeletingDtorPointer: - GD = GlobalDecl(Component.getDestructorDecl(), Dtor_Deleting); - break; - } - - if (CGM.getLangOpts().CUDA) { - // Emit NULL for methods we can't codegen on this - // side. Otherwise we'd end up with vtable with unresolved - // references. - const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); - // OK on device side: functions w/ __device__ attribute - // OK on host side: anything except __device__-only functions. - bool CanEmitMethod = CGM.getLangOpts().CUDAIsDevice - ? MD->hasAttr<CUDADeviceAttr>() - : (MD->hasAttr<CUDAHostAttr>() || - !MD->hasAttr<CUDADeviceAttr>()); - if (!CanEmitMethod) { - Init = llvm::ConstantExpr::getNullValue(Int8PtrTy); - break; - } - // Method is acceptable, continue processing as usual. - } + } - if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) { - // We have a pure virtual member function. - if (!PureVirtualFn) { - llvm::FunctionType *Ty = - llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); - StringRef PureCallName = CGM.getCXXABI().GetPureVirtualCallName(); - PureVirtualFn = CGM.CreateRuntimeFunction(Ty, PureCallName); - if (auto *F = dyn_cast<llvm::Function>(PureVirtualFn)) - F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - PureVirtualFn = llvm::ConstantExpr::getBitCast(PureVirtualFn, - CGM.Int8PtrTy); - } - Init = PureVirtualFn; - } else if (cast<CXXMethodDecl>(GD.getDecl())->isDeleted()) { - if (!DeletedVirtualFn) { - llvm::FunctionType *Ty = - llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); - StringRef DeletedCallName = - CGM.getCXXABI().GetDeletedVirtualCallName(); - DeletedVirtualFn = CGM.CreateRuntimeFunction(Ty, DeletedCallName); - if (auto *F = dyn_cast<llvm::Function>(DeletedVirtualFn)) - F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - DeletedVirtualFn = llvm::ConstantExpr::getBitCast(DeletedVirtualFn, - CGM.Int8PtrTy); - } - Init = DeletedVirtualFn; - } else { - // Check if we should use a thunk. - if (NextVTableThunkIndex < NumVTableThunks && - VTableThunks[NextVTableThunkIndex].first == I) { - const ThunkInfo &Thunk = VTableThunks[NextVTableThunkIndex].second; - - maybeEmitThunkForVTable(GD, Thunk); - Init = CGM.GetAddrOfThunk(GD, Thunk); - - NextVTableThunkIndex++; - } else { - llvm::Type *Ty = CGM.getTypes().GetFunctionTypeForVTable(GD); - - Init = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true); - } - - Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy); - } - break; + if (CGM.getLangOpts().CUDA) { + // Emit NULL for methods we can't codegen on this + // side. Otherwise we'd end up with vtable with unresolved + // references. + const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); + // OK on device side: functions w/ __device__ attribute + // OK on host side: anything except __device__-only functions. + bool CanEmitMethod = + CGM.getLangOpts().CUDAIsDevice + ? MD->hasAttr<CUDADeviceAttr>() + : (MD->hasAttr<CUDAHostAttr>() || !MD->hasAttr<CUDADeviceAttr>()); + if (!CanEmitMethod) + return builder.addNullPointer(CGM.Int8PtrTy); + // Method is acceptable, continue processing as usual. } - case VTableComponent::CK_UnusedFunctionPointer: - Init = llvm::ConstantExpr::getNullValue(Int8PtrTy); - break; + auto getSpecialVirtualFn = [&](StringRef name) { + llvm::FunctionType *fnTy = + llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); + llvm::Constant *fn = CGM.CreateRuntimeFunction(fnTy, name); + if (auto f = dyn_cast<llvm::Function>(fn)) + f->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); + return llvm::ConstantExpr::getBitCast(fn, CGM.Int8PtrTy); }; - - Inits.push_back(Init); + + llvm::Constant *fnPtr; + + // Pure virtual member functions. + if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) { + if (!PureVirtualFn) + PureVirtualFn = + getSpecialVirtualFn(CGM.getCXXABI().GetPureVirtualCallName()); + fnPtr = PureVirtualFn; + + // Deleted virtual member functions. + } else if (cast<CXXMethodDecl>(GD.getDecl())->isDeleted()) { + if (!DeletedVirtualFn) + DeletedVirtualFn = + getSpecialVirtualFn(CGM.getCXXABI().GetDeletedVirtualCallName()); + fnPtr = DeletedVirtualFn; + + // Thunks. + } else if (nextVTableThunkIndex < layout.vtable_thunks().size() && + layout.vtable_thunks()[nextVTableThunkIndex].first == idx) { + auto &thunkInfo = layout.vtable_thunks()[nextVTableThunkIndex].second; + + maybeEmitThunkForVTable(GD, thunkInfo); + nextVTableThunkIndex++; + fnPtr = CGM.GetAddrOfThunk(GD, thunkInfo); + + // Otherwise we can use the method definition directly. + } else { + llvm::Type *fnTy = CGM.getTypes().GetFunctionTypeForVTable(GD); + fnPtr = CGM.GetAddrOfFunction(GD, fnTy, /*ForVTable=*/true); + } + + fnPtr = llvm::ConstantExpr::getBitCast(fnPtr, CGM.Int8PtrTy); + builder.add(fnPtr); + return; + } + + case VTableComponent::CK_UnusedFunctionPointer: + return builder.addNullPointer(CGM.Int8PtrTy); + } + + llvm_unreachable("Unexpected vtable component kind"); +} + +llvm::Type *CodeGenVTables::getVTableType(const VTableLayout &layout) { + SmallVector<llvm::Type *, 4> tys; + for (unsigned i = 0, e = layout.getNumVTables(); i != e; ++i) { + tys.push_back(llvm::ArrayType::get(CGM.Int8PtrTy, layout.getVTableSize(i))); + } + + return llvm::StructType::get(CGM.getLLVMContext(), tys); +} + +void CodeGenVTables::createVTableInitializer(ConstantStructBuilder &builder, + const VTableLayout &layout, + llvm::Constant *rtti) { + unsigned nextVTableThunkIndex = 0; + for (unsigned i = 0, e = layout.getNumVTables(); i != e; ++i) { + auto vtableElem = builder.beginArray(CGM.Int8PtrTy); + size_t thisIndex = layout.getVTableOffset(i); + size_t nextIndex = thisIndex + layout.getVTableSize(i); + for (unsigned i = thisIndex; i != nextIndex; ++i) { + addVTableComponent(vtableElem, layout, i, rtti, nextVTableThunkIndex); + } + vtableElem.finishAndAddTo(builder); } - - llvm::ArrayType *ArrayType = llvm::ArrayType::get(Int8PtrTy, NumComponents); - return llvm::ConstantArray::get(ArrayType, Inits); } llvm::GlobalVariable * -CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, - const BaseSubobject &Base, - bool BaseIsVirtual, +CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, + const BaseSubobject &Base, + bool BaseIsVirtual, llvm::GlobalVariable::LinkageTypes Linkage, VTableAddressPointsMapTy& AddressPoints) { if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) @@ -680,8 +685,7 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, Base.getBase(), Out); StringRef Name = OutName.str(); - llvm::ArrayType *ArrayType = - llvm::ArrayType::get(CGM.Int8PtrTy, VTLayout->getNumVTableComponents()); + llvm::Type *VTType = getVTableType(*VTLayout); // Construction vtable symbols are not part of the Itanium ABI, so we cannot // guarantee that they actually will be available externally. Instead, when @@ -692,8 +696,8 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, Linkage = llvm::GlobalVariable::InternalLinkage; // Create the variable that will hold the construction vtable. - llvm::GlobalVariable *VTable = - CGM.CreateOrReplaceCXXRuntimeVariable(Name, ArrayType, Linkage); + llvm::GlobalVariable *VTable = + CGM.CreateOrReplaceCXXRuntimeVariable(Name, VTType, Linkage); CGM.setGlobalVisibility(VTable, RD); // V-tables are always unnamed_addr. @@ -703,12 +707,11 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, CGM.getContext().getTagDeclType(Base.getBase())); // Create and set the initializer. - llvm::Constant *Init = CreateVTableInitializer( - Base.getBase(), VTLayout->vtable_component_begin(), - VTLayout->getNumVTableComponents(), VTLayout->vtable_thunk_begin(), - VTLayout->getNumVTableThunks(), RTTI); - VTable->setInitializer(Init); - + ConstantInitBuilder builder(CGM); + auto components = builder.beginStruct(); + createVTableInitializer(components, *VTLayout, RTTI); + components.finishAndSetAsInitializer(VTable); + CGM.EmitVTableTypeMetadata(VTable, *VTLayout.get()); return VTable; @@ -723,7 +726,7 @@ static bool shouldEmitAvailableExternallyVTable(const CodeGenModule &CGM, /// Compute the required linkage of the vtable for the given class. /// /// Note that we only call this at the end of the translation unit. -llvm::GlobalVariable::LinkageTypes +llvm::GlobalVariable::LinkageTypes CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { if (!RD->isExternallyVisible()) return llvm::GlobalVariable::InternalLinkage; @@ -737,7 +740,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { const FunctionDecl *def = nullptr; if (keyFunction->hasBody(def)) keyFunction = cast<CXXMethodDecl>(def); - + switch (keyFunction->getTemplateSpecializationKind()) { case TSK_Undeclared: case TSK_ExplicitSpecialization: @@ -751,7 +754,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { return !Context.getLangOpts().AppleKext ? llvm::GlobalVariable::LinkOnceODRLinkage : llvm::Function::InternalLinkage; - + return llvm::GlobalVariable::ExternalLinkage; case TSK_ImplicitInstantiation: @@ -763,7 +766,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { return !Context.getLangOpts().AppleKext ? llvm::GlobalVariable::WeakODRLinkage : llvm::Function::InternalLinkage; - + case TSK_ExplicitInstantiationDeclaration: llvm_unreachable("Should not have been asked to emit this"); } @@ -819,7 +822,7 @@ void CodeGenModule::EmitVTable(CXXRecordDecl *theClass) { VTables.GenerateClassData(theClass); } -void +void CodeGenVTables::GenerateClassData(const CXXRecordDecl *RD) { if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) DI->completeClassData(RD); @@ -949,7 +952,10 @@ void CodeGenModule::EmitVTableTypeMetadata(llvm::GlobalVariable *VTable, std::vector<BSEntry> BitsetEntries; // Create a bit set entry for each address point. for (auto &&AP : VTLayout.getAddressPoints()) - BitsetEntries.push_back(std::make_pair(AP.first.getBase(), AP.second)); + BitsetEntries.push_back( + std::make_pair(AP.first.getBase(), + VTLayout.getVTableOffset(AP.second.VTableIndex) + + AP.second.AddressPointIndex)); // Sort the bit set entries for determinism. std::sort(BitsetEntries.begin(), BitsetEntries.end(), diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h index c27e54a..b92212c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h @@ -27,6 +27,8 @@ namespace clang { namespace CodeGen { class CodeGenModule; + class ConstantArrayBuilder; + class ConstantStructBuilder; class CodeGenVTables { CodeGenModule &CGM; @@ -34,7 +36,7 @@ class CodeGenVTables { VTableContextBase *VTContext; /// VTableAddressPointsMapTy - Address points for a single vtable. - typedef llvm::DenseMap<BaseSubobject, uint64_t> VTableAddressPointsMapTy; + typedef VTableLayout::AddressPointsMapTy VTableAddressPointsMapTy; typedef std::pair<const CXXRecordDecl *, BaseSubobject> BaseSubobjectPairTy; typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SubVTTIndiciesMapTy; @@ -49,6 +51,12 @@ class CodeGenVTables { /// indices. SecondaryVirtualPointerIndicesMapTy SecondaryVirtualPointerIndices; + /// Cache for the pure virtual member call function. + llvm::Constant *PureVirtualFn = nullptr; + + /// Cache for the deleted virtual member call function. + llvm::Constant *DeletedVirtualFn = nullptr; + /// emitThunk - Emit a single thunk. void emitThunk(GlobalDecl GD, const ThunkInfo &Thunk, bool ForVTable); @@ -56,15 +64,17 @@ class CodeGenVTables { /// the ABI. void maybeEmitThunkForVTable(GlobalDecl GD, const ThunkInfo &Thunk); + void addVTableComponent(ConstantArrayBuilder &builder, + const VTableLayout &layout, unsigned idx, + llvm::Constant *rtti, + unsigned &nextVTableThunkIndex); + public: - /// CreateVTableInitializer - Create a vtable initializer for the given record - /// decl. - /// \param Components - The vtable components; this is really an array of - /// VTableComponents. - llvm::Constant *CreateVTableInitializer( - const CXXRecordDecl *RD, const VTableComponent *Components, - unsigned NumComponents, const VTableLayout::VTableThunkTy *VTableThunks, - unsigned NumVTableThunks, llvm::Constant *RTTI); + /// Add vtable components for the given vtable layout to the given + /// global initializer. + void createVTableInitializer(ConstantStructBuilder &builder, + const VTableLayout &layout, + llvm::Constant *rtti); CodeGenVTables(CodeGenModule &CGM); @@ -112,6 +122,11 @@ public: void GenerateClassData(const CXXRecordDecl *RD); bool isVTableExternal(const CXXRecordDecl *RD); + + /// Returns the type of a vtable with the given layout. Normally a struct of + /// arrays of pointers, with one struct element for each vtable in the vtable + /// group. + llvm::Type *getVTableType(const VTableLayout &layout); }; } // end namespace CodeGen diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp index 49738a2..5f74141 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp @@ -21,8 +21,7 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Lex/Preprocessor.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" @@ -34,6 +33,8 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/YAMLTraits.h" #include <memory> using namespace clang; using namespace llvm; @@ -43,6 +44,7 @@ namespace clang { virtual void anchor(); DiagnosticsEngine &Diags; BackendAction Action; + const HeaderSearchOptions &HeaderSearchOpts; const CodeGenOptions &CodeGenOpts; const TargetOptions &TargetOpts; const LangOptions &LangOpts; @@ -50,6 +52,12 @@ namespace clang { ASTContext *Context; Timer LLVMIRGeneration; + unsigned LLVMIRGenerationRefCount; + + /// True if we've finished generating IR. This prevents us from generating + /// additional LLVM IR after emitting output in HandleTranslationUnit. This + /// can happen when Clang plugins trigger additional AST deserialization. + bool IRGenFinished = false; std::unique_ptr<CodeGenerator> Gen; @@ -70,10 +78,11 @@ namespace clang { const SmallVectorImpl<std::pair<unsigned, llvm::Module *>> &LinkModules, std::unique_ptr<raw_pwrite_stream> OS, LLVMContext &C, CoverageSourceInfo *CoverageInfo = nullptr) - : Diags(Diags), Action(Action), CodeGenOpts(CodeGenOpts), - TargetOpts(TargetOpts), LangOpts(LangOpts), + : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts), + CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts), AsmOutStream(std::move(OS)), Context(nullptr), - LLVMIRGeneration("LLVM IR Generation Time"), + LLVMIRGeneration("irgen", "LLVM IR Generation Time"), + LLVMIRGenerationRefCount(0), Gen(CreateLLVMCodeGen(Diags, InFile, HeaderSearchOpts, PPOpts, CodeGenOpts, C, CoverageInfo)) { llvm::TimePassesIsEnabled = TimePasses; @@ -113,13 +122,20 @@ namespace clang { Context->getSourceManager(), "LLVM IR generation of declaration"); - if (llvm::TimePassesIsEnabled) - LLVMIRGeneration.startTimer(); + // Recurse. + if (llvm::TimePassesIsEnabled) { + LLVMIRGenerationRefCount += 1; + if (LLVMIRGenerationRefCount == 1) + LLVMIRGeneration.startTimer(); + } Gen->HandleTopLevelDecl(D); - if (llvm::TimePassesIsEnabled) - LLVMIRGeneration.stopTimer(); + if (llvm::TimePassesIsEnabled) { + LLVMIRGenerationRefCount -= 1; + if (LLVMIRGenerationRefCount == 0) + LLVMIRGeneration.stopTimer(); + } return true; } @@ -137,16 +153,30 @@ namespace clang { LLVMIRGeneration.stopTimer(); } + void HandleInterestingDecl(DeclGroupRef D) override { + // Ignore interesting decls from the AST reader after IRGen is finished. + if (!IRGenFinished) + HandleTopLevelDecl(D); + } + void HandleTranslationUnit(ASTContext &C) override { { PrettyStackTraceString CrashInfo("Per-file LLVM IR generation"); - if (llvm::TimePassesIsEnabled) - LLVMIRGeneration.startTimer(); + if (llvm::TimePassesIsEnabled) { + LLVMIRGenerationRefCount += 1; + if (LLVMIRGenerationRefCount == 1) + LLVMIRGeneration.startTimer(); + } Gen->HandleTranslationUnit(C); - if (llvm::TimePassesIsEnabled) - LLVMIRGeneration.stopTimer(); + if (llvm::TimePassesIsEnabled) { + LLVMIRGenerationRefCount -= 1; + if (LLVMIRGenerationRefCount == 0) + LLVMIRGeneration.stopTimer(); + } + + IRGenFinished = true; } // Silently ignore if we weren't initialized for some reason. @@ -165,6 +195,26 @@ namespace clang { Ctx.getDiagnosticHandler(); void *OldDiagnosticContext = Ctx.getDiagnosticContext(); Ctx.setDiagnosticHandler(DiagnosticHandler, this); + Ctx.setDiagnosticHotnessRequested(CodeGenOpts.DiagnosticsWithHotness); + + std::unique_ptr<llvm::tool_output_file> OptRecordFile; + if (!CodeGenOpts.OptRecordFile.empty()) { + std::error_code EC; + OptRecordFile = + llvm::make_unique<llvm::tool_output_file>(CodeGenOpts.OptRecordFile, + EC, sys::fs::F_None); + if (EC) { + Diags.Report(diag::err_cannot_open_file) << + CodeGenOpts.OptRecordFile << EC.message(); + return; + } + + Ctx.setDiagnosticsOutputFile( + llvm::make_unique<yaml::Output>(OptRecordFile->os())); + + if (CodeGenOpts.getProfileUse() != CodeGenOptions::ProfileNone) + Ctx.setDiagnosticHotnessRequested(true); + } // Link LinkModule into this module if present, preserving its validity. for (auto &I : LinkModules) { @@ -176,13 +226,16 @@ namespace clang { EmbedBitcode(getModule(), CodeGenOpts, llvm::MemoryBufferRef()); - EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, - C.getTargetInfo().getDataLayout(), + EmitBackendOutput(Diags, HeaderSearchOpts, CodeGenOpts, TargetOpts, + LangOpts, C.getTargetInfo().getDataLayout(), getModule(), Action, std::move(AsmOutStream)); Ctx.setInlineAsmDiagnosticHandler(OldHandler, OldContext); Ctx.setDiagnosticHandler(OldDiagnosticHandler, OldDiagnosticContext); + + if (OptRecordFile) + OptRecordFile->keep(); } void HandleTagDeclDefinition(TagDecl *D) override { @@ -245,16 +298,13 @@ namespace clang { /// them. void EmitOptimizationMessage(const llvm::DiagnosticInfoOptimizationBase &D, unsigned DiagID); - void - OptimizationRemarkHandler(const llvm::DiagnosticInfoOptimizationRemark &D); - void OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkMissed &D); - void OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkAnalysis &D); + void OptimizationRemarkHandler(const llvm::OptimizationRemark &D); + void OptimizationRemarkHandler(const llvm::OptimizationRemarkMissed &D); + void OptimizationRemarkHandler(const llvm::OptimizationRemarkAnalysis &D); void OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkAnalysisFPCommute &D); + const llvm::OptimizationRemarkAnalysisFPCommute &D); void OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkAnalysisAliasing &D); + const llvm::OptimizationRemarkAnalysisAliasing &D); void OptimizationFailureHandler( const llvm::DiagnosticInfoOptimizationFailure &D); }; @@ -497,9 +547,16 @@ void BackendConsumer::EmitOptimizationMessage( FullSourceLoc Loc = getBestLocationFromDebugLoc(D, BadDebugInfo, Filename, Line, Column); + std::string Msg; + raw_string_ostream MsgStream(Msg); + MsgStream << D.getMsg(); + + if (D.getHotness()) + MsgStream << " (hotness: " << *D.getHotness() << ")"; + Diags.Report(Loc, DiagID) - << AddFlagValue(D.getPassName() ? D.getPassName() : "") - << D.getMsg().str(); + << AddFlagValue(D.getPassName()) + << MsgStream.str(); if (BadDebugInfo) // If we were not able to translate the file:line:col information @@ -511,7 +568,7 @@ void BackendConsumer::EmitOptimizationMessage( } void BackendConsumer::OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemark &D) { + const llvm::OptimizationRemark &D) { // Optimization remarks are active only if the -Rpass flag has a regular // expression that matches the name of the pass name in \p D. if (CodeGenOpts.OptimizationRemarkPattern && @@ -520,7 +577,7 @@ void BackendConsumer::OptimizationRemarkHandler( } void BackendConsumer::OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkMissed &D) { + const llvm::OptimizationRemarkMissed &D) { // Missed optimization remarks are active only if the -Rpass-missed // flag has a regular expression that matches the name of the pass // name in \p D. @@ -531,7 +588,7 @@ void BackendConsumer::OptimizationRemarkHandler( } void BackendConsumer::OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkAnalysis &D) { + const llvm::OptimizationRemarkAnalysis &D) { // Optimization analysis remarks are active if the pass name is set to // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a // regular expression that matches the name of the pass name in \p D. @@ -544,7 +601,7 @@ void BackendConsumer::OptimizationRemarkHandler( } void BackendConsumer::OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkAnalysisFPCommute &D) { + const llvm::OptimizationRemarkAnalysisFPCommute &D) { // Optimization analysis remarks are active if the pass name is set to // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a // regular expression that matches the name of the pass name in \p D. @@ -557,7 +614,7 @@ void BackendConsumer::OptimizationRemarkHandler( } void BackendConsumer::OptimizationRemarkHandler( - const llvm::DiagnosticInfoOptimizationRemarkAnalysisAliasing &D) { + const llvm::OptimizationRemarkAnalysisAliasing &D) { // Optimization analysis remarks are active if the pass name is set to // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a // regular expression that matches the name of the pass name in \p D. @@ -601,30 +658,27 @@ void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) { case llvm::DK_OptimizationRemark: // Optimization remarks are always handled completely by this // handler. There is no generic way of emitting them. - OptimizationRemarkHandler(cast<DiagnosticInfoOptimizationRemark>(DI)); + OptimizationRemarkHandler(cast<OptimizationRemark>(DI)); return; case llvm::DK_OptimizationRemarkMissed: // Optimization remarks are always handled completely by this // handler. There is no generic way of emitting them. - OptimizationRemarkHandler(cast<DiagnosticInfoOptimizationRemarkMissed>(DI)); + OptimizationRemarkHandler(cast<OptimizationRemarkMissed>(DI)); return; case llvm::DK_OptimizationRemarkAnalysis: // Optimization remarks are always handled completely by this // handler. There is no generic way of emitting them. - OptimizationRemarkHandler( - cast<DiagnosticInfoOptimizationRemarkAnalysis>(DI)); + OptimizationRemarkHandler(cast<OptimizationRemarkAnalysis>(DI)); return; case llvm::DK_OptimizationRemarkAnalysisFPCommute: // Optimization remarks are always handled completely by this // handler. There is no generic way of emitting them. - OptimizationRemarkHandler( - cast<DiagnosticInfoOptimizationRemarkAnalysisFPCommute>(DI)); + OptimizationRemarkHandler(cast<OptimizationRemarkAnalysisFPCommute>(DI)); return; case llvm::DK_OptimizationRemarkAnalysisAliasing: // Optimization remarks are always handled completely by this // handler. There is no generic way of emitting them. - OptimizationRemarkHandler( - cast<DiagnosticInfoOptimizationRemarkAnalysisAliasing>(DI)); + OptimizationRemarkHandler(cast<OptimizationRemarkAnalysisAliasing>(DI)); return; case llvm::DK_OptimizationFailure: // Optimization failures are always handled completely by this @@ -732,11 +786,13 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { return nullptr; } - ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr = - getLazyBitcodeModule(std::move(*BCBuf), *VMContext); - if (std::error_code EC = ModuleOrErr.getError()) { - CI.getDiagnostics().Report(diag::err_cannot_open_file) << LinkBCFile - << EC.message(); + Expected<std::unique_ptr<llvm::Module>> ModuleOrErr = + getOwningLazyBitcodeModule(std::move(*BCBuf), *VMContext); + if (!ModuleOrErr) { + handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) { + CI.getDiagnostics().Report(diag::err_cannot_open_file) + << LinkBCFile << EIB.message(); + }); LinkModules.clear(); return nullptr; } @@ -843,9 +899,10 @@ void CodeGenAction::ExecuteAction() { Ctx.setInlineAsmDiagnosticHandler(BitcodeInlineAsmDiagHandler, &CI.getDiagnostics()); - EmitBackendOutput(CI.getDiagnostics(), CI.getCodeGenOpts(), TargetOpts, - CI.getLangOpts(), CI.getTarget().getDataLayout(), - TheModule.get(), BA, std::move(OS)); + EmitBackendOutput(CI.getDiagnostics(), CI.getHeaderSearchOpts(), + CI.getCodeGenOpts(), TargetOpts, CI.getLangOpts(), + CI.getTarget().getDataLayout(), TheModule.get(), BA, + std::move(OS)); return; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp index 11e4ad9..e142a21 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp @@ -38,20 +38,38 @@ using namespace clang; using namespace CodeGen; +/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time +/// markers. +static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts, + const LangOptions &LangOpts) { + if (CGOpts.DisableLifetimeMarkers) + return false; + + // Asan uses markers for use-after-scope checks. + if (CGOpts.SanitizeAddressUseAfterScope) + return true; + + // Disable lifetime markers in msan builds. + // FIXME: Remove this when msan works with lifetime markers. + if (LangOpts.Sanitize.has(SanitizerKind::Memory)) + return false; + + // For now, only in optimized builds. + return CGOpts.OptimizationLevel != 0; +} + CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()), Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(), CGBuilderInserterTy(this)), CurFn(nullptr), ReturnValue(Address::invalid()), - CapturedStmtInfo(nullptr), - SanOpts(CGM.getLangOpts().Sanitize), IsSanitizerScope(false), - CurFuncIsThunk(false), AutoreleaseResult(false), SawAsmBlock(false), - IsOutlinedSEHHelper(false), - BlockInfo(nullptr), BlockPointer(nullptr), - LambdaThisCaptureField(nullptr), NormalCleanupDest(nullptr), - NextCleanupDestIndex(1), FirstBlockInfo(nullptr), EHResumeBlock(nullptr), - ExceptionSlot(nullptr), EHSelectorSlot(nullptr), - DebugInfo(CGM.getModuleDebugInfo()), + CapturedStmtInfo(nullptr), SanOpts(CGM.getLangOpts().Sanitize), + IsSanitizerScope(false), CurFuncIsThunk(false), AutoreleaseResult(false), + SawAsmBlock(false), IsOutlinedSEHHelper(false), BlockInfo(nullptr), + BlockPointer(nullptr), LambdaThisCaptureField(nullptr), + NormalCleanupDest(nullptr), NextCleanupDestIndex(1), + FirstBlockInfo(nullptr), EHResumeBlock(nullptr), ExceptionSlot(nullptr), + EHSelectorSlot(nullptr), DebugInfo(CGM.getModuleDebugInfo()), DisableDebugInfo(false), DidCallStackSave(false), IndirectBranch(nullptr), PGO(cgm), SwitchInsn(nullptr), SwitchWeights(nullptr), CaseRangeBlock(nullptr), UnreachableBlock(nullptr), NumReturnExprs(0), @@ -60,7 +78,9 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) CXXStructorImplicitParamDecl(nullptr), CXXStructorImplicitParamValue(nullptr), OutermostConditional(nullptr), CurLexicalScope(nullptr), TerminateLandingPad(nullptr), - TerminateHandler(nullptr), TrapBB(nullptr) { + TerminateHandler(nullptr), TrapBB(nullptr), + ShouldEmitLifetimeMarkers( + shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) { if (!suppressNewContext) CGM.getCXXABI().getMangleContext().startNewFunction(); @@ -92,9 +112,8 @@ CodeGenFunction::~CodeGenFunction() { if (FirstBlockInfo) destroyBlockInfos(FirstBlockInfo); - if (getLangOpts().OpenMP) { + if (getLangOpts().OpenMP && CurFn) CGM.getOpenMPRuntime().functionFinished(*this); - } } CharUnits CodeGenFunction::getNaturalPointeeTypeAlignment(QualType T, @@ -429,12 +448,24 @@ void CodeGenFunction::EmitFunctionInstrumentation(const char *Fn) { EmitNounwindRuntimeCall(F, args); } -void CodeGenFunction::EmitMCountInstrumentation() { - llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); - - llvm::Constant *MCountFn = - CGM.CreateRuntimeFunction(FTy, getTarget().getMCountName()); - EmitNounwindRuntimeCall(MCountFn); +static void removeImageAccessQualifier(std::string& TyName) { + std::string ReadOnlyQual("__read_only"); + std::string::size_type ReadOnlyPos = TyName.find(ReadOnlyQual); + if (ReadOnlyPos != std::string::npos) + // "+ 1" for the space after access qualifier. + TyName.erase(ReadOnlyPos, ReadOnlyQual.size() + 1); + else { + std::string WriteOnlyQual("__write_only"); + std::string::size_type WriteOnlyPos = TyName.find(WriteOnlyQual); + if (WriteOnlyPos != std::string::npos) + TyName.erase(WriteOnlyPos, WriteOnlyQual.size() + 1); + else { + std::string ReadWriteQual("__read_write"); + std::string::size_type ReadWritePos = TyName.find(ReadWriteQual); + if (ReadWritePos != std::string::npos) + TyName.erase(ReadWritePos, ReadWriteQual.size() + 1); + } + } } // Returns the address space id that should be produced to the @@ -549,8 +580,6 @@ static void GenOpenCLArgMetadata(const FunctionDecl *FD, llvm::Function *Fn, if (ty.isCanonical() && pos != std::string::npos) typeName.erase(pos+1, 8); - argTypeNames.push_back(llvm::MDString::get(Context, typeName)); - std::string baseTypeName; if (isPipe) baseTypeName = ty.getCanonicalType()->getAs<PipeType>() @@ -560,6 +589,17 @@ static void GenOpenCLArgMetadata(const FunctionDecl *FD, llvm::Function *Fn, baseTypeName = ty.getUnqualifiedType().getCanonicalType().getAsString(Policy); + // Remove access qualifiers on images + // (as they are inseparable from type in clang implementation, + // but OpenCL spec provides a special query to get access qualifier + // via clGetKernelArgInfo with CL_KERNEL_ARG_ACCESS_QUALIFIER): + if (ty->isImageType()) { + removeImageAccessQualifier(typeName); + removeImageAccessQualifier(baseTypeName); + } + + argTypeNames.push_back(llvm::MDString::get(Context, typeName)); + // Turn "unsigned type" to "utype" pos = baseTypeName.find("unsigned"); if (pos != std::string::npos) @@ -709,6 +749,20 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (SanOpts.has(SanitizerKind::SafeStack)) Fn->addFnAttr(llvm::Attribute::SafeStack); + // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize, + // .cxx_destruct and all of their calees at run time. + if (SanOpts.has(SanitizerKind::Thread)) { + if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) { + IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0); + if (OMD->getMethodFamily() == OMF_dealloc || + OMD->getMethodFamily() == OMF_initialize || + (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) { + Fn->addFnAttr("sanitize_thread_no_checking_at_run_time"); + Fn->removeFnAttr(llvm::Attribute::SanitizeThread); + } + } + } + // Apply xray attributes to the function (as a string, for now) if (D && ShouldXRayInstrumentFunction()) { if (const auto *XRayAttr = D->getAttr<XRayInstrumentAttr>()) { @@ -723,27 +777,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, } } - // Pass inline keyword to optimizer if it appears explicitly on any - // declaration. Also, in the case of -fno-inline attach NoInline - // attribute to all functions that are not marked AlwaysInline, or - // to all functions that are not marked inline or implicitly inline - // in the case of -finline-hint-functions. - if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) { - const CodeGenOptions& CodeGenOpts = CGM.getCodeGenOpts(); - if (!CodeGenOpts.NoInline) { - for (auto RI : FD->redecls()) - if (RI->isInlineSpecified()) { - Fn->addFnAttr(llvm::Attribute::InlineHint); - break; - } - if (CodeGenOpts.getInlining() == CodeGenOptions::OnlyHintInlining && - !FD->isInlined() && !Fn->hasFnAttribute(llvm::Attribute::InlineHint)) - Fn->addFnAttr(llvm::Attribute::NoInline); - } else if (!FD->hasAttr<AlwaysInlineAttr>()) - Fn->addFnAttr(llvm::Attribute::NoInline); + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) if (CGM.getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>()) CGM.getOpenMPRuntime().emitDeclareSimdFunction(FD, Fn); - } // Add no-jump-tables value. Fn->addFnAttr("no-jump-tables", @@ -778,7 +814,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) if (FD->isMain()) Fn->addFnAttr(llvm::Attribute::NoRecurse); - + llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn); // Create a marker to make it easy to insert allocas into the entryblock @@ -811,8 +847,12 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (ShouldInstrumentFunction()) EmitFunctionInstrumentation("__cyg_profile_func_enter"); + // Since emitting the mcount call here impacts optimizations such as function + // inlining, we just add an attribute to insert a mcount call in backend. + // The attribute "counting-function" is set to mcount function name which is + // architecture dependent. if (CGM.getCodeGenOpts().InstrumentForProfiling) - EmitMCountInstrumentation(); + Fn->addFnAttr("counting-function", getTarget().getMCountName()); if (RetTy->isVoidType()) { // Void type; nothing to return. @@ -1011,6 +1051,19 @@ QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD, return ResTy; } +static bool +shouldUseUndefinedBehaviorReturnOptimization(const FunctionDecl *FD, + const ASTContext &Context) { + QualType T = FD->getReturnType(); + // Avoid the optimization for functions that return a record type with a + // trivial destructor or another trivially copyable type. + if (const RecordType *RT = T.getCanonicalType()->getAs<RecordType>()) { + if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) + return !ClassDecl->hasTrivialDestructor(); + } + return !T.isTriviallyCopyableType(Context); +} + void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, const CGFunctionInfo &FnInfo) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); @@ -1040,6 +1093,13 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, if (SpecDecl->hasBody(SpecDecl)) Loc = SpecDecl->getLocation(); + Stmt *Body = FD->getBody(); + + // Initialize helper which will detect jumps which can cause invalid lifetime + // markers. + if (Body && ShouldEmitLifetimeMarkers) + Bypasses.Init(Body); + // Emit the standard function prologue. StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin()); @@ -1069,7 +1129,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, // Implicit copy-assignment gets the same special treatment as implicit // copy-constructors. emitImplicitAssignmentOperatorBody(Args); - } else if (Stmt *Body = FD->getBody()) { + } else if (Body) { EmitFunctionBody(Args, Body); } else llvm_unreachable("no definition for emitted function"); @@ -1082,17 +1142,23 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, // function call is used by the caller, the behavior is undefined. if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock && !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) { + bool ShouldEmitUnreachable = + CGM.getCodeGenOpts().StrictReturn || + shouldUseUndefinedBehaviorReturnOptimization(FD, getContext()); if (SanOpts.has(SanitizerKind::Return)) { SanitizerScope SanScope(this); llvm::Value *IsFalse = Builder.getFalse(); EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return), - "missing_return", EmitCheckSourceLocation(FD->getLocation()), - None); - } else if (CGM.getCodeGenOpts().OptimizationLevel == 0) { - EmitTrapCall(llvm::Intrinsic::trap); + SanitizerHandler::MissingReturn, + EmitCheckSourceLocation(FD->getLocation()), None); + } else if (ShouldEmitUnreachable) { + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + EmitTrapCall(llvm::Intrinsic::trap); + } + if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) { + Builder.CreateUnreachable(); + Builder.ClearInsertionPoint(); } - Builder.CreateUnreachable(); - Builder.ClearInsertionPoint(); } // Emit the standard function epilogue. @@ -1731,6 +1797,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { case Type::Enum: case Type::Elaborated: case Type::TemplateSpecialization: + case Type::ObjCTypeParam: case Type::ObjCObject: case Type::ObjCInterface: case Type::ObjCObjectPointer: @@ -1794,7 +1861,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { }; EmitCheck(std::make_pair(Builder.CreateICmpSGT(Size, Zero), SanitizerKind::VLABound), - "vla_bound_not_positive", StaticArgs, Size); + SanitizerHandler::VLABoundNotPositive, StaticArgs, Size); } // Always zexting here would be wrong if it weren't @@ -1854,8 +1921,8 @@ Address CodeGenFunction::EmitMSVAListRef(const Expr *E) { } void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E, - llvm::Constant *Init) { - assert (Init && "Invalid DeclRefExpr initializer!"); + const APValue &Init) { + assert(!Init.isUninit() && "Invalid DeclRefExpr initializer!"); if (CGDebugInfo *Dbg = getDebugInfo()) if (CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo) Dbg->EmitGlobalVariable(E->getDecl(), Init); @@ -2046,3 +2113,10 @@ void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) { IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation()); CGM.getSanStats().create(IRB, SSK); } + +llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) { + if (CGDebugInfo *DI = getDebugInfo()) + return DI->SourceLocToDebugLoc(Location); + + return llvm::DebugLoc(); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h index fb19a26..5861340 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h @@ -21,6 +21,7 @@ #include "CodeGenModule.h" #include "CodeGenPGO.h" #include "EHScopeStack.h" +#include "VarBypassDetector.h" #include "clang/AST/CharUnits.h" #include "clang/AST/ExprCXX.h" #include "clang/AST/ExprObjC.h" @@ -77,6 +78,7 @@ class ObjCAutoreleasePoolStmt; namespace CodeGen { class CodeGenTypes; +class CGCallee; class CGFunctionInfo; class CGRecordLayout; class CGBlockInfo; @@ -88,6 +90,7 @@ class BlockFieldFlags; class RegionCodeGenTy; class TargetCodeGenInfo; struct OMPTaskDataTy; +struct CGCoroData; /// The kind of evaluation to perform on values of a particular /// type. Basically, is the code in CGExprScalar, CGExprComplex, or @@ -100,6 +103,32 @@ enum TypeEvaluationKind { TEK_Aggregate }; +#define LIST_SANITIZER_CHECKS \ + SANITIZER_CHECK(AddOverflow, add_overflow, 0) \ + SANITIZER_CHECK(BuiltinUnreachable, builtin_unreachable, 0) \ + SANITIZER_CHECK(CFICheckFail, cfi_check_fail, 0) \ + SANITIZER_CHECK(DivremOverflow, divrem_overflow, 0) \ + SANITIZER_CHECK(DynamicTypeCacheMiss, dynamic_type_cache_miss, 0) \ + SANITIZER_CHECK(FloatCastOverflow, float_cast_overflow, 0) \ + SANITIZER_CHECK(FunctionTypeMismatch, function_type_mismatch, 0) \ + SANITIZER_CHECK(LoadInvalidValue, load_invalid_value, 0) \ + SANITIZER_CHECK(MissingReturn, missing_return, 0) \ + SANITIZER_CHECK(MulOverflow, mul_overflow, 0) \ + SANITIZER_CHECK(NegateOverflow, negate_overflow, 0) \ + SANITIZER_CHECK(NonnullArg, nonnull_arg, 0) \ + SANITIZER_CHECK(NonnullReturn, nonnull_return, 0) \ + SANITIZER_CHECK(OutOfBounds, out_of_bounds, 0) \ + SANITIZER_CHECK(ShiftOutOfBounds, shift_out_of_bounds, 0) \ + SANITIZER_CHECK(SubOverflow, sub_overflow, 0) \ + SANITIZER_CHECK(TypeMismatch, type_mismatch, 1) \ + SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0) + +enum SanitizerHandler { +#define SANITIZER_CHECK(Enum, Name, Version) Enum, + LIST_SANITIZER_CHECKS +#undef SANITIZER_CHECK +}; + /// CodeGenFunction - This class organizes the per-function state that is used /// while generating LLVM code. class CodeGenFunction : public CodeGenTypeCache { @@ -140,6 +169,10 @@ public: LoopInfoStack LoopStack; CGBuilderTy Builder; + // Stores variables for which we can't generate correct lifetime markers + // because of jumps. + VarBypassDetector Bypasses; + /// \brief CGBuilder insert helper. This function is called after an /// instruction is created using Builder. void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name, @@ -155,6 +188,16 @@ public: QualType FnRetTy; llvm::Function *CurFn; + // Holds coroutine data if the current function is a coroutine. We use a + // wrapper to manage its lifetime, so that we don't have to define CGCoroData + // in this header. + struct CGCoroInfo { + std::unique_ptr<CGCoroData> Data; + CGCoroInfo(); + ~CGCoroInfo(); + }; + CGCoroInfo CurCoro; + /// CurGD - The GlobalDecl for the current function being compiled. GlobalDecl CurGD; @@ -430,7 +473,7 @@ public: LifetimeExtendedCleanupStack.resize( LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size); - static_assert(sizeof(Header) % llvm::AlignOf<T>::Alignment == 0, + static_assert(sizeof(Header) % alignof(T) == 0, "Cleanup will be allocated on misaligned address"); char *Buffer = &LifetimeExtendedCleanupStack[OldSize]; new (Buffer) LifetimeExtendedCleanupHeader(Header); @@ -901,6 +944,17 @@ public: e->getCommon()); } + /// Build the opaque value mapping for an OpaqueValueExpr whose source + /// expression is set to the expression the OVE represents. + OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV) + : CGF(CGF) { + if (OV) { + assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used " + "for OVE with no source expression"); + Data = OpaqueValueMappingData::bind(CGF, OV, OV->getSourceExpr()); + } + } + OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue, LValue lvalue) @@ -1166,6 +1220,23 @@ public: CharUnits OldCXXThisAlignment; }; + /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the + /// current loop index is overridden. + class ArrayInitLoopExprScope { + public: + ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index) + : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) { + CGF.ArrayInitIndex = Index; + } + ~ArrayInitLoopExprScope() { + CGF.ArrayInitIndex = OldArrayInitIndex; + } + + private: + CodeGenFunction &CGF; + llvm::Value *OldArrayInitIndex; + }; + class InlinedInheritingConstructorScope { public: InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD) @@ -1234,6 +1305,10 @@ private: /// this expression. Address CXXDefaultInitExprThis = Address::invalid(); + /// The current array initialization index when evaluating an + /// ArrayInitIndexExpr within an ArrayInitLoopExpr. + llvm::Value *ArrayInitIndex = nullptr; + /// The values of function arguments to use when evaluating /// CXXInheritedCtorInitExprs within this context. CallArgList CXXInheritedCtorInitExprArgs; @@ -1263,6 +1338,9 @@ private: llvm::BasicBlock *TerminateHandler; llvm::BasicBlock *TrapBB; + /// True if we need emit the life-time markers. + const bool ShouldEmitLifetimeMarkers; + /// Add a kernel metadata node to the named metadata node 'opencl.kernels'. /// In the kernel metadata node, reference the kernel function and metadata /// nodes for its optional attribute qualifiers (OpenCL 1.1 6.7.2): @@ -1421,7 +1499,6 @@ public: //===--------------------------------------------------------------------===// llvm::Value *EmitBlockLiteral(const BlockExpr *); - llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info); static void destroyBlockInfos(CGBlockInfo *info); llvm::Function *GenerateBlockFunction(GlobalDecl GD, @@ -1503,7 +1580,8 @@ public: void StartThunk(llvm::Function *Fn, GlobalDecl GD, const CGFunctionInfo &FnInfo); - void EmitCallAndReturnForThunk(llvm::Value *Callee, const ThunkInfo *Thunk); + void EmitCallAndReturnForThunk(llvm::Constant *Callee, + const ThunkInfo *Thunk); void FinishThunk(); @@ -1522,8 +1600,7 @@ public: void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type, FunctionArgList &Args); - void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init, - ArrayRef<VarDecl *> ArrayIndexes); + void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init); /// Struct with all informations about dynamic [sub]class needed to set vptr. struct VPtr { @@ -1932,6 +2009,9 @@ public: return it->second; } + /// Get the index of the current ArrayInitLoopExpr, if any. + llvm::Value *getArrayInitIndex() { return ArrayInitIndex; } + /// getAccessedFieldNo - Given an encoded value and a result number, return /// the input field number being accessed. static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts); @@ -2110,7 +2190,8 @@ public: void EmitCXXDeleteExpr(const CXXDeleteExpr *E); void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr, - QualType DeleteTy); + QualType DeleteTy, llvm::Value *NumElements = nullptr, + CharUnits CookieSize = CharUnits()); RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const Expr *Arg, bool IsDelete); @@ -2179,6 +2260,10 @@ public: OffsetValue); } + /// Converts Location to a DebugLoc, if debug information is enabled. + llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location); + + //===--------------------------------------------------------------------===// // Declaration Emission //===--------------------------------------------------------------------===// @@ -2195,7 +2280,6 @@ public: void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit); - void EmitScalarInit(llvm::Value *init, LValue lvalue); typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D, llvm::Value *Address); @@ -2378,6 +2462,9 @@ public: void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S); void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S); + void EmitCoroutineBody(const CoroutineBodyStmt &S); + RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID); + void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false); void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false); @@ -2480,6 +2567,9 @@ public: OMPPrivateScope &PrivateScope); void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope); + void EmitOMPUseDevicePtrClause( + const OMPClause &C, OMPPrivateScope &PrivateScope, + const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap); /// \brief Emit code for copyin clause in \a D directive. The next code is /// generated at the start of outlined functions for directives: /// \code @@ -2596,6 +2686,23 @@ public: void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S); void EmitOMPTargetParallelForSimdDirective( const OMPTargetParallelForSimdDirective &S); + void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S); + void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S); + void + EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S); + void EmitOMPTeamsDistributeParallelForSimdDirective( + const OMPTeamsDistributeParallelForSimdDirective &S); + void EmitOMPTeamsDistributeParallelForDirective( + const OMPTeamsDistributeParallelForDirective &S); + void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S); + void EmitOMPTargetTeamsDistributeDirective( + const OMPTargetTeamsDistributeDirective &S); + void EmitOMPTargetTeamsDistributeParallelForDirective( + const OMPTargetTeamsDistributeParallelForDirective &S); + void EmitOMPTargetTeamsDistributeParallelForSimdDirective( + const OMPTargetTeamsDistributeParallelForSimdDirective &S); + void EmitOMPTargetTeamsDistributeSimdDirective( + const OMPTargetTeamsDistributeSimdDirective &S); /// Emit outlined function for the target directive. static std::pair<llvm::Function * /*OutlinedFn*/, @@ -2626,6 +2733,9 @@ public: OMPPrivateScope &LoopScope); private: + /// Helpers for blocks + llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info); + /// Helpers for the OpenMP loop directives. void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit); void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic = false); @@ -2893,7 +3003,7 @@ public: LValue EmitStmtExprLValue(const StmtExpr *E); LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E); LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E); - void EmitDeclRefExprDbgValue(const DeclRefExpr *E, llvm::Constant *Init); + void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init); //===--------------------------------------------------------------------===// // Scalar Expression Emission @@ -2902,17 +3012,17 @@ public: /// EmitCall - Generate a call of the given function, expecting the given /// result type, and using the given argument list which specifies both the /// LLVM arguments and the types they were derived from. - RValue EmitCall(const CGFunctionInfo &FnInfo, llvm::Value *Callee, + RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, - CGCalleeInfo CalleeInfo = CGCalleeInfo(), llvm::Instruction **callOrInvoke = nullptr); - RValue EmitCall(QualType FnType, llvm::Value *Callee, const CallExpr *E, + RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E, ReturnValueSlot ReturnValue, - CGCalleeInfo CalleeInfo = CGCalleeInfo(), llvm::Value *Chain = nullptr); RValue EmitCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue = ReturnValueSlot()); + RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue); + CGCallee EmitCallee(const Expr *E); void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl); @@ -2938,20 +3048,23 @@ public: void EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee, ArrayRef<llvm::Value*> args); - llvm::Value *BuildAppleKextVirtualCall(const CXXMethodDecl *MD, - NestedNameSpecifier *Qual, - llvm::Type *Ty); + CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD, + NestedNameSpecifier *Qual, + llvm::Type *Ty); - llvm::Value *BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD, - CXXDtorType Type, - const CXXRecordDecl *RD); + CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD, + CXXDtorType Type, + const CXXRecordDecl *RD); RValue - EmitCXXMemberOrOperatorCall(const CXXMethodDecl *MD, llvm::Value *Callee, + EmitCXXMemberOrOperatorCall(const CXXMethodDecl *Method, + const CGCallee &Callee, ReturnValueSlot ReturnValue, llvm::Value *This, llvm::Value *ImplicitParam, - QualType ImplicitParamTy, const CallExpr *E); - RValue EmitCXXDestructorCall(const CXXDestructorDecl *DD, llvm::Value *Callee, + QualType ImplicitParamTy, const CallExpr *E, + CallArgList *RtlArgs); + RValue EmitCXXDestructorCall(const CXXDestructorDecl *DD, + const CGCallee &Callee, llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E, StructorType Type); @@ -2974,6 +3087,7 @@ public: RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue); + RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E); RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E, ReturnValueSlot ReturnValue); @@ -3029,6 +3143,12 @@ public: llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E); +private: + enum class MSVCIntrin; + +public: + llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E); + llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E); llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E); llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E); @@ -3292,7 +3412,7 @@ public: /// sanitizer runtime with the provided arguments, and create a conditional /// branch to it. void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, - StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, + SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs); /// \brief Emit a slow path cross-DSO CFI check which calls __cfi_slowpath @@ -3400,12 +3520,22 @@ public: static bool isObjCMethodWithTypeParams(const T *) { return false; } #endif + enum class EvaluationOrder { + ///! No language constraints on evaluation order. + Default, + ///! Language semantics require left-to-right evaluation. + ForceLeftToRight, + ///! Language semantics require right-to-left evaluation. + ForceRightToLeft + }; + /// EmitCallArgs - Emit call arguments for a function. template <typename T> void EmitCallArgs(CallArgList &Args, const T *CallArgTypeInfo, llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, const FunctionDecl *CalleeDecl = nullptr, - unsigned ParamsToSkip = 0) { + unsigned ParamsToSkip = 0, + EvaluationOrder Order = EvaluationOrder::Default) { SmallVector<QualType, 16> ArgTypes; CallExpr::const_arg_iterator Arg = ArgRange.begin(); @@ -3443,15 +3573,16 @@ public: // If we still have any arguments, emit them using the type of the argument. for (auto *A : llvm::make_range(Arg, ArgRange.end())) - ArgTypes.push_back(getVarArgType(A)); + ArgTypes.push_back(CallArgTypeInfo ? getVarArgType(A) : A->getType()); - EmitCallArgs(Args, ArgTypes, ArgRange, CalleeDecl, ParamsToSkip); + EmitCallArgs(Args, ArgTypes, ArgRange, CalleeDecl, ParamsToSkip, Order); } void EmitCallArgs(CallArgList &Args, ArrayRef<QualType> ArgTypes, llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, const FunctionDecl *CalleeDecl = nullptr, - unsigned ParamsToSkip = 0); + unsigned ParamsToSkip = 0, + EvaluationOrder Order = EvaluationOrder::Default); /// EmitPointerWithAlignment - Given an expression with a pointer /// type, emit the value and compute our best estimate of the diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp index 0161cfb..3600543 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp @@ -24,6 +24,7 @@ #include "CodeGenFunction.h" #include "CodeGenPGO.h" #include "CodeGenTBAA.h" +#include "ConstantBuilder.h" #include "CoverageMappingGen.h" #include "TargetInfo.h" #include "clang/AST/ASTContext.h" @@ -43,7 +44,6 @@ #include "clang/Basic/Version.h" #include "clang/Frontend/CodeGenOptions.h" #include "clang/Sema/SemaDiagnostic.h" -#include "llvm/ADT/APSInt.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" @@ -102,10 +102,13 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, PointerWidthInBits = C.getTargetInfo().getPointerWidth(0); PointerAlignInBytes = C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity(); + SizeSizeInBytes = + C.toCharUnitsFromBits(C.getTargetInfo().getMaxPointerWidth()).getQuantity(); IntAlignInBytes = C.toCharUnitsFromBits(C.getTargetInfo().getIntAlign()).getQuantity(); IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth()); - IntPtrTy = llvm::IntegerType::get(LLVMContext, PointerWidthInBits); + IntPtrTy = llvm::IntegerType::get(LLVMContext, + C.getTargetInfo().getMaxPointerWidth()); Int8PtrTy = Int8Ty->getPointerTo(0); Int8PtrPtrTy = Int8PtrTy->getPointerTo(0); @@ -187,8 +190,7 @@ void CodeGenModule::createOpenCLRuntime() { void CodeGenModule::createOpenMPRuntime() { // Select a specialized code generation class based on the target, if any. // If it does not exist use the default implementation. - switch (getTarget().getTriple().getArch()) { - + switch (getTriple().getArch()) { case llvm::Triple::nvptx: case llvm::Triple::nvptx64: assert(getLangOpts().OpenMPIsDevice && @@ -469,7 +471,7 @@ void CodeGenModule::Release() { getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1); } - if (LangOpts.CUDAIsDevice && getTarget().getTriple().isNVPTX()) { + if (LangOpts.CUDAIsDevice && getTriple().isNVPTX()) { // Indicate whether __nvvm_reflect should be configured to flush denormal // floating point values to 0. (This corresponds to its "__CUDA_FTZ" // property.) @@ -672,7 +674,16 @@ StringRef CodeGenModule::getMangledName(GlobalDecl GD) { } else { IdentifierInfo *II = ND->getIdentifier(); assert(II && "Attempt to mangle unnamed decl."); - Str = II->getName(); + const auto *FD = dyn_cast<FunctionDecl>(ND); + + if (FD && + FD->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) { + llvm::raw_svector_ostream Out(Buffer); + Out << "__regcall3__" << II->getName(); + Str = Out.str(); + } else { + Str = II->getName(); + } } // Keep the first result in the case of a mangling collision. @@ -720,7 +731,9 @@ void CodeGenModule::AddGlobalDtor(llvm::Function *Dtor, int Priority) { GlobalDtors.push_back(Structor(Priority, Dtor, nullptr)); } -void CodeGenModule::EmitCtorList(const CtorList &Fns, const char *GlobalName) { +void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) { + if (Fns.empty()) return; + // Ctor function type is void()*. llvm::FunctionType* CtorFTy = llvm::FunctionType::get(VoidTy, false); llvm::Type *CtorPFTy = llvm::PointerType::getUnqual(CtorFTy); @@ -730,24 +743,29 @@ void CodeGenModule::EmitCtorList(const CtorList &Fns, const char *GlobalName) { Int32Ty, llvm::PointerType::getUnqual(CtorFTy), VoidPtrTy, nullptr); // Construct the constructor and destructor arrays. - SmallVector<llvm::Constant *, 8> Ctors; + ConstantInitBuilder builder(*this); + auto ctors = builder.beginArray(CtorStructTy); for (const auto &I : Fns) { - llvm::Constant *S[] = { - llvm::ConstantInt::get(Int32Ty, I.Priority, false), - llvm::ConstantExpr::getBitCast(I.Initializer, CtorPFTy), - (I.AssociatedData - ? llvm::ConstantExpr::getBitCast(I.AssociatedData, VoidPtrTy) - : llvm::Constant::getNullValue(VoidPtrTy))}; - Ctors.push_back(llvm::ConstantStruct::get(CtorStructTy, S)); + auto ctor = ctors.beginStruct(CtorStructTy); + ctor.addInt(Int32Ty, I.Priority); + ctor.add(llvm::ConstantExpr::getBitCast(I.Initializer, CtorPFTy)); + if (I.AssociatedData) + ctor.add(llvm::ConstantExpr::getBitCast(I.AssociatedData, VoidPtrTy)); + else + ctor.addNullPointer(VoidPtrTy); + ctor.finishAndAddTo(ctors); } - if (!Ctors.empty()) { - llvm::ArrayType *AT = llvm::ArrayType::get(CtorStructTy, Ctors.size()); - new llvm::GlobalVariable(TheModule, AT, false, - llvm::GlobalValue::AppendingLinkage, - llvm::ConstantArray::get(AT, Ctors), - GlobalName); - } + auto list = + ctors.finishAndCreateGlobal(GlobalName, getPointerAlign(), + /*constant*/ false, + llvm::GlobalValue::AppendingLinkage); + + // The LTO linker doesn't seem to like it when we set an alignment + // on appending variables. Take it off as a workaround. + list->setAlignment(0); + + Fns.clear(); } llvm::GlobalValue::LinkageTypes @@ -800,14 +818,7 @@ llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) { llvm::MDString *MDS = dyn_cast<llvm::MDString>(MD); if (!MDS) return nullptr; - llvm::MD5 md5; - llvm::MD5::MD5Result result; - md5.update(MDS->getString()); - md5.final(result); - uint64_t id = 0; - for (int i = 0; i < 8; ++i) - id |= static_cast<uint64_t>(result[i]) << (i * 8); - return llvm::ConstantInt::get(Int64Ty, id); + return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString())); } void CodeGenModule::setFunctionDefinitionAttributes(const FunctionDecl *D, @@ -864,6 +875,13 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, B.addAttribute(llvm::Attribute::StackProtectReq); if (!D) { + // If we don't have a declaration to control inlining, the function isn't + // explicitly marked as alwaysinline for semantic reasons, and inlining is + // disabled, mark the function as noinline. + if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline) && + CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) + B.addAttribute(llvm::Attribute::NoInline); + F->addAttributes(llvm::AttributeSet::FunctionIndex, llvm::AttributeSet::get( F->getContext(), @@ -871,7 +889,23 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, return; } - if (D->hasAttr<NakedAttr>()) { + if (D->hasAttr<OptimizeNoneAttr>()) { + B.addAttribute(llvm::Attribute::OptimizeNone); + + // OptimizeNone implies noinline; we should not be inlining such functions. + B.addAttribute(llvm::Attribute::NoInline); + assert(!F->hasFnAttribute(llvm::Attribute::AlwaysInline) && + "OptimizeNone and AlwaysInline on same function!"); + + // We still need to handle naked functions even though optnone subsumes + // much of their semantics. + if (D->hasAttr<NakedAttr>()) + B.addAttribute(llvm::Attribute::Naked); + + // OptimizeNone wins over OptimizeForSize and MinSize. + F->removeFnAttr(llvm::Attribute::OptimizeForSize); + F->removeFnAttr(llvm::Attribute::MinSize); + } else if (D->hasAttr<NakedAttr>()) { // Naked implies noinline: we should not be inlining such functions. B.addAttribute(llvm::Attribute::Naked); B.addAttribute(llvm::Attribute::NoInline); @@ -880,41 +914,47 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, } else if (D->hasAttr<NoInlineAttr>()) { B.addAttribute(llvm::Attribute::NoInline); } else if (D->hasAttr<AlwaysInlineAttr>() && - !F->getAttributes().hasAttribute(llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoInline)) { + !F->hasFnAttribute(llvm::Attribute::NoInline)) { // (noinline wins over always_inline, and we can't specify both in IR) B.addAttribute(llvm::Attribute::AlwaysInline); + } else if (CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) { + // If we're not inlining, then force everything that isn't always_inline to + // carry an explicit noinline attribute. + if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline)) + B.addAttribute(llvm::Attribute::NoInline); + } else { + // Otherwise, propagate the inline hint attribute and potentially use its + // absence to mark things as noinline. + if (auto *FD = dyn_cast<FunctionDecl>(D)) { + if (any_of(FD->redecls(), [&](const FunctionDecl *Redecl) { + return Redecl->isInlineSpecified(); + })) { + B.addAttribute(llvm::Attribute::InlineHint); + } else if (CodeGenOpts.getInlining() == + CodeGenOptions::OnlyHintInlining && + !FD->isInlined() && + !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) { + B.addAttribute(llvm::Attribute::NoInline); + } + } } - if (D->hasAttr<ColdAttr>()) { - if (!D->hasAttr<OptimizeNoneAttr>()) + // Add other optimization related attributes if we are optimizing this + // function. + if (!D->hasAttr<OptimizeNoneAttr>()) { + if (D->hasAttr<ColdAttr>()) { B.addAttribute(llvm::Attribute::OptimizeForSize); - B.addAttribute(llvm::Attribute::Cold); - } + B.addAttribute(llvm::Attribute::Cold); + } - if (D->hasAttr<MinSizeAttr>()) - B.addAttribute(llvm::Attribute::MinSize); + if (D->hasAttr<MinSizeAttr>()) + B.addAttribute(llvm::Attribute::MinSize); + } F->addAttributes(llvm::AttributeSet::FunctionIndex, llvm::AttributeSet::get( F->getContext(), llvm::AttributeSet::FunctionIndex, B)); - if (D->hasAttr<OptimizeNoneAttr>()) { - // OptimizeNone implies noinline; we should not be inlining such functions. - F->addFnAttr(llvm::Attribute::OptimizeNone); - F->addFnAttr(llvm::Attribute::NoInline); - - // OptimizeNone wins over OptimizeForSize, MinSize, AlwaysInline. - F->removeFnAttr(llvm::Attribute::OptimizeForSize); - F->removeFnAttr(llvm::Attribute::MinSize); - assert(!F->hasFnAttribute(llvm::Attribute::AlwaysInline) && - "OptimizeNone and AlwaysInline on same function!"); - - // Attribute 'inlinehint' has no effect on 'optnone' functions. - // Explicitly remove it from the set of function attributes. - F->removeFnAttr(llvm::Attribute::InlineHint); - } - unsigned alignment = D->getMaxAlignment() / Context.getCharWidth(); if (alignment) F->setAlignment(alignment); @@ -927,6 +967,11 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D)) F->setAlignment(2); } + + // In the cross-dso CFI mode, we want !type attributes on definitions only. + if (CodeGenOpts.SanitizeCfiCrossDso) + if (auto *FD = dyn_cast<FunctionDecl>(D)) + CreateFunctionTypeMetadata(FD, F); } void CodeGenModule::SetCommonAttributes(const Decl *D, @@ -1009,10 +1054,6 @@ void CodeGenModule::CreateFunctionTypeMetadata(const FunctionDecl *FD, // Additionally, if building with cross-DSO support... if (CodeGenOpts.SanitizeCfiCrossDso) { - // Don't emit entries for function declarations. In cross-DSO mode these are - // handled with better precision at run time. - if (!FD->hasBody()) - return; // Skip available_externally functions. They won't be codegen'ed in the // current module anyway. if (getContext().GetGVALinkageForFunction(FD) == GVA_AvailableExternally) @@ -1047,8 +1088,7 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, // where substantial code, including the libstdc++ dylib, was compiled with // GCC and does not actually return "this". if (!IsThunk && getCXXABI().HasThisReturn(GD) && - !(getTarget().getTriple().isiOS() && - getTarget().getTriple().isOSVersionLT(6))) { + !(getTriple().isiOS() && getTriple().isOSVersionLT(6))) { assert(!F->arg_empty() && F->arg_begin()->getType() ->canLosslesslyBitCastTo(F->getReturnType()) && @@ -1086,7 +1126,10 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, if (MD->isVirtual()) F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - CreateFunctionTypeMetadata(FD, F); + // Don't emit entries for function declarations in the cross-DSO mode. This + // is handled with better precision by the receiving DSO. + if (!CodeGenOpts.SanitizeCfiCrossDso) + CreateFunctionTypeMetadata(FD, F); } void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) { @@ -1200,9 +1243,15 @@ void CodeGenModule::EmitModuleLinkOptions() { SmallVector<clang::Module *, 16> Stack; // Seed the stack with imported modules. - for (Module *M : ImportedModules) + for (Module *M : ImportedModules) { + // Do not add any link flags when an implementation TU of a module imports + // a header of that same module. + if (M->getTopLevelModuleName() == getLangOpts().CurrentModule && + !getLangOpts().isCompilingModule()) + continue; if (Visited.insert(M).second) Stack.push_back(M); + } // Find all of the modules to import, making a little effort to prune // non-leaf modules. @@ -1282,7 +1331,7 @@ void CodeGenModule::EmitDeferred() { // might had been created for another decl with the same mangled name but // different type. llvm::GlobalValue *GV = dyn_cast<llvm::GlobalValue>( - GetAddrOfGlobal(D, /*IsForDefinition=*/true)); + GetAddrOfGlobal(D, ForDefinition)); // In case of different address spaces, we may still get a cast, even with // IsForDefinition equal to true. Query mangled names table to get @@ -1681,6 +1730,8 @@ namespace { : public RecursiveASTVisitor<DLLImportFunctionVisitor> { bool SafeToInline = true; + bool shouldVisitImplicitCode() const { return true; } + bool VisitVarDecl(VarDecl *VD) { // A thread-local variable cannot be imported. SafeToInline = !VD->getTLSKind(); @@ -1696,6 +1747,10 @@ namespace { SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>(); return SafeToInline; } + bool VisitCXXConstructExpr(CXXConstructExpr *E) { + SafeToInline = E->getConstructor()->hasAttr<DLLImportAttr>(); + return SafeToInline; + } bool VisitCXXDeleteExpr(CXXDeleteExpr *E) { SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>(); return SafeToInline; @@ -1728,8 +1783,17 @@ CodeGenModule::isTriviallyRecursive(const FunctionDecl *FD) { return Walker.Result; } -bool -CodeGenModule::shouldEmitFunction(GlobalDecl GD) { +// Check if T is a class type with a destructor that's not dllimport. +static bool HasNonDllImportDtor(QualType T) { + if (const RecordType *RT = dyn_cast<RecordType>(T)) + if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) + if (RD->getDestructor() && !RD->getDestructor()->hasAttr<DLLImportAttr>()) + return true; + + return false; +} + +bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) { if (getFunctionLinkage(GD) != llvm::Function::AvailableExternallyLinkage) return true; const auto *F = cast<FunctionDecl>(GD.getDecl()); @@ -1742,6 +1806,18 @@ CodeGenModule::shouldEmitFunction(GlobalDecl GD) { Visitor.TraverseFunctionDecl(const_cast<FunctionDecl*>(F)); if (!Visitor.SafeToInline) return false; + + if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(F)) { + // Implicit destructor invocations aren't captured in the AST, so the + // check above can't see them. Check for them manually here. + for (const Decl *Member : Dtor->getParent()->decls()) + if (isa<FieldDecl>(Member)) + if (HasNonDllImportDtor(cast<FieldDecl>(Member)->getType())) + return false; + for (const CXXBaseSpecifier &B : Dtor->getParent()->bases()) + if (HasNonDllImportDtor(B.getType())) + return false; + } } // PR9614. Avoid cases where the source code is lying to us. An available @@ -1823,7 +1899,7 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, GlobalDecl GD, bool ForVTable, bool DontDefer, bool IsThunk, llvm::AttributeSet ExtraAttrs, - bool IsForDefinition) { + ForDefinition_t IsForDefinition) { const Decl *D = GD.getDecl(); // Lookup the entry, lazily creating it if necessary. @@ -1983,7 +2059,7 @@ llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable, bool DontDefer, - bool IsForDefinition) { + ForDefinition_t IsForDefinition) { // If there was no specific requested type, just convert it now. if (!Ty) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); @@ -1997,18 +2073,70 @@ llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD, IsForDefinition); } +static const FunctionDecl * +GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) { + TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl(); + DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl); + + IdentifierInfo &CII = C.Idents.get(Name); + for (const auto &Result : DC->lookup(&CII)) + if (const auto FD = dyn_cast<FunctionDecl>(Result)) + return FD; + + if (!C.getLangOpts().CPlusPlus) + return nullptr; + + // Demangle the premangled name from getTerminateFn() + IdentifierInfo &CXXII = + (Name == "_ZSt9terminatev" || Name == "\01?terminate@@YAXXZ") + ? C.Idents.get("terminate") + : C.Idents.get(Name); + + for (const auto &N : {"__cxxabiv1", "std"}) { + IdentifierInfo &NS = C.Idents.get(N); + for (const auto &Result : DC->lookup(&NS)) { + NamespaceDecl *ND = dyn_cast<NamespaceDecl>(Result); + if (auto LSD = dyn_cast<LinkageSpecDecl>(Result)) + for (const auto &Result : LSD->lookup(&NS)) + if ((ND = dyn_cast<NamespaceDecl>(Result))) + break; + + if (ND) + for (const auto &Result : ND->lookup(&CXXII)) + if (const auto *FD = dyn_cast<FunctionDecl>(Result)) + return FD; + } + } + + return nullptr; +} + /// CreateRuntimeFunction - Create a new runtime function with the specified /// type and name. llvm::Constant * -CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, - StringRef Name, - llvm::AttributeSet ExtraAttrs) { +CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name, + llvm::AttributeSet ExtraAttrs, + bool Local) { llvm::Constant *C = GetOrCreateLLVMFunction(Name, FTy, GlobalDecl(), /*ForVTable=*/false, - /*DontDefer=*/false, /*IsThunk=*/false, ExtraAttrs); - if (auto *F = dyn_cast<llvm::Function>(C)) - if (F->empty()) + /*DontDefer=*/false, /*IsThunk=*/false, + ExtraAttrs); + + if (auto *F = dyn_cast<llvm::Function>(C)) { + if (F->empty()) { F->setCallingConv(getRuntimeCC()); + + if (!Local && getTriple().isOSBinFormatCOFF() && + !getCodeGenOpts().LTOVisibilityPublicStd) { + const FunctionDecl *FD = GetRuntimeFunctionDecl(Context, Name); + if (!FD || FD->hasAttr<DLLImportAttr>()) { + F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + F->setLinkage(llvm::GlobalValue::ExternalLinkage); + } + } + } + } + return C; } @@ -2062,7 +2190,7 @@ llvm::Constant * CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::PointerType *Ty, const VarDecl *D, - bool IsForDefinition) { + ForDefinition_t IsForDefinition) { // Lookup the entry, lazily creating it if necessary. llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (Entry) { @@ -2162,7 +2290,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, } // Handle XCore specific ABI requirements. - if (getTarget().getTriple().getArch() == llvm::Triple::xcore && + if (getTriple().getArch() == llvm::Triple::xcore && D->getLanguageLinkage() == CLanguageLinkage && D->getType().isConstant(Context) && isExternallyVisible(D->getLinkageAndVisibility().getLinkage())) @@ -2177,30 +2305,31 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Constant * CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, - bool IsForDefinition) { - if (isa<CXXConstructorDecl>(GD.getDecl())) - return getAddrOfCXXStructor(cast<CXXConstructorDecl>(GD.getDecl()), + ForDefinition_t IsForDefinition) { + const Decl *D = GD.getDecl(); + if (isa<CXXConstructorDecl>(D)) + return getAddrOfCXXStructor(cast<CXXConstructorDecl>(D), getFromCtorType(GD.getCtorType()), /*FnInfo=*/nullptr, /*FnType=*/nullptr, /*DontDefer=*/false, IsForDefinition); - else if (isa<CXXDestructorDecl>(GD.getDecl())) - return getAddrOfCXXStructor(cast<CXXDestructorDecl>(GD.getDecl()), + else if (isa<CXXDestructorDecl>(D)) + return getAddrOfCXXStructor(cast<CXXDestructorDecl>(D), getFromDtorType(GD.getDtorType()), /*FnInfo=*/nullptr, /*FnType=*/nullptr, /*DontDefer=*/false, IsForDefinition); - else if (isa<CXXMethodDecl>(GD.getDecl())) { + else if (isa<CXXMethodDecl>(D)) { auto FInfo = &getTypes().arrangeCXXMethodDeclaration( - cast<CXXMethodDecl>(GD.getDecl())); + cast<CXXMethodDecl>(D)); auto Ty = getTypes().GetFunctionType(*FInfo); return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false, IsForDefinition); - } else if (isa<FunctionDecl>(GD.getDecl())) { + } else if (isa<FunctionDecl>(D)) { const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false, IsForDefinition); } else - return GetAddrOfGlobalVar(cast<VarDecl>(GD.getDecl()), /*Ty=*/nullptr, + return GetAddrOfGlobalVar(cast<VarDecl>(D), /*Ty=*/nullptr, IsForDefinition); } @@ -2254,7 +2383,7 @@ CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name, /// variable with the same mangled name but some other type. llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D, llvm::Type *Ty, - bool IsForDefinition) { + ForDefinition_t IsForDefinition) { assert(D->hasGlobalStorage() && "Not a global variable"); QualType ASTTy = D->getType(); if (!Ty) @@ -2385,8 +2514,13 @@ void CodeGenModule::maybeSetTrivialComdat(const Decl &D, /// Pass IsTentative as true if you want to create a tentative definition. void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, bool IsTentative) { - llvm::Constant *Init = nullptr; + // OpenCL global variables of sampler type are translated to function calls, + // therefore no need to be translated. QualType ASTTy = D->getType(); + if (getLangOpts().OpenCL && ASTTy->isSamplerT()) + return; + + llvm::Constant *Init = nullptr; CXXRecordDecl *RD = ASTTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl(); bool NeedsGlobalCtor = false; bool NeedsGlobalDtor = RD && !RD->hasTrivialDestructor(); @@ -2439,7 +2573,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, llvm::Type* InitType = Init->getType(); llvm::Constant *Entry = - GetAddrOfGlobalVar(D, InitType, /*IsForDefinition=*/!IsTentative); + GetAddrOfGlobalVar(D, InitType, ForDefinition_t(!IsTentative)); // Strip off a bitcast if we got one back. if (auto *CE = dyn_cast<llvm::ConstantExpr>(Entry)) { @@ -2472,7 +2606,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, // Make a new global with the correct type, this is now guaranteed to work. GV = cast<llvm::GlobalVariable>( - GetAddrOfGlobalVar(D, InitType, /*IsForDefinition=*/!IsTentative)); + GetAddrOfGlobalVar(D, InitType, ForDefinition_t(!IsTentative))); // Replace all uses of the old global with the new global llvm::Constant *NewPtrForOldDecl = @@ -2565,9 +2699,16 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, else GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass); - if (Linkage == llvm::GlobalVariable::CommonLinkage) + if (Linkage == llvm::GlobalVariable::CommonLinkage) { // common vars aren't constant even if declared const. GV->setConstant(false); + // Tentative definition of global variables may be initialized with + // non-zero null pointers. In this case they should have weak linkage + // since common linkage must have zero initializer and must not have + // explicit section therefore cannot have non-zero initial value. + if (!GV->getInitializer()->isNullValue()) + GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage); + } setNonAliasAttributes(D, GV); @@ -2876,7 +3017,7 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD, if (!GV || (GV->getType()->getElementType() != Ty)) GV = cast<llvm::GlobalValue>(GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/true, - /*IsForDefinition=*/true)); + ForDefinition)); // Already emitted. if (!GV->isDeclaration()) @@ -3067,13 +3208,12 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, // Otherwise, convert the UTF8 literals into a string of shorts. IsUTF16 = true; - SmallVector<UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls. - const UTF8 *FromPtr = (const UTF8 *)String.data(); - UTF16 *ToPtr = &ToBuf[0]; + SmallVector<llvm::UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls. + const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data(); + llvm::UTF16 *ToPtr = &ToBuf[0]; - (void)ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes, - &ToPtr, ToPtr + NumBytes, - strictConversion); + (void)llvm::ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes, &ToPtr, + ToPtr + NumBytes, llvm::strictConversion); // ConvertUTF8toUTF16 returns the length in ToPtr. StringLength = ToPtr - &ToBuf[0]; @@ -3086,14 +3226,6 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, nullptr)).first; } -static llvm::StringMapEntry<llvm::GlobalVariable *> & -GetConstantStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, - const StringLiteral *Literal, unsigned &StringLength) { - StringRef String = Literal->getString(); - StringLength = String.size(); - return *Map.insert(std::make_pair(String, nullptr)).first; -} - ConstantAddress CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { unsigned StringLength = 0; @@ -3108,7 +3240,6 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); llvm::Constant *Zeros[] = { Zero, Zero }; - llvm::Value *V; // If we don't already have it, get __CFConstantStringClassReference. if (!CFConstantStringClassRef) { @@ -3117,7 +3248,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { llvm::Constant *GV = CreateRuntimeVariable(Ty, "__CFConstantStringClassReference"); - if (getTarget().getTriple().isOSBinFormatCOFF()) { + if (getTriple().isOSBinFormatCOFF()) { IdentifierInfo &II = getContext().Idents.get(GV->getName()); TranslationUnitDecl *TUDecl = getContext().getTranslationUnitDecl(); DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl); @@ -3138,25 +3269,22 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { } // Decay array -> ptr - V = llvm::ConstantExpr::getGetElementPtr(Ty, GV, Zeros); - CFConstantStringClassRef = V; - } else { - V = CFConstantStringClassRef; + CFConstantStringClassRef = + llvm::ConstantExpr::getGetElementPtr(Ty, GV, Zeros); } QualType CFTy = getContext().getCFConstantStringType(); auto *STy = cast<llvm::StructType>(getTypes().ConvertType(CFTy)); - llvm::Constant *Fields[4]; + ConstantInitBuilder Builder(*this); + auto Fields = Builder.beginStruct(STy); // Class pointer. - Fields[0] = cast<llvm::ConstantExpr>(V); + Fields.add(cast<llvm::ConstantExpr>(CFConstantStringClassRef)); // Flags. - llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy); - Fields[1] = isUTF16 ? llvm::ConstantInt::get(Ty, 0x07d0) - : llvm::ConstantInt::get(Ty, 0x07C8); + Fields.addInt(IntTy, isUTF16 ? 0x07d0 : 0x07C8); // String pointer. llvm::Constant *C = nullptr; @@ -3185,31 +3313,30 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { // FIXME: We set the section explicitly to avoid a bug in ld64 224.1. // Without it LLVM can merge the string with a non unnamed_addr one during // LTO. Doing that changes the section it ends in, which surprises ld64. - if (getTarget().getTriple().isOSBinFormatMachO()) + if (getTriple().isOSBinFormatMachO()) GV->setSection(isUTF16 ? "__TEXT,__ustring" : "__TEXT,__cstring,cstring_literals"); // String. - Fields[2] = + llvm::Constant *Str = llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); if (isUTF16) // Cast the UTF16 string to the correct type. - Fields[2] = llvm::ConstantExpr::getBitCast(Fields[2], Int8PtrTy); + Str = llvm::ConstantExpr::getBitCast(Str, Int8PtrTy); + Fields.add(Str); // String length. - Ty = getTypes().ConvertType(getContext().LongTy); - Fields[3] = llvm::ConstantInt::get(Ty, StringLength); + auto Ty = getTypes().ConvertType(getContext().LongTy); + Fields.addInt(cast<llvm::IntegerType>(Ty), StringLength); CharUnits Alignment = getPointerAlign(); // The struct. - C = llvm::ConstantStruct::get(STy, Fields); - GV = new llvm::GlobalVariable(getModule(), C->getType(), true, - llvm::GlobalVariable::PrivateLinkage, C, - "_unnamed_cfstring_"); - GV->setAlignment(Alignment.getQuantity()); - switch (getTarget().getTriple().getObjectFormat()) { + GV = Fields.finishAndCreateGlobal("_unnamed_cfstring_", Alignment, + /*isConstant=*/false, + llvm::GlobalVariable::PrivateLinkage); + switch (getTriple().getObjectFormat()) { case llvm::Triple::UnknownObjectFormat: llvm_unreachable("unknown file format"); case llvm::Triple::COFF: @@ -3225,124 +3352,6 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { return ConstantAddress(GV, Alignment); } -ConstantAddress -CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { - unsigned StringLength = 0; - llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = - GetConstantStringEntry(CFConstantStringMap, Literal, StringLength); - - if (auto *C = Entry.second) - return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); - - llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); - llvm::Constant *Zeros[] = { Zero, Zero }; - llvm::Value *V; - // If we don't already have it, get _NSConstantStringClassReference. - if (!ConstantStringClassRef) { - std::string StringClass(getLangOpts().ObjCConstantStringClass); - llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy); - llvm::Constant *GV; - if (LangOpts.ObjCRuntime.isNonFragile()) { - std::string str = - StringClass.empty() ? "OBJC_CLASS_$_NSConstantString" - : "OBJC_CLASS_$_" + StringClass; - GV = getObjCRuntime().GetClassGlobal(str); - // Make sure the result is of the correct type. - llvm::Type *PTy = llvm::PointerType::getUnqual(Ty); - V = llvm::ConstantExpr::getBitCast(GV, PTy); - ConstantStringClassRef = V; - } else { - std::string str = - StringClass.empty() ? "_NSConstantStringClassReference" - : "_" + StringClass + "ClassReference"; - llvm::Type *PTy = llvm::ArrayType::get(Ty, 0); - GV = CreateRuntimeVariable(PTy, str); - // Decay array -> ptr - V = llvm::ConstantExpr::getGetElementPtr(PTy, GV, Zeros); - ConstantStringClassRef = V; - } - } else - V = ConstantStringClassRef; - - if (!NSConstantStringType) { - // Construct the type for a constant NSString. - RecordDecl *D = Context.buildImplicitRecord("__builtin_NSString"); - D->startDefinition(); - - QualType FieldTypes[3]; - - // const int *isa; - FieldTypes[0] = Context.getPointerType(Context.IntTy.withConst()); - // const char *str; - FieldTypes[1] = Context.getPointerType(Context.CharTy.withConst()); - // unsigned int length; - FieldTypes[2] = Context.UnsignedIntTy; - - // Create fields - for (unsigned i = 0; i < 3; ++i) { - FieldDecl *Field = FieldDecl::Create(Context, D, - SourceLocation(), - SourceLocation(), nullptr, - FieldTypes[i], /*TInfo=*/nullptr, - /*BitWidth=*/nullptr, - /*Mutable=*/false, - ICIS_NoInit); - Field->setAccess(AS_public); - D->addDecl(Field); - } - - D->completeDefinition(); - QualType NSTy = Context.getTagDeclType(D); - NSConstantStringType = cast<llvm::StructType>(getTypes().ConvertType(NSTy)); - } - - llvm::Constant *Fields[3]; - - // Class pointer. - Fields[0] = cast<llvm::ConstantExpr>(V); - - // String pointer. - llvm::Constant *C = - llvm::ConstantDataArray::getString(VMContext, Entry.first()); - - llvm::GlobalValue::LinkageTypes Linkage; - bool isConstant; - Linkage = llvm::GlobalValue::PrivateLinkage; - isConstant = !LangOpts.WritableStrings; - - auto *GV = new llvm::GlobalVariable(getModule(), C->getType(), isConstant, - Linkage, C, ".str"); - GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - // Don't enforce the target's minimum global alignment, since the only use - // of the string is via this class initializer. - CharUnits Align = getContext().getTypeAlignInChars(getContext().CharTy); - GV->setAlignment(Align.getQuantity()); - Fields[1] = - llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); - - // String length. - llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy); - Fields[2] = llvm::ConstantInt::get(Ty, StringLength); - - // The struct. - CharUnits Alignment = getPointerAlign(); - C = llvm::ConstantStruct::get(NSConstantStringType, Fields); - GV = new llvm::GlobalVariable(getModule(), C->getType(), true, - llvm::GlobalVariable::PrivateLinkage, C, - "_unnamed_nsstring_"); - GV->setAlignment(Alignment.getQuantity()); - const char *NSStringSection = "__OBJC,__cstring_object,regular,no_dead_strip"; - const char *NSStringNonFragileABISection = - "__DATA,__objc_stringobj,regular,no_dead_strip"; - // FIXME. Fix section. - GV->setSection(LangOpts.ObjCRuntime.isNonFragile() - ? NSStringNonFragileABISection - : NSStringSection); - Entry.second = GV; - - return ConstantAddress(GV, Alignment); -} - QualType CodeGenModule::getObjCFastEnumerationStateType() { if (ObjCFastEnumerationStateType.isNull()) { RecordDecl *D = Context.buildImplicitRecord("__objcFastEnumerationState"); @@ -3710,17 +3719,6 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { D->setHasNonZeroConstructors(true); } -/// EmitNamespace - Emit all declarations in a namespace. -void CodeGenModule::EmitNamespace(const NamespaceDecl *ND) { - for (auto *I : ND->decls()) { - if (const auto *VD = dyn_cast<VarDecl>(I)) - if (VD->getTemplateSpecializationKind() != TSK_ExplicitSpecialization && - VD->getTemplateSpecializationKind() != TSK_Undeclared) - continue; - EmitTopLevelDecl(I); - } -} - // EmitLinkageSpec - Emit all declarations in a linkage spec. void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) { if (LSD->getLanguage() != LinkageSpecDecl::lang_c && @@ -3729,13 +3727,21 @@ void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) { return; } - for (auto *I : LSD->decls()) { - // Meta-data for ObjC class includes references to implemented methods. - // Generate class's method definitions first. + EmitDeclContext(LSD); +} + +void CodeGenModule::EmitDeclContext(const DeclContext *DC) { + for (auto *I : DC->decls()) { + // Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope + // are themselves considered "top-level", so EmitTopLevelDecl on an + // ObjCImplDecl does not recursively visit them. We need to do that in + // case they're nested inside another construct (LinkageSpecDecl / + // ExportDecl) that does stop them from being considered "top-level". if (auto *OID = dyn_cast<ObjCImplDecl>(I)) { for (auto *M : OID->methods()) EmitTopLevelDecl(M); } + EmitTopLevelDecl(I); } } @@ -3762,11 +3768,16 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { break; case Decl::Var: + case Decl::Decomposition: // Skip variable templates if (cast<VarDecl>(D)->getDescribedVarTemplate()) return; case Decl::VarTemplateSpecialization: EmitGlobal(cast<VarDecl>(D)); + if (auto *DD = dyn_cast<DecompositionDecl>(D)) + for (auto *B : DD->bindings()) + if (auto *HD = B->getHoldingVar()) + EmitGlobal(HD); break; // Indirect fields from global anonymous structs and unions can be @@ -3776,7 +3787,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { // C++ Decls case Decl::Namespace: - EmitNamespace(cast<NamespaceDecl>(D)); + EmitDeclContext(cast<NamespaceDecl>(D)); break; case Decl::CXXRecord: // Emit any static data members, they may be definitions. @@ -3911,16 +3922,50 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { case Decl::Import: { auto *Import = cast<ImportDecl>(D); - // Ignore import declarations that come from imported modules. - if (Import->getImportedOwningModule()) + // If we've already imported this module, we're done. + if (!ImportedModules.insert(Import->getImportedModule())) break; - if (CGDebugInfo *DI = getModuleDebugInfo()) - DI->EmitImportDecl(*Import); - ImportedModules.insert(Import->getImportedModule()); + // Emit debug information for direct imports. + if (!Import->getImportedOwningModule()) { + if (CGDebugInfo *DI = getModuleDebugInfo()) + DI->EmitImportDecl(*Import); + } + + // Find all of the submodules and emit the module initializers. + llvm::SmallPtrSet<clang::Module *, 16> Visited; + SmallVector<clang::Module *, 16> Stack; + Visited.insert(Import->getImportedModule()); + Stack.push_back(Import->getImportedModule()); + + while (!Stack.empty()) { + clang::Module *Mod = Stack.pop_back_val(); + if (!EmittedModuleInitializers.insert(Mod).second) + continue; + + for (auto *D : Context.getModuleInitializers(Mod)) + EmitTopLevelDecl(D); + + // Visit the submodules of this module. + for (clang::Module::submodule_iterator Sub = Mod->submodule_begin(), + SubEnd = Mod->submodule_end(); + Sub != SubEnd; ++Sub) { + // Skip explicit children; they need to be explicitly imported to emit + // the initializers. + if ((*Sub)->IsExplicit) + continue; + + if (Visited.insert(*Sub).second) + Stack.push_back(*Sub); + } + } break; } + case Decl::Export: + EmitDeclContext(cast<ExportDecl>(D)); + break; + case Decl::OMPThreadPrivate: EmitOMPThreadPrivateDecl(cast<OMPThreadPrivateDecl>(D)); break; @@ -4153,18 +4198,24 @@ void CodeGenModule::EmitTargetMetadata() { } void CodeGenModule::EmitCoverageFile() { - if (!getCodeGenOpts().CoverageFile.empty()) { - if (llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata("llvm.dbg.cu")) { - llvm::NamedMDNode *GCov = TheModule.getOrInsertNamedMetadata("llvm.gcov"); - llvm::LLVMContext &Ctx = TheModule.getContext(); - llvm::MDString *CoverageFile = - llvm::MDString::get(Ctx, getCodeGenOpts().CoverageFile); - for (int i = 0, e = CUNode->getNumOperands(); i != e; ++i) { - llvm::MDNode *CU = CUNode->getOperand(i); - llvm::Metadata *Elts[] = {CoverageFile, CU}; - GCov->addOperand(llvm::MDNode::get(Ctx, Elts)); - } - } + if (getCodeGenOpts().CoverageDataFile.empty() && + getCodeGenOpts().CoverageNotesFile.empty()) + return; + + llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata("llvm.dbg.cu"); + if (!CUNode) + return; + + llvm::NamedMDNode *GCov = TheModule.getOrInsertNamedMetadata("llvm.gcov"); + llvm::LLVMContext &Ctx = TheModule.getContext(); + auto *CoverageDataFile = + llvm::MDString::get(Ctx, getCodeGenOpts().CoverageDataFile); + auto *CoverageNotesFile = + llvm::MDString::get(Ctx, getCodeGenOpts().CoverageNotesFile); + for (int i = 0, e = CUNode->getNumOperands(); i != e; ++i) { + llvm::MDNode *CU = CUNode->getOperand(i); + llvm::Metadata *Elts[] = {CoverageNotesFile, CoverageDataFile, CU}; + GCov->addOperand(llvm::MDNode::get(Ctx, Elts)); } } @@ -4311,3 +4362,13 @@ llvm::SanitizerStatReport &CodeGenModule::getSanStats() { return *SanStats; } +llvm::Value * +CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E, + CodeGenFunction &CGF) { + llvm::Constant *C = EmitConstantExpr(E, E->getType(), &CGF); + auto SamplerT = getOpenCLRuntime().getSamplerType(); + auto FTy = llvm::FunctionType::get(SamplerT, {C->getType()}, false); + return CGF.Builder.CreateCall(CreateRuntimeFunction(FTy, + "__translate_sampler_initializer"), + {C}); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h index 9490499..36f6785 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h @@ -94,6 +94,11 @@ class FunctionArgList; class CoverageMappingModuleGen; class TargetCodeGenInfo; +enum ForDefinition_t : bool { + NotForDefinition = false, + ForDefinition = true +}; + struct OrderGlobalInits { unsigned int priority; unsigned int lex_order; @@ -161,7 +166,7 @@ struct ObjCEntrypoints { /// void objc_release(id); llvm::Constant *objc_release; - /// id objc_storeStrong(id*, id); + /// void objc_storeStrong(id*, id); llvm::Constant *objc_storeStrong; /// id objc_storeWeak(id*, id); @@ -420,6 +425,10 @@ private: /// \brief The complete set of modules that has been imported. llvm::SetVector<clang::Module *> ImportedModules; + /// \brief The set of modules for which the module initializers + /// have been emitted. + llvm::SmallPtrSet<clang::Module *, 16> EmittedModuleInitializers; + /// \brief A vector of metadata strings. SmallVector<llvm::Metadata *, 16> LinkerOptionsMetadata; @@ -430,13 +439,6 @@ private: /// int * but is actually an Obj-C class pointer. llvm::WeakVH CFConstantStringClassRef; - /// Cached reference to the class for constant strings. This value has type - /// int * but is actually an Obj-C class pointer. - llvm::WeakVH ConstantStringClassRef; - - /// \brief The LLVM type corresponding to NSConstantString. - llvm::StructType *NSConstantStringType = nullptr; - /// \brief The type used to describe the state of a fast enumeration in /// Objective-C's for..in loop. QualType ObjCFastEnumerationStateType; @@ -453,6 +455,14 @@ private: bool isTriviallyRecursive(const FunctionDecl *F); bool shouldEmitFunction(GlobalDecl GD); + /// Map used to be sure we don't emit the same CompoundLiteral twice. + llvm::DenseMap<const CompoundLiteralExpr *, llvm::GlobalVariable *> + EmittedCompoundLiterals; + + /// Map of the global blocks we've emitted, so that we don't have to re-emit + /// them if the constexpr evaluator gets aggressive. + llvm::DenseMap<const BlockExpr *, llvm::Constant *> EmittedGlobalBlocks; + /// @name Cache for Blocks Runtime Globals /// @{ @@ -610,7 +620,7 @@ public: return TheModule.getDataLayout(); } const TargetInfo &getTarget() const { return Target; } - const llvm::Triple &getTriple() const; + const llvm::Triple &getTriple() const { return Target.getTriple(); } bool supportsCOMDAT() const; void maybeSetTrivialComdat(const Decl &D, llvm::GlobalObject &GO); @@ -679,7 +689,9 @@ public: llvm_unreachable("unknown visibility!"); } - llvm::Constant *GetAddrOfGlobal(GlobalDecl GD, bool IsForDefinition = false); + llvm::Constant *GetAddrOfGlobal(GlobalDecl GD, + ForDefinition_t IsForDefinition + = NotForDefinition); /// Will return a global variable of the given type. If a variable with a /// different type already exists then a new variable with the right type @@ -709,14 +721,16 @@ public: /// the same mangled name but some other type. llvm::Constant *GetAddrOfGlobalVar(const VarDecl *D, llvm::Type *Ty = nullptr, - bool IsForDefinition = false); + ForDefinition_t IsForDefinition + = NotForDefinition); /// Return the address of the given function. If Ty is non-null, then this /// function will use the specified type if it has to create it. llvm::Constant *GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty = nullptr, bool ForVTable = false, bool DontDefer = false, - bool IsForDefinition = false); + ForDefinition_t IsForDefinition + = NotForDefinition); /// Get the address of the RTTI descriptor for the given type. llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false); @@ -769,7 +783,17 @@ public: llvm::Type *getGenericBlockLiteralType(); /// Gets the address of a block which requires no captures. - llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, const char *); + llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, StringRef Name); + + /// Returns the address of a block which requires no caputres, or null if + /// we've yet to emit the block for BE. + llvm::Constant *getAddrOfGlobalBlockIfEmitted(const BlockExpr *BE) { + return EmittedGlobalBlocks.lookup(BE); + } + + /// Notes that BE's global block is available via Addr. Asserts that BE + /// isn't already emitted. + void setAddrOfGlobalBlock(const BlockExpr *BE, llvm::Constant *Addr); /// Return a pointer to a constant CFString object for the given string. ConstantAddress GetAddrOfConstantCFString(const StringLiteral *Literal); @@ -804,6 +828,16 @@ public: /// compound literal expression. ConstantAddress GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr*E); + /// If it's been emitted already, returns the GlobalVariable corresponding to + /// a compound literal. Otherwise, returns null. + llvm::GlobalVariable * + getAddrOfConstantCompoundLiteralIfEmitted(const CompoundLiteralExpr *E); + + /// Notes that CLE's GlobalVariable is GV. Asserts that CLE isn't already + /// emitted. + void setAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *CLE, + llvm::GlobalVariable *GV); + /// \brief Returns a pointer to a global variable representing a temporary /// with static or thread storage duration. ConstantAddress GetAddrOfGlobalTemporary(const MaterializeTemporaryExpr *E, @@ -824,12 +858,13 @@ public: getAddrOfCXXStructor(const CXXMethodDecl *MD, StructorType Type, const CGFunctionInfo *FnInfo = nullptr, llvm::FunctionType *FnType = nullptr, - bool DontDefer = false, bool IsForDefinition = false); + bool DontDefer = false, + ForDefinition_t IsForDefinition = NotForDefinition); /// Given a builtin id for a function like "__builtin_fabsf", return a /// Function* for "fabsf". - llvm::Value *getBuiltinLibFunction(const FunctionDecl *FD, - unsigned BuiltinID); + llvm::Constant *getBuiltinLibFunction(const FunctionDecl *FD, + unsigned BuiltinID); llvm::Function *getIntrinsic(unsigned IID, ArrayRef<llvm::Type*> Tys = None); @@ -869,10 +904,11 @@ public: } /// Create a new runtime function with the specified type and name. - llvm::Constant *CreateRuntimeFunction(llvm::FunctionType *Ty, - StringRef Name, - llvm::AttributeSet ExtraAttrs = - llvm::AttributeSet()); + llvm::Constant * + CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, + llvm::AttributeSet ExtraAttrs = llvm::AttributeSet(), + bool Local = false); + /// Create a new compiler builtin function with the specified type and name. llvm::Constant *CreateBuiltinFunction(llvm::FunctionType *Ty, StringRef Name, @@ -1145,18 +1181,27 @@ public: llvm::SanitizerStatReport &getSanStats(); + llvm::Value * + createOpenCLIntToSamplerConversion(const Expr *E, CodeGenFunction &CGF); + + /// Get target specific null pointer. + /// \param T is the LLVM type of the null pointer. + /// \param QT is the clang QualType of the null pointer. + llvm::Constant *getNullPointer(llvm::PointerType *T, QualType QT); + private: llvm::Constant * GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable, bool DontDefer = false, bool IsThunk = false, llvm::AttributeSet ExtraAttrs = llvm::AttributeSet(), - bool IsForDefinition = false); + ForDefinition_t IsForDefinition = NotForDefinition); llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName, llvm::PointerType *PTy, const VarDecl *D, - bool IsForDefinition = false); + ForDefinition_t IsForDefinition + = NotForDefinition); void setNonAliasAttributes(const Decl *D, llvm::GlobalObject *GO); @@ -1175,7 +1220,7 @@ private: // C++ related functions. - void EmitNamespace(const NamespaceDecl *D); + void EmitDeclContext(const DeclContext *DC); void EmitLinkageSpec(const LinkageSpecDecl *D); void CompleteDIClassType(const CXXMethodDecl* D); @@ -1202,10 +1247,10 @@ private: llvm::Constant *AssociatedData = nullptr); void AddGlobalDtor(llvm::Function *Dtor, int Priority = 65535); - /// Generates a global array of functions and priorities using the given list - /// and name. This array will have appending linkage and is suitable for use - /// as a LLVM constructor or destructor array. - void EmitCtorList(const CtorList &Fns, const char *GlobalName); + /// EmitCtorList - Generates a global array of functions and priorities using + /// the given list and name. This array will have appending linkage and is + /// suitable for use as a LLVM constructor or destructor array. Clears Fns. + void EmitCtorList(CtorList &Fns, const char *GlobalName); /// Emit any needed decls for which code generation was deferred. void EmitDeferred(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp index 4eefdd7..c6c3fa4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp @@ -458,6 +458,8 @@ struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { void VisitSwitchStmt(const SwitchStmt *S) { RecordStmtCount(S); + if (S->getInit()) + Visit(S->getInit()); Visit(S->getCond()); CurrentCount = 0; BreakContinueStack.push_back(BreakContinue()); @@ -488,6 +490,8 @@ struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { void VisitIfStmt(const IfStmt *S) { RecordStmtCount(S); uint64_t ParentCount = CurrentCount; + if (S->getInit()) + Visit(S->getInit()); Visit(S->getCond()); // Counter tracks the "then" part of an if statement. The count for diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h index d03f235..4f229cd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h @@ -18,9 +18,7 @@ #include "CodeGenModule.h" #include "CodeGenTypes.h" #include "clang/Frontend/CodeGenOptions.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ProfileData/InstrProfReader.h" -#include "llvm/Support/MemoryBuffer.h" #include <array> #include <memory> diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h index c32b66d..47e26bc 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h @@ -80,9 +80,14 @@ struct CodeGenTypeCache { union { unsigned char PointerAlignInBytes; unsigned char PointerSizeInBytes; + }; + + /// The size and alignment of size_t. + union { unsigned char SizeSizeInBytes; // sizeof(size_t) unsigned char SizeAlignInBytes; }; + CharUnits getSizeSize() const { return CharUnits::fromQuantity(SizeSizeInBytes); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp index ebe55c7..adb40c8 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp @@ -286,21 +286,21 @@ void CodeGenTypes::RefreshTypeCacheForClass(const CXXRecordDecl *RD) { static llvm::Type *getTypeForFormat(llvm::LLVMContext &VMContext, const llvm::fltSemantics &format, bool UseNativeHalf = false) { - if (&format == &llvm::APFloat::IEEEhalf) { + if (&format == &llvm::APFloat::IEEEhalf()) { if (UseNativeHalf) return llvm::Type::getHalfTy(VMContext); else return llvm::Type::getInt16Ty(VMContext); } - if (&format == &llvm::APFloat::IEEEsingle) + if (&format == &llvm::APFloat::IEEEsingle()) return llvm::Type::getFloatTy(VMContext); - if (&format == &llvm::APFloat::IEEEdouble) + if (&format == &llvm::APFloat::IEEEdouble()) return llvm::Type::getDoubleTy(VMContext); - if (&format == &llvm::APFloat::IEEEquad) + if (&format == &llvm::APFloat::IEEEquad()) return llvm::Type::getFP128Ty(VMContext); - if (&format == &llvm::APFloat::PPCDoubleDouble) + if (&format == &llvm::APFloat::PPCDoubleDouble()) return llvm::Type::getPPC_FP128Ty(VMContext); - if (&format == &llvm::APFloat::x87DoubleExtended) + if (&format == &llvm::APFloat::x87DoubleExtended()) return llvm::Type::getX86_FP80Ty(VMContext); llvm_unreachable("Unknown float format!"); } @@ -736,10 +736,14 @@ CodeGenTypes::getCGRecordLayout(const RecordDecl *RD) { return *Layout; } +bool CodeGenTypes::isPointerZeroInitializable(QualType T) { + assert((T->isAnyPointerType() || T->isBlockPointerType()) && "Invalid type"); + return isZeroInitializable(T); +} + bool CodeGenTypes::isZeroInitializable(QualType T) { - // No need to check for member pointers when not compiling C++. - if (!Context.getLangOpts().CPlusPlus) - return true; + if (T->getAs<PointerType>()) + return Context.getTargetNullPointerValue(T) == 0; if (const auto *AT = Context.getAsArrayType(T)) { if (isa<IncompleteArrayType>(AT)) @@ -753,7 +757,7 @@ bool CodeGenTypes::isZeroInitializable(QualType T) { // Records are non-zero-initializable if they contain any // non-zero-initializable subobjects. if (const RecordType *RT = T->getAs<RecordType>()) { - const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl()); + auto RD = cast<RecordDecl>(RT->getDecl()); return isZeroInitializable(RD); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h index 5796ab8..2ce6591 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h @@ -15,15 +15,14 @@ #define LLVM_CLANG_LIB_CODEGEN_CODEGENTYPES_H #include "CGCall.h" -#include "clang/AST/GlobalDecl.h" +#include "clang/Basic/ABI.h" #include "clang/CodeGen/CGFunctionInfo.h" +#include "clang/Sema/Sema.h" #include "llvm/ADT/DenseMap.h" #include "llvm/IR/Module.h" -#include <vector> namespace llvm { class FunctionType; -class Module; class DataLayout; class Type; class LLVMContext; @@ -48,6 +47,7 @@ class TagDecl; class TargetInfo; class Type; typedef CanQual<Type> CanQualType; +class GlobalDecl; namespace CodeGen { class ABIInfo; @@ -352,6 +352,10 @@ public: // These are internal details of CGT that shouldn't be used externally. /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer. bool isZeroInitializable(QualType T); + /// Check if the pointer type can be zero-initialized (in the C++ sense) + /// with an LLVM zeroinitializer. + bool isPointerZeroInitializable(QualType T); + /// IsZeroInitializable - Return whether a record type can be /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer. bool isZeroInitializable(const RecordDecl *RD); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ConstantBuilder.h b/contrib/llvm/tools/clang/lib/CodeGen/ConstantBuilder.h new file mode 100644 index 0000000..40b34a9 --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/ConstantBuilder.h @@ -0,0 +1,444 @@ +//===----- ConstantBuilder.h - Builder for LLVM IR constants ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This class provides a convenient interface for building complex +// global initializers. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_CODEGEN_CONSTANTBUILDER_H +#define LLVM_CLANG_LIB_CODEGEN_CONSTANTBUILDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Constants.h" + +#include "CodeGenModule.h" + +#include <vector> + +namespace clang { +namespace CodeGen { + +class ConstantStructBuilder; +class ConstantArrayBuilder; + +/// A convenience builder class for complex constant initializers, +/// especially for anonymous global structures used by various language +/// runtimes. +/// +/// The basic usage pattern is expected to be something like: +/// ConstantInitBuilder builder(CGM); +/// auto toplevel = builder.beginStruct(); +/// toplevel.addInt(CGM.SizeTy, widgets.size()); +/// auto widgetArray = builder.beginArray(); +/// for (auto &widget : widgets) { +/// auto widgetDesc = widgetArray.beginStruct(); +/// widgetDesc.addInt(CGM.SizeTy, widget.getPower()); +/// widgetDesc.add(CGM.GetAddrOfConstantString(widget.getName())); +/// widgetDesc.add(CGM.GetAddrOfGlobal(widget.getInitializerDecl())); +/// widgetArray.add(widgetDesc.finish()); +/// } +/// toplevel.add(widgetArray.finish()); +/// auto global = toplevel.finishAndCreateGlobal("WIDGET_LIST", Align, +/// /*constant*/ true); +class ConstantInitBuilder { + struct SelfReference { + llvm::GlobalVariable *Dummy; + llvm::SmallVector<llvm::Constant*, 4> Indices; + + SelfReference(llvm::GlobalVariable *dummy) : Dummy(dummy) {} + }; + CodeGenModule &CGM; + llvm::SmallVector<llvm::Constant*, 16> Buffer; + std::vector<SelfReference> SelfReferences; + bool Frozen = false; + +public: + explicit ConstantInitBuilder(CodeGenModule &CGM) : CGM(CGM) {} + + ~ConstantInitBuilder() { + assert(Buffer.empty() && "didn't claim all values out of buffer"); + } + + class AggregateBuilderBase { + protected: + ConstantInitBuilder &Builder; + AggregateBuilderBase *Parent; + size_t Begin; + bool Finished = false; + bool Frozen = false; + + llvm::SmallVectorImpl<llvm::Constant*> &getBuffer() { + return Builder.Buffer; + } + + const llvm::SmallVectorImpl<llvm::Constant*> &getBuffer() const { + return Builder.Buffer; + } + + AggregateBuilderBase(ConstantInitBuilder &builder, + AggregateBuilderBase *parent) + : Builder(builder), Parent(parent), Begin(builder.Buffer.size()) { + if (parent) { + assert(!parent->Frozen && "parent already has child builder active"); + parent->Frozen = true; + } else { + assert(!builder.Frozen && "builder already has child builder active"); + builder.Frozen = true; + } + } + + ~AggregateBuilderBase() { + assert(Finished && "didn't finish aggregate builder"); + } + + void markFinished() { + assert(!Frozen && "child builder still active"); + assert(!Finished && "builder already finished"); + Finished = true; + if (Parent) { + assert(Parent->Frozen && + "parent not frozen while child builder active"); + Parent->Frozen = false; + } else { + assert(Builder.Frozen && + "builder not frozen while child builder active"); + Builder.Frozen = false; + } + } + + public: + // Not copyable. + AggregateBuilderBase(const AggregateBuilderBase &) = delete; + AggregateBuilderBase &operator=(const AggregateBuilderBase &) = delete; + + // Movable, mostly to allow returning. But we have to write this out + // properly to satisfy the assert in the destructor. + AggregateBuilderBase(AggregateBuilderBase &&other) + : Builder(other.Builder), Parent(other.Parent), Begin(other.Begin), + Finished(other.Finished), Frozen(other.Frozen) { + other.Finished = false; + } + AggregateBuilderBase &operator=(AggregateBuilderBase &&other) = delete; + + /// Abandon this builder completely. + void abandon() { + markFinished(); + auto &buffer = Builder.Buffer; + buffer.erase(buffer.begin() + Begin, buffer.end()); + } + + /// Add a new value to this initializer. + void add(llvm::Constant *value) { + assert(value && "adding null value to constant initializer"); + assert(!Finished && "cannot add more values after finishing builder"); + assert(!Frozen && "cannot add values while subbuilder is active"); + Builder.Buffer.push_back(value); + } + + /// Add an integer value of type size_t. + void addSize(CharUnits size) { + add(Builder.CGM.getSize(size)); + } + + /// Add an integer value of a specific type. + void addInt(llvm::IntegerType *intTy, uint64_t value, + bool isSigned = false) { + add(llvm::ConstantInt::get(intTy, value, isSigned)); + } + + /// Add a null pointer of a specific type. + void addNullPointer(llvm::PointerType *ptrTy) { + add(llvm::ConstantPointerNull::get(ptrTy)); + } + + /// Add a bitcast of a value to a specific type. + void addBitCast(llvm::Constant *value, llvm::Type *type) { + add(llvm::ConstantExpr::getBitCast(value, type)); + } + + /// Add a bunch of new values to this initializer. + void addAll(ArrayRef<llvm::Constant *> values) { + assert(!Finished && "cannot add more values after finishing builder"); + assert(!Frozen && "cannot add values while subbuilder is active"); + Builder.Buffer.append(values.begin(), values.end()); + } + + /// An opaque class to hold the abstract position of a placeholder. + class PlaceholderPosition { + size_t Index; + friend class AggregateBuilderBase; + PlaceholderPosition(size_t index) : Index(index) {} + }; + + /// Add a placeholder value to the structure. The returned position + /// can be used to set the value later; it will not be invalidated by + /// any intermediate operations except (1) filling the same position or + /// (2) finishing the entire builder. + /// + /// This is useful for emitting certain kinds of structure which + /// contain some sort of summary field, generaly a count, before any + /// of the data. By emitting a placeholder first, the structure can + /// be emitted eagerly. + PlaceholderPosition addPlaceholder() { + assert(!Finished && "cannot add more values after finishing builder"); + assert(!Frozen && "cannot add values while subbuilder is active"); + Builder.Buffer.push_back(nullptr); + return Builder.Buffer.size() - 1; + } + + /// Fill a previously-added placeholder. + void fillPlaceholderWithInt(PlaceholderPosition position, + llvm::IntegerType *type, uint64_t value, + bool isSigned = false) { + fillPlaceholder(position, llvm::ConstantInt::get(type, value, isSigned)); + } + + /// Fill a previously-added placeholder. + void fillPlaceholder(PlaceholderPosition position, llvm::Constant *value) { + assert(!Finished && "cannot change values after finishing builder"); + assert(!Frozen && "cannot add values while subbuilder is active"); + llvm::Constant *&slot = Builder.Buffer[position.Index]; + assert(slot == nullptr && "placeholder already filled"); + slot = value; + } + + /// Produce an address which will eventually point to the the next + /// position to be filled. This is computed with an indexed + /// getelementptr rather than by computing offsets. + /// + /// The returned pointer will have type T*, where T is the given + /// position. + llvm::Constant *getAddrOfCurrentPosition(llvm::Type *type) { + // Make a global variable. We will replace this with a GEP to this + // position after installing the initializer. + auto dummy = + new llvm::GlobalVariable(Builder.CGM.getModule(), type, true, + llvm::GlobalVariable::PrivateLinkage, + nullptr, ""); + Builder.SelfReferences.emplace_back(dummy); + auto &entry = Builder.SelfReferences.back(); + (void) getGEPIndicesToCurrentPosition(entry.Indices); + return dummy; + } + + ArrayRef<llvm::Constant*> getGEPIndicesToCurrentPosition( + llvm::SmallVectorImpl<llvm::Constant*> &indices) { + getGEPIndicesTo(indices, Builder.Buffer.size()); + return indices; + } + + ConstantArrayBuilder beginArray(llvm::Type *eltTy = nullptr); + ConstantStructBuilder beginStruct(llvm::StructType *structTy = nullptr); + + private: + void getGEPIndicesTo(llvm::SmallVectorImpl<llvm::Constant*> &indices, + size_t position) const { + // Recurse on the parent builder if present. + if (Parent) { + Parent->getGEPIndicesTo(indices, Begin); + + // Otherwise, add an index to drill into the first level of pointer. + } else { + assert(indices.empty()); + indices.push_back(llvm::ConstantInt::get(Builder.CGM.Int32Ty, 0)); + } + + assert(position >= Begin); + // We have to use i32 here because struct GEPs demand i32 indices. + // It's rather unlikely to matter in practice. + indices.push_back(llvm::ConstantInt::get(Builder.CGM.Int32Ty, + position - Begin)); + } + }; + + template <class Impl> + class AggregateBuilder : public AggregateBuilderBase { + protected: + AggregateBuilder(ConstantInitBuilder &builder, + AggregateBuilderBase *parent) + : AggregateBuilderBase(builder, parent) {} + + Impl &asImpl() { return *static_cast<Impl*>(this); } + + public: + /// Given that this builder was created by beginning an array or struct + /// component on the given parent builder, finish the array/struct + /// component and add it to the parent. + /// + /// It is an intentional choice that the parent is passed in explicitly + /// despite it being redundant with information already kept in the + /// builder. This aids in readability by making it easier to find the + /// places that add components to a builder, as well as "bookending" + /// the sub-builder more explicitly. + void finishAndAddTo(AggregateBuilderBase &parent) { + assert(Parent == &parent && "adding to non-parent builder"); + parent.add(asImpl().finishImpl()); + } + + /// Given that this builder was created by beginning an array or struct + /// directly on a ConstantInitBuilder, finish the array/struct and + /// create a global variable with it as the initializer. + template <class... As> + llvm::GlobalVariable *finishAndCreateGlobal(As &&...args) { + assert(!Parent && "finishing non-root builder"); + return Builder.createGlobal(asImpl().finishImpl(), + std::forward<As>(args)...); + } + + /// Given that this builder was created by beginning an array or struct + /// directly on a ConstantInitBuilder, finish the array/struct and + /// set it as the initializer of the given global variable. + void finishAndSetAsInitializer(llvm::GlobalVariable *global) { + assert(!Parent && "finishing non-root builder"); + return Builder.setGlobalInitializer(global, asImpl().finishImpl()); + } + }; + + ConstantArrayBuilder beginArray(llvm::Type *eltTy = nullptr); + + ConstantStructBuilder beginStruct(llvm::StructType *structTy = nullptr); + +private: + llvm::GlobalVariable *createGlobal(llvm::Constant *initializer, + const llvm::Twine &name, + CharUnits alignment, + bool constant = false, + llvm::GlobalValue::LinkageTypes linkage + = llvm::GlobalValue::InternalLinkage, + unsigned addressSpace = 0) { + auto GV = new llvm::GlobalVariable(CGM.getModule(), + initializer->getType(), + constant, + linkage, + initializer, + name, + /*insert before*/ nullptr, + llvm::GlobalValue::NotThreadLocal, + addressSpace); + GV->setAlignment(alignment.getQuantity()); + resolveSelfReferences(GV); + return GV; + } + + void setGlobalInitializer(llvm::GlobalVariable *GV, + llvm::Constant *initializer) { + GV->setInitializer(initializer); + resolveSelfReferences(GV); + } + + void resolveSelfReferences(llvm::GlobalVariable *GV) { + for (auto &entry : SelfReferences) { + llvm::Constant *resolvedReference = + llvm::ConstantExpr::getInBoundsGetElementPtr( + GV->getValueType(), GV, entry.Indices); + entry.Dummy->replaceAllUsesWith(resolvedReference); + entry.Dummy->eraseFromParent(); + } + } +}; + +/// A helper class of ConstantInitBuilder, used for building constant +/// array initializers. +class ConstantArrayBuilder + : public ConstantInitBuilder::AggregateBuilder<ConstantArrayBuilder> { + llvm::Type *EltTy; + friend class ConstantInitBuilder; + template <class Impl> friend class ConstantInitBuilder::AggregateBuilder; + ConstantArrayBuilder(ConstantInitBuilder &builder, + AggregateBuilderBase *parent, llvm::Type *eltTy) + : AggregateBuilder(builder, parent), EltTy(eltTy) {} +public: + size_t size() const { + assert(!Finished); + assert(!Frozen); + assert(Begin <= getBuffer().size()); + return getBuffer().size() - Begin; + } + + bool empty() const { + return size() == 0; + } + +private: + /// Form an array constant from the values that have been added to this + /// builder. + llvm::Constant *finishImpl() { + markFinished(); + + auto &buffer = getBuffer(); + assert((Begin < buffer.size() || + (Begin == buffer.size() && EltTy)) + && "didn't add any array elements without element type"); + auto elts = llvm::makeArrayRef(buffer).slice(Begin); + auto eltTy = EltTy ? EltTy : elts[0]->getType(); + auto type = llvm::ArrayType::get(eltTy, elts.size()); + auto constant = llvm::ConstantArray::get(type, elts); + buffer.erase(buffer.begin() + Begin, buffer.end()); + return constant; + } +}; + +inline ConstantArrayBuilder +ConstantInitBuilder::beginArray(llvm::Type *eltTy) { + return ConstantArrayBuilder(*this, nullptr, eltTy); +} + +inline ConstantArrayBuilder +ConstantInitBuilder::AggregateBuilderBase::beginArray(llvm::Type *eltTy) { + return ConstantArrayBuilder(Builder, this, eltTy); +} + +/// A helper class of ConstantInitBuilder, used for building constant +/// struct initializers. +class ConstantStructBuilder + : public ConstantInitBuilder::AggregateBuilder<ConstantStructBuilder> { + llvm::StructType *Ty; + friend class ConstantInitBuilder; + template <class Impl> friend class ConstantInitBuilder::AggregateBuilder; + ConstantStructBuilder(ConstantInitBuilder &builder, + AggregateBuilderBase *parent, llvm::StructType *ty) + : AggregateBuilder(builder, parent), Ty(ty) {} + + /// Finish the struct. + llvm::Constant *finishImpl() { + markFinished(); + + auto &buffer = getBuffer(); + assert(Begin < buffer.size() && "didn't add any struct elements?"); + auto elts = llvm::makeArrayRef(buffer).slice(Begin); + + llvm::Constant *constant; + if (Ty) { + constant = llvm::ConstantStruct::get(Ty, elts); + } else { + constant = llvm::ConstantStruct::getAnon(elts, /*packed*/ false); + } + + buffer.erase(buffer.begin() + Begin, buffer.end()); + return constant; + } +}; + +inline ConstantStructBuilder +ConstantInitBuilder::beginStruct(llvm::StructType *structTy) { + return ConstantStructBuilder(*this, nullptr, structTy); +} + +inline ConstantStructBuilder +ConstantInitBuilder::AggregateBuilderBase::beginStruct( + llvm::StructType *structTy) { + return ConstantStructBuilder(Builder, this, structTy); +} + +} // end namespace CodeGen +} // end namespace clang + +#endif diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp index b011a0f..5bc9e50 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -23,6 +23,7 @@ #include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" using namespace clang; using namespace CodeGen; @@ -91,6 +92,14 @@ public: /// \brief The source mapping regions for this function. std::vector<SourceMappingRegion> SourceRegions; + /// \brief A set of regions which can be used as a filter. + /// + /// It is produced by emitExpansionRegions() and is used in + /// emitSourceRegions() to suppress producing code regions if + /// the same area is covered by expansion regions. + typedef llvm::SmallSet<std::pair<SourceLocation, SourceLocation>, 8> + SourceRegionFilter; + CoverageMappingBuilder(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts) : CVM(CVM), SM(SM), LangOpts(LangOpts) {} @@ -127,7 +136,7 @@ public: /// \brief Return true if \c Loc is a location in a built-in macro. bool isInBuiltin(SourceLocation Loc) { - return strcmp(SM.getBufferName(SM.getSpellingLoc(Loc)), "<built-in>") == 0; + return SM.getBufferName(SM.getSpellingLoc(Loc)) == "<built-in>"; } /// \brief Check whether \c Loc is included or expanded from \c Parent. @@ -248,7 +257,7 @@ public: /// \brief Generate the coverage counter mapping regions from collected /// source regions. - void emitSourceRegions() { + void emitSourceRegions(const SourceRegionFilter &Filter) { for (const auto &Region : SourceRegions) { assert(Region.hasEndLoc() && "incomplete region"); @@ -268,6 +277,13 @@ public: assert(SM.isWrittenInSameFile(LocStart, LocEnd) && "region spans multiple files"); + // Don't add code regions for the area covered by expansion regions. + // This not only suppresses redundant regions, but sometimes prevents + // creating regions with wrong counters if, for example, a statement's + // body ends at the end of a nested macro. + if (Filter.count(std::make_pair(LocStart, LocEnd))) + continue; + // Find the spilling locations for the mapping region. unsigned LineStart = SM.getSpellingLineNumber(LocStart); unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); @@ -282,7 +298,8 @@ public: } /// \brief Generate expansion regions for each virtual file we've seen. - void emitExpansionRegions() { + SourceRegionFilter emitExpansionRegions() { + SourceRegionFilter Filter; for (const auto &FM : FileIDMapping) { SourceLocation ExpandedLoc = FM.second.second; SourceLocation ParentLoc = getIncludeOrExpansionLoc(ExpandedLoc); @@ -298,6 +315,7 @@ public: SourceLocation LocEnd = getPreciseTokenLocEnd(ParentLoc); assert(SM.isWrittenInSameFile(ParentLoc, LocEnd) && "region spans multiple files"); + Filter.insert(std::make_pair(ParentLoc, LocEnd)); unsigned LineStart = SM.getSpellingLineNumber(ParentLoc); unsigned ColumnStart = SM.getSpellingColumnNumber(ParentLoc); @@ -308,6 +326,7 @@ public: *ParentFileID, *ExpandedFileID, LineStart, ColumnStart, LineEnd, ColumnEnd)); } + return Filter; } }; @@ -349,7 +368,7 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder { void write(llvm::raw_ostream &OS) { SmallVector<unsigned, 16> FileIDMapping; gatherFileIDs(FileIDMapping); - emitSourceRegions(); + emitSourceRegions(SourceRegionFilter()); if (MappingRegions.empty()) return; @@ -431,7 +450,8 @@ struct CounterCoverageMappingBuilder SourceLocation NestedLoc = getStartOfFileOrMacro(EndLoc); assert(SM.isWrittenInSameFile(NestedLoc, EndLoc)); - SourceRegions.emplace_back(Region.getCounter(), NestedLoc, EndLoc); + if (!isRegionAlreadyAdded(NestedLoc, EndLoc)) + SourceRegions.emplace_back(Region.getCounter(), NestedLoc, EndLoc); EndLoc = getPreciseTokenLocEnd(getIncludeOrExpansionLoc(EndLoc)); if (EndLoc.isInvalid()) @@ -603,8 +623,8 @@ struct CounterCoverageMappingBuilder void write(llvm::raw_ostream &OS) { llvm::SmallVector<unsigned, 8> VirtualFileMapping; gatherFileIDs(VirtualFileMapping); - emitSourceRegions(); - emitExpansionRegions(); + SourceRegionFilter Filter = emitExpansionRegions(); + emitSourceRegions(Filter); gatherSkippedRegions(); if (MappingRegions.empty()) @@ -793,6 +813,8 @@ struct CounterCoverageMappingBuilder void VisitSwitchStmt(const SwitchStmt *S) { extendRegion(S); + if (S->getInit()) + Visit(S->getInit()); Visit(S->getCond()); BreakContinueStack.push_back(BreakContinue()); @@ -822,7 +844,11 @@ struct CounterCoverageMappingBuilder Counter ExitCount = getRegionCounter(S); SourceLocation ExitLoc = getEnd(S); - pushRegion(ExitCount, getStart(S), ExitLoc); + pushRegion(ExitCount); + + // Ensure that handleFileExit recognizes when the end location is located + // in a different file. + MostRecentLocation = getStart(S); handleFileExit(ExitLoc); } @@ -849,6 +875,9 @@ struct CounterCoverageMappingBuilder void VisitIfStmt(const IfStmt *S) { extendRegion(S); + if (S->getInit()) + Visit(S->getInit()); + // Extend into the condition before we propagate through it below - this is // needed to handle macros that generate the "if" but not the condition. extendRegion(S->getCond()); @@ -931,16 +960,24 @@ struct CounterCoverageMappingBuilder // propagate counts into them. } }; -} -static bool isMachO(const CodeGenModule &CGM) { +bool isMachO(const CodeGenModule &CGM) { return CGM.getTarget().getTriple().isOSBinFormatMachO(); } -static StringRef getCoverageSection(const CodeGenModule &CGM) { +StringRef getCoverageSection(const CodeGenModule &CGM) { return llvm::getInstrProfCoverageSectionName(isMachO(CGM)); } +std::string normalizeFilename(StringRef Filename) { + llvm::SmallString<256> Path(Filename); + llvm::sys::fs::make_absolute(Path); + llvm::sys::path::remove_dots(Path, /*remove_dot_dots=*/true); + return Path.str().str(); +} + +} // end anonymous namespace + static void dump(llvm::raw_ostream &OS, StringRef FunctionName, ArrayRef<CounterExpression> Expressions, ArrayRef<CounterMappingRegion> Regions) { @@ -1002,10 +1039,15 @@ void CoverageMappingModuleGen::addFunctionMappingRecord( std::vector<StringRef> Filenames; std::vector<CounterExpression> Expressions; std::vector<CounterMappingRegion> Regions; + llvm::SmallVector<std::string, 16> FilenameStrs; llvm::SmallVector<StringRef, 16> FilenameRefs; + FilenameStrs.resize(FileEntries.size()); FilenameRefs.resize(FileEntries.size()); - for (const auto &Entry : FileEntries) - FilenameRefs[Entry.second] = Entry.first->getName(); + for (const auto &Entry : FileEntries) { + auto I = Entry.second; + FilenameStrs[I] = normalizeFilename(Entry.first->getName()); + FilenameRefs[I] = FilenameStrs[I]; + } RawCoverageMappingReader Reader(CoverageMapping, FilenameRefs, Filenames, Expressions, Regions); if (Reader.read()) @@ -1026,11 +1068,8 @@ void CoverageMappingModuleGen::emit() { FilenameStrs.resize(FileEntries.size()); FilenameRefs.resize(FileEntries.size()); for (const auto &Entry : FileEntries) { - llvm::SmallString<256> Path(Entry.first->getName()); - llvm::sys::fs::make_absolute(Path); - auto I = Entry.second; - FilenameStrs[I] = std::string(Path.begin(), Path.end()); + FilenameStrs[I] = normalizeFilename(Entry.first->getName()); FilenameRefs[I] = FilenameStrs[I]; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.h b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.h index c202fe8..b6789c2 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.h @@ -19,7 +19,6 @@ #include "clang/Frontend/CodeGenOptions.h" #include "clang/Lex/PPCallbacks.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringMap.h" #include "llvm/IR/GlobalValue.h" #include "llvm/Support/raw_ostream.h" diff --git a/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h b/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h index 4717a66..2435830 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h @@ -271,7 +271,7 @@ public: /// Push a lazily-created cleanup on the stack. template <class T, class... As> void pushCleanup(CleanupKind Kind, As... A) { - static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment, + static_assert(alignof(T) <= ScopeStackAlignment, "Cleanup's alignment is too large."); void *Buffer = pushCleanup(Kind, sizeof(T)); Cleanup *Obj = new (Buffer) T(A...); @@ -281,7 +281,7 @@ public: /// Push a lazily-created cleanup on the stack. Tuple version. template <class T, class... As> void pushCleanupTuple(CleanupKind Kind, std::tuple<As...> A) { - static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment, + static_assert(alignof(T) <= ScopeStackAlignment, "Cleanup's alignment is too large."); void *Buffer = pushCleanup(Kind, sizeof(T)); Cleanup *Obj = new (Buffer) T(std::move(A)); @@ -303,7 +303,7 @@ public: /// stack is modified. template <class T, class... As> T *pushCleanupWithExtra(CleanupKind Kind, size_t N, As... A) { - static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment, + static_assert(alignof(T) <= ScopeStackAlignment, "Cleanup's alignment is too large."); void *Buffer = pushCleanup(Kind, sizeof(T) + T::getExtraSize(N)); return new (Buffer) T(N, A...); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp index 6051594..f7a8dd6 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -24,6 +24,7 @@ #include "CGVTables.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "ConstantBuilder.h" #include "TargetInfo.h" #include "clang/AST/Mangle.h" #include "clang/AST/Type.h" @@ -45,6 +46,7 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI { protected: bool UseARMMethodPtrABI; bool UseARMGuardVarABI; + bool Use32BitVTableOffsetABI; ItaniumMangleContext &getMangleContext() { return cast<ItaniumMangleContext>(CodeGen::CGCXXABI::getMangleContext()); @@ -55,7 +57,8 @@ public: bool UseARMMethodPtrABI = false, bool UseARMGuardVarABI = false) : CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI), - UseARMGuardVarABI(UseARMGuardVarABI) { } + UseARMGuardVarABI(UseARMGuardVarABI), + Use32BitVTableOffsetABI(false) { } bool classifyReturnType(CGFunctionInfo &FI) const override; @@ -112,7 +115,7 @@ public: llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override; - llvm::Value * + CGCallee EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E, Address This, @@ -168,8 +171,8 @@ public: emitTerminateForUnexpectedException(CodeGenFunction &CGF, llvm::Value *Exn) override; - void EmitFundamentalRTTIDescriptor(QualType Type); - void EmitFundamentalRTTIDescriptors(); + void EmitFundamentalRTTIDescriptor(QualType Type, bool DLLExport); + void EmitFundamentalRTTIDescriptors(bool DLLExport); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; CatchTypeInfo getAddrOfCXXCatchHandlerType(QualType Ty, @@ -261,9 +264,9 @@ public: llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; - llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - Address This, llvm::Type *Ty, - SourceLocation Loc) override; + CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, + Address This, llvm::Type *Ty, + SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, @@ -363,11 +366,12 @@ public: void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override; private: - bool hasAnyUsedVirtualInlineFunction(const CXXRecordDecl *RD) const { + bool hasAnyVirtualInlineFunction(const CXXRecordDecl *RD) const { const auto &VtableLayout = CGM.getItaniumVTableContext().getVTableLayout(RD); for (const auto &VtableComponent : VtableLayout.vtable_components()) { + // Skip empty slot. if (!VtableComponent.isUsedFunctionPointerKind()) continue; @@ -425,7 +429,9 @@ public: class iOS64CXXABI : public ARMCXXABI { public: - iOS64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {} + iOS64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) { + Use32BitVTableOffsetABI = true; + } // ARM64 libraries are prepared for non-unique RTTI. bool shouldRTTIBeUnique() const override { return false; } @@ -516,7 +522,7 @@ ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) { /// /// If the member is non-virtual, memptr.ptr is the address of /// the function to call. -llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( +CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( CodeGenFunction &CGF, const Expr *E, Address ThisAddr, llvm::Value *&ThisPtrForCall, llvm::Value *MemFnPtr, const MemberPointerType *MPT) { @@ -579,9 +585,15 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy, RD); // Apply the offset. + // On ARM64, to reserve extra space in virtual member function pointers, + // we only pay attention to the low 32 bits of the offset. llvm::Value *VTableOffset = FnAsInt; if (!UseARMMethodPtrABI) VTableOffset = Builder.CreateSub(VTableOffset, ptrdiff_1); + if (Use32BitVTableOffsetABI) { + VTableOffset = Builder.CreateTrunc(VTableOffset, CGF.Int32Ty); + VTableOffset = Builder.CreateZExt(VTableOffset, CGM.PtrDiffTy); + } VTable = Builder.CreateGEP(VTable, VTableOffset); // Load the virtual function to call. @@ -599,9 +611,11 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // We're done. CGF.EmitBlock(FnEnd); - llvm::PHINode *Callee = Builder.CreatePHI(FTy->getPointerTo(), 2); - Callee->addIncoming(VirtualFn, FnVirtual); - Callee->addIncoming(NonVirtualFn, FnNonVirtual); + llvm::PHINode *CalleePtr = Builder.CreatePHI(FTy->getPointerTo(), 2); + CalleePtr->addIncoming(VirtualFn, FnVirtual); + CalleePtr->addIncoming(NonVirtualFn, FnNonVirtual); + + CGCallee Callee(FPT, CalleePtr); return Callee; } @@ -1390,6 +1404,10 @@ void ItaniumCXXABI::addImplicitStructorParams(CodeGenFunction &CGF, } void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) { + // Naked functions have no prolog. + if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>()) + return; + /// Initialize the 'this' slot. EmitThisParam(CGF); @@ -1434,15 +1452,18 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, llvm::Value *VTT = CGF.GetVTTParameter(GD, ForVirtualBase, Delegating); QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy); - llvm::Value *Callee = nullptr; - if (getContext().getLangOpts().AppleKext) + CGCallee Callee; + if (getContext().getLangOpts().AppleKext && + Type != Dtor_Base && DD->isVirtual()) Callee = CGF.BuildAppleKextVirtualDestructorCall(DD, Type, DD->getParent()); - - if (!Callee) - Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type)); + else + Callee = + CGCallee::forDirect(CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type)), + DD); CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(), - This.getPointer(), VTT, VTTTy, nullptr); + This.getPointer(), VTT, VTTTy, + nullptr, nullptr); } void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1458,10 +1479,10 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, CGM.GetAddrOfRTTIDescriptor(CGM.getContext().getTagDeclType(RD)); // Create and set the initializer. - llvm::Constant *Init = CGVT.CreateVTableInitializer( - RD, VTLayout.vtable_component_begin(), VTLayout.getNumVTableComponents(), - VTLayout.vtable_thunk_begin(), VTLayout.getNumVTableThunks(), RTTI); - VTable->setInitializer(Init); + ConstantInitBuilder Builder(CGM); + auto Components = Builder.beginStruct(); + CGVT.createVTableInitializer(Components, VTLayout, RTTI); + Components.finishAndSetAsInitializer(VTable); // Set the correct linkage. VTable->setLinkage(Linkage); @@ -1487,7 +1508,7 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, isa<NamespaceDecl>(DC) && cast<NamespaceDecl>(DC)->getIdentifier() && cast<NamespaceDecl>(DC)->getIdentifier()->isStr("__cxxabiv1") && DC->getParent()->isTranslationUnit()) - EmitFundamentalRTTIDescriptors(); + EmitFundamentalRTTIDescriptors(RD->hasAttr<DLLExportAttr>()); if (!VTable->isDeclarationForLinker()) CGM.EmitVTableTypeMetadata(VTable, VTLayout); @@ -1517,17 +1538,21 @@ ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base, const CXXRecordDecl *VTableClass) { llvm::GlobalValue *VTable = getAddrOfVTable(VTableClass, CharUnits()); - // Find the appropriate vtable within the vtable group. - uint64_t AddressPoint = CGM.getItaniumVTableContext() - .getVTableLayout(VTableClass) - .getAddressPoint(Base); + // Find the appropriate vtable within the vtable group, and the address point + // within that vtable. + VTableLayout::AddressPointLocation AddressPoint = + CGM.getItaniumVTableContext() + .getVTableLayout(VTableClass) + .getAddressPoint(Base); llvm::Value *Indices[] = { llvm::ConstantInt::get(CGM.Int32Ty, 0), - llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint) + llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.VTableIndex), + llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.AddressPointIndex), }; - return llvm::ConstantExpr::getInBoundsGetElementPtr(VTable->getValueType(), - VTable, Indices); + return llvm::ConstantExpr::getGetElementPtr(VTable->getValueType(), VTable, + Indices, /*InBounds=*/true, + /*InRangeIndex=*/1); } llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( @@ -1569,12 +1594,12 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, llvm::raw_svector_ostream Out(Name); getMangleContext().mangleCXXVTable(RD, Out); - ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext(); - llvm::ArrayType *ArrayType = llvm::ArrayType::get( - CGM.Int8PtrTy, VTContext.getVTableLayout(RD).getNumVTableComponents()); + const VTableLayout &VTLayout = + CGM.getItaniumVTableContext().getVTableLayout(RD); + llvm::Type *VTableType = CGM.getVTables().getVTableType(VTLayout); VTable = CGM.CreateOrReplaceCXXRuntimeVariable( - Name, ArrayType, llvm::GlobalValue::ExternalLinkage); + Name, VTableType, llvm::GlobalValue::ExternalLinkage); VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); if (RD->hasAttr<DLLImportAttr>()) @@ -1585,19 +1610,20 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, return VTable; } -llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, - GlobalDecl GD, - Address This, - llvm::Type *Ty, - SourceLocation Loc) { +CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, + GlobalDecl GD, + Address This, + llvm::Type *Ty, + SourceLocation Loc) { GD = GD.getCanonicalDecl(); Ty = Ty->getPointerTo()->getPointerTo(); auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl()); llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); + llvm::Value *VFunc; if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { - return CGF.EmitVTableTypeCheckedLoad( + VFunc = CGF.EmitVTableTypeCheckedLoad( MethodDecl->getParent(), VTable, VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); } else { @@ -1605,8 +1631,26 @@ llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); - return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); - } + auto *VFuncLoad = + CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); + + // Add !invariant.load md to virtual function load to indicate that + // function didn't change inside vtable. + // It's safe to add it without -fstrict-vtable-pointers, but it would not + // help in devirtualization because it will only matter if we will have 2 + // the same virtual function loads from the same vtable load, which won't + // happen without enabled devirtualization with -fstrict-vtable-pointers. + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGM.getCodeGenOpts().StrictVTablePointers) + VFuncLoad->setMetadata( + llvm::LLVMContext::MD_invariant_load, + llvm::MDNode::get(CGM.getLLVMContext(), + llvm::ArrayRef<llvm::Metadata *>())); + VFunc = VFuncLoad; + } + + CGCallee Callee(MethodDecl, VFunc); + return Callee; } llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( @@ -1618,13 +1662,13 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( const CGFunctionInfo *FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration( Dtor, getFromDtorType(DtorType)); llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); - llvm::Value *Callee = + CGCallee Callee = getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty, CE ? CE->getLocStart() : SourceLocation()); CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This.getPointer(), /*ImplicitParam=*/nullptr, - QualType(), CE); + QualType(), CE, nullptr); return nullptr; } @@ -1644,7 +1688,7 @@ bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const { // then we are safe to emit available_externally copy of vtable. // FIXME we can still emit a copy of the vtable if we // can emit definition of the inline functions. - return !hasAnyUsedVirtualInlineFunction(RD) && !isVTableHidden(RD); + return !hasAnyVirtualInlineFunction(RD) && !isVTableHidden(RD); } static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, Address InitialPtr, @@ -2228,7 +2272,21 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( ArrayRef<llvm::Function *> CXXThreadLocalInits, ArrayRef<const VarDecl *> CXXThreadLocalInitVars) { llvm::Function *InitFunc = nullptr; - if (!CXXThreadLocalInits.empty()) { + + // Separate initializers into those with ordered (or partially-ordered) + // initialization and those with unordered initialization. + llvm::SmallVector<llvm::Function *, 8> OrderedInits; + llvm::SmallDenseMap<const VarDecl *, llvm::Function *> UnorderedInits; + for (unsigned I = 0; I != CXXThreadLocalInits.size(); ++I) { + if (isTemplateInstantiation( + CXXThreadLocalInitVars[I]->getTemplateSpecializationKind())) + UnorderedInits[CXXThreadLocalInitVars[I]->getCanonicalDecl()] = + CXXThreadLocalInits[I]; + else + OrderedInits.push_back(CXXThreadLocalInits[I]); + } + + if (!OrderedInits.empty()) { // Generate a guarded initialization function. llvm::FunctionType *FTy = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); @@ -2245,24 +2303,28 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( CharUnits GuardAlign = CharUnits::One(); Guard->setAlignment(GuardAlign.getQuantity()); - CodeGenFunction(CGM) - .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, - Address(Guard, GuardAlign)); + CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(InitFunc, OrderedInits, + Address(Guard, GuardAlign)); // On Darwin platforms, use CXX_FAST_TLS calling convention. if (CGM.getTarget().getTriple().isOSDarwin()) { InitFunc->setCallingConv(llvm::CallingConv::CXX_FAST_TLS); InitFunc->addFnAttr(llvm::Attribute::NoUnwind); } } + + // Emit thread wrappers. for (const VarDecl *VD : CXXThreadLocals) { llvm::GlobalVariable *Var = cast<llvm::GlobalVariable>(CGM.GetGlobalValue(CGM.getMangledName(VD))); + llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var); // Some targets require that all access to thread local variables go through // the thread wrapper. This means that we cannot attempt to create a thread // wrapper or a thread helper. - if (isThreadWrapperReplaceable(VD, CGM) && !VD->hasDefinition()) + if (isThreadWrapperReplaceable(VD, CGM) && !VD->hasDefinition()) { + Wrapper->setLinkage(llvm::Function::ExternalLinkage); continue; + } // Mangle the name for the thread_local initialization function. SmallString<256> InitFnName; @@ -2278,18 +2340,21 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( bool InitIsInitFunc = false; if (VD->hasDefinition()) { InitIsInitFunc = true; - if (InitFunc) + llvm::Function *InitFuncToUse = InitFunc; + if (isTemplateInstantiation(VD->getTemplateSpecializationKind())) + InitFuncToUse = UnorderedInits.lookup(VD->getCanonicalDecl()); + if (InitFuncToUse) Init = llvm::GlobalAlias::create(Var->getLinkage(), InitFnName.str(), - InitFunc); + InitFuncToUse); } else { // Emit a weak global function referring to the initialization function. // This function will not exist if the TU defining the thread_local // variable in question does not need any dynamic initialization for // its thread_local variables. llvm::FunctionType *FnTy = llvm::FunctionType::get(CGM.VoidTy, false); - Init = llvm::Function::Create( - FnTy, llvm::GlobalVariable::ExternalWeakLinkage, InitFnName.str(), - &CGM.getModule()); + Init = llvm::Function::Create(FnTy, + llvm::GlobalVariable::ExternalWeakLinkage, + InitFnName.str(), &CGM.getModule()); const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction(); CGM.SetLLVMFunctionAttributes(nullptr, FI, cast<llvm::Function>(Init)); } @@ -2297,7 +2362,6 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( if (Init) Init->setVisibility(Var->getVisibility()); - llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var); llvm::LLVMContext &Context = CGM.getModule().getContext(); llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper); CGBuilderTy Builder(CGM, Entry); @@ -2342,8 +2406,7 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val); llvm::CallInst *CallVal = CGF.Builder.CreateCall(Wrapper); - if (isThreadWrapperReplaceable(VD, CGF.CGM)) - CallVal->setCallingConv(llvm::CallingConv::CXX_FAST_TLS); + CallVal->setCallingConv(Wrapper->getCallingConv()); LValue LV; if (VD->getType()->isReferenceType()) @@ -2436,7 +2499,13 @@ public: /// PTI_ContainingClassIncomplete - Containing class is incomplete. /// (in pointer to member). - PTI_ContainingClassIncomplete = 0x10 + PTI_ContainingClassIncomplete = 0x10, + + /// PTI_TransactionSafe - Pointee is transaction_safe function (C++ TM TS). + //PTI_TransactionSafe = 0x20, + + /// PTI_Noexcept - Pointee is noexcept function (C++1z). + PTI_Noexcept = 0x40, }; // VMI type info flags. @@ -2460,7 +2529,9 @@ public: /// BuildTypeInfo - Build the RTTI type info struct for the given type. /// /// \param Force - true to force the creation of this RTTI value - llvm::Constant *BuildTypeInfo(QualType Ty, bool Force = false); + /// \param DLLExport - true to mark the RTTI value as DLLExport + llvm::Constant *BuildTypeInfo(QualType Ty, bool Force = false, + bool DLLExport = false); }; } @@ -2865,16 +2936,18 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM, case VisibleNoLinkage: case ExternalLinkage: - if (!CGM.getLangOpts().RTTI) { - // RTTI is not enabled, which means that this type info struct is going - // to be used for exception handling. Give it linkonce_odr linkage. + // RTTI is not enabled, which means that this type info struct is going + // to be used for exception handling. Give it linkonce_odr linkage. + if (!CGM.getLangOpts().RTTI) return llvm::GlobalValue::LinkOnceODRLinkage; - } if (const RecordType *Record = dyn_cast<RecordType>(Ty)) { const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl()); if (RD->hasAttr<WeakAttr>()) return llvm::GlobalValue::WeakODRLinkage; + if (CGM.getTriple().isWindowsItaniumEnvironment()) + if (RD->hasAttr<DLLImportAttr>()) + return llvm::GlobalValue::ExternalLinkage; if (RD->isDynamicClass()) { llvm::GlobalValue::LinkageTypes LT = CGM.getVTableLinkage(RD); // MinGW won't export the RTTI information when there is a key function. @@ -2892,7 +2965,8 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM, llvm_unreachable("Invalid linkage!"); } -llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { +llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force, + bool DLLExport) { // We want to operate on the canonical type. Ty = Ty.getCanonicalType(); @@ -3075,25 +3149,28 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { llvmVisibility = llvm::GlobalValue::HiddenVisibility; else llvmVisibility = CodeGenModule::GetLLVMVisibility(Ty->getVisibility()); + TypeName->setVisibility(llvmVisibility); GV->setVisibility(llvmVisibility); - return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); -} - -/// ComputeQualifierFlags - Compute the pointer type info flags from the -/// given qualifier. -static unsigned ComputeQualifierFlags(Qualifiers Quals) { - unsigned Flags = 0; - - if (Quals.hasConst()) - Flags |= ItaniumRTTIBuilder::PTI_Const; - if (Quals.hasVolatile()) - Flags |= ItaniumRTTIBuilder::PTI_Volatile; - if (Quals.hasRestrict()) - Flags |= ItaniumRTTIBuilder::PTI_Restrict; + if (CGM.getTriple().isWindowsItaniumEnvironment()) { + auto RD = Ty->getAsCXXRecordDecl(); + if (DLLExport || (RD && RD->hasAttr<DLLExportAttr>())) { + TypeName->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + } else if (CGM.getLangOpts().RTTI && RD && RD->hasAttr<DLLImportAttr>()) { + TypeName->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + + // Because the typename and the typeinfo are DLL import, convert them to + // declarations rather than definitions. The initializers still need to + // be constructed to calculate the type for the declarations. + TypeName->setInitializer(nullptr); + GV->setInitializer(nullptr); + } + } - return Flags; + return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); } /// BuildObjCObjectTypeInfo - Build the appropriate kind of type_info @@ -3214,9 +3291,6 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) { if (!RD->getNumBases()) return; - llvm::Type *LongLTy = - CGM.getTypes().ConvertType(CGM.getContext().LongTy); - // Now add the base class descriptions. // Itanium C++ ABI 2.9.5p6c: @@ -3234,6 +3308,19 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) { // __offset_shift = 8 // }; // }; + + // If we're in mingw and 'long' isn't wide enough for a pointer, use 'long + // long' instead of 'long' for __offset_flags. libstdc++abi uses long long on + // LLP64 platforms. + // FIXME: Consider updating libc++abi to match, and extend this logic to all + // LLP64 platforms. + QualType OffsetFlagsTy = CGM.getContext().LongTy; + const TargetInfo &TI = CGM.getContext().getTargetInfo(); + if (TI.getTriple().isOSCygMing() && TI.getPointerWidth(0) > TI.getLongWidth()) + OffsetFlagsTy = CGM.getContext().LongLongTy; + llvm::Type *OffsetFlagsLTy = + CGM.getTypes().ConvertType(OffsetFlagsTy); + for (const auto &Base : RD->bases()) { // The __base_type member points to the RTTI for the base type. Fields.push_back(ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Base.getType())); @@ -3265,27 +3352,48 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) { if (Base.getAccessSpecifier() == AS_public) OffsetFlags |= BCTI_Public; - Fields.push_back(llvm::ConstantInt::get(LongLTy, OffsetFlags)); + Fields.push_back(llvm::ConstantInt::get(OffsetFlagsLTy, OffsetFlags)); + } +} + +/// Compute the flags for a __pbase_type_info, and remove the corresponding +/// pieces from \p Type. +static unsigned extractPBaseFlags(ASTContext &Ctx, QualType &Type) { + unsigned Flags = 0; + + if (Type.isConstQualified()) + Flags |= ItaniumRTTIBuilder::PTI_Const; + if (Type.isVolatileQualified()) + Flags |= ItaniumRTTIBuilder::PTI_Volatile; + if (Type.isRestrictQualified()) + Flags |= ItaniumRTTIBuilder::PTI_Restrict; + Type = Type.getUnqualifiedType(); + + // Itanium C++ ABI 2.9.5p7: + // When the abi::__pbase_type_info is for a direct or indirect pointer to an + // incomplete class type, the incomplete target type flag is set. + if (ContainsIncompleteClassType(Type)) + Flags |= ItaniumRTTIBuilder::PTI_Incomplete; + + if (auto *Proto = Type->getAs<FunctionProtoType>()) { + if (Proto->isNothrow(Ctx)) { + Flags |= ItaniumRTTIBuilder::PTI_Noexcept; + Type = Ctx.getFunctionType( + Proto->getReturnType(), Proto->getParamTypes(), + Proto->getExtProtoInfo().withExceptionSpec(EST_None)); + } } + + return Flags; } /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct, /// used for pointer types. void ItaniumRTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) { - Qualifiers Quals; - QualType UnqualifiedPointeeTy = - CGM.getContext().getUnqualifiedArrayType(PointeeTy, Quals); - // Itanium C++ ABI 2.9.5p7: // __flags is a flag word describing the cv-qualification and other // attributes of the type pointed to - unsigned Flags = ComputeQualifierFlags(Quals); - - // Itanium C++ ABI 2.9.5p7: - // When the abi::__pbase_type_info is for a direct or indirect pointer to an - // incomplete class type, the incomplete target type flag is set. - if (ContainsIncompleteClassType(UnqualifiedPointeeTy)) - Flags |= PTI_Incomplete; + unsigned Flags = extractPBaseFlags(CGM.getContext(), PointeeTy); llvm::Type *UnsignedIntLTy = CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy); @@ -3295,7 +3403,7 @@ void ItaniumRTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) { // __pointee is a pointer to the std::type_info derivation for the // unqualified type being pointed to. llvm::Constant *PointeeTypeInfo = - ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(UnqualifiedPointeeTy); + ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(PointeeTy); Fields.push_back(PointeeTypeInfo); } @@ -3305,23 +3413,12 @@ void ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) { QualType PointeeTy = Ty->getPointeeType(); - Qualifiers Quals; - QualType UnqualifiedPointeeTy = - CGM.getContext().getUnqualifiedArrayType(PointeeTy, Quals); - // Itanium C++ ABI 2.9.5p7: // __flags is a flag word describing the cv-qualification and other // attributes of the type pointed to. - unsigned Flags = ComputeQualifierFlags(Quals); + unsigned Flags = extractPBaseFlags(CGM.getContext(), PointeeTy); const RecordType *ClassType = cast<RecordType>(Ty->getClass()); - - // Itanium C++ ABI 2.9.5p7: - // When the abi::__pbase_type_info is for a direct or indirect pointer to an - // incomplete class type, the incomplete target type flag is set. - if (ContainsIncompleteClassType(UnqualifiedPointeeTy)) - Flags |= PTI_Incomplete; - if (IsIncompleteClassType(ClassType)) Flags |= PTI_ContainingClassIncomplete; @@ -3333,7 +3430,7 @@ ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) { // __pointee is a pointer to the std::type_info derivation for the // unqualified type being pointed to. llvm::Constant *PointeeTypeInfo = - ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(UnqualifiedPointeeTy); + ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(PointeeTy); Fields.push_back(PointeeTypeInfo); // Itanium C++ ABI 2.9.5p9: @@ -3348,15 +3445,18 @@ llvm::Constant *ItaniumCXXABI::getAddrOfRTTIDescriptor(QualType Ty) { return ItaniumRTTIBuilder(*this).BuildTypeInfo(Ty); } -void ItaniumCXXABI::EmitFundamentalRTTIDescriptor(QualType Type) { +void ItaniumCXXABI::EmitFundamentalRTTIDescriptor(QualType Type, + bool DLLExport) { QualType PointerType = getContext().getPointerType(Type); QualType PointerTypeConst = getContext().getPointerType(Type.withConst()); - ItaniumRTTIBuilder(*this).BuildTypeInfo(Type, true); - ItaniumRTTIBuilder(*this).BuildTypeInfo(PointerType, true); - ItaniumRTTIBuilder(*this).BuildTypeInfo(PointerTypeConst, true); + ItaniumRTTIBuilder(*this).BuildTypeInfo(Type, /*Force=*/true, DLLExport); + ItaniumRTTIBuilder(*this).BuildTypeInfo(PointerType, /*Force=*/true, + DLLExport); + ItaniumRTTIBuilder(*this).BuildTypeInfo(PointerTypeConst, /*Force=*/true, + DLLExport); } -void ItaniumCXXABI::EmitFundamentalRTTIDescriptors() { +void ItaniumCXXABI::EmitFundamentalRTTIDescriptors(bool DLLExport) { // Types added here must also be added to TypeInfoIsInStandardLibrary. QualType FundamentalTypes[] = { getContext().VoidTy, getContext().NullPtrTy, @@ -3373,7 +3473,7 @@ void ItaniumCXXABI::EmitFundamentalRTTIDescriptors() { getContext().Char16Ty, getContext().Char32Ty }; for (const QualType &FundamentalType : FundamentalTypes) - EmitFundamentalRTTIDescriptor(FundamentalType); + EmitFundamentalRTTIDescriptor(FundamentalType, DLLExport); } /// What sort of uniqueness rules should we use for the RTTI for the @@ -3820,7 +3920,8 @@ static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) { llvm::FunctionType *fnTy = llvm::FunctionType::get(CGM.VoidTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); llvm::Constant *fnRef = - CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate"); + CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate", + llvm::AttributeSet(), /*Local=*/true); llvm::Function *fn = dyn_cast<llvm::Function>(fnRef); if (fn && fn->empty()) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 41cd53c..38df455 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -19,6 +19,7 @@ #include "CGVTables.h" #include "CodeGenModule.h" #include "CodeGenTypes.h" +#include "ConstantBuilder.h" #include "TargetInfo.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" @@ -96,9 +97,9 @@ public: const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); const VBTableGlobals &VBGlobals = enumerateVBTables(RD); - for (const VPtrInfo *VBT : *VBGlobals.VBTables) { + for (const std::unique_ptr<VPtrInfo> &VBT : *VBGlobals.VBTables) { const ASTRecordLayout &SubobjectLayout = - Context.getASTRecordLayout(VBT->BaseWithVPtr); + Context.getASTRecordLayout(VBT->IntroducingObject); CharUnits Offs = VBT->NonVirtualOffset; Offs += SubobjectLayout.getVBPtrOffset(); if (VBT->getVBaseWithVPtr()) @@ -122,7 +123,7 @@ public: void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override; llvm::GlobalVariable *getMSCompleteObjectLocator(const CXXRecordDecl *RD, - const VPtrInfo *Info); + const VPtrInfo &Info); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; CatchTypeInfo @@ -164,6 +165,9 @@ public: llvm::BasicBlock * EmitCtorCompleteObjectHandler(CodeGenFunction &CGF, const CXXRecordDecl *RD) override; + + llvm::BasicBlock * + EmitDtorCompleteObjectHandler(CodeGenFunction &CGF); void initializeHiddenVirtualInheritanceMembers(CodeGenFunction &CGF, const CXXRecordDecl *RD) override; @@ -254,7 +258,7 @@ public: CXXDtorType Type, bool ForVirtualBase, bool Delegating, Address This) override; - void emitVTableTypeMetadata(VPtrInfo *Info, const CXXRecordDecl *RD, + void emitVTableTypeMetadata(const VPtrInfo &Info, const CXXRecordDecl *RD, llvm::GlobalVariable *VTable); void emitVTableDefinitions(CodeGenVTables &CGVT, @@ -284,9 +288,9 @@ public: llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; - llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - Address This, llvm::Type *Ty, - SourceLocation Loc) override; + CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, + Address This, llvm::Type *Ty, + SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, @@ -660,7 +664,7 @@ public: CastKind CK, CastExpr::path_const_iterator PathBegin, CastExpr::path_const_iterator PathEnd, llvm::Constant *Src); - llvm::Value * + CGCallee EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E, Address This, llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, @@ -794,6 +798,12 @@ MicrosoftCXXABI::getRecordArgABI(const CXXRecordDecl *RD) const { // FIXME: Implement for other architectures. return RAA_Default; + case llvm::Triple::thumb: + // Use the simple Itanium rules for now. + // FIXME: This is incompatible with MSVC for arguments with a dtor and no + // copy ctor. + return !canCopyArgument(RD) ? RAA_Indirect : RAA_Default; + case llvm::Triple::x86: // All record arguments are passed in memory on x86. Decide whether to // construct the object directly in argument memory, or to construct the @@ -824,25 +834,32 @@ MicrosoftCXXABI::getRecordArgABI(const CXXRecordDecl *RD) const { getContext().getTypeSize(RD->getTypeForDecl()) > 64) return RAA_Indirect; - // We have a trivial copy constructor or no copy constructors, but we have - // to make sure it isn't deleted. - bool CopyDeleted = false; + // If this is true, the implicit copy constructor that Sema would have + // created would not be deleted. FIXME: We should provide a more direct way + // for CodeGen to ask whether the constructor was deleted. + if (!RD->hasUserDeclaredCopyConstructor() && + !RD->hasUserDeclaredMoveConstructor() && + !RD->needsOverloadResolutionForMoveConstructor() && + !RD->hasUserDeclaredMoveAssignment() && + !RD->needsOverloadResolutionForMoveAssignment()) + return RAA_Default; + + // Otherwise, Sema should have created an implicit copy constructor if + // needed. + assert(!RD->needsImplicitCopyConstructor()); + + // We have to make sure the trivial copy constructor isn't deleted. for (const CXXConstructorDecl *CD : RD->ctors()) { if (CD->isCopyConstructor()) { assert(CD->isTrivial()); // We had at least one undeleted trivial copy ctor. Return directly. if (!CD->isDeleted()) return RAA_Default; - CopyDeleted = true; } } // The trivial copy constructor was deleted. Return indirectly. - if (CopyDeleted) - return RAA_Indirect; - - // There were no copy ctors. Return in RAX. - return RAA_Default; + return RAA_Indirect; } llvm_unreachable("invalid enum"); @@ -1121,6 +1138,25 @@ MicrosoftCXXABI::EmitCtorCompleteObjectHandler(CodeGenFunction &CGF, return SkipVbaseCtorsBB; } +llvm::BasicBlock * +MicrosoftCXXABI::EmitDtorCompleteObjectHandler(CodeGenFunction &CGF) { + llvm::Value *IsMostDerivedClass = getStructorImplicitParamValue(CGF); + assert(IsMostDerivedClass && + "ctor for a class with virtual bases must have an implicit parameter"); + llvm::Value *IsCompleteObject = + CGF.Builder.CreateIsNotNull(IsMostDerivedClass, "is_complete_object"); + + llvm::BasicBlock *CallVbaseDtorsBB = CGF.createBasicBlock("Dtor.dtor_vbases"); + llvm::BasicBlock *SkipVbaseDtorsBB = CGF.createBasicBlock("Dtor.skip_vbases"); + CGF.Builder.CreateCondBr(IsCompleteObject, + CallVbaseDtorsBB, SkipVbaseDtorsBB); + + CGF.EmitBlock(CallVbaseDtorsBB); + // CGF will put the base dtor calls in this basic block for us later. + + return SkipVbaseDtorsBB; +} + void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers( CodeGenFunction &CGF, const CXXRecordDecl *RD) { // In most cases, an override for a vbase virtual method can adjust @@ -1208,10 +1244,10 @@ void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF, const VBTableGlobals &VBGlobals = enumerateVBTables(RD); for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) { - const VPtrInfo *VBT = (*VBGlobals.VBTables)[I]; + const std::unique_ptr<VPtrInfo> &VBT = (*VBGlobals.VBTables)[I]; llvm::GlobalVariable *GV = VBGlobals.Globals[I]; const ASTRecordLayout &SubobjectLayout = - Context.getASTRecordLayout(VBT->BaseWithVPtr); + Context.getASTRecordLayout(VBT->IntroducingObject); CharUnits Offs = VBT->NonVirtualOffset; Offs += SubobjectLayout.getVBPtrOffset(); if (VBT->getVBaseWithVPtr()) @@ -1220,7 +1256,7 @@ void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF, llvm::Value *GVPtr = CGF.Builder.CreateConstInBoundsGEP2_32(GV->getValueType(), GV, 0, 0); VBPtr = CGF.Builder.CreateElementBitCast(VBPtr, GVPtr->getType(), - "vbptr." + VBT->ReusingBase->getName()); + "vbptr." + VBT->ObjectWithVPtr->getName()); CGF.Builder.CreateStore(GVPtr, VBPtr); } } @@ -1417,6 +1453,10 @@ llvm::Value *MicrosoftCXXABI::adjustThisParameterInVirtualFunctionPrologue( } void MicrosoftCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) { + // Naked functions have no prolog. + if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>()) + return; + EmitThisParam(CGF); /// If this is a function that the ABI specifies returns 'this', initialize @@ -1484,7 +1524,9 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, bool Delegating, Address This) { - llvm::Value *Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type)); + CGCallee Callee = CGCallee::forDirect( + CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type)), + DD); if (DD->isVirtual()) { assert(Type != CXXDtorType::Dtor_Deleting && @@ -1492,14 +1534,24 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, This = adjustThisArgumentForVirtualFunctionCall(CGF, GlobalDecl(DD, Type), This, false); } + + llvm::BasicBlock *BaseDtorEndBB = nullptr; + if (ForVirtualBase && isa<CXXConstructorDecl>(CGF.CurCodeDecl)) { + BaseDtorEndBB = EmitDtorCompleteObjectHandler(CGF); + } CGF.EmitCXXDestructorCall(DD, Callee, This.getPointer(), /*ImplicitParam=*/nullptr, /*ImplicitParamTy=*/QualType(), nullptr, getFromDtorType(Type)); + if (BaseDtorEndBB) { + // Complete object handler should continue to be the remaining + CGF.Builder.CreateBr(BaseDtorEndBB); + CGF.EmitBlock(BaseDtorEndBB); + } } -void MicrosoftCXXABI::emitVTableTypeMetadata(VPtrInfo *Info, +void MicrosoftCXXABI::emitVTableTypeMetadata(const VPtrInfo &Info, const CXXRecordDecl *RD, llvm::GlobalVariable *VTable) { if (!CGM.getCodeGenOpts().PrepareForLTO) @@ -1514,20 +1566,20 @@ void MicrosoftCXXABI::emitVTableTypeMetadata(VPtrInfo *Info, getContext().getTargetInfo().getPointerWidth(0)) : CharUnits::Zero(); - if (Info->PathToBaseWithVPtr.empty()) { + if (Info.PathToIntroducingObject.empty()) { CGM.AddVTableTypeMetadata(VTable, AddressPoint, RD); return; } // Add a bitset entry for the least derived base belonging to this vftable. CGM.AddVTableTypeMetadata(VTable, AddressPoint, - Info->PathToBaseWithVPtr.back()); + Info.PathToIntroducingObject.back()); // Add a bitset entry for each derived class that is laid out at the same // offset as the least derived base. - for (unsigned I = Info->PathToBaseWithVPtr.size() - 1; I != 0; --I) { - const CXXRecordDecl *DerivedRD = Info->PathToBaseWithVPtr[I - 1]; - const CXXRecordDecl *BaseRD = Info->PathToBaseWithVPtr[I]; + for (unsigned I = Info.PathToIntroducingObject.size() - 1; I != 0; --I) { + const CXXRecordDecl *DerivedRD = Info.PathToIntroducingObject[I - 1]; + const CXXRecordDecl *BaseRD = Info.PathToIntroducingObject[I]; const ASTRecordLayout &Layout = getContext().getASTRecordLayout(DerivedRD); @@ -1543,7 +1595,7 @@ void MicrosoftCXXABI::emitVTableTypeMetadata(VPtrInfo *Info, } // Finally do the same for the most derived class. - if (Info->FullOffsetInMDC.isZero()) + if (Info.FullOffsetInMDC.isZero()) CGM.AddVTableTypeMetadata(VTable, AddressPoint, RD); } @@ -1552,7 +1604,7 @@ void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext(); const VPtrInfoVector &VFPtrs = VFTContext.getVFPtrOffsets(RD); - for (VPtrInfo *Info : VFPtrs) { + for (const std::unique_ptr<VPtrInfo>& Info : VFPtrs) { llvm::GlobalVariable *VTable = getAddrOfVTable(RD, Info->FullOffsetInMDC); if (VTable->hasInitializer()) continue; @@ -1563,16 +1615,14 @@ void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, llvm::Constant *RTTI = nullptr; if (any_of(VTLayout.vtable_components(), [](const VTableComponent &VTC) { return VTC.isRTTIKind(); })) - RTTI = getMSCompleteObjectLocator(RD, Info); + RTTI = getMSCompleteObjectLocator(RD, *Info); - llvm::Constant *Init = CGVT.CreateVTableInitializer( - RD, VTLayout.vtable_component_begin(), - VTLayout.getNumVTableComponents(), VTLayout.vtable_thunk_begin(), - VTLayout.getNumVTableThunks(), RTTI); + ConstantInitBuilder Builder(CGM); + auto Components = Builder.beginStruct(); + CGVT.createVTableInitializer(Components, VTLayout, RTTI); + Components.finishAndSetAsInitializer(VTable); - VTable->setInitializer(Init); - - emitVTableTypeMetadata(Info, RD, VTable); + emitVTableTypeMetadata(*Info, RD, VTable); } } @@ -1593,10 +1643,10 @@ llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor( } static void mangleVFTableName(MicrosoftMangleContext &MangleContext, - const CXXRecordDecl *RD, const VPtrInfo *VFPtr, + const CXXRecordDecl *RD, const VPtrInfo &VFPtr, SmallString<256> &Name) { llvm::raw_svector_ostream Out(Name); - MangleContext.mangleCXXVFTable(RD, VFPtr->MangledPath, Out); + MangleContext.mangleCXXVFTable(RD, VFPtr.MangledPath, Out); } llvm::Constant * @@ -1643,25 +1693,25 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, llvm::StringSet<> ObservedMangledNames; for (size_t J = 0, F = VFPtrs.size(); J != F; ++J) { SmallString<256> Name; - mangleVFTableName(getMangleContext(), RD, VFPtrs[J], Name); + mangleVFTableName(getMangleContext(), RD, *VFPtrs[J], Name); if (!ObservedMangledNames.insert(Name.str()).second) llvm_unreachable("Already saw this mangling before?"); } #endif } - VPtrInfo *const *VFPtrI = - std::find_if(VFPtrs.begin(), VFPtrs.end(), [&](VPtrInfo *VPI) { + const std::unique_ptr<VPtrInfo> *VFPtrI = std::find_if( + VFPtrs.begin(), VFPtrs.end(), [&](const std::unique_ptr<VPtrInfo>& VPI) { return VPI->FullOffsetInMDC == VPtrOffset; }); if (VFPtrI == VFPtrs.end()) { VFTablesMap[ID] = nullptr; return nullptr; } - VPtrInfo *VFPtr = *VFPtrI; + const std::unique_ptr<VPtrInfo> &VFPtr = *VFPtrI; SmallString<256> VFTableName; - mangleVFTableName(getMangleContext(), RD, VFPtr, VFTableName); + mangleVFTableName(getMangleContext(), RD, *VFPtr, VFTableName); // Classes marked __declspec(dllimport) need vftables generated on the // import-side in order to support features like constexpr. No other @@ -1689,16 +1739,14 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, return VTable; } - uint64_t NumVTableSlots = - VTContext.getVFTableLayout(RD, VFPtr->FullOffsetInMDC) - .getNumVTableComponents(); + const VTableLayout &VTLayout = + VTContext.getVFTableLayout(RD, VFPtr->FullOffsetInMDC); llvm::GlobalValue::LinkageTypes VTableLinkage = VTableAliasIsRequred ? llvm::GlobalValue::PrivateLinkage : VFTableLinkage; StringRef VTableName = VTableAliasIsRequred ? StringRef() : VFTableName.str(); - llvm::ArrayType *VTableType = - llvm::ArrayType::get(CGM.Int8PtrTy, NumVTableSlots); + llvm::Type *VTableType = CGM.getVTables().getVTableType(VTLayout); // Create a backing variable for the contents of VTable. The VTable may // or may not include space for a pointer to RTTI data. @@ -1719,8 +1767,9 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, // importing it. We never reference the RTTI data directly so there is no // need to make room for it. if (VTableAliasIsRequred) { - llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(CGM.IntTy, 0), - llvm::ConstantInt::get(CGM.IntTy, 1)}; + llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(CGM.Int32Ty, 0), + llvm::ConstantInt::get(CGM.Int32Ty, 0), + llvm::ConstantInt::get(CGM.Int32Ty, 1)}; // Create a GEP which points just after the first entry in the VFTable, // this should be the location of the first virtual method. llvm::Constant *VTableGEP = llvm::ConstantExpr::getInBoundsGetElementPtr( @@ -1752,54 +1801,11 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, return VTable; } -// Compute the identity of the most derived class whose virtual table is located -// at the given offset into RD. -static const CXXRecordDecl *getClassAtVTableLocation(ASTContext &Ctx, - const CXXRecordDecl *RD, - CharUnits Offset) { - if (Offset.isZero()) - return RD; - - const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD); - const CXXRecordDecl *MaxBase = nullptr; - CharUnits MaxBaseOffset; - for (auto &&B : RD->bases()) { - const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl(); - CharUnits BaseOffset = Layout.getBaseClassOffset(Base); - if (BaseOffset <= Offset && BaseOffset >= MaxBaseOffset) { - MaxBase = Base; - MaxBaseOffset = BaseOffset; - } - } - for (auto &&B : RD->vbases()) { - const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl(); - CharUnits BaseOffset = Layout.getVBaseClassOffset(Base); - if (BaseOffset <= Offset && BaseOffset >= MaxBaseOffset) { - MaxBase = Base; - MaxBaseOffset = BaseOffset; - } - } - assert(MaxBase); - return getClassAtVTableLocation(Ctx, MaxBase, Offset - MaxBaseOffset); -} - -// Compute the identity of the most derived class whose virtual table is located -// at the MethodVFTableLocation ML. -static const CXXRecordDecl * -getClassAtVTableLocation(ASTContext &Ctx, GlobalDecl GD, - MicrosoftVTableContext::MethodVFTableLocation &ML) { - const CXXRecordDecl *RD = ML.VBase; - if (!RD) - RD = cast<CXXMethodDecl>(GD.getDecl())->getParent(); - - return getClassAtVTableLocation(Ctx, RD, ML.VFPtrOffset); -} - -llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, - GlobalDecl GD, - Address This, - llvm::Type *Ty, - SourceLocation Loc) { +CGCallee MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, + GlobalDecl GD, + Address This, + llvm::Type *Ty, + SourceLocation Loc) { GD = GD.getCanonicalDecl(); CGBuilderTy &Builder = CGF.Builder; @@ -1810,22 +1816,38 @@ llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl()); llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent()); + MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext(); MicrosoftVTableContext::MethodVFTableLocation ML = - CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); + VFTContext.getMethodVFTableLocation(GD); + + // Compute the identity of the most derived class whose virtual table is + // located at the MethodVFTableLocation ML. + auto getObjectWithVPtr = [&] { + return llvm::find_if(VFTContext.getVFPtrOffsets( + ML.VBase ? ML.VBase : MethodDecl->getParent()), + [&](const std::unique_ptr<VPtrInfo> &Info) { + return Info->FullOffsetInMDC == ML.VFPtrOffset; + }) + ->get() + ->ObjectWithVPtr; + }; + llvm::Value *VFunc; if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { - return CGF.EmitVTableTypeCheckedLoad( - getClassAtVTableLocation(getContext(), GD, ML), VTable, + VFunc = CGF.EmitVTableTypeCheckedLoad( + getObjectWithVPtr(), VTable, ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); } else { if (CGM.getCodeGenOpts().PrepareForLTO) - CGF.EmitTypeMetadataCodeForVCall( - getClassAtVTableLocation(getContext(), GD, ML), VTable, Loc); + CGF.EmitTypeMetadataCodeForVCall(getObjectWithVPtr(), VTable, Loc); llvm::Value *VFuncPtr = Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); - return Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); + VFunc = Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); } + + CGCallee Callee(MethodDecl, VFunc); + return Callee; } llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( @@ -1840,7 +1862,7 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( const CGFunctionInfo *FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration( Dtor, StructorType::Deleting); llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); - llvm::Value *Callee = getVirtualFunctionPointer( + CGCallee Callee = getVirtualFunctionPointer( CGF, GD, This, Ty, CE ? CE->getLocStart() : SourceLocation()); ASTContext &Context = getContext(); @@ -1956,7 +1978,7 @@ llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk( void MicrosoftCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) { const VBTableGlobals &VBGlobals = enumerateVBTables(RD); for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) { - const VPtrInfo *VBT = (*VBGlobals.VBTables)[I]; + const std::unique_ptr<VPtrInfo>& VBT = (*VBGlobals.VBTables)[I]; llvm::GlobalVariable *GV = VBGlobals.Globals[I]; if (GV->isDeclaration()) emitVBTableDefinition(*VBT, RD, GV); @@ -1972,7 +1994,7 @@ MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD, StringRef Name = OutName.str(); llvm::ArrayType *VBTableType = - llvm::ArrayType::get(CGM.IntTy, 1 + VBT.ReusingBase->getNumVBases()); + llvm::ArrayType::get(CGM.IntTy, 1 + VBT.ObjectWithVPtr->getNumVBases()); assert(!CGM.getModule().getNamedGlobal(Name) && "vbtable with this name already exists: mangling bug?"); @@ -1994,24 +2016,24 @@ MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD, void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, const CXXRecordDecl *RD, llvm::GlobalVariable *GV) const { - const CXXRecordDecl *ReusingBase = VBT.ReusingBase; + const CXXRecordDecl *ObjectWithVPtr = VBT.ObjectWithVPtr; - assert(RD->getNumVBases() && ReusingBase->getNumVBases() && + assert(RD->getNumVBases() && ObjectWithVPtr->getNumVBases() && "should only emit vbtables for classes with vbtables"); const ASTRecordLayout &BaseLayout = - getContext().getASTRecordLayout(VBT.BaseWithVPtr); + getContext().getASTRecordLayout(VBT.IntroducingObject); const ASTRecordLayout &DerivedLayout = getContext().getASTRecordLayout(RD); - SmallVector<llvm::Constant *, 4> Offsets(1 + ReusingBase->getNumVBases(), + SmallVector<llvm::Constant *, 4> Offsets(1 + ObjectWithVPtr->getNumVBases(), nullptr); - // The offset from ReusingBase's vbptr to itself always leads. + // The offset from ObjectWithVPtr's vbptr to itself always leads. CharUnits VBPtrOffset = BaseLayout.getVBPtrOffset(); Offsets[0] = llvm::ConstantInt::get(CGM.IntTy, -VBPtrOffset.getQuantity()); MicrosoftVTableContext &Context = CGM.getMicrosoftVTableContext(); - for (const auto &I : ReusingBase->vbases()) { + for (const auto &I : ObjectWithVPtr->vbases()) { const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl(); CharUnits Offset = DerivedLayout.getVBaseClassOffset(VBase); assert(!Offset.isNegative()); @@ -2023,7 +2045,7 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, DerivedLayout.getVBaseClassOffset(VBT.getVBaseWithVPtr()); Offset -= CompleteVBPtrOffset; - unsigned VBIndex = Context.getVBTableIndex(ReusingBase, VBase); + unsigned VBIndex = Context.getVBTableIndex(ObjectWithVPtr, VBase); assert(Offsets[VBIndex] == nullptr && "The same vbindex seen twice?"); Offsets[VBIndex] = llvm::ConstantInt::get(CGM.IntTy, Offset.getQuantity()); } @@ -2182,7 +2204,8 @@ static void emitGlobalDtorWithTLRegDtor(CodeGenFunction &CGF, const VarDecl &VD, CGF.IntTy, DtorStub->getType(), /*IsVarArg=*/false); llvm::Constant *TLRegDtor = - CGF.CGM.CreateRuntimeFunction(TLRegDtorTy, "__tlregdtor"); + CGF.CGM.CreateRuntimeFunction(TLRegDtorTy, "__tlregdtor", + llvm::AttributeSet(), /*Local=*/true); if (llvm::Function *TLRegDtorFn = dyn_cast<llvm::Function>(TLRegDtor)) TLRegDtorFn->setDoesNotThrow(); @@ -2203,6 +2226,14 @@ void MicrosoftCXXABI::EmitThreadLocalInitFuncs( CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits, ArrayRef<const VarDecl *> CXXThreadLocalInitVars) { + if (CXXThreadLocalInits.empty()) + return; + + CGM.AppendLinkerOptions(CGM.getTarget().getTriple().getArch() == + llvm::Triple::x86 + ? "/include:___dyn_tls_init@12" + : "/include:__dyn_tls_init"); + // This will create a GV in the .CRT$XDU section. It will point to our // initialization function. The CRT will call all of these function // pointers at start-up time and, eventually, at thread-creation time. @@ -2272,7 +2303,8 @@ static llvm::Constant *getInitThreadHeaderFn(CodeGenModule &CGM) { FTy, "_Init_thread_header", llvm::AttributeSet::get(CGM.getLLVMContext(), llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoUnwind)); + llvm::Attribute::NoUnwind), + /*Local=*/true); } static llvm::Constant *getInitThreadFooterFn(CodeGenModule &CGM) { @@ -2283,7 +2315,8 @@ static llvm::Constant *getInitThreadFooterFn(CodeGenModule &CGM) { FTy, "_Init_thread_footer", llvm::AttributeSet::get(CGM.getLLVMContext(), llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoUnwind)); + llvm::Attribute::NoUnwind), + /*Local=*/true); } static llvm::Constant *getInitThreadAbortFn(CodeGenModule &CGM) { @@ -2294,7 +2327,8 @@ static llvm::Constant *getInitThreadAbortFn(CodeGenModule &CGM) { FTy, "_Init_thread_abort", llvm::AttributeSet::get(CGM.getLLVMContext(), llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoUnwind)); + llvm::Attribute::NoUnwind), + /*Local=*/true); } namespace { @@ -3222,7 +3256,7 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion( return Dst; } -llvm::Value *MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( +CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( CodeGenFunction &CGF, const Expr *E, Address This, llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, const MemberPointerType *MPT) { @@ -3269,7 +3303,10 @@ llvm::Value *MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( "this.adjusted"); } - return Builder.CreateBitCast(FunctionPointer, FTy->getPointerTo()); + FunctionPointer = + Builder.CreateBitCast(FunctionPointer, FTy->getPointerTo()); + CGCallee Callee(FPT, FunctionPointer); + return Callee; } CGCXXABI *clang::CodeGen::CreateMicrosoftCXXABI(CodeGenModule &CGM) { @@ -3410,7 +3447,7 @@ struct MSRTTIBuilder { llvm::GlobalVariable * getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes); llvm::GlobalVariable *getClassHierarchyDescriptor(); - llvm::GlobalVariable *getCompleteObjectLocator(const VPtrInfo *Info); + llvm::GlobalVariable *getCompleteObjectLocator(const VPtrInfo &Info); CodeGenModule &CGM; ASTContext &Context; @@ -3499,7 +3536,7 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() { // Initialize the base class ClassHierarchyDescriptor. llvm::Constant *Fields[] = { - llvm::ConstantInt::get(CGM.IntTy, 0), // Unknown + llvm::ConstantInt::get(CGM.IntTy, 0), // reserved by the runtime llvm::ConstantInt::get(CGM.IntTy, Flags), llvm::ConstantInt::get(CGM.IntTy, Classes.size()), ABI.getImageRelativeConstant(llvm::ConstantExpr::getInBoundsGetElementPtr( @@ -3592,11 +3629,11 @@ MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) { } llvm::GlobalVariable * -MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { +MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo &Info) { SmallString<256> MangledName; { llvm::raw_svector_ostream Out(MangledName); - ABI.getMangleContext().mangleCXXRTTICompleteObjectLocator(RD, Info->MangledPath, Out); + ABI.getMangleContext().mangleCXXRTTICompleteObjectLocator(RD, Info.MangledPath, Out); } // Check to see if we've already computed this complete object locator. @@ -3604,15 +3641,15 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { return COL; // Compute the fields of the complete object locator. - int OffsetToTop = Info->FullOffsetInMDC.getQuantity(); + int OffsetToTop = Info.FullOffsetInMDC.getQuantity(); int VFPtrOffset = 0; // The offset includes the vtordisp if one exists. - if (const CXXRecordDecl *VBase = Info->getVBaseWithVPtr()) + if (const CXXRecordDecl *VBase = Info.getVBaseWithVPtr()) if (Context.getASTRecordLayout(RD) .getVBaseOffsetsMap() .find(VBase) ->second.hasVtorDisp()) - VFPtrOffset = Info->NonVirtualOffset.getQuantity() + 4; + VFPtrOffset = Info.NonVirtualOffset.getQuantity() + 4; // Forward-declare the complete object locator. llvm::StructType *Type = ABI.getCompleteObjectLocatorType(); @@ -3740,7 +3777,7 @@ llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { /// \brief Gets or a creates a Microsoft CompleteObjectLocator. llvm::GlobalVariable * MicrosoftCXXABI::getMSCompleteObjectLocator(const CXXRecordDecl *RD, - const VPtrInfo *Info) { + const VPtrInfo &Info) { return MSRTTIBuilder(*this, RD).getCompleteObjectLocator(Info); } @@ -3846,8 +3883,11 @@ MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, FunctionArgs.push_back(&IsMostDerived); // Start defining the function. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); CGF.StartFunction(GlobalDecl(), FnInfo.getReturnType(), ThunkFn, FnInfo, FunctionArgs, CD->getLocation(), SourceLocation()); + // Create a scope with an artificial location for the body of this function. + auto AL = ApplyDebugLocation::CreateArtificial(CGF); EmitThisParam(CGF); llvm::Value *This = getThisValue(CGF); @@ -3865,11 +3905,11 @@ MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, Args.add(RValue::get(SrcVal), SrcParam.getType()); // Add the rest of the default arguments. - std::vector<Stmt *> ArgVec; - for (unsigned I = IsCopy ? 1 : 0, E = CD->getNumParams(); I != E; ++I) { - Stmt *DefaultArg = getContext().getDefaultArgExprForConstructor(CD, I); - assert(DefaultArg && "sema forgot to instantiate default args"); - ArgVec.push_back(DefaultArg); + SmallVector<const Stmt *, 4> ArgVec; + ArrayRef<ParmVarDecl *> params = CD->parameters().drop_front(IsCopy ? 1 : 0); + for (const ParmVarDecl *PD : params) { + assert(PD->hasDefaultArg() && "ctor closure lacks default args"); + ArgVec.push_back(PD->getDefaultArg()); } CodeGenFunction::RunCleanupsScope Cleanups(CGF); @@ -3883,10 +3923,12 @@ MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, /*Delegating=*/false, Args); // Call the destructor with our arguments. - llvm::Value *CalleeFn = CGM.getAddrOfCXXStructor(CD, StructorType::Complete); + llvm::Constant *CalleePtr = + CGM.getAddrOfCXXStructor(CD, StructorType::Complete); + CGCallee Callee = CGCallee::forDirect(CalleePtr, CD); const CGFunctionInfo &CalleeInfo = CGM.getTypes().arrangeCXXConstructorCall( Args, CD, Ctor_Complete, ExtraArgs); - CGF.EmitCall(CalleeInfo, CalleeFn, ReturnValueSlot(), Args, CD); + CGF.EmitCall(CalleeInfo, Callee, ReturnValueSlot(), Args); Cleanups.ForceCleanup(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp index 952d162..f925c25 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp @@ -112,7 +112,7 @@ namespace { } llvm::Constant *GetAddrOfGlobal(GlobalDecl global, bool isForDefinition) { - return Builder->GetAddrOfGlobal(global, isForDefinition); + return Builder->GetAddrOfGlobal(global, ForDefinition_t(isForDefinition)); } void Initialize(ASTContext &Context) override { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp index de40e41..754f996 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp @@ -282,7 +282,7 @@ public: // Print the IR for the PCH container to the debug output. llvm::SmallString<0> Buffer; clang::EmitBackendOutput( - Diags, CodeGenOpts, TargetOpts, LangOpts, + Diags, HeaderSearchOpts, CodeGenOpts, TargetOpts, LangOpts, Ctx.getTargetInfo().getDataLayout(), M.get(), BackendAction::Backend_EmitLL, llvm::make_unique<llvm::raw_svector_ostream>(Buffer)); @@ -290,9 +290,10 @@ public: }); // Use the LLVM backend to emit the pch container. - clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, - Ctx.getTargetInfo().getDataLayout(), M.get(), - BackendAction::Backend_EmitObj, std::move(OS)); + clang::EmitBackendOutput(Diags, HeaderSearchOpts, CodeGenOpts, TargetOpts, + LangOpts, Ctx.getTargetInfo().getDataLayout(), + M.get(), BackendAction::Backend_EmitObj, + std::move(OS)); // Free the memory for the temporary buffer. llvm::SmallVector<char, 0> Empty; @@ -312,27 +313,30 @@ ObjectFilePCHContainerWriter::CreatePCHContainerGenerator( CI, MainFileName, OutputFileName, std::move(OS), Buffer); } -void ObjectFilePCHContainerReader::ExtractPCH( - llvm::MemoryBufferRef Buffer, llvm::BitstreamReader &StreamFile) const { - if (auto OF = llvm::object::ObjectFile::createObjectFile(Buffer)) { - auto *Obj = OF.get().get(); - bool IsCOFF = isa<llvm::object::COFFObjectFile>(Obj); +StringRef +ObjectFilePCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const { + StringRef PCH; + auto OFOrErr = llvm::object::ObjectFile::createObjectFile(Buffer); + if (OFOrErr) { + auto &OF = OFOrErr.get(); + bool IsCOFF = isa<llvm::object::COFFObjectFile>(*OF); // Find the clang AST section in the container. - for (auto &Section : OF->get()->sections()) { + for (auto &Section : OF->sections()) { StringRef Name; Section.getName(Name); - if ((!IsCOFF && Name == "__clangast") || - ( IsCOFF && Name == "clangast")) { - StringRef Buf; - Section.getContents(Buf); - StreamFile.init((const unsigned char *)Buf.begin(), - (const unsigned char *)Buf.end()); - return; + if ((!IsCOFF && Name == "__clangast") || (IsCOFF && Name == "clangast")) { + Section.getContents(PCH); + return PCH; } } } - - // As a fallback, treat the buffer as a raw AST. - StreamFile.init((const unsigned char *)Buffer.getBufferStart(), - (const unsigned char *)Buffer.getBufferEnd()); + handleAllErrors(OFOrErr.takeError(), [&](const llvm::ErrorInfoBase &EIB) { + if (EIB.convertToErrorCode() == + llvm::object::object_error::invalid_file_type) + // As a fallback, treat the buffer as a raw AST. + PCH = Buffer.getBuffer(); + else + EIB.log(llvm::errs()); + }); + return PCH; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp index 2a338ba..9848e3e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp @@ -63,7 +63,13 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, std::string QualName; llvm::raw_string_ostream OS(QualName); D.printQualifiedName(OS); - reportGlobalToASan(GV, D.getLocation(), OS.str(), D.getType(), IsDynInit); + + bool IsBlacklisted = false; + for (auto Attr : D.specific_attrs<NoSanitizeAttr>()) + if (Attr->getMask() & SanitizerKind::Address) + IsBlacklisted = true; + reportGlobalToASan(GV, D.getLocation(), OS.str(), D.getType(), IsDynInit, + IsBlacklisted); } void SanitizerMetadata::disableSanitizerForGlobal(llvm::GlobalVariable *GV) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/SwiftCallingConv.cpp b/contrib/llvm/tools/clang/lib/CodeGen/SwiftCallingConv.cpp index 6c20f8c..0bfe30a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/SwiftCallingConv.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/SwiftCallingConv.cpp @@ -384,7 +384,7 @@ void SwiftAggLowering::splitVectorEntry(unsigned index) { auto eltTy = split.first; CharUnits eltSize = getTypeStoreSize(CGM, eltTy); auto numElts = split.second; - Entries.insert(&Entries[index + 1], numElts - 1, StorageEntry()); + Entries.insert(Entries.begin() + index + 1, numElts - 1, StorageEntry()); CharUnits begin = Entries[index].Begin; for (unsigned i = 0; i != numElts; ++i) { @@ -506,7 +506,7 @@ void SwiftAggLowering::enumerateComponents(EnumerationCallback callback) const { assert(Finished && "haven't yet finished lowering"); for (auto &entry : Entries) { - callback(entry.Begin, entry.Type); + callback(entry.Begin, entry.End, entry.Type); } } @@ -828,3 +828,8 @@ void swiftcall::computeABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) { argInfo.info = classifyArgumentType(CGM, argInfo.type); } } + +// Is swifterror lowered to a register by the target ABI. +bool swiftcall::isSwiftErrorLoweredInRegister(CodeGenModule &CGM) { + return getSwiftABIInfo(CGM).isSwiftErrorInRegister(); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp index bc03616..d2fc388 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp @@ -31,6 +31,31 @@ using namespace clang; using namespace CodeGen; +// Helper for coercing an aggregate argument or return value into an integer +// array of the same size (including padding) and alignment. This alternate +// coercion happens only for the RenderScript ABI and can be removed after +// runtimes that rely on it are no longer supported. +// +// RenderScript assumes that the size of the argument / return value in the IR +// is the same as the size of the corresponding qualified type. This helper +// coerces the aggregate type into an array of the same size (including +// padding). This coercion is used in lieu of expansion of struct members or +// other canonical coercions that return a coerced-type of larger size. +// +// Ty - The argument / return value type +// Context - The associated ASTContext +// LLVMContext - The associated LLVMContext +static ABIArgInfo coerceToIntArray(QualType Ty, + ASTContext &Context, + llvm::LLVMContext &LLVMContext) { + // Alignment and Size are measured in bits. + const uint64_t Size = Context.getTypeSize(Ty); + const uint64_t Alignment = Context.getTypeAlign(Ty); + llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment); + const uint64_t NumElements = (Size + Alignment - 1) / Alignment; + return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements)); +} + static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder, llvm::Value *Array, llvm::Value *Value, @@ -375,6 +400,21 @@ TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib, unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const { return llvm::CallingConv::C; } + +llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM, + llvm::PointerType *T, QualType QT) const { + return llvm::ConstantPointerNull::get(T); +} + +llvm::Value *TargetCodeGenInfo::performAddrSpaceCast( + CodeGen::CodeGenFunction &CGF, llvm::Value *Src, QualType SrcTy, + QualType DestTy) const { + // Since target may map different address spaces in AST to the same address + // space, an address space conversion may end up as a bitcast. + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, + CGF.ConvertType(DestTy)); +} + static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays); /// isEmptyField - Return true iff a the field is "empty", that is it @@ -831,6 +871,14 @@ static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) { return NumMembers <= 4; } +/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86. +static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) { + auto AI = ABIArgInfo::getDirect(T); + AI.setInReg(true); + AI.setCanBeFlattened(false); + return AI; +} + //===----------------------------------------------------------------------===// // X86-32 ABI Implementation //===----------------------------------------------------------------------===// @@ -844,6 +892,11 @@ struct CCState { unsigned FreeSSERegs; }; +enum { + // Vectorcall only allows the first 6 parameters to be passed in registers. + VectorcallMaxParamNumAsReg = 6 +}; + /// X86_32ABIInfo - The X86-32 ABI information. class X86_32ABIInfo : public SwiftABIInfo { enum Class { @@ -889,6 +942,8 @@ class X86_32ABIInfo : public SwiftABIInfo { Class classify(QualType Ty) const; ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const; ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const; + ABIArgInfo reclassifyHvaArgType(QualType RetTy, CCState &State, + const ABIArgInfo& current) const; /// \brief Updates the number of available free registers, returns /// true if any registers were allocated. bool updateFreeRegs(QualType Ty, CCState &State) const; @@ -906,6 +961,8 @@ class X86_32ABIInfo : public SwiftABIInfo { void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields, CharUnits &StackOffset, ABIArgInfo &Info, QualType Type) const; + void computeVectorCallArgs(CGFunctionInfo &FI, CCState &State, + bool &UsedInAlloca) const; public: @@ -932,6 +989,11 @@ public: // scalar registers. return occupiesMoreThan(CGT, scalars, /*total*/ 3); } + + bool isSwiftErrorInRegister() const override { + // x86-32 lowering does not support passing swifterror in a register. + return false; + } }; class X86_32TargetCodeGenInfo : public TargetCodeGenInfo { @@ -1203,7 +1265,8 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, const Type *Base = nullptr; uint64_t NumElts = 0; - if (State.CC == llvm::CallingConv::X86_VectorCall && + if ((State.CC == llvm::CallingConv::X86_VectorCall || + State.CC == llvm::CallingConv::X86_RegCall) && isHomogeneousAggregate(RetTy, Base, NumElts)) { // The LLVM struct type for such an aggregate should lower properly. return ABIArgInfo::getDirect(); @@ -1417,7 +1480,8 @@ bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State, return true; if (State.CC == llvm::CallingConv::X86_FastCall || - State.CC == llvm::CallingConv::X86_VectorCall) { + State.CC == llvm::CallingConv::X86_VectorCall || + State.CC == llvm::CallingConv::X86_RegCall) { if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs) NeedsPadding = true; @@ -1435,7 +1499,8 @@ bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const { return false; if (State.CC == llvm::CallingConv::X86_FastCall || - State.CC == llvm::CallingConv::X86_VectorCall) { + State.CC == llvm::CallingConv::X86_VectorCall || + State.CC == llvm::CallingConv::X86_RegCall) { if (getContext().getTypeSize(Ty) > 32) return false; @@ -1446,6 +1511,27 @@ bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const { return true; } +ABIArgInfo +X86_32ABIInfo::reclassifyHvaArgType(QualType Ty, CCState &State, + const ABIArgInfo ¤t) const { + // Assumes vectorCall calling convention. + const Type *Base = nullptr; + uint64_t NumElts = 0; + + if (!Ty->isBuiltinType() && !Ty->isVectorType() && + isHomogeneousAggregate(Ty, Base, NumElts)) { + if (State.FreeSSERegs >= NumElts) { + // HVA types get passed directly in registers if there is room. + State.FreeSSERegs -= NumElts; + return getDirectX86Hva(); + } + // If there's no room, the HVA gets passed as normal indirect + // structure. + return getIndirectResult(Ty, /*ByVal=*/false, State); + } + return current; +} + ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State) const { // FIXME: Set alignment on indirect arguments. @@ -1465,18 +1551,34 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, } // vectorcall adds the concept of a homogenous vector aggregate, similar - // to other targets. + // to other targets, regcall uses some of the HVA rules. const Type *Base = nullptr; uint64_t NumElts = 0; - if (State.CC == llvm::CallingConv::X86_VectorCall && + if ((State.CC == llvm::CallingConv::X86_VectorCall || + State.CC == llvm::CallingConv::X86_RegCall) && isHomogeneousAggregate(Ty, Base, NumElts)) { - if (State.FreeSSERegs >= NumElts) { - State.FreeSSERegs -= NumElts; - if (Ty->isBuiltinType() || Ty->isVectorType()) + + if (State.CC == llvm::CallingConv::X86_RegCall) { + if (State.FreeSSERegs >= NumElts) { + State.FreeSSERegs -= NumElts; + if (Ty->isBuiltinType() || Ty->isVectorType()) + return ABIArgInfo::getDirect(); + return ABIArgInfo::getExpand(); + + } + return getIndirectResult(Ty, /*ByVal=*/false, State); + } else if (State.CC == llvm::CallingConv::X86_VectorCall) { + if (State.FreeSSERegs >= NumElts && (Ty->isBuiltinType() || Ty->isVectorType())) { + // Actual floating-point types get registers first time through if + // there is registers available + State.FreeSSERegs -= NumElts; return ABIArgInfo::getDirect(); - return ABIArgInfo::getExpand(); + } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) { + // HVA Types only get registers after everything else has been + // set, so it gets set as indirect for now. + return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty)); + } } - return getIndirectResult(Ty, /*ByVal=*/false, State); } if (isAggregateTypeForABI(Ty)) { @@ -1514,7 +1616,8 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, (!IsMCUABI || State.FreeRegs == 0) && canExpandIndirectArgument(Ty)) return ABIArgInfo::getExpandWithPadding( State.CC == llvm::CallingConv::X86_FastCall || - State.CC == llvm::CallingConv::X86_VectorCall, + State.CC == llvm::CallingConv::X86_VectorCall || + State.CC == llvm::CallingConv::X86_RegCall, PaddingType); return getIndirectResult(Ty, true, State); @@ -1554,6 +1657,36 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, return ABIArgInfo::getDirect(); } +void X86_32ABIInfo::computeVectorCallArgs(CGFunctionInfo &FI, CCState &State, + bool &UsedInAlloca) const { + // Vectorcall only allows the first 6 parameters to be passed in registers, + // and homogeneous vector aggregates are only put into registers as a second + // priority. + unsigned Count = 0; + CCState ZeroState = State; + ZeroState.FreeRegs = ZeroState.FreeSSERegs = 0; + // HVAs must be done as a second priority for registers, so the deferred + // items are dealt with by going through the pattern a second time. + for (auto &I : FI.arguments()) { + if (Count < VectorcallMaxParamNumAsReg) + I.info = classifyArgumentType(I.type, State); + else + // Parameters after the 6th cannot be passed in registers, + // so pretend there are no registers left for them. + I.info = classifyArgumentType(I.type, ZeroState); + UsedInAlloca |= (I.info.getKind() == ABIArgInfo::InAlloca); + ++Count; + } + Count = 0; + // Go through the arguments a second time to get HVAs registers if there + // are still some available. + for (auto &I : FI.arguments()) { + if (Count < VectorcallMaxParamNumAsReg) + I.info = reclassifyHvaArgType(I.type, State, I.info); + ++Count; + } +} + void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { CCState State(FI.getCallingConvention()); if (IsMCUABI) @@ -1565,7 +1698,10 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { State.FreeSSERegs = 6; } else if (FI.getHasRegParm()) State.FreeRegs = FI.getRegParm(); - else + else if (State.CC == llvm::CallingConv::X86_RegCall) { + State.FreeRegs = 5; + State.FreeSSERegs = 8; + } else State.FreeRegs = DefaultNumRegisterParameters; if (!getCXXABI().classifyReturnType(FI)) { @@ -1585,9 +1721,14 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { ++State.FreeRegs; bool UsedInAlloca = false; - for (auto &I : FI.arguments()) { - I.info = classifyArgumentType(I.type, State); - UsedInAlloca |= (I.info.getKind() == ABIArgInfo::InAlloca); + if (State.CC == llvm::CallingConv::X86_VectorCall) { + computeVectorCallArgs(FI, State, UsedInAlloca); + } else { + // If not vectorcall, revert to normal behavior. + for (auto &I : FI.arguments()) { + I.info = classifyArgumentType(I.type, State); + UsedInAlloca |= (I.info.getKind() == ABIArgInfo::InAlloca); + } } // If we needed to use inalloca for any argument, do a second pass and rewrite @@ -1906,12 +2047,16 @@ class X86_64ABIInfo : public SwiftABIInfo { ABIArgInfo classifyReturnType(QualType RetTy) const; - ABIArgInfo classifyArgumentType(QualType Ty, - unsigned freeIntRegs, - unsigned &neededInt, - unsigned &neededSSE, + ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs, + unsigned &neededInt, unsigned &neededSSE, bool isNamedArg) const; + ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt, + unsigned &NeededSSE) const; + + ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt, + unsigned &NeededSSE) const; + bool IsIllegalVectorType(QualType Ty) const; /// The 0.98 ABI revision clarified a lot of ambiguities, @@ -1974,13 +2119,16 @@ public: bool asReturnValue) const override { return occupiesMoreThan(CGT, scalars, /*total*/ 4); } + bool isSwiftErrorInRegister() const override { + return true; + } }; /// WinX86_64ABIInfo - The Windows X86_64 ABI information. -class WinX86_64ABIInfo : public ABIInfo { +class WinX86_64ABIInfo : public SwiftABIInfo { public: WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT) - : ABIInfo(CGT), + : SwiftABIInfo(CGT), IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {} void computeInfo(CGFunctionInfo &FI) const override; @@ -1999,11 +2147,25 @@ public: return isX86VectorCallAggregateSmallEnough(NumMembers); } -private: - ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, - bool IsReturnType) const; + bool shouldPassIndirectlyForSwift(CharUnits totalSize, + ArrayRef<llvm::Type *> scalars, + bool asReturnValue) const override { + return occupiesMoreThan(CGT, scalars, /*total*/ 4); + } + + bool isSwiftErrorInRegister() const override { + return true; + } - bool IsMingw64; +private: + ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType, + bool IsVectorCall, bool IsRegCall) const; + ABIArgInfo reclassifyHvaArgType(QualType Ty, unsigned &FreeSSERegs, + const ABIArgInfo ¤t) const; + void computeVectorCallArgs(CGFunctionInfo &FI, unsigned FreeSSERegs, + bool IsVectorCall, bool IsRegCall) const; + + bool IsMingw64; }; class X86_64TargetCodeGenInfo : public TargetCodeGenInfo { @@ -2315,13 +2477,13 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Current = SSE; } else if (k == BuiltinType::LongDouble) { const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat(); - if (LDF == &llvm::APFloat::IEEEquad) { + if (LDF == &llvm::APFloat::IEEEquad()) { Lo = SSE; Hi = SSEUp; - } else if (LDF == &llvm::APFloat::x87DoubleExtended) { + } else if (LDF == &llvm::APFloat::x87DoubleExtended()) { Lo = X87; Hi = X87Up; - } else if (LDF == &llvm::APFloat::IEEEdouble) { + } else if (LDF == &llvm::APFloat::IEEEdouble()) { Current = SSE; } else llvm_unreachable("unexpected long double representation!"); @@ -2440,11 +2602,11 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Lo = Hi = SSE; } else if (ET == getContext().LongDoubleTy) { const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat(); - if (LDF == &llvm::APFloat::IEEEquad) + if (LDF == &llvm::APFloat::IEEEquad()) Current = Memory; - else if (LDF == &llvm::APFloat::x87DoubleExtended) + else if (LDF == &llvm::APFloat::x87DoubleExtended()) Current = ComplexX87; - else if (LDF == &llvm::APFloat::IEEEdouble) + else if (LDF == &llvm::APFloat::IEEEdouble()) Lo = Hi = SSE; else llvm_unreachable("unexpected long double representation!"); @@ -2466,8 +2628,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, uint64_t Size = getContext().getTypeSize(Ty); // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger - // than four eightbytes, ..., it has class MEMORY. - if (Size > 256) + // than eight eightbytes, ..., it has class MEMORY. + if (Size > 512) return; // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned @@ -2486,7 +2648,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // The only case a 256-bit wide vector could be used is when the array // contains a single 256-bit element. Since Lo and Hi logic isn't extended // to work for sizes wider than 128, early check and fallback to memory. - if (Size > 128 && EltSize != 256) + // + if (Size > 128 && + (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel))) return; for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) { @@ -2507,8 +2671,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, uint64_t Size = getContext().getTypeSize(Ty); // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger - // than four eightbytes, ..., it has class MEMORY. - if (Size > 256) + // than eight eightbytes, ..., it has class MEMORY. + if (Size > 512) return; // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial @@ -2561,6 +2725,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx); bool BitField = i->isBitField(); + // Ignore padding bit-fields. + if (BitField && i->isUnnamedBitfield()) + continue; + // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than // four eightbytes, or it contains unaligned fields, it has class MEMORY. // @@ -2568,7 +2736,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // contains a single 256-bit element. Since Lo and Hi logic isn't extended // to work for sizes wider than 128, early check and fallback to memory. // - if (Size > 128 && getContext().getTypeSize(i->getType()) != 256) { + if (Size > 128 && (Size != getContext().getTypeSize(i->getType()) || + Size > getNativeVectorSizeForAVXABI(AVXLevel))) { Lo = Memory; postMerge(Size, Lo, Hi); return; @@ -2592,10 +2761,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // structure to be passed in memory even if unaligned, and // therefore they can straddle an eightbyte. if (BitField) { - // Ignore padding bit-fields. - if (i->isUnnamedBitfield()) - continue; - + assert(!i->isUnnamedBitfield()); uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx); uint64_t Size = i->getBitWidthValue(getContext()); @@ -2723,7 +2889,7 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const { // We couldn't find the preferred IR vector type for 'Ty'. uint64_t Size = getContext().getTypeSize(Ty); - assert((Size == 128 || Size == 256) && "Invalid type found!"); + assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!"); // Return a LLVM IR vector type based on the size of 'Ty'. return llvm::VectorType::get(llvm::Type::getDoubleTy(getVMContext()), @@ -3247,22 +3413,94 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType( return ABIArgInfo::getDirect(ResType); } +ABIArgInfo +X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt, + unsigned &NeededSSE) const { + auto RT = Ty->getAs<RecordType>(); + assert(RT && "classifyRegCallStructType only valid with struct types"); + + if (RT->getDecl()->hasFlexibleArrayMember()) + return getIndirectReturnResult(Ty); + + // Sum up bases + if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) { + if (CXXRD->isDynamicClass()) { + NeededInt = NeededSSE = 0; + return getIndirectReturnResult(Ty); + } + + for (const auto &I : CXXRD->bases()) + if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE) + .isIndirect()) { + NeededInt = NeededSSE = 0; + return getIndirectReturnResult(Ty); + } + } + + // Sum up members + for (const auto *FD : RT->getDecl()->fields()) { + if (FD->getType()->isRecordType() && !FD->getType()->isUnionType()) { + if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE) + .isIndirect()) { + NeededInt = NeededSSE = 0; + return getIndirectReturnResult(Ty); + } + } else { + unsigned LocalNeededInt, LocalNeededSSE; + if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt, + LocalNeededSSE, true) + .isIndirect()) { + NeededInt = NeededSSE = 0; + return getIndirectReturnResult(Ty); + } + NeededInt += LocalNeededInt; + NeededSSE += LocalNeededSSE; + } + } + + return ABIArgInfo::getDirect(); +} + +ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty, + unsigned &NeededInt, + unsigned &NeededSSE) const { + + NeededInt = 0; + NeededSSE = 0; + + return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE); +} + void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { - if (!getCXXABI().classifyReturnType(FI)) - FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + bool IsRegCall = FI.getCallingConvention() == llvm::CallingConv::X86_RegCall; // Keep track of the number of assigned registers. - unsigned freeIntRegs = 6, freeSSERegs = 8; + unsigned FreeIntRegs = IsRegCall ? 11 : 6; + unsigned FreeSSERegs = IsRegCall ? 16 : 8; + unsigned NeededInt, NeededSSE; + + if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() && + !FI.getReturnType()->getTypePtr()->isUnionType()) { + FI.getReturnInfo() = + classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE); + if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) { + FreeIntRegs -= NeededInt; + FreeSSERegs -= NeededSSE; + } else { + FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType()); + } + } else if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); // If the return value is indirect, then the hidden argument is consuming one // integer register. if (FI.getReturnInfo().isIndirect()) - --freeIntRegs; + --FreeIntRegs; // The chain argument effectively gives us another free register. if (FI.isChainCall()) - ++freeIntRegs; + ++FreeIntRegs; unsigned NumRequiredArgs = FI.getNumRequiredArgs(); // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers @@ -3272,19 +3510,21 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { it != ie; ++it, ++ArgNo) { bool IsNamedArg = ArgNo < NumRequiredArgs; - unsigned neededInt, neededSSE; - it->info = classifyArgumentType(it->type, freeIntRegs, neededInt, - neededSSE, IsNamedArg); + if (IsRegCall && it->type->isStructureOrClassType()) + it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE); + else + it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt, + NeededSSE, IsNamedArg); // AMD64-ABI 3.2.3p3: If there are no registers available for any // eightbyte of an argument, the whole argument is passed on the // stack. If registers have already been assigned for some // eightbytes of such an argument, the assignments get reverted. - if (freeIntRegs >= neededInt && freeSSERegs >= neededSSE) { - freeIntRegs -= neededInt; - freeSSERegs -= neededSSE; + if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) { + FreeIntRegs -= NeededInt; + FreeSSERegs -= NeededSSE; } else { - it->info = getIndirectResult(it->type, freeIntRegs); + it->info = getIndirectResult(it->type, FreeIntRegs); } } } @@ -3426,15 +3666,17 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr; // Copy the first element. - llvm::Value *V = - CGF.Builder.CreateDefaultAlignedLoad( - CGF.Builder.CreateBitCast(RegLoAddr, PTyLo)); + // FIXME: Our choice of alignment here and below is probably pessimistic. + llvm::Value *V = CGF.Builder.CreateAlignedLoad( + TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo), + CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo))); CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0, CharUnits::Zero())); // Copy the second element. - V = CGF.Builder.CreateDefaultAlignedLoad( - CGF.Builder.CreateBitCast(RegHiAddr, PTyHi)); + V = CGF.Builder.CreateAlignedLoad( + TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi), + CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi))); CharUnits Offset = CharUnits::fromQuantity( getDataLayout().getStructLayout(ST)->getElementOffset(1)); CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1, Offset)); @@ -3529,8 +3771,24 @@ Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, /*allowHigherAlign*/ false); } +ABIArgInfo +WinX86_64ABIInfo::reclassifyHvaArgType(QualType Ty, unsigned &FreeSSERegs, + const ABIArgInfo ¤t) const { + // Assumes vectorCall calling convention. + const Type *Base = nullptr; + uint64_t NumElts = 0; + + if (!Ty->isBuiltinType() && !Ty->isVectorType() && + isHomogeneousAggregate(Ty, Base, NumElts) && FreeSSERegs >= NumElts) { + FreeSSERegs -= NumElts; + return getDirectX86Hva(); + } + return current; +} + ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, - bool IsReturnType) const { + bool IsReturnType, bool IsVectorCall, + bool IsRegCall) const { if (Ty->isVoidType()) return ABIArgInfo::getIgnore(); @@ -3554,21 +3812,34 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, } - // vectorcall adds the concept of a homogenous vector aggregate, similar to - // other targets. const Type *Base = nullptr; uint64_t NumElts = 0; - if (FreeSSERegs && isHomogeneousAggregate(Ty, Base, NumElts)) { - if (FreeSSERegs >= NumElts) { - FreeSSERegs -= NumElts; - if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType()) + // vectorcall adds the concept of a homogenous vector aggregate, similar to + // other targets. + if ((IsVectorCall || IsRegCall) && + isHomogeneousAggregate(Ty, Base, NumElts)) { + if (IsRegCall) { + if (FreeSSERegs >= NumElts) { + FreeSSERegs -= NumElts; + if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType()) + return ABIArgInfo::getDirect(); + return ABIArgInfo::getExpand(); + } + return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); + } else if (IsVectorCall) { + if (FreeSSERegs >= NumElts && + (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) { + FreeSSERegs -= NumElts; return ABIArgInfo::getDirect(); - return ABIArgInfo::getExpand(); + } else if (IsReturnType) { + return ABIArgInfo::getExpand(); + } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) { + // HVAs are delayed and reclassified in the 2nd step. + return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); + } } - return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); } - if (Ty->isMemberPointerType()) { // If the member pointer is represented by an LLVM int or ptr, pass it // directly. @@ -3597,31 +3868,87 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, // passes them indirectly through memory. if (IsMingw64 && BT && BT->getKind() == BuiltinType::LongDouble) { const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat(); - if (LDF == &llvm::APFloat::x87DoubleExtended) + if (LDF == &llvm::APFloat::x87DoubleExtended()) return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); } return ABIArgInfo::getDirect(); } +void WinX86_64ABIInfo::computeVectorCallArgs(CGFunctionInfo &FI, + unsigned FreeSSERegs, + bool IsVectorCall, + bool IsRegCall) const { + unsigned Count = 0; + for (auto &I : FI.arguments()) { + if (Count < VectorcallMaxParamNumAsReg) + I.info = classify(I.type, FreeSSERegs, false, IsVectorCall, IsRegCall); + else { + // Since these cannot be passed in registers, pretend no registers + // are left. + unsigned ZeroSSERegsAvail = 0; + I.info = classify(I.type, /*FreeSSERegs=*/ZeroSSERegsAvail, false, + IsVectorCall, IsRegCall); + } + ++Count; + } + + Count = 0; + for (auto &I : FI.arguments()) { + if (Count < VectorcallMaxParamNumAsReg) + I.info = reclassifyHvaArgType(I.type, FreeSSERegs, I.info); + ++Count; + } +} + void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { bool IsVectorCall = FI.getCallingConvention() == llvm::CallingConv::X86_VectorCall; + bool IsRegCall = FI.getCallingConvention() == llvm::CallingConv::X86_RegCall; + + unsigned FreeSSERegs = 0; + if (IsVectorCall) { + // We can use up to 4 SSE return registers with vectorcall. + FreeSSERegs = 4; + } else if (IsRegCall) { + // RegCall gives us 16 SSE registers. + FreeSSERegs = 16; + } - // We can use up to 4 SSE return registers with vectorcall. - unsigned FreeSSERegs = IsVectorCall ? 4 : 0; if (!getCXXABI().classifyReturnType(FI)) - FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true); + FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true, + IsVectorCall, IsRegCall); + + if (IsVectorCall) { + // We can use up to 6 SSE register parameters with vectorcall. + FreeSSERegs = 6; + } else if (IsRegCall) { + // RegCall gives us 16 SSE registers, we can reuse the return registers. + FreeSSERegs = 16; + } + + if (IsVectorCall) { + computeVectorCallArgs(FI, FreeSSERegs, IsVectorCall, IsRegCall); + } else { + for (auto &I : FI.arguments()) + I.info = classify(I.type, FreeSSERegs, false, IsVectorCall, IsRegCall); + } - // We can use up to 6 SSE register parameters with vectorcall. - FreeSSERegs = IsVectorCall ? 6 : 0; - for (auto &I : FI.arguments()) - I.info = classify(I.type, FreeSSERegs, false); } Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty) const { - return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, + + bool IsIndirect = false; + + // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is + // not 1, 2, 4, or 8 bytes, must be passed by reference." + if (isAggregateTypeForABI(Ty) || Ty->isMemberPointerType()) { + uint64_t Width = getContext().getTypeSize(Ty); + IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width); + } + + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, CGF.getContext().getTypeInfoInChars(Ty), CharUnits::fromQuantity(8), /*allowHigherAlign*/ false); @@ -4381,14 +4708,17 @@ PPC64_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, // 32-63: fp0-31, the 8-byte floating-point registers AssignToArrayRange(Builder, Address, Eight8, 32, 63); - // 64-76 are various 4-byte special-purpose registers: + // 64-67 are various 8-byte special-purpose registers: // 64: mq // 65: lr // 66: ctr // 67: ap + AssignToArrayRange(Builder, Address, Eight8, 64, 67); + + // 68-76 are various 4-byte special-purpose registers: // 68-75 cr0-7 // 76: xer - AssignToArrayRange(Builder, Address, Four8, 64, 76); + AssignToArrayRange(Builder, Address, Four8, 68, 76); // 77-108: v0-31, the 16-byte vector registers AssignToArrayRange(Builder, Address, Sixteen8, 77, 108); @@ -4398,7 +4728,10 @@ PPC64_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, // 111: spe_acc // 112: spefscr // 113: sfp - AssignToArrayRange(Builder, Address, Four8, 109, 113); + // 114: tfhar + // 115: tfiar + // 116: texasr + AssignToArrayRange(Builder, Address, Eight8, 109, 116); return false; } @@ -4475,6 +4808,9 @@ private: bool asReturnValue) const override { return occupiesMoreThan(CGT, scalars, /*total*/ 4); } + bool isSwiftErrorInRegister() const override { + return true; + } }; class AArch64TargetCodeGenInfo : public TargetCodeGenInfo { @@ -4559,6 +4895,11 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { // Aggregates <= 16 bytes are passed directly in registers or on the stack. uint64_t Size = getContext().getTypeSize(Ty); if (Size <= 128) { + // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of + // same size and alignment. + if (getTarget().isRenderScriptTarget()) { + return coerceToIntArray(Ty, getContext(), getVMContext()); + } unsigned Alignment = getContext().getTypeAlign(Ty); Size = 64 * ((Size + 63) / 64); // round up to multiple of 8 bytes @@ -4604,6 +4945,11 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { // Aggregates <= 16 bytes are returned directly in registers or on the stack. uint64_t Size = getContext().getTypeSize(RetTy); if (Size <= 128) { + // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of + // same size and alignment. + if (getTarget().isRenderScriptTarget()) { + return coerceToIntArray(RetTy, getContext(), getVMContext()); + } unsigned Alignment = getContext().getTypeAlign(RetTy); Size = 64 * ((Size + 63) / 64); // round up to multiple of 8 bytes @@ -5018,6 +5364,9 @@ private: bool asReturnValue) const override { return occupiesMoreThan(CGT, scalars, /*total*/ 4); } + bool isSwiftErrorInRegister() const override { + return true; + } }; class ARMTargetCodeGenInfo : public TargetCodeGenInfo { @@ -5294,6 +5643,12 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, /*Realign=*/TyAlign > ABIAlign); } + // On RenderScript, coerce Aggregates <= 64 bytes to an integer array of + // same size and alignment. + if (getTarget().isRenderScriptTarget()) { + return coerceToIntArray(Ty, getContext(), getVMContext()); + } + // Otherwise, pass by coercing to a structure of the appropriate size. llvm::Type* ElemTy; unsigned SizeRegs; @@ -5475,6 +5830,11 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, // are returned indirectly. uint64_t Size = getContext().getTypeSize(RetTy); if (Size <= 32) { + // On RenderScript, coerce Aggregates <= 4 bytes to an integer array of + // same size and alignment. + if (getTarget().isRenderScriptTarget()) { + return coerceToIntArray(RetTy, getContext(), getVMContext()); + } if (getDataLayout().isBigEndian()) // Return in 32 bit integer integer type (as if loaded by LDR, AAPCS 5.4) return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext())); @@ -5775,6 +6135,9 @@ public: bool asReturnValue) const override { return occupiesMoreThan(CGT, scalars, /*total*/ 4); } + bool isSwiftErrorInRegister() const override { + return true; + } }; class SystemZTargetCodeGenInfo : public TargetCodeGenInfo { @@ -6833,45 +7196,138 @@ public: namespace { +class AMDGPUABIInfo final : public DefaultABIInfo { +public: + explicit AMDGPUABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} + +private: + ABIArgInfo classifyArgumentType(QualType Ty) const; + + void computeInfo(CGFunctionInfo &FI) const override; +}; + +void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const { + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + + unsigned CC = FI.getCallingConvention(); + for (auto &Arg : FI.arguments()) + if (CC == llvm::CallingConv::AMDGPU_KERNEL) + Arg.info = classifyArgumentType(Arg.type); + else + Arg.info = DefaultABIInfo::classifyArgumentType(Arg.type); +} + +/// \brief Classify argument of given type \p Ty. +ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty) const { + llvm::StructType *StrTy = dyn_cast<llvm::StructType>(CGT.ConvertType(Ty)); + if (!StrTy) { + return DefaultABIInfo::classifyArgumentType(Ty); + } + + // Coerce single element structs to its element. + if (StrTy->getNumElements() == 1) { + return ABIArgInfo::getDirect(); + } + + // If we set CanBeFlattened to true, CodeGen will expand the struct to its + // individual elements, which confuses the Clover OpenCL backend; therefore we + // have to set it to false here. Other args of getDirect() are just defaults. + return ABIArgInfo::getDirect(nullptr, 0, nullptr, false); +} + class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo { public: AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} + : TargetCodeGenInfo(new AMDGPUABIInfo(CGT)) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; unsigned getOpenCLKernelCallingConv() const override; -}; + llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM, + llvm::PointerType *T, QualType QT) const override; +}; } +static void appendOpenCLVersionMD (CodeGen::CodeGenModule &CGM); + void AMDGPUTargetCodeGenInfo::setTargetAttributes( - const Decl *D, - llvm::GlobalValue *GV, - CodeGen::CodeGenModule &M) const { + const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &M) const { const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; - if (const auto Attr = FD->getAttr<AMDGPUNumVGPRAttr>()) { - llvm::Function *F = cast<llvm::Function>(GV); - uint32_t NumVGPR = Attr->getNumVGPR(); - if (NumVGPR != 0) - F->addFnAttr("amdgpu_num_vgpr", llvm::utostr(NumVGPR)); + llvm::Function *F = cast<llvm::Function>(GV); + + if (const auto *Attr = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) { + unsigned Min = Attr->getMin(); + unsigned Max = Attr->getMax(); + + if (Min != 0) { + assert(Min <= Max && "Min must be less than or equal Max"); + + std::string AttrVal = llvm::utostr(Min) + "," + llvm::utostr(Max); + F->addFnAttr("amdgpu-flat-work-group-size", AttrVal); + } else + assert(Max == 0 && "Max must be zero"); + } + + if (const auto *Attr = FD->getAttr<AMDGPUWavesPerEUAttr>()) { + unsigned Min = Attr->getMin(); + unsigned Max = Attr->getMax(); + + if (Min != 0) { + assert((Max == 0 || Min <= Max) && "Min must be less than or equal Max"); + + std::string AttrVal = llvm::utostr(Min); + if (Max != 0) + AttrVal = AttrVal + "," + llvm::utostr(Max); + F->addFnAttr("amdgpu-waves-per-eu", AttrVal); + } else + assert(Max == 0 && "Max must be zero"); } - if (const auto Attr = FD->getAttr<AMDGPUNumSGPRAttr>()) { - llvm::Function *F = cast<llvm::Function>(GV); + if (const auto *Attr = FD->getAttr<AMDGPUNumSGPRAttr>()) { unsigned NumSGPR = Attr->getNumSGPR(); + if (NumSGPR != 0) - F->addFnAttr("amdgpu_num_sgpr", llvm::utostr(NumSGPR)); + F->addFnAttr("amdgpu-num-sgpr", llvm::utostr(NumSGPR)); } -} + if (const auto *Attr = FD->getAttr<AMDGPUNumVGPRAttr>()) { + uint32_t NumVGPR = Attr->getNumVGPR(); + + if (NumVGPR != 0) + F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR)); + } + + appendOpenCLVersionMD(M); +} unsigned AMDGPUTargetCodeGenInfo::getOpenCLKernelCallingConv() const { return llvm::CallingConv::AMDGPU_KERNEL; } +// Currently LLVM assumes null pointers always have value 0, +// which results in incorrectly transformed IR. Therefore, instead of +// emitting null pointers in private and local address spaces, a null +// pointer in generic address space is emitted which is casted to a +// pointer in local or private address space. +llvm::Constant *AMDGPUTargetCodeGenInfo::getNullPointer( + const CodeGen::CodeGenModule &CGM, llvm::PointerType *PT, + QualType QT) const { + if (CGM.getContext().getTargetNullPointerValue(QT) == 0) + return llvm::ConstantPointerNull::get(PT); + + auto &Ctx = CGM.getContext(); + auto NPT = llvm::PointerType::get(PT->getElementType(), + Ctx.getTargetAddressSpace(LangAS::opencl_generic)); + return llvm::ConstantExpr::getAddrSpaceCast( + llvm::ConstantPointerNull::get(NPT), PT); +} + //===----------------------------------------------------------------------===// // SPARC v8 ABI Implementation. // Based on the SPARC Compliance Definition version 2.4.1. @@ -7311,7 +7767,7 @@ class FieldEncoding { std::string Enc; public: FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {} - StringRef str() {return Enc.c_str();} + StringRef str() { return Enc; } bool operator<(const FieldEncoding &rhs) const { if (HasName != rhs.HasName) return HasName; return Enc < rhs.Enc; @@ -7477,7 +7933,7 @@ StringRef TypeStringCache::lookupStr(const IdentifierInfo *ID) { E.State = IncompleteUsed; ++IncompleteUsedCount; } - return E.Str.c_str(); + return E.Str; } /// The XCore ABI includes a type information section that communicates symbol @@ -7533,11 +7989,20 @@ void SPIRTargetCodeGenInfo::emitTargetMD(const Decl *D, llvm::GlobalValue *GV, // SPIR v2.0 s2.12 - The SPIR version used by the module is stored in the // opencl.spir.version named metadata. llvm::Metadata *SPIRVerElts[] = { - llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(Int32Ty, 2)), - llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(Int32Ty, 0))}; + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int32Ty, CGM.getLangOpts().OpenCLVersion / 100)), + llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( + Int32Ty, (CGM.getLangOpts().OpenCLVersion / 100 > 1) ? 0 : 2))}; llvm::NamedMDNode *SPIRVerMD = M.getOrInsertNamedMetadata("opencl.spir.version"); SPIRVerMD->addOperand(llvm::MDNode::get(Ctx, SPIRVerElts)); + appendOpenCLVersionMD(CGM); +} + +static void appendOpenCLVersionMD(CodeGen::CodeGenModule &CGM) { + llvm::LLVMContext &Ctx = CGM.getModule().getContext(); + llvm::Type *Int32Ty = llvm::Type::getInt32Ty(Ctx); + llvm::Module &M = CGM.getModule(); // SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the // opencl.ocl.version named metadata node. llvm::Metadata *OCLVerElts[] = { @@ -7890,10 +8355,6 @@ static bool getTypeString(SmallStringEnc &Enc, const Decl *D, // Driver code //===----------------------------------------------------------------------===// -const llvm::Triple &CodeGenModule::getTriple() const { - return getTarget().getTriple(); -} - bool CodeGenModule::supportsCOMDAT() const { return getTriple().supportsCOMDAT(); } @@ -8003,6 +8464,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { } case llvm::Triple::tce: + case llvm::Triple::tcele: return SetCGInfo(new TCETargetCodeGenInfo(Types)); case llvm::Triple::x86: { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h index e463825..223d6d0 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h @@ -220,6 +220,22 @@ public: /// Get LLVM calling convention for OpenCL kernel. virtual unsigned getOpenCLKernelCallingConv() const; + + /// Get target specific null pointer. + /// \param T is the LLVM type of the null pointer. + /// \param QT is the clang QualType of the null pointer. + /// \return ConstantPointerNull with the given type \p T. + /// Each target can override it to return its own desired constant value. + virtual llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM, + llvm::PointerType *T, QualType QT) const; + + /// Perform address space cast of an expression of pointer type. + /// \param V is the LLVM value to be casted to another address space. + /// \param SrcTy is the QualType of \p V. + /// \param DestTy is the destination QualType. + virtual llvm::Value *performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, + llvm::Value *V, QualType SrcTy, QualType DestTy) const; + }; } // namespace CodeGen diff --git a/contrib/llvm/tools/clang/lib/CodeGen/VarBypassDetector.cpp b/contrib/llvm/tools/clang/lib/CodeGen/VarBypassDetector.cpp new file mode 100644 index 0000000..cfb93d6 --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/VarBypassDetector.cpp @@ -0,0 +1,168 @@ +//===--- VarBypassDetector.h - Bypass jumps detector --------------*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "VarBypassDetector.h" + +#include "clang/AST/Decl.h" +#include "clang/AST/Expr.h" +#include "clang/AST/Stmt.h" + +using namespace clang; +using namespace CodeGen; + +/// Clear the object and pre-process for the given statement, usually function +/// body statement. +void VarBypassDetector::Init(const Stmt *Body) { + FromScopes.clear(); + ToScopes.clear(); + Bypasses.clear(); + Scopes = {{~0U, nullptr}}; + unsigned ParentScope = 0; + AlwaysBypassed = !BuildScopeInformation(Body, ParentScope); + if (!AlwaysBypassed) + Detect(); +} + +/// Build scope information for a declaration that is part of a DeclStmt. +/// Returns false if we failed to build scope information and can't tell for +/// which vars are being bypassed. +bool VarBypassDetector::BuildScopeInformation(const Decl *D, + unsigned &ParentScope) { + const VarDecl *VD = dyn_cast<VarDecl>(D); + if (VD && VD->hasLocalStorage()) { + Scopes.push_back({ParentScope, VD}); + ParentScope = Scopes.size() - 1; + } + + if (const VarDecl *VD = dyn_cast<VarDecl>(D)) + if (const Expr *Init = VD->getInit()) + return BuildScopeInformation(Init, ParentScope); + + return true; +} + +/// Walk through the statements, adding any labels or gotos to +/// LabelAndGotoScopes and recursively walking the AST as needed. +/// Returns false if we failed to build scope information and can't tell for +/// which vars are being bypassed. +bool VarBypassDetector::BuildScopeInformation(const Stmt *S, + unsigned &origParentScope) { + // If this is a statement, rather than an expression, scopes within it don't + // propagate out into the enclosing scope. Otherwise we have to worry about + // block literals, which have the lifetime of their enclosing statement. + unsigned independentParentScope = origParentScope; + unsigned &ParentScope = + ((isa<Expr>(S) && !isa<StmtExpr>(S)) ? origParentScope + : independentParentScope); + + unsigned StmtsToSkip = 0u; + + switch (S->getStmtClass()) { + case Stmt::IndirectGotoStmtClass: + return false; + + case Stmt::SwitchStmtClass: + if (const Stmt *Init = cast<SwitchStmt>(S)->getInit()) { + if (!BuildScopeInformation(Init, ParentScope)) + return false; + ++StmtsToSkip; + } + if (const VarDecl *Var = cast<SwitchStmt>(S)->getConditionVariable()) { + if (!BuildScopeInformation(Var, ParentScope)) + return false; + ++StmtsToSkip; + } + // Fall through + + case Stmt::GotoStmtClass: + FromScopes.push_back({S, ParentScope}); + break; + + case Stmt::DeclStmtClass: { + const DeclStmt *DS = cast<DeclStmt>(S); + for (auto *I : DS->decls()) + if (!BuildScopeInformation(I, origParentScope)) + return false; + return true; + } + + case Stmt::CaseStmtClass: + case Stmt::DefaultStmtClass: + case Stmt::LabelStmtClass: + llvm_unreachable("the loop bellow handles labels and cases"); + break; + + default: + break; + } + + for (const Stmt *SubStmt : S->children()) { + if (!SubStmt) + continue; + if (StmtsToSkip) { + --StmtsToSkip; + continue; + } + + // Cases, labels, and defaults aren't "scope parents". It's also + // important to handle these iteratively instead of recursively in + // order to avoid blowing out the stack. + while (true) { + const Stmt *Next; + if (const SwitchCase *SC = dyn_cast<SwitchCase>(SubStmt)) + Next = SC->getSubStmt(); + else if (const LabelStmt *LS = dyn_cast<LabelStmt>(SubStmt)) + Next = LS->getSubStmt(); + else + break; + + ToScopes[SubStmt] = ParentScope; + SubStmt = Next; + } + + // Recursively walk the AST. + if (!BuildScopeInformation(SubStmt, ParentScope)) + return false; + } + return true; +} + +/// Checks each jump and stores each variable declaration they bypass. +void VarBypassDetector::Detect() { + for (const auto &S : FromScopes) { + const Stmt *St = S.first; + unsigned from = S.second; + if (const GotoStmt *GS = dyn_cast<GotoStmt>(St)) { + if (const LabelStmt *LS = GS->getLabel()->getStmt()) + Detect(from, ToScopes[LS]); + } else if (const SwitchStmt *SS = dyn_cast<SwitchStmt>(St)) { + for (const SwitchCase *SC = SS->getSwitchCaseList(); SC; + SC = SC->getNextSwitchCase()) { + Detect(from, ToScopes[SC]); + } + } else { + llvm_unreachable("goto or switch was expected"); + } + } +} + +/// Checks the jump and stores each variable declaration it bypasses. +void VarBypassDetector::Detect(unsigned From, unsigned To) { + while (From != To) { + if (From < To) { + assert(Scopes[To].first < To); + const auto &ScopeTo = Scopes[To]; + To = ScopeTo.first; + Bypasses.insert(ScopeTo.second); + } else { + assert(Scopes[From].first < From); + From = Scopes[From].first; + } + } +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/VarBypassDetector.h b/contrib/llvm/tools/clang/lib/CodeGen/VarBypassDetector.h new file mode 100644 index 0000000..f50baf4 --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/VarBypassDetector.h @@ -0,0 +1,70 @@ +//===--- VarBypassDetector.cpp - Bypass jumps detector ------------*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains VarBypassDetector class, which is used to detect +// local variable declarations which can be bypassed by jumps. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_CODEGEN_VARBYPASSDETECTOR_H +#define LLVM_CLANG_LIB_CODEGEN_VARBYPASSDETECTOR_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" + +namespace clang { + +class Decl; +class Stmt; +class VarDecl; + +namespace CodeGen { + +/// The class detects jumps which bypass local variables declaration: +/// goto L; +/// int a; +/// L: +/// +/// This is simplified version of JumpScopeChecker. Primary differences: +/// * Detects only jumps into the scope local variables. +/// * Does not detect jumps out of the scope of local variables. +/// * Not limited to variables with initializers, JumpScopeChecker is limited. +class VarBypassDetector { + // Scope information. Contains a parent scope and related variable + // declaration. + llvm::SmallVector<std::pair<unsigned, const VarDecl *>, 48> Scopes; + // List of jumps with scopes. + llvm::SmallVector<std::pair<const Stmt *, unsigned>, 16> FromScopes; + // Lookup map to find scope for destinations. + llvm::DenseMap<const Stmt *, unsigned> ToScopes; + // Set of variables which were bypassed by some jump. + llvm::DenseSet<const VarDecl *> Bypasses; + // If true assume that all variables are being bypassed. + bool AlwaysBypassed = false; + +public: + void Init(const Stmt *Body); + + /// Returns true if the variable declaration was by bypassed by any goto or + /// switch statement. + bool IsBypassed(const VarDecl *D) const { + return AlwaysBypassed || Bypasses.find(D) != Bypasses.end(); + } + +private: + bool BuildScopeInformation(const Decl *D, unsigned &ParentScope); + bool BuildScopeInformation(const Stmt *S, unsigned &origParentScope); + void Detect(); + void Detect(unsigned From, unsigned To); +}; +} +} + +#endif |
