diff options
Diffstat (limited to 'contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp')
| -rw-r--r-- | contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp | 1083 |
1 files changed, 409 insertions, 674 deletions
diff --git a/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp b/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp index d892996..1094e6d 100644 --- a/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp +++ b/contrib/llvm/tools/clang/lib/Frontend/ASTUnit.cpp @@ -18,6 +18,7 @@ #include "clang/AST/StmtVisitor.h" #include "clang/AST/TypeOrdering.h" #include "clang/Basic/Diagnostic.h" +#include "clang/Basic/MemoryBufferCache.h" #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" #include "clang/Basic/VirtualFileSystem.h" @@ -78,106 +79,83 @@ namespace { } } }; - - struct OnDiskData { - /// \brief The file in which the precompiled preamble is stored. - std::string PreambleFile; - - /// \brief Temporary files that should be removed when the ASTUnit is - /// destroyed. - SmallVector<std::string, 4> TemporaryFiles; - - /// \brief Erase temporary files. - void CleanTemporaryFiles(); - - /// \brief Erase the preamble file. - void CleanPreambleFile(); - - /// \brief Erase temporary files and the preamble file. - void Cleanup(); - }; -} - -static llvm::sys::SmartMutex<false> &getOnDiskMutex() { - static llvm::sys::SmartMutex<false> M(/* recursive = */ true); - return M; -} - -static void cleanupOnDiskMapAtExit(); -typedef llvm::DenseMap<const ASTUnit *, - std::unique_ptr<OnDiskData>> OnDiskDataMap; -static OnDiskDataMap &getOnDiskDataMap() { - static OnDiskDataMap M; - static bool hasRegisteredAtExit = false; - if (!hasRegisteredAtExit) { - hasRegisteredAtExit = true; - atexit(cleanupOnDiskMapAtExit); + template <class T> + std::unique_ptr<T> valueOrNull(llvm::ErrorOr<std::unique_ptr<T>> Val) { + if (!Val) + return nullptr; + return std::move(*Val); } - return M; -} -static void cleanupOnDiskMapAtExit() { - // Use the mutex because there can be an alive thread destroying an ASTUnit. - llvm::MutexGuard Guard(getOnDiskMutex()); - for (const auto &I : getOnDiskDataMap()) { - // We don't worry about freeing the memory associated with OnDiskDataMap. - // All we care about is erasing stale files. - I.second->Cleanup(); + template <class T> + bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) { + if (!Val) + return false; + Output = std::move(*Val); + return true; } -} -static OnDiskData &getOnDiskData(const ASTUnit *AU) { - // We require the mutex since we are modifying the structure of the - // DenseMap. - llvm::MutexGuard Guard(getOnDiskMutex()); - OnDiskDataMap &M = getOnDiskDataMap(); - auto &D = M[AU]; - if (!D) - D = llvm::make_unique<OnDiskData>(); - return *D; -} - -static void erasePreambleFile(const ASTUnit *AU) { - getOnDiskData(AU).CleanPreambleFile(); -} +/// \brief Get a source buffer for \p MainFilePath, handling all file-to-file +/// and file-to-buffer remappings inside \p Invocation. +static std::unique_ptr<llvm::MemoryBuffer> +getBufferForFileHandlingRemapping(const CompilerInvocation &Invocation, + vfs::FileSystem *VFS, + StringRef FilePath) { + const auto &PreprocessorOpts = Invocation.getPreprocessorOpts(); -static void removeOnDiskEntry(const ASTUnit *AU) { - // We require the mutex since we are modifying the structure of the - // DenseMap. - llvm::MutexGuard Guard(getOnDiskMutex()); - OnDiskDataMap &M = getOnDiskDataMap(); - OnDiskDataMap::iterator I = M.find(AU); - if (I != M.end()) { - I->second->Cleanup(); - M.erase(I); - } -} - -static void setPreambleFile(const ASTUnit *AU, StringRef preambleFile) { - getOnDiskData(AU).PreambleFile = preambleFile; -} + // Try to determine if the main file has been remapped, either from the + // command line (to another file) or directly through the compiler + // invocation (to a memory buffer). + llvm::MemoryBuffer *Buffer = nullptr; + std::unique_ptr<llvm::MemoryBuffer> BufferOwner; + auto FileStatus = VFS->status(FilePath); + if (FileStatus) { + llvm::sys::fs::UniqueID MainFileID = FileStatus->getUniqueID(); -static const std::string &getPreambleFile(const ASTUnit *AU) { - return getOnDiskData(AU).PreambleFile; -} + // Check whether there is a file-file remapping of the main file + for (const auto &RF : PreprocessorOpts.RemappedFiles) { + std::string MPath(RF.first); + auto MPathStatus = VFS->status(MPath); + if (MPathStatus) { + llvm::sys::fs::UniqueID MID = MPathStatus->getUniqueID(); + if (MainFileID == MID) { + // We found a remapping. Try to load the resulting, remapped source. + BufferOwner = valueOrNull(VFS->getBufferForFile(RF.second)); + if (!BufferOwner) + return nullptr; + } + } + } -void OnDiskData::CleanTemporaryFiles() { - for (StringRef File : TemporaryFiles) - llvm::sys::fs::remove(File); - TemporaryFiles.clear(); -} + // Check whether there is a file-buffer remapping. It supercedes the + // file-file remapping. + for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) { + std::string MPath(RB.first); + auto MPathStatus = VFS->status(MPath); + if (MPathStatus) { + llvm::sys::fs::UniqueID MID = MPathStatus->getUniqueID(); + if (MainFileID == MID) { + // We found a remapping. + BufferOwner.reset(); + Buffer = const_cast<llvm::MemoryBuffer *>(RB.second); + } + } + } + } -void OnDiskData::CleanPreambleFile() { - if (!PreambleFile.empty()) { - llvm::sys::fs::remove(PreambleFile); - PreambleFile.clear(); + // If the main source file was not remapped, load it now. + if (!Buffer && !BufferOwner) { + BufferOwner = valueOrNull(VFS->getBufferForFile(FilePath)); + if (!BufferOwner) + return nullptr; } -} -void OnDiskData::Cleanup() { - CleanTemporaryFiles(); - CleanPreambleFile(); + if (BufferOwner) + return BufferOwner; + if (!Buffer) + return nullptr; + return llvm::MemoryBuffer::getMemBufferCopy(Buffer->getBuffer(), FilePath); +} } struct ASTUnit::ASTWriterData { @@ -185,21 +163,14 @@ struct ASTUnit::ASTWriterData { llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { } + ASTWriterData(MemoryBufferCache &PCMCache) + : Stream(Buffer), Writer(Stream, Buffer, PCMCache, {}) {} }; void ASTUnit::clearFileLevelDecls() { llvm::DeleteContainerSeconds(FileDecls); } -void ASTUnit::CleanTemporaryFiles() { - getOnDiskData(this).CleanTemporaryFiles(); -} - -void ASTUnit::addTemporaryFile(StringRef TempFile) { - getOnDiskData(this).TemporaryFiles.push_back(TempFile); -} - /// \brief After failing to build a precompiled preamble (due to /// errors in the source that occurs in the preamble), the number of /// reparses during which we'll skip even trying to precompile the @@ -238,9 +209,6 @@ ASTUnit::~ASTUnit() { clearFileLevelDecls(); - // Clean up the temporary files and the preamble file. - removeOnDiskEntry(this); - // Free the buffers associated with remapped files. We are required to // perform this operation here because we explicitly request that the // compiler instance *not* free these buffers for each invocation of the @@ -490,7 +458,9 @@ namespace { /// a Preprocessor. class ASTInfoCollector : public ASTReaderListener { Preprocessor &PP; - ASTContext &Context; + ASTContext *Context; + HeaderSearchOptions &HSOpts; + PreprocessorOptions &PPOpts; LangOptions &LangOpt; std::shared_ptr<TargetOptions> &TargetOpts; IntrusiveRefCntPtr<TargetInfo> &Target; @@ -498,11 +468,14 @@ class ASTInfoCollector : public ASTReaderListener { bool InitializedLanguage; public: - ASTInfoCollector(Preprocessor &PP, ASTContext &Context, LangOptions &LangOpt, + ASTInfoCollector(Preprocessor &PP, ASTContext *Context, + HeaderSearchOptions &HSOpts, PreprocessorOptions &PPOpts, + LangOptions &LangOpt, std::shared_ptr<TargetOptions> &TargetOpts, IntrusiveRefCntPtr<TargetInfo> &Target, unsigned &Counter) - : PP(PP), Context(Context), LangOpt(LangOpt), TargetOpts(TargetOpts), - Target(Target), Counter(Counter), InitializedLanguage(false) {} + : PP(PP), Context(Context), HSOpts(HSOpts), PPOpts(PPOpts), + LangOpt(LangOpt), TargetOpts(TargetOpts), Target(Target), + Counter(Counter), InitializedLanguage(false) {} bool ReadLanguageOptions(const LangOptions &LangOpts, bool Complain, bool AllowCompatibleDifferences) override { @@ -516,6 +489,20 @@ public: return false; } + virtual bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, + StringRef SpecificModuleCachePath, + bool Complain) override { + this->HSOpts = HSOpts; + return false; + } + + virtual bool + ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, bool Complain, + std::string &SuggestedPredefines) override { + this->PPOpts = PPOpts; + return false; + } + bool ReadTargetOptions(const TargetOptions &TargetOpts, bool Complain, bool AllowCompatibleDifferences) override { // If we've already initialized the target, don't do it again. @@ -549,28 +536,39 @@ private: // Initialize the preprocessor. PP.Initialize(*Target); + if (!Context) + return; + // Initialize the ASTContext - Context.InitBuiltinTypes(*Target); + Context->InitBuiltinTypes(*Target); // We didn't have access to the comment options when the ASTContext was // constructed, so register them now. - Context.getCommentCommandTraits().registerCommentOptions( + Context->getCommentCommandTraits().registerCommentOptions( LangOpt.CommentOpts); } }; /// \brief Diagnostic consumer that saves each diagnostic it is given. class StoredDiagnosticConsumer : public DiagnosticConsumer { - SmallVectorImpl<StoredDiagnostic> &StoredDiags; + SmallVectorImpl<StoredDiagnostic> *StoredDiags; + SmallVectorImpl<ASTUnit::StandaloneDiagnostic> *StandaloneDiags; + const LangOptions *LangOpts; SourceManager *SourceMgr; public: - explicit StoredDiagnosticConsumer( - SmallVectorImpl<StoredDiagnostic> &StoredDiags) - : StoredDiags(StoredDiags), SourceMgr(nullptr) {} + StoredDiagnosticConsumer( + SmallVectorImpl<StoredDiagnostic> *StoredDiags, + SmallVectorImpl<ASTUnit::StandaloneDiagnostic> *StandaloneDiags) + : StoredDiags(StoredDiags), StandaloneDiags(StandaloneDiags), + LangOpts(nullptr), SourceMgr(nullptr) { + assert((StoredDiags || StandaloneDiags) && + "No output collections were passed to StoredDiagnosticConsumer."); + } void BeginSourceFile(const LangOptions &LangOpts, const Preprocessor *PP = nullptr) override { + this->LangOpts = &LangOpts; if (PP) SourceMgr = &PP->getSourceManager(); } @@ -589,8 +587,9 @@ class CaptureDroppedDiagnostics { public: CaptureDroppedDiagnostics(bool RequestCapture, DiagnosticsEngine &Diags, - SmallVectorImpl<StoredDiagnostic> &StoredDiags) - : Diags(Diags), Client(StoredDiags), PreviousClient(nullptr) + SmallVectorImpl<StoredDiagnostic> *StoredDiags, + SmallVectorImpl<ASTUnit::StandaloneDiagnostic> *StandaloneDiags) + : Diags(Diags), Client(StoredDiags, StandaloneDiags), PreviousClient(nullptr) { if (RequestCapture || Diags.getClient() == nullptr) { OwningPreviousClient = Diags.takeClient(); @@ -607,16 +606,39 @@ public: } // anonymous namespace +static ASTUnit::StandaloneDiagnostic +makeStandaloneDiagnostic(const LangOptions &LangOpts, + const StoredDiagnostic &InDiag); + void StoredDiagnosticConsumer::HandleDiagnostic(DiagnosticsEngine::Level Level, - const Diagnostic &Info) { + const Diagnostic &Info) { // Default implementation (Warnings/errors count). DiagnosticConsumer::HandleDiagnostic(Level, Info); // Only record the diagnostic if it's part of the source manager we know // about. This effectively drops diagnostics from modules we're building. // FIXME: In the long run, ee don't want to drop source managers from modules. - if (!Info.hasSourceManager() || &Info.getSourceManager() == SourceMgr) - StoredDiags.emplace_back(Level, Info); + if (!Info.hasSourceManager() || &Info.getSourceManager() == SourceMgr) { + StoredDiagnostic *ResultDiag = nullptr; + if (StoredDiags) { + StoredDiags->emplace_back(Level, Info); + ResultDiag = &StoredDiags->back(); + } + + if (StandaloneDiags) { + llvm::Optional<StoredDiagnostic> StoredDiag = llvm::None; + if (!ResultDiag) { + StoredDiag.emplace(Level, Info); + ResultDiag = StoredDiag.getPointer(); + } + StandaloneDiags->push_back( + makeStandaloneDiagnostic(*LangOpts, *ResultDiag)); + } + } +} + +IntrusiveRefCntPtr<ASTReader> ASTUnit::getASTReader() const { + return Reader; } ASTMutationListener *ASTUnit::getASTMutationListener() { @@ -647,12 +669,12 @@ void ASTUnit::ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> Diags, ASTUnit &AST, bool CaptureDiagnostics) { assert(Diags.get() && "no DiagnosticsEngine was provided"); if (CaptureDiagnostics) - Diags->setClient(new StoredDiagnosticConsumer(AST.StoredDiagnostics)); + Diags->setClient(new StoredDiagnosticConsumer(&AST.StoredDiagnostics, nullptr)); } std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( const std::string &Filename, const PCHContainerReader &PCHContainerRdr, - IntrusiveRefCntPtr<DiagnosticsEngine> Diags, + WhatToLoad ToLoad, IntrusiveRefCntPtr<DiagnosticsEngine> Diags, const FileSystemOptions &FileSystemOpts, bool UseDebugInfo, bool OnlyLocalDecls, ArrayRef<RemappedFile> RemappedFiles, bool CaptureDiagnostics, bool AllowPCHWithCompilerErrors, @@ -668,6 +690,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( ConfigureDiags(Diags, *AST, CaptureDiagnostics); + AST->LangOpts = std::make_shared<LangOptions>(); AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->Diagnostics = Diags; @@ -677,18 +700,18 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( AST->SourceMgr = new SourceManager(AST->getDiagnostics(), AST->getFileManager(), UserFilesAreVolatile); + AST->PCMCache = new MemoryBufferCache; AST->HSOpts = std::make_shared<HeaderSearchOptions>(); AST->HSOpts->ModuleFormat = PCHContainerRdr.getFormat(); AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts, AST->getSourceManager(), AST->getDiagnostics(), - AST->ASTFileLangOpts, + AST->getLangOpts(), /*Target=*/nullptr)); - - auto PPOpts = std::make_shared<PreprocessorOptions>(); + AST->PPOpts = std::make_shared<PreprocessorOptions>(); for (const auto &RemappedFile : RemappedFiles) - PPOpts->addRemappedFile(RemappedFile.first, RemappedFile.second); + AST->PPOpts->addRemappedFile(RemappedFile.first, RemappedFile.second); // Gather Info for preprocessor construction later on. @@ -696,35 +719,36 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( unsigned Counter; AST->PP = std::make_shared<Preprocessor>( - std::move(PPOpts), AST->getDiagnostics(), AST->ASTFileLangOpts, - AST->getSourceManager(), HeaderInfo, *AST, + AST->PPOpts, AST->getDiagnostics(), *AST->LangOpts, + AST->getSourceManager(), *AST->PCMCache, HeaderInfo, AST->ModuleLoader, /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false); Preprocessor &PP = *AST->PP; - AST->Ctx = new ASTContext(AST->ASTFileLangOpts, AST->getSourceManager(), - PP.getIdentifierTable(), PP.getSelectorTable(), - PP.getBuiltinInfo()); - ASTContext &Context = *AST->Ctx; + if (ToLoad >= LoadASTOnly) + AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(), + PP.getIdentifierTable(), PP.getSelectorTable(), + PP.getBuiltinInfo()); bool disableValid = false; if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = true; - AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, { }, + AST->Reader = new ASTReader(PP, AST->Ctx.get(), PCHContainerRdr, { }, /*isysroot=*/"", /*DisableValidation=*/disableValid, AllowPCHWithCompilerErrors); AST->Reader->setListener(llvm::make_unique<ASTInfoCollector>( - *AST->PP, Context, AST->ASTFileLangOpts, AST->TargetOpts, AST->Target, - Counter)); + *AST->PP, AST->Ctx.get(), *AST->HSOpts, *AST->PPOpts, *AST->LangOpts, + AST->TargetOpts, AST->Target, Counter)); // Attach the AST reader to the AST context as an external AST // source, so that declarations will be deserialized from the // AST file as needed. // We need the external source to be set up before we read the AST, because // eagerly-deserialized declarations may use it. - Context.setExternalSource(AST->Reader); + if (AST->Ctx) + AST->Ctx->setExternalSource(AST->Reader); switch (AST->Reader->ReadAST(Filename, serialization::MK_MainFile, SourceLocation(), ASTReader::ARR_None)) { @@ -746,21 +770,29 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( PP.setCounterValue(Counter); // Create an AST consumer, even though it isn't used. - AST->Consumer.reset(new ASTConsumer); - + if (ToLoad >= LoadASTOnly) + AST->Consumer.reset(new ASTConsumer); + // Create a semantic analysis object and tell the AST reader about it. - AST->TheSema.reset(new Sema(PP, Context, *AST->Consumer)); - AST->TheSema->Initialize(); - AST->Reader->InitializeSema(*AST->TheSema); + if (ToLoad >= LoadEverything) { + AST->TheSema.reset(new Sema(PP, *AST->Ctx, *AST->Consumer)); + AST->TheSema->Initialize(); + AST->Reader->InitializeSema(*AST->TheSema); + } // Tell the diagnostic client that we have started a source file. - AST->getDiagnostics().getClient()->BeginSourceFile(Context.getLangOpts(),&PP); + AST->getDiagnostics().getClient()->BeginSourceFile(PP.getLangOpts(), &PP); return AST; } namespace { +/// \brief Add the given macro to the hash of all top-level entities. +void AddDefinedMacroToHash(const Token &MacroNameTok, unsigned &Hash) { + Hash = llvm::HashString(MacroNameTok.getIdentifierInfo()->getName(), Hash); +} + /// \brief Preprocessor callback class that updates a hash value with the names /// of all macros that have been defined by the translation unit. class MacroDefinitionTrackerPPCallbacks : public PPCallbacks { @@ -771,7 +803,7 @@ public: void MacroDefined(const Token &MacroNameTok, const MacroDirective *MD) override { - Hash = llvm::HashString(MacroNameTok.getIdentifierInfo()->getName(), Hash); + AddDefinedMacroToHash(MacroNameTok, Hash); } }; @@ -897,45 +929,27 @@ public: } }; -class PrecompilePreambleAction : public ASTFrontendAction { - ASTUnit &Unit; - bool HasEmittedPreamblePCH; - +class ASTUnitPreambleCallbacks : public PreambleCallbacks { public: - explicit PrecompilePreambleAction(ASTUnit &Unit) - : Unit(Unit), HasEmittedPreamblePCH(false) {} + unsigned getHash() const { return Hash; } - std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI, - StringRef InFile) override; - bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; } - void setHasEmittedPreamblePCH() { HasEmittedPreamblePCH = true; } - bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); } - - bool hasCodeCompletionSupport() const override { return false; } - bool hasASTFileSupport() const override { return false; } - TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; } -}; + std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); } -class PrecompilePreambleConsumer : public PCHGenerator { - ASTUnit &Unit; - unsigned &Hash; - std::vector<Decl *> TopLevelDecls; - PrecompilePreambleAction *Action; - std::unique_ptr<raw_ostream> Out; + std::vector<serialization::DeclID> takeTopLevelDeclIDs() { + return std::move(TopLevelDeclIDs); + } -public: - PrecompilePreambleConsumer(ASTUnit &Unit, PrecompilePreambleAction *Action, - const Preprocessor &PP, StringRef isysroot, - std::unique_ptr<raw_ostream> Out) - : PCHGenerator(PP, "", isysroot, std::make_shared<PCHBuffer>(), - ArrayRef<std::shared_ptr<ModuleFileExtension>>(), - /*AllowASTWithErrors=*/true), - Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action), - Out(std::move(Out)) { - Hash = 0; + void AfterPCHEmitted(ASTWriter &Writer) override { + TopLevelDeclIDs.reserve(TopLevelDecls.size()); + for (Decl *D : TopLevelDecls) { + // Invalid top-level decls may not have been serialized. + if (D->isInvalidDecl()) + continue; + TopLevelDeclIDs.push_back(Writer.getDeclID(D)); + } } - bool HandleTopLevelDecl(DeclGroupRef DG) override { + void HandleTopLevelDecl(DeclGroupRef DG) override { for (Decl *D : DG) { // FIXME: Currently ObjC method declarations are incorrectly being // reported as top-level declarations, even though their DeclContext @@ -946,59 +960,22 @@ public: AddTopLevelDeclarationToHash(D, Hash); TopLevelDecls.push_back(D); } - return true; } - void HandleTranslationUnit(ASTContext &Ctx) override { - PCHGenerator::HandleTranslationUnit(Ctx); - if (hasEmittedPCH()) { - // Write the generated bitstream to "Out". - *Out << getPCH(); - // Make sure it hits disk now. - Out->flush(); - // Free the buffer. - llvm::SmallVector<char, 0> Empty; - getPCH() = std::move(Empty); - - // Translate the top-level declarations we captured during - // parsing into declaration IDs in the precompiled - // preamble. This will allow us to deserialize those top-level - // declarations when requested. - for (Decl *D : TopLevelDecls) { - // Invalid top-level decls may not have been serialized. - if (D->isInvalidDecl()) - continue; - Unit.addTopLevelDeclFromPreamble(getWriter().getDeclID(D)); - } - - Action->setHasEmittedPreamblePCH(); - } + void HandleMacroDefined(const Token &MacroNameTok, + const MacroDirective *MD) override { + AddDefinedMacroToHash(MacroNameTok, Hash); } + +private: + unsigned Hash = 0; + std::vector<Decl *> TopLevelDecls; + std::vector<serialization::DeclID> TopLevelDeclIDs; + llvm::SmallVector<ASTUnit::StandaloneDiagnostic, 4> PreambleDiags; }; } // anonymous namespace -std::unique_ptr<ASTConsumer> -PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI, - StringRef InFile) { - std::string Sysroot; - std::string OutputFile; - std::unique_ptr<raw_ostream> OS = - GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot, - OutputFile); - if (!OS) - return nullptr; - - if (!CI.getFrontendOpts().RelocatablePCH) - Sysroot.clear(); - - CI.getPreprocessor().addPPCallbacks( - llvm::make_unique<MacroDefinitionTrackerPPCallbacks>( - Unit.getCurrentTopLevelHashValue())); - return llvm::make_unique<PrecompilePreambleConsumer>( - Unit, this, CI.getPreprocessor(), Sysroot, std::move(OS)); -} - static bool isNonDriverDiag(const StoredDiagnostic &StoredDiag) { return StoredDiag.getLocation().isValid(); } @@ -1034,15 +1011,20 @@ static void checkAndSanitizeDiags(SmallVectorImpl<StoredDiagnostic> & /// \returns True if a failure occurred that causes the ASTUnit not to /// contain any translation-unit information, false otherwise. bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, - std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer) { - SavedMainFileBuffer.reset(); - + std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer, + IntrusiveRefCntPtr<vfs::FileSystem> VFS) { if (!Invocation) return true; // Create the compiler instance to use for building the AST. std::unique_ptr<CompilerInstance> Clang( new CompilerInstance(std::move(PCHContainerOps))); + if (FileMgr && VFS) { + assert(VFS == FileMgr->getVirtualFileSystem() && + "VFS passed to Parse and VFS in FileMgr are different"); + } else if (VFS) { + Clang->setVirtualFileSystem(VFS); + } // Recover resources if we crash before exiting this method. llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> @@ -1069,9 +1051,11 @@ bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, assert(Clang->getFrontendOpts().Inputs.size() == 1 && "Invocation must have exactly one source file!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST && + assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() == + InputKind::Source && "FIXME: AST inputs not yet supported here!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR && + assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() != + InputKind::LLVM_IR && "IR inputs not support here!"); // Configure the various subsystems. @@ -1081,18 +1065,11 @@ bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, Clang->createFileManager(); FileMgr = &Clang->getFileManager(); } - SourceMgr = new SourceManager(getDiagnostics(), *FileMgr, - UserFilesAreVolatile); - TheSema.reset(); - Ctx = nullptr; - PP = nullptr; - Reader = nullptr; - // Clear out old caches and data. - TopLevelDecls.clear(); - clearFileLevelDecls(); - CleanTemporaryFiles(); + ResetForParse(); + SourceMgr = new SourceManager(getDiagnostics(), *FileMgr, + UserFilesAreVolatile); if (!OverrideMainBuffer) { checkAndRemoveNonDriverDiags(StoredDiagnostics); TopLevelDeclsInPreamble.clear(); @@ -1106,15 +1083,9 @@ bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, // If the main file has been overridden due to the use of a preamble, // make that override happen and introduce the preamble. - PreprocessorOptions &PreprocessorOpts = Clang->getPreprocessorOpts(); if (OverrideMainBuffer) { - PreprocessorOpts.addRemappedFile(OriginalSourceFile, - OverrideMainBuffer.get()); - PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size(); - PreprocessorOpts.PrecompiledPreambleBytes.second - = PreambleEndsAtStartOfLine; - PreprocessorOpts.ImplicitPCHInclude = getPreambleFile(this); - PreprocessorOpts.DisablePCHValidation = true; + assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null"); + Preamble->AddImplicitPreamble(Clang->getInvocation(), OverrideMainBuffer.get()); // The stored diagnostic has the old source manager in it; update // the locations to refer into the new source manager. Since we've @@ -1140,6 +1111,8 @@ bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, if (SavedMainFileBuffer) TranslateStoredDiagnostics(getFileManager(), getSourceManager(), PreambleDiagnostics, StoredDiagnostics); + else + PreambleSrcLocCache.clear(); if (!Act->Execute()) goto error; @@ -1165,111 +1138,6 @@ error: return true; } -/// \brief Simple function to retrieve a path for a preamble precompiled header. -static std::string GetPreamblePCHPath() { - // FIXME: This is a hack so that we can override the preamble file during - // crash-recovery testing, which is the only case where the preamble files - // are not necessarily cleaned up. - const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE"); - if (TmpFile) - return TmpFile; - - SmallString<128> Path; - llvm::sys::fs::createTemporaryFile("preamble", "pch", Path); - - return Path.str(); -} - -/// \brief Compute the preamble for the main file, providing the source buffer -/// that corresponds to the main file along with a pair (bytes, start-of-line) -/// that describes the preamble. -ASTUnit::ComputedPreamble -ASTUnit::ComputePreamble(CompilerInvocation &Invocation, unsigned MaxLines) { - FrontendOptions &FrontendOpts = Invocation.getFrontendOpts(); - PreprocessorOptions &PreprocessorOpts = Invocation.getPreprocessorOpts(); - - // Try to determine if the main file has been remapped, either from the - // command line (to another file) or directly through the compiler invocation - // (to a memory buffer). - llvm::MemoryBuffer *Buffer = nullptr; - std::unique_ptr<llvm::MemoryBuffer> BufferOwner; - std::string MainFilePath(FrontendOpts.Inputs[0].getFile()); - llvm::sys::fs::UniqueID MainFileID; - if (!llvm::sys::fs::getUniqueID(MainFilePath, MainFileID)) { - // Check whether there is a file-file remapping of the main file - for (const auto &RF : PreprocessorOpts.RemappedFiles) { - std::string MPath(RF.first); - llvm::sys::fs::UniqueID MID; - if (!llvm::sys::fs::getUniqueID(MPath, MID)) { - if (MainFileID == MID) { - // We found a remapping. Try to load the resulting, remapped source. - BufferOwner = getBufferForFile(RF.second); - if (!BufferOwner) - return ComputedPreamble(nullptr, nullptr, 0, true); - } - } - } - - // Check whether there is a file-buffer remapping. It supercedes the - // file-file remapping. - for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) { - std::string MPath(RB.first); - llvm::sys::fs::UniqueID MID; - if (!llvm::sys::fs::getUniqueID(MPath, MID)) { - if (MainFileID == MID) { - // We found a remapping. - BufferOwner.reset(); - Buffer = const_cast<llvm::MemoryBuffer *>(RB.second); - } - } - } - } - - // If the main source file was not remapped, load it now. - if (!Buffer && !BufferOwner) { - BufferOwner = getBufferForFile(FrontendOpts.Inputs[0].getFile()); - if (!BufferOwner) - return ComputedPreamble(nullptr, nullptr, 0, true); - } - - if (!Buffer) - Buffer = BufferOwner.get(); - auto Pre = Lexer::ComputePreamble(Buffer->getBuffer(), - *Invocation.getLangOpts(), MaxLines); - return ComputedPreamble(Buffer, std::move(BufferOwner), Pre.first, - Pre.second); -} - -ASTUnit::PreambleFileHash -ASTUnit::PreambleFileHash::createForFile(off_t Size, time_t ModTime) { - PreambleFileHash Result; - Result.Size = Size; - Result.ModTime = ModTime; - memset(Result.MD5, 0, sizeof(Result.MD5)); - return Result; -} - -ASTUnit::PreambleFileHash ASTUnit::PreambleFileHash::createForMemoryBuffer( - const llvm::MemoryBuffer *Buffer) { - PreambleFileHash Result; - Result.Size = Buffer->getBufferSize(); - Result.ModTime = 0; - - llvm::MD5 MD5Ctx; - MD5Ctx.update(Buffer->getBuffer().data()); - MD5Ctx.final(Result.MD5); - - return Result; -} - -namespace clang { -bool operator==(const ASTUnit::PreambleFileHash &LHS, - const ASTUnit::PreambleFileHash &RHS) { - return LHS.Size == RHS.Size && LHS.ModTime == RHS.ModTime && - memcmp(LHS.MD5, RHS.MD5, sizeof(LHS.MD5)) == 0; -} -} // namespace clang - static std::pair<unsigned, unsigned> makeStandaloneRange(CharSourceRange Range, const SourceManager &SM, const LangOptions &LangOpts) { @@ -1338,135 +1206,44 @@ makeStandaloneDiagnostic(const LangOptions &LangOpts, std::unique_ptr<llvm::MemoryBuffer> ASTUnit::getMainBufferWithPrecompiledPreamble( std::shared_ptr<PCHContainerOperations> PCHContainerOps, - const CompilerInvocation &PreambleInvocationIn, bool AllowRebuild, + const CompilerInvocation &PreambleInvocationIn, + IntrusiveRefCntPtr<vfs::FileSystem> VFS, bool AllowRebuild, unsigned MaxLines) { - auto PreambleInvocation = - std::make_shared<CompilerInvocation>(PreambleInvocationIn); - FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts(); - PreprocessorOptions &PreprocessorOpts - = PreambleInvocation->getPreprocessorOpts(); - - ComputedPreamble NewPreamble = ComputePreamble(*PreambleInvocation, MaxLines); - - if (!NewPreamble.Size) { - // We couldn't find a preamble in the main source. Clear out the current - // preamble, if we have one. It's obviously no good any more. - Preamble.clear(); - erasePreambleFile(this); - - // The next time we actually see a preamble, precompile it. - PreambleRebuildCounter = 1; + auto MainFilePath = + PreambleInvocationIn.getFrontendOpts().Inputs[0].getFile(); + std::unique_ptr<llvm::MemoryBuffer> MainFileBuffer = + getBufferForFileHandlingRemapping(PreambleInvocationIn, VFS.get(), + MainFilePath); + if (!MainFileBuffer) return nullptr; - } - - if (!Preamble.empty()) { - // We've previously computed a preamble. Check whether we have the same - // preamble now that we did before, and that there's enough space in - // the main-file buffer within the precompiled preamble to fit the - // new main file. - if (Preamble.size() == NewPreamble.Size && - PreambleEndsAtStartOfLine == NewPreamble.PreambleEndsAtStartOfLine && - memcmp(Preamble.getBufferStart(), NewPreamble.Buffer->getBufferStart(), - NewPreamble.Size) == 0) { - // The preamble has not changed. We may be able to re-use the precompiled - // preamble. - - // Check that none of the files used by the preamble have changed. - bool AnyFileChanged = false; - - // First, make a record of those files that have been overridden via - // remapping or unsaved_files. - std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles; - for (const auto &R : PreprocessorOpts.RemappedFiles) { - if (AnyFileChanged) - break; - - vfs::Status Status; - if (FileMgr->getNoncachedStatValue(R.second, Status)) { - // If we can't stat the file we're remapping to, assume that something - // horrible happened. - AnyFileChanged = true; - break; - } - OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile( - Status.getSize(), - llvm::sys::toTimeT(Status.getLastModificationTime())); - } + PreambleBounds Bounds = + ComputePreambleBounds(*PreambleInvocationIn.getLangOpts(), + MainFileBuffer.get(), MaxLines); + if (!Bounds.Size) + return nullptr; - for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) { - if (AnyFileChanged) - break; + if (Preamble) { + if (Preamble->CanReuse(PreambleInvocationIn, MainFileBuffer.get(), Bounds, + VFS.get())) { + // Okay! We can re-use the precompiled preamble. - vfs::Status Status; - if (FileMgr->getNoncachedStatValue(RB.first, Status)) { - AnyFileChanged = true; - break; - } + // Set the state of the diagnostic object to mimic its state + // after parsing the preamble. + getDiagnostics().Reset(); + ProcessWarningOptions(getDiagnostics(), + PreambleInvocationIn.getDiagnosticOpts()); + getDiagnostics().setNumWarnings(NumWarningsInPreamble); - OverriddenFiles[Status.getUniqueID()] = - PreambleFileHash::createForMemoryBuffer(RB.second); - } - - // Check whether anything has changed. - for (llvm::StringMap<PreambleFileHash>::iterator - F = FilesInPreamble.begin(), FEnd = FilesInPreamble.end(); - !AnyFileChanged && F != FEnd; - ++F) { - vfs::Status Status; - if (FileMgr->getNoncachedStatValue(F->first(), Status)) { - // If we can't stat the file, assume that something horrible happened. - AnyFileChanged = true; - break; - } - - std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden - = OverriddenFiles.find(Status.getUniqueID()); - if (Overridden != OverriddenFiles.end()) { - // This file was remapped; check whether the newly-mapped file - // matches up with the previous mapping. - if (Overridden->second != F->second) - AnyFileChanged = true; - continue; - } - - // The file was not remapped; check whether it has changed on disk. - if (Status.getSize() != uint64_t(F->second.Size) || - llvm::sys::toTimeT(Status.getLastModificationTime()) != - F->second.ModTime) - AnyFileChanged = true; - } - - if (!AnyFileChanged) { - // Okay! We can re-use the precompiled preamble. - - // Set the state of the diagnostic object to mimic its state - // after parsing the preamble. - getDiagnostics().Reset(); - ProcessWarningOptions(getDiagnostics(), - PreambleInvocation->getDiagnosticOpts()); - getDiagnostics().setNumWarnings(NumWarningsInPreamble); - - return llvm::MemoryBuffer::getMemBufferCopy( - NewPreamble.Buffer->getBuffer(), FrontendOpts.Inputs[0].getFile()); - } + PreambleRebuildCounter = 1; + return MainFileBuffer; + } else { + Preamble.reset(); + PreambleDiagnostics.clear(); + TopLevelDeclsInPreamble.clear(); + PreambleRebuildCounter = 1; } - - // If we aren't allowed to rebuild the precompiled preamble, just - // return now. - if (!AllowRebuild) - return nullptr; - - // We can't reuse the previously-computed preamble. Build a new one. - Preamble.clear(); - PreambleDiagnostics.clear(); - erasePreambleFile(this); - PreambleRebuildCounter = 1; - } else if (!AllowRebuild) { - // We aren't allowed to rebuild the precompiled preamble; just - // return now. - return nullptr; } // If the preamble rebuild counter > 1, it's because we previously @@ -1477,162 +1254,61 @@ ASTUnit::getMainBufferWithPrecompiledPreamble( return nullptr; } - // Create a temporary file for the precompiled preamble. In rare - // circumstances, this can fail. - std::string PreamblePCHPath = GetPreamblePCHPath(); - if (PreamblePCHPath.empty()) { - // Try again next time. - PreambleRebuildCounter = 1; - return nullptr; - } - - // We did not previously compute a preamble, or it can't be reused anyway. - SimpleTimer PreambleTimer(WantTiming); - PreambleTimer.setOutput("Precompiling preamble"); - - // Save the preamble text for later; we'll need to compare against it for - // subsequent reparses. - StringRef MainFilename = FrontendOpts.Inputs[0].getFile(); - Preamble.assign(FileMgr->getFile(MainFilename), - NewPreamble.Buffer->getBufferStart(), - NewPreamble.Buffer->getBufferStart() + NewPreamble.Size); - PreambleEndsAtStartOfLine = NewPreamble.PreambleEndsAtStartOfLine; - - PreambleBuffer = llvm::MemoryBuffer::getMemBufferCopy( - NewPreamble.Buffer->getBuffer().slice(0, Preamble.size()), MainFilename); - - // Remap the main source file to the preamble buffer. - StringRef MainFilePath = FrontendOpts.Inputs[0].getFile(); - PreprocessorOpts.addRemappedFile(MainFilePath, PreambleBuffer.get()); - - // Tell the compiler invocation to generate a temporary precompiled header. - FrontendOpts.ProgramAction = frontend::GeneratePCH; - // FIXME: Generate the precompiled header into memory? - FrontendOpts.OutputFile = PreamblePCHPath; - PreprocessorOpts.PrecompiledPreambleBytes.first = 0; - PreprocessorOpts.PrecompiledPreambleBytes.second = false; - - // Create the compiler instance to use for building the precompiled preamble. - std::unique_ptr<CompilerInstance> Clang( - new CompilerInstance(std::move(PCHContainerOps))); - - // Recover resources if we crash before exiting this method. - llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> - CICleanup(Clang.get()); - - Clang->setInvocation(std::move(PreambleInvocation)); - OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile(); - - // Set up diagnostics, capturing all of the diagnostics produced. - Clang->setDiagnostics(&getDiagnostics()); - - // Create the target instance. - Clang->setTarget(TargetInfo::CreateTargetInfo( - Clang->getDiagnostics(), Clang->getInvocation().TargetOpts)); - if (!Clang->hasTarget()) { - llvm::sys::fs::remove(FrontendOpts.OutputFile); - Preamble.clear(); - PreambleRebuildCounter = DefaultPreambleRebuildInterval; - PreprocessorOpts.RemappedFileBuffers.pop_back(); - return nullptr; - } - - // Inform the target of the language options. - // - // FIXME: We shouldn't need to do this, the target should be immutable once - // created. This complexity should be lifted elsewhere. - Clang->getTarget().adjust(Clang->getLangOpts()); - - assert(Clang->getFrontendOpts().Inputs.size() == 1 && - "Invocation must have exactly one source file!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST && - "FIXME: AST inputs not yet supported here!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR && - "IR inputs not support here!"); - - // Clear out old caches and data. - getDiagnostics().Reset(); - ProcessWarningOptions(getDiagnostics(), Clang->getDiagnosticOpts()); - checkAndRemoveNonDriverDiags(StoredDiagnostics); - TopLevelDecls.clear(); - TopLevelDeclsInPreamble.clear(); - PreambleDiagnostics.clear(); - - IntrusiveRefCntPtr<vfs::FileSystem> VFS = - createVFSFromCompilerInvocation(Clang->getInvocation(), getDiagnostics()); - if (!VFS) + assert(!Preamble && "No Preamble should be stored at that point"); + // If we aren't allowed to rebuild the precompiled preamble, just + // return now. + if (!AllowRebuild) return nullptr; - // Create a file manager object to provide access to and cache the filesystem. - Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS)); - - // Create the source manager. - Clang->setSourceManager(new SourceManager(getDiagnostics(), - Clang->getFileManager())); - - auto PreambleDepCollector = std::make_shared<DependencyCollector>(); - Clang->addDependencyCollector(PreambleDepCollector); - - std::unique_ptr<PrecompilePreambleAction> Act; - Act.reset(new PrecompilePreambleAction(*this)); - if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0])) { - llvm::sys::fs::remove(FrontendOpts.OutputFile); - Preamble.clear(); - PreambleRebuildCounter = DefaultPreambleRebuildInterval; - PreprocessorOpts.RemappedFileBuffers.pop_back(); - return nullptr; + SmallVector<StandaloneDiagnostic, 4> NewPreambleDiagsStandalone; + SmallVector<StoredDiagnostic, 4> NewPreambleDiags; + ASTUnitPreambleCallbacks Callbacks; + { + llvm::Optional<CaptureDroppedDiagnostics> Capture; + if (CaptureDiagnostics) + Capture.emplace(/*RequestCapture=*/true, *Diagnostics, &NewPreambleDiags, + &NewPreambleDiagsStandalone); + + // We did not previously compute a preamble, or it can't be reused anyway. + SimpleTimer PreambleTimer(WantTiming); + PreambleTimer.setOutput("Precompiling preamble"); + + llvm::ErrorOr<PrecompiledPreamble> NewPreamble = PrecompiledPreamble::Build( + PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS, + PCHContainerOps, Callbacks); + if (NewPreamble) { + Preamble = std::move(*NewPreamble); + PreambleRebuildCounter = 1; + } else { + switch (static_cast<BuildPreambleError>(NewPreamble.getError().value())) { + case BuildPreambleError::CouldntCreateTempFile: + case BuildPreambleError::PreambleIsEmpty: + // Try again next time. + PreambleRebuildCounter = 1; + return nullptr; + case BuildPreambleError::CouldntCreateTargetInfo: + case BuildPreambleError::BeginSourceFileFailed: + case BuildPreambleError::CouldntEmitPCH: + case BuildPreambleError::CouldntCreateVFSOverlay: + // These erros are more likely to repeat, retry after some period. + PreambleRebuildCounter = DefaultPreambleRebuildInterval; + return nullptr; + } + llvm_unreachable("unexpected BuildPreambleError"); + } } - - Act->Execute(); - - // Transfer any diagnostics generated when parsing the preamble into the set - // of preamble diagnostics. - for (stored_diag_iterator I = stored_diag_afterDriver_begin(), - E = stored_diag_end(); - I != E; ++I) - PreambleDiagnostics.push_back( - makeStandaloneDiagnostic(Clang->getLangOpts(), *I)); - Act->EndSourceFile(); + assert(Preamble && "Preamble wasn't built"); - checkAndRemoveNonDriverDiags(StoredDiagnostics); + TopLevelDecls.clear(); + TopLevelDeclsInPreamble = Callbacks.takeTopLevelDeclIDs(); + PreambleTopLevelHashValue = Callbacks.getHash(); - if (!Act->hasEmittedPreamblePCH()) { - // The preamble PCH failed (e.g. there was a module loading fatal error), - // so no precompiled header was generated. Forget that we even tried. - // FIXME: Should we leave a note for ourselves to try again? - llvm::sys::fs::remove(FrontendOpts.OutputFile); - Preamble.clear(); - TopLevelDeclsInPreamble.clear(); - PreambleRebuildCounter = DefaultPreambleRebuildInterval; - PreprocessorOpts.RemappedFileBuffers.pop_back(); - return nullptr; - } - - // Keep track of the preamble we precompiled. - setPreambleFile(this, FrontendOpts.OutputFile); NumWarningsInPreamble = getDiagnostics().getNumWarnings(); - - // Keep track of all of the files that the source manager knows about, - // so we can verify whether they have changed or not. - FilesInPreamble.clear(); - SourceManager &SourceMgr = Clang->getSourceManager(); - for (auto &Filename : PreambleDepCollector->getDependencies()) { - const FileEntry *File = Clang->getFileManager().getFile(Filename); - if (!File || File == SourceMgr.getFileEntryForID(SourceMgr.getMainFileID())) - continue; - if (time_t ModTime = File->getModificationTime()) { - FilesInPreamble[File->getName()] = PreambleFileHash::createForFile( - File->getSize(), ModTime); - } else { - llvm::MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File); - FilesInPreamble[File->getName()] = - PreambleFileHash::createForMemoryBuffer(Buffer); - } - } - PreambleRebuildCounter = 1; - PreprocessorOpts.RemappedFileBuffers.pop_back(); + checkAndRemoveNonDriverDiags(NewPreambleDiags); + StoredDiagnostics = std::move(NewPreambleDiags); + PreambleDiagnostics = std::move(NewPreambleDiagsStandalone); // If the hash of top-level entities differs from the hash of the top-level // entities the last time we rebuilt the preamble, clear out the completion @@ -1642,11 +1318,12 @@ ASTUnit::getMainBufferWithPrecompiledPreamble( PreambleTopLevelHashValue = CurrentTopLevelHashValue; } - return llvm::MemoryBuffer::getMemBufferCopy(NewPreamble.Buffer->getBuffer(), - MainFilename); + return MainFileBuffer; } void ASTUnit::RealizeTopLevelDeclsFromPreamble() { + assert(Preamble && "Should only be called when preamble was built"); + std::vector<Decl *> Resolved; Resolved.reserve(TopLevelDeclsInPreamble.size()); ExternalASTSource &Source = *getASTContext().getExternalSource(); @@ -1723,6 +1400,7 @@ ASTUnit::create(std::shared_ptr<CompilerInvocation> CI, AST->UserFilesAreVolatile = UserFilesAreVolatile; AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr, UserFilesAreVolatile); + AST->PCMCache = new MemoryBufferCache; return AST; } @@ -1802,10 +1480,12 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction( assert(Clang->getFrontendOpts().Inputs.size() == 1 && "Invocation must have exactly one source file!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST && + assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() == + InputKind::Source && "FIXME: AST inputs not yet supported here!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR && - "IR inputs not supported here!"); + assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() != + InputKind::LLVM_IR && + "IR inputs not support here!"); // Configure the various subsystems. AST->TheSema.reset(); @@ -1872,30 +1552,36 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction( bool ASTUnit::LoadFromCompilerInvocation( std::shared_ptr<PCHContainerOperations> PCHContainerOps, - unsigned PrecompilePreambleAfterNParses) { + unsigned PrecompilePreambleAfterNParses, + IntrusiveRefCntPtr<vfs::FileSystem> VFS) { if (!Invocation) return true; - + + assert(VFS && "VFS is null"); + // We'll manage file buffers ourselves. Invocation->getPreprocessorOpts().RetainRemappedFileBuffers = true; Invocation->getFrontendOpts().DisableFree = false; + getDiagnostics().Reset(); ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts()); std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer; if (PrecompilePreambleAfterNParses > 0) { PreambleRebuildCounter = PrecompilePreambleAfterNParses; OverrideMainBuffer = - getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation); + getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS); + getDiagnostics().Reset(); + ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts()); } - + SimpleTimer ParsingTimer(WantTiming); ParsingTimer.setOutput("Parsing " + getMainFileName()); - + // Recover resources if we crash before exiting this method. llvm::CrashRecoveryContextCleanupRegistrar<llvm::MemoryBuffer> MemBufferCleanup(OverrideMainBuffer.get()); - return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer)); + return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS); } std::unique_ptr<ASTUnit> ASTUnit::LoadFromCompilerInvocation( @@ -1929,7 +1615,8 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCompilerInvocation( DiagCleanup(Diags.get()); if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps), - PrecompilePreambleAfterNParses)) + PrecompilePreambleAfterNParses, + AST->FileMgr->getVirtualFileSystem())) return nullptr; return AST; } @@ -1943,8 +1630,9 @@ ASTUnit *ASTUnit::LoadFromCommandLine( unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind, bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion, bool AllowPCHWithCompilerErrors, bool SkipFunctionBodies, - bool UserFilesAreVolatile, bool ForSerialization, - llvm::Optional<StringRef> ModuleFormat, std::unique_ptr<ASTUnit> *ErrAST) { + bool SingleFileParse, bool UserFilesAreVolatile, bool ForSerialization, + llvm::Optional<StringRef> ModuleFormat, std::unique_ptr<ASTUnit> *ErrAST, + IntrusiveRefCntPtr<vfs::FileSystem> VFS) { assert(Diags.get() && "no DiagnosticsEngine was provided"); SmallVector<StoredDiagnostic, 4> StoredDiagnostics; @@ -1953,11 +1641,11 @@ ASTUnit *ASTUnit::LoadFromCommandLine( { - CaptureDroppedDiagnostics Capture(CaptureDiagnostics, *Diags, - StoredDiagnostics); + CaptureDroppedDiagnostics Capture(CaptureDiagnostics, *Diags, + &StoredDiagnostics, nullptr); CI = clang::createInvocationFromCommandLine( - llvm::makeArrayRef(ArgBegin, ArgEnd), Diags); + llvm::makeArrayRef(ArgBegin, ArgEnd), Diags, VFS); if (!CI) return nullptr; } @@ -1970,6 +1658,8 @@ ASTUnit *ASTUnit::LoadFromCommandLine( PreprocessorOptions &PPOpts = CI->getPreprocessorOpts(); PPOpts.RemappedFilesKeepOriginalName = RemappedFilesKeepOriginalName; PPOpts.AllowPCHWithCompilerErrors = AllowPCHWithCompilerErrors; + PPOpts.GeneratePreamble = PrecompilePreambleAfterNParses != 0; + PPOpts.SingleFileParseMode = SingleFileParse; // Override the resources path. CI->getHeaderSearchOpts().ResourceDir = ResourceFilesPath; @@ -1985,11 +1675,13 @@ ASTUnit *ASTUnit::LoadFromCommandLine( ConfigureDiags(Diags, *AST, CaptureDiagnostics); AST->Diagnostics = Diags; AST->FileSystemOpts = CI->getFileSystemOpts(); - IntrusiveRefCntPtr<vfs::FileSystem> VFS = - createVFSFromCompilerInvocation(*CI, *Diags); + if (!VFS) + VFS = vfs::getRealFileSystem(); + VFS = createVFSFromCompilerInvocation(*CI, *Diags, VFS); if (!VFS) return nullptr; AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS); + AST->PCMCache = new MemoryBufferCache; AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->TUKind = TUKind; @@ -2001,7 +1693,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine( AST->StoredDiagnostics.swap(StoredDiagnostics); AST->Invocation = CI; if (ForSerialization) - AST->WriterData.reset(new ASTWriterData()); + AST->WriterData.reset(new ASTWriterData(*AST->PCMCache)); // Zero out now to ease cleanup during crash recovery. CI = nullptr; Diags = nullptr; @@ -2011,7 +1703,8 @@ ASTUnit *ASTUnit::LoadFromCommandLine( ASTUnitCleanup(AST.get()); if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps), - PrecompilePreambleAfterNParses)) { + PrecompilePreambleAfterNParses, + VFS)) { // Some error occurred, if caller wants to examine diagnostics, pass it the // ASTUnit. if (ErrAST) { @@ -2025,10 +1718,16 @@ ASTUnit *ASTUnit::LoadFromCommandLine( } bool ASTUnit::Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, - ArrayRef<RemappedFile> RemappedFiles) { + ArrayRef<RemappedFile> RemappedFiles, + IntrusiveRefCntPtr<vfs::FileSystem> VFS) { if (!Invocation) return true; + if (!VFS) { + assert(FileMgr && "FileMgr is null on Reparse call"); + VFS = FileMgr->getVirtualFileSystem(); + } + clearFileLevelDecls(); SimpleTimer ParsingTimer(WantTiming); @@ -2048,9 +1747,10 @@ bool ASTUnit::Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, // If we have a preamble file lying around, or if we might try to // build a precompiled preamble, do so now. std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer; - if (!getPreambleFile(this).empty() || PreambleRebuildCounter > 0) + if (Preamble || PreambleRebuildCounter > 0) OverrideMainBuffer = - getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation); + getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS); + // Clear out the diagnostics state. FileMgr.reset(); @@ -2061,7 +1761,7 @@ bool ASTUnit::Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, // Parse the sources bool Result = - Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer)); + Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS); // If we're caching global code-completion results, and the top-level // declarations have changed, clear out the code-completion cache. @@ -2076,6 +1776,19 @@ bool ASTUnit::Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps, return Result; } +void ASTUnit::ResetForParse() { + SavedMainFileBuffer.reset(); + + SourceMgr.reset(); + TheSema.reset(); + Ctx.reset(); + PP.reset(); + Reader.reset(); + + TopLevelDecls.clear(); + clearFileLevelDecls(); +} + //----------------------------------------------------------------------------// // Code completion //----------------------------------------------------------------------------// @@ -2368,7 +2081,7 @@ void ASTUnit::CodeComplete( Clang->setDiagnostics(&Diag); CaptureDroppedDiagnostics Capture(true, Clang->getDiagnostics(), - StoredDiagnostics); + &StoredDiagnostics, nullptr); ProcessWarningOptions(Diag, Inv.getDiagnosticOpts()); // Create the target instance. @@ -2387,11 +2100,12 @@ void ASTUnit::CodeComplete( assert(Clang->getFrontendOpts().Inputs.size() == 1 && "Invocation must have exactly one source file!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST && + assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() == + InputKind::Source && "FIXME: AST inputs not yet supported here!"); - assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR && + assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() != + InputKind::LLVM_IR && "IR inputs not support here!"); - // Use the source and file managers that we were given. Clang->setFileManager(&FileMgr); @@ -2416,17 +2130,21 @@ void ASTUnit::CodeComplete( // point is within the main file, after the end of the precompiled // preamble. std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer; - if (!getPreambleFile(this).empty()) { + if (Preamble) { std::string CompleteFilePath(File); - llvm::sys::fs::UniqueID CompleteFileID; - if (!llvm::sys::fs::getUniqueID(CompleteFilePath, CompleteFileID)) { + auto VFS = FileMgr.getVirtualFileSystem(); + auto CompleteFileStatus = VFS->status(CompleteFilePath); + if (CompleteFileStatus) { + llvm::sys::fs::UniqueID CompleteFileID = CompleteFileStatus->getUniqueID(); + std::string MainPath(OriginalSourceFile); - llvm::sys::fs::UniqueID MainID; - if (!llvm::sys::fs::getUniqueID(MainPath, MainID)) { + auto MainStatus = VFS->status(MainPath); + if (MainStatus) { + llvm::sys::fs::UniqueID MainID = MainStatus->getUniqueID(); if (CompleteFileID == MainID && Line > 1) OverrideMainBuffer = getMainBufferWithPrecompiledPreamble( - PCHContainerOps, Inv, false, Line - 1); + PCHContainerOps, Inv, VFS, false, Line - 1); } } } @@ -2434,14 +2152,8 @@ void ASTUnit::CodeComplete( // If the main file has been overridden due to the use of a preamble, // make that override happen and introduce the preamble. if (OverrideMainBuffer) { - PreprocessorOpts.addRemappedFile(OriginalSourceFile, - OverrideMainBuffer.get()); - PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size(); - PreprocessorOpts.PrecompiledPreambleBytes.second - = PreambleEndsAtStartOfLine; - PreprocessorOpts.ImplicitPCHInclude = getPreambleFile(this); - PreprocessorOpts.DisablePCHValidation = true; - + assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null"); + Preamble->AddImplicitPreamble(Clang->getInvocation(), OverrideMainBuffer.get()); OwnedBuffers.push_back(OverrideMainBuffer.release()); } else { PreprocessorOpts.PrecompiledPreambleBytes.first = 0; @@ -2516,7 +2228,8 @@ bool ASTUnit::serialize(raw_ostream &OS) { SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - ASTWriter Writer(Stream, { }); + MemoryBufferCache PCMCache; + ASTWriter Writer(Stream, Buffer, PCMCache, {}); return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS); } @@ -2531,9 +2244,9 @@ void ASTUnit::TranslateStoredDiagnostics( // remap all the locations to the new view. This includes the diag location, // any associated source ranges, and the source ranges of associated fix-its. // FIXME: There should be a cleaner way to do this. - SmallVector<StoredDiagnostic, 4> Result; Result.reserve(Diags.size()); + for (const StandaloneDiagnostic &SD : Diags) { // Rebuild the StoredDiagnostic. if (SD.Filename.empty()) @@ -2541,8 +2254,16 @@ void ASTUnit::TranslateStoredDiagnostics( const FileEntry *FE = FileMgr.getFile(SD.Filename); if (!FE) continue; - FileID FID = SrcMgr.translateFile(FE); - SourceLocation FileLoc = SrcMgr.getLocForStartOfFile(FID); + SourceLocation FileLoc; + auto ItFileID = PreambleSrcLocCache.find(SD.Filename); + if (ItFileID == PreambleSrcLocCache.end()) { + FileID FID = SrcMgr.translateFile(FE); + FileLoc = SrcMgr.getLocForStartOfFile(FID); + PreambleSrcLocCache[SD.Filename] = FileLoc; + } else { + FileLoc = ItFileID->getValue(); + } + if (FileLoc.isInvalid()) continue; SourceLocation L = FileLoc.getLocWithOffset(SD.LocOffset); @@ -2679,11 +2400,11 @@ SourceLocation ASTUnit::mapLocationFromPreamble(SourceLocation Loc) { if (SourceMgr) PreambleID = SourceMgr->getPreambleFileID(); - if (Loc.isInvalid() || Preamble.empty() || PreambleID.isInvalid()) + if (Loc.isInvalid() || !Preamble || PreambleID.isInvalid()) return Loc; unsigned Offs; - if (SourceMgr->isInFileID(Loc, PreambleID, &Offs) && Offs < Preamble.size()) { + if (SourceMgr->isInFileID(Loc, PreambleID, &Offs) && Offs < Preamble->getBounds().Size) { SourceLocation FileLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID()); return FileLoc.getLocWithOffset(Offs); @@ -2700,12 +2421,12 @@ SourceLocation ASTUnit::mapLocationToPreamble(SourceLocation Loc) { if (SourceMgr) PreambleID = SourceMgr->getPreambleFileID(); - if (Loc.isInvalid() || Preamble.empty() || PreambleID.isInvalid()) + if (Loc.isInvalid() || !Preamble || PreambleID.isInvalid()) return Loc; unsigned Offs; if (SourceMgr->isInFileID(Loc, SourceMgr->getMainFileID(), &Offs) && - Offs < Preamble.size()) { + Offs < Preamble->getBounds().Size) { SourceLocation FileLoc = SourceMgr->getLocForStartOfFile(PreambleID); return FileLoc.getLocWithOffset(Offs); } @@ -2823,18 +2544,32 @@ const FileEntry *ASTUnit::getPCHFile() { } bool ASTUnit::isModuleFile() { - return isMainFileAST() && ASTFileLangOpts.isCompilingModule(); + return isMainFileAST() && getLangOpts().isCompilingModule(); } -void ASTUnit::PreambleData::countLines() const { - NumLines = 0; - if (empty()) - return; +InputKind ASTUnit::getInputKind() const { + auto &LangOpts = getLangOpts(); + + InputKind::Language Lang; + if (LangOpts.OpenCL) + Lang = InputKind::OpenCL; + else if (LangOpts.CUDA) + Lang = InputKind::CUDA; + else if (LangOpts.RenderScript) + Lang = InputKind::RenderScript; + else if (LangOpts.CPlusPlus) + Lang = LangOpts.ObjC1 ? InputKind::ObjCXX : InputKind::CXX; + else + Lang = LangOpts.ObjC1 ? InputKind::ObjC : InputKind::C; + + InputKind::Format Fmt = InputKind::Source; + if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap) + Fmt = InputKind::ModuleMap; - NumLines = std::count(Buffer.begin(), Buffer.end(), '\n'); + // We don't know if input was preprocessed. Assume not. + bool PP = false; - if (Buffer.back() != '\n') - ++NumLines; + return InputKind(Lang, Fmt, PP); } #ifndef NDEBUG |
