diff options
Diffstat (limited to 'contrib/llvm/lib/Bitcode')
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitReader.cpp | 59 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 3425 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp | 89 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp | 1850 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h | 85 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/ValueList.cpp | 199 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/ValueList.h | 76 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitWriter.cpp | 3 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 636 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp | 15 |
10 files changed, 3730 insertions, 2707 deletions
diff --git a/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp index 9ac3cb9..f64785b 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp @@ -9,7 +9,7 @@ #include "llvm-c/BitReader.h" #include "llvm-c/Core.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -34,13 +34,6 @@ LLVMBool LLVMParseBitcode2(LLVMMemoryBufferRef MemBuf, return LLVMParseBitcodeInContext2(LLVMGetGlobalContext(), MemBuf, OutModule); } -static void diagnosticHandler(const DiagnosticInfo &DI, void *C) { - auto *Message = reinterpret_cast<std::string *>(C); - raw_string_ostream Stream(*Message); - DiagnosticPrinterRawOStream DP(Stream); - DI.print(DP); -} - LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef, LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutModule, @@ -48,17 +41,12 @@ LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef, MemoryBufferRef Buf = unwrap(MemBuf)->getMemBufferRef(); LLVMContext &Ctx = *unwrap(ContextRef); - LLVMContext::DiagnosticHandlerTy OldDiagnosticHandler = - Ctx.getDiagnosticHandler(); - void *OldDiagnosticContext = Ctx.getDiagnosticContext(); - std::string Message; - Ctx.setDiagnosticHandler(diagnosticHandler, &Message, true); - - ErrorOr<std::unique_ptr<Module>> ModuleOrErr = parseBitcodeFile(Buf, Ctx); - - Ctx.setDiagnosticHandler(OldDiagnosticHandler, OldDiagnosticContext, true); - - if (ModuleOrErr.getError()) { + Expected<std::unique_ptr<Module>> ModuleOrErr = parseBitcodeFile(Buf, Ctx); + if (Error Err = ModuleOrErr.takeError()) { + std::string Message; + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + Message = EIB.message(); + }); if (OutMessage) *OutMessage = strdup(Message.c_str()); *OutModule = wrap((Module *)nullptr); @@ -75,7 +63,8 @@ LLVMBool LLVMParseBitcodeInContext2(LLVMContextRef ContextRef, MemoryBufferRef Buf = unwrap(MemBuf)->getMemBufferRef(); LLVMContext &Ctx = *unwrap(ContextRef); - ErrorOr<std::unique_ptr<Module>> ModuleOrErr = parseBitcodeFile(Buf, Ctx); + ErrorOr<std::unique_ptr<Module>> ModuleOrErr = + expectedToErrorOrAndEmitErrors(Ctx, parseBitcodeFile(Buf, Ctx)); if (ModuleOrErr.getError()) { *OutModule = wrap((Module *)nullptr); return 1; @@ -92,23 +81,21 @@ LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef, LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutM, char **OutMessage) { LLVMContext &Ctx = *unwrap(ContextRef); - LLVMContext::DiagnosticHandlerTy OldDiagnosticHandler = - Ctx.getDiagnosticHandler(); - void *OldDiagnosticContext = Ctx.getDiagnosticContext(); - - std::string Message; - Ctx.setDiagnosticHandler(diagnosticHandler, &Message, true); std::unique_ptr<MemoryBuffer> Owner(unwrap(MemBuf)); - - ErrorOr<std::unique_ptr<Module>> ModuleOrErr = - getLazyBitcodeModule(std::move(Owner), Ctx); - Owner.release(); - Ctx.setDiagnosticHandler(OldDiagnosticHandler, OldDiagnosticContext, true); - - if (ModuleOrErr.getError()) { - *OutM = wrap((Module *)nullptr); + Expected<std::unique_ptr<Module>> ModuleOrErr = + getOwningLazyBitcodeModule(std::move(Owner), Ctx); + // Release the buffer if we didn't take ownership of it since we never owned + // it anyway. + (void)Owner.release(); + + if (Error Err = ModuleOrErr.takeError()) { + std::string Message; + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + Message = EIB.message(); + }); if (OutMessage) *OutMessage = strdup(Message.c_str()); + *OutM = wrap((Module *)nullptr); return 1; } @@ -123,8 +110,8 @@ LLVMBool LLVMGetBitcodeModuleInContext2(LLVMContextRef ContextRef, LLVMContext &Ctx = *unwrap(ContextRef); std::unique_ptr<MemoryBuffer> Owner(unwrap(MemBuf)); - ErrorOr<std::unique_ptr<Module>> ModuleOrErr = - getLazyBitcodeModule(std::move(Owner), Ctx); + ErrorOr<std::unique_ptr<Module>> ModuleOrErr = expectedToErrorOrAndEmitErrors( + Ctx, getOwningLazyBitcodeModule(std::move(Owner), Ctx)); Owner.release(); if (ModuleOrErr.getError()) { diff --git a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 73a30c6..a46e49c 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -7,38 +7,84 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Bitcode/BitcodeReader.h" +#include "MetadataLoader.h" +#include "ValueList.h" + +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" +#include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" #include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CallingConv.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" +#include "llvm/IR/GlobalIndirectSymbol.h" +#include "llvm/IR/GlobalObject.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/GVMaterializer.h" #include "llvm/IR/InlineAsm.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/OperandTraits.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/TrackingMDRef.h" +#include "llvm/IR/Type.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/DataStream.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> #include <deque> +#include <limits> +#include <map> +#include <memory> +#include <string> +#include <system_error> +#include <tuple> #include <utility> +#include <vector> using namespace llvm; @@ -48,159 +94,320 @@ static cl::opt<bool> PrintSummaryGUIDs( "Print the global id for each value when reading the module summary")); namespace { + enum { SWITCH_INST_MAGIC = 0x4B5 // May 2012 => 1205 => Hex }; -class BitcodeReaderValueList { - std::vector<WeakVH> ValuePtrs; - - /// As we resolve forward-referenced constants, we add information about them - /// to this vector. This allows us to resolve them in bulk instead of - /// resolving each reference at a time. See the code in - /// ResolveConstantForwardRefs for more information about this. - /// - /// The key of this vector is the placeholder constant, the value is the slot - /// number that holds the resolved value. - typedef std::vector<std::pair<Constant*, unsigned> > ResolveConstantsTy; - ResolveConstantsTy ResolveConstants; - LLVMContext &Context; -public: - BitcodeReaderValueList(LLVMContext &C) : Context(C) {} - ~BitcodeReaderValueList() { - assert(ResolveConstants.empty() && "Constants not resolved?"); - } +Error error(const Twine &Message) { + return make_error<StringError>( + Message, make_error_code(BitcodeError::CorruptedBitcode)); +} + +/// Helper to read the header common to all bitcode files. +bool hasValidBitcodeHeader(BitstreamCursor &Stream) { + // Sniff for the signature. + if (!Stream.canSkipToPos(4) || + Stream.Read(8) != 'B' || + Stream.Read(8) != 'C' || + Stream.Read(4) != 0x0 || + Stream.Read(4) != 0xC || + Stream.Read(4) != 0xE || + Stream.Read(4) != 0xD) + return false; + return true; +} + +Expected<BitstreamCursor> initStream(MemoryBufferRef Buffer) { + const unsigned char *BufPtr = (const unsigned char *)Buffer.getBufferStart(); + const unsigned char *BufEnd = BufPtr + Buffer.getBufferSize(); + + if (Buffer.getBufferSize() & 3) + return error("Invalid bitcode signature"); + + // If we have a wrapper header, parse it and ignore the non-bc file contents. + // The magic number is 0x0B17C0DE stored in little endian. + if (isBitcodeWrapper(BufPtr, BufEnd)) + if (SkipBitcodeWrapperHeader(BufPtr, BufEnd, true)) + return error("Invalid bitcode wrapper header"); + + BitstreamCursor Stream(ArrayRef<uint8_t>(BufPtr, BufEnd)); + if (!hasValidBitcodeHeader(Stream)) + return error("Invalid bitcode signature"); + + return std::move(Stream); +} - // vector compatibility methods - unsigned size() const { return ValuePtrs.size(); } - void resize(unsigned N) { ValuePtrs.resize(N); } - void push_back(Value *V) { ValuePtrs.emplace_back(V); } +/// Convert a string from a record into an std::string, return true on failure. +template <typename StrTy> +static bool convertToString(ArrayRef<uint64_t> Record, unsigned Idx, + StrTy &Result) { + if (Idx > Record.size()) + return true; - void clear() { - assert(ResolveConstants.empty() && "Constants not resolved?"); - ValuePtrs.clear(); + for (unsigned i = Idx, e = Record.size(); i != e; ++i) + Result += (char)Record[i]; + return false; +} + +// Strip all the TBAA attachment for the module. +void stripTBAA(Module *M) { + for (auto &F : *M) { + if (F.isMaterializable()) + continue; + for (auto &I : instructions(F)) + I.setMetadata(LLVMContext::MD_tbaa, nullptr); } +} + +/// Read the "IDENTIFICATION_BLOCK_ID" block, do some basic enforcement on the +/// "epoch" encoded in the bitcode, and return the producer name if any. +Expected<std::string> readIdentificationBlock(BitstreamCursor &Stream) { + if (Stream.EnterSubBlock(bitc::IDENTIFICATION_BLOCK_ID)) + return error("Invalid record"); - Value *operator[](unsigned i) const { - assert(i < ValuePtrs.size()); - return ValuePtrs[i]; + // Read all the records. + SmallVector<uint64_t, 64> Record; + + std::string ProducerIdentification; + + while (true) { + BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + default: + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return ProducerIdentification; + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read a record. + Record.clear(); + unsigned BitCode = Stream.readRecord(Entry.ID, Record); + switch (BitCode) { + default: // Default behavior: reject + return error("Invalid value"); + case bitc::IDENTIFICATION_CODE_STRING: // IDENTIFICATION: [strchr x N] + convertToString(Record, 0, ProducerIdentification); + break; + case bitc::IDENTIFICATION_CODE_EPOCH: { // EPOCH: [epoch#] + unsigned epoch = (unsigned)Record[0]; + if (epoch != bitc::BITCODE_CURRENT_EPOCH) { + return error( + Twine("Incompatible epoch: Bitcode '") + Twine(epoch) + + "' vs current: '" + Twine(bitc::BITCODE_CURRENT_EPOCH) + "'"); + } + } + } } +} + +Expected<std::string> readIdentificationCode(BitstreamCursor &Stream) { + // We expect a number of well-defined blocks, though we don't necessarily + // need to understand them all. + while (true) { + if (Stream.AtEndOfStream()) + return ""; + + BitstreamEntry Entry = Stream.advance(); + switch (Entry.Kind) { + case BitstreamEntry::EndBlock: + case BitstreamEntry::Error: + return error("Malformed block"); - Value *back() const { return ValuePtrs.back(); } - void pop_back() { ValuePtrs.pop_back(); } - bool empty() const { return ValuePtrs.empty(); } - void shrinkTo(unsigned N) { - assert(N <= size() && "Invalid shrinkTo request!"); - ValuePtrs.resize(N); + case BitstreamEntry::SubBlock: + if (Entry.ID == bitc::IDENTIFICATION_BLOCK_ID) + return readIdentificationBlock(Stream); + + // Ignore other sub-blocks. + if (Stream.SkipBlock()) + return error("Malformed block"); + continue; + case BitstreamEntry::Record: + Stream.skipRecord(Entry.ID); + continue; + } } +} - Constant *getConstantFwdRef(unsigned Idx, Type *Ty); - Value *getValueFwdRef(unsigned Idx, Type *Ty); +Expected<bool> hasObjCCategoryInModule(BitstreamCursor &Stream) { + if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return error("Invalid record"); - void assignValue(Value *V, unsigned Idx); + SmallVector<uint64_t, 64> Record; + // Read all the records for this module. - /// Once all constants are read, this method bulk resolves any forward - /// references. - void resolveConstantForwardRefs(); -}; + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); -class BitcodeReaderMetadataList { - unsigned NumFwdRefs; - bool AnyFwdRefs; - unsigned MinFwdRef; - unsigned MaxFwdRef; - - /// Array of metadata references. - /// - /// Don't use std::vector here. Some versions of libc++ copy (instead of - /// move) on resize, and TrackingMDRef is very expensive to copy. - SmallVector<TrackingMDRef, 1> MetadataPtrs; - - /// Structures for resolving old type refs. - struct { - SmallDenseMap<MDString *, TempMDTuple, 1> Unknown; - SmallDenseMap<MDString *, DICompositeType *, 1> Final; - SmallDenseMap<MDString *, DICompositeType *, 1> FwdDecls; - SmallVector<std::pair<TrackingMDRef, TempMDTuple>, 1> Arrays; - } OldTypeRefs; + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return false; + case BitstreamEntry::Record: + // The interesting case. + break; + } - LLVMContext &Context; -public: - BitcodeReaderMetadataList(LLVMContext &C) - : NumFwdRefs(0), AnyFwdRefs(false), Context(C) {} - - // vector compatibility methods - unsigned size() const { return MetadataPtrs.size(); } - void resize(unsigned N) { MetadataPtrs.resize(N); } - void push_back(Metadata *MD) { MetadataPtrs.emplace_back(MD); } - void clear() { MetadataPtrs.clear(); } - Metadata *back() const { return MetadataPtrs.back(); } - void pop_back() { MetadataPtrs.pop_back(); } - bool empty() const { return MetadataPtrs.empty(); } - - Metadata *operator[](unsigned i) const { - assert(i < MetadataPtrs.size()); - return MetadataPtrs[i]; + // Read a record. + switch (Stream.readRecord(Entry.ID, Record)) { + default: + break; // Default behavior, ignore unknown content. + case bitc::MODULE_CODE_SECTIONNAME: { // SECTIONNAME: [strchr x N] + std::string S; + if (convertToString(Record, 0, S)) + return error("Invalid record"); + // Check for the i386 and other (x86_64, ARM) conventions + if (S.find("__DATA, __objc_catlist") != std::string::npos || + S.find("__OBJC,__category") != std::string::npos) + return true; + break; + } + } + Record.clear(); } + llvm_unreachable("Exit infinite loop"); +} - Metadata *lookup(unsigned I) const { - if (I < MetadataPtrs.size()) - return MetadataPtrs[I]; - return nullptr; +Expected<bool> hasObjCCategory(BitstreamCursor &Stream) { + // We expect a number of well-defined blocks, though we don't necessarily + // need to understand them all. + while (true) { + BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return false; + + case BitstreamEntry::SubBlock: + if (Entry.ID == bitc::MODULE_BLOCK_ID) + return hasObjCCategoryInModule(Stream); + + // Ignore other sub-blocks. + if (Stream.SkipBlock()) + return error("Malformed block"); + continue; + + case BitstreamEntry::Record: + Stream.skipRecord(Entry.ID); + continue; + } } +} + +Expected<std::string> readModuleTriple(BitstreamCursor &Stream) { + if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return error("Invalid record"); + + SmallVector<uint64_t, 64> Record; + + std::string Triple; + + // Read all the records for this module. + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - void shrinkTo(unsigned N) { - assert(N <= size() && "Invalid shrinkTo request!"); - assert(!AnyFwdRefs && "Unexpected forward refs"); - MetadataPtrs.resize(N); + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return Triple; + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read a record. + switch (Stream.readRecord(Entry.ID, Record)) { + default: break; // Default behavior, ignore unknown content. + case bitc::MODULE_CODE_TRIPLE: { // TRIPLE: [strchr x N] + std::string S; + if (convertToString(Record, 0, S)) + return error("Invalid record"); + Triple = S; + break; + } + } + Record.clear(); } + llvm_unreachable("Exit infinite loop"); +} - /// Return the given metadata, creating a replaceable forward reference if - /// necessary. - Metadata *getMetadataFwdRef(unsigned Idx); +Expected<std::string> readTriple(BitstreamCursor &Stream) { + // We expect a number of well-defined blocks, though we don't necessarily + // need to understand them all. + while (true) { + BitstreamEntry Entry = Stream.advance(); - /// Return the the given metadata only if it is fully resolved. - /// - /// Gives the same result as \a lookup(), unless \a MDNode::isResolved() - /// would give \c false. - Metadata *getMetadataIfResolved(unsigned Idx); + switch (Entry.Kind) { + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return ""; - MDNode *getMDNodeFwdRefOrNull(unsigned Idx); - void assignValue(Metadata *MD, unsigned Idx); - void tryToResolveCycles(); - bool hasFwdRefs() const { return AnyFwdRefs; } + case BitstreamEntry::SubBlock: + if (Entry.ID == bitc::MODULE_BLOCK_ID) + return readModuleTriple(Stream); - /// Upgrade a type that had an MDString reference. - void addTypeRef(MDString &UUID, DICompositeType &CT); + // Ignore other sub-blocks. + if (Stream.SkipBlock()) + return error("Malformed block"); + continue; - /// Upgrade a type that had an MDString reference. - Metadata *upgradeTypeRef(Metadata *MaybeUUID); + case BitstreamEntry::Record: + Stream.skipRecord(Entry.ID); + continue; + } + } +} - /// Upgrade a type ref array that may have MDString references. - Metadata *upgradeTypeRefArray(Metadata *MaybeTuple); +class BitcodeReaderBase { +protected: + BitcodeReaderBase(BitstreamCursor Stream) : Stream(std::move(Stream)) { + this->Stream.setBlockInfo(&BlockInfo); + } -private: - Metadata *resolveTypeRefArray(Metadata *MaybeTuple); + BitstreamBlockInfo BlockInfo; + BitstreamCursor Stream; + + bool readBlockInfo(); + + // Contains an arbitrary and optional string identifying the bitcode producer + std::string ProducerIdentification; + + Error error(const Twine &Message); }; -class BitcodeReader : public GVMaterializer { +Error BitcodeReaderBase::error(const Twine &Message) { + std::string FullMsg = Message.str(); + if (!ProducerIdentification.empty()) + FullMsg += " (Producer: '" + ProducerIdentification + "' Reader: 'LLVM " + + LLVM_VERSION_STRING "')"; + return ::error(FullMsg); +} + +class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { LLVMContext &Context; Module *TheModule = nullptr; - std::unique_ptr<MemoryBuffer> Buffer; - std::unique_ptr<BitstreamReader> StreamFile; - BitstreamCursor Stream; // Next offset to start scanning for lazy parsing of function bodies. uint64_t NextUnreadBit = 0; // Last function offset found in the VST. uint64_t LastFunctionBlockBit = 0; bool SeenValueSymbolTable = false; uint64_t VSTOffset = 0; - // Contains an arbitrary and optional string identifying the bitcode producer - std::string ProducerIdentification; std::vector<Type*> TypeList; BitcodeReaderValueList ValueList; - BitcodeReaderMetadataList MetadataList; + Optional<MetadataLoader> MDLoader; std::vector<Comdat *> ComdatList; SmallVector<Instruction *, 64> InstructionList; @@ -210,10 +417,6 @@ class BitcodeReader : public GVMaterializer { std::vector<std::pair<Function*, unsigned> > FunctionPrologues; std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFns; - SmallVector<Instruction*, 64> InstsWithTBAATag; - - bool HasSeenOldLoopTags = false; - /// The set of attributes by index. Index zero in the file is for null, and /// is thus not represented here. As such all indices are off by one. std::vector<AttributeSet> MAttributes; @@ -236,9 +439,6 @@ class BitcodeReader : public GVMaterializer { // Intrinsics which were remangled because of types rename UpdatedIntrinsicMap RemangledIntrinsics; - // Map the bitcode's custom MDKind ID to the Module's MDKind ID. - DenseMap<unsigned, unsigned> MDKindMap; - // Several operations happen after the module header has been read, but // before function bodies are processed. This keeps track of whether // we've done this yet. @@ -271,82 +471,55 @@ class BitcodeReader : public GVMaterializer { /// (e.g.) blockaddress forward references. bool WillMaterializeAllForwardRefs = false; - /// True if any Metadata block has been materialized. - bool IsMetadataMaterialized = false; - bool StripDebugInfo = false; - - /// Functions that need to be matched with subprograms when upgrading old - /// metadata. - SmallDenseMap<Function *, DISubprogram *, 16> FunctionsWithSPs; + TBAAVerifier TBAAVerifyHelper; std::vector<std::string> BundleTags; public: - std::error_code error(BitcodeError E, const Twine &Message); - std::error_code error(const Twine &Message); - - BitcodeReader(MemoryBuffer *Buffer, LLVMContext &Context); - BitcodeReader(LLVMContext &Context); - ~BitcodeReader() override { freeState(); } - - std::error_code materializeForwardReferencedFunctions(); - - void freeState(); + BitcodeReader(BitstreamCursor Stream, StringRef ProducerIdentification, + LLVMContext &Context); - void releaseBuffer(); + Error materializeForwardReferencedFunctions(); - std::error_code materialize(GlobalValue *GV) override; - std::error_code materializeModule() override; + Error materialize(GlobalValue *GV) override; + Error materializeModule() override; std::vector<StructType *> getIdentifiedStructTypes() const override; /// \brief Main interface to parsing a bitcode buffer. /// \returns true if an error occurred. - std::error_code parseBitcodeInto(std::unique_ptr<DataStreamer> Streamer, - Module *M, - bool ShouldLazyLoadMetadata = false); - - /// \brief Cheap mechanism to just extract module triple - /// \returns true if an error occurred. - ErrorOr<std::string> parseTriple(); - - /// Cheap mechanism to just extract the identification block out of bitcode. - ErrorOr<std::string> parseIdentificationBlock(); - - /// Peak at the module content and return true if any ObjC category or class - /// is found. - ErrorOr<bool> hasObjCCategory(); + Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata = false, + bool IsImporting = false); static uint64_t decodeSignRotatedValue(uint64_t V); /// Materialize any deferred Metadata block. - std::error_code materializeMetadata() override; + Error materializeMetadata() override; void setStripDebugInfo() override; private: - /// Parse the "IDENTIFICATION_BLOCK_ID" block, populate the - // ProducerIdentification data member, and do some basic enforcement on the - // "epoch" encoded in the bitcode. - std::error_code parseBitcodeVersion(); - std::vector<StructType *> IdentifiedStructTypes; StructType *createIdentifiedStructType(LLVMContext &Context, StringRef Name); StructType *createIdentifiedStructType(LLVMContext &Context); Type *getTypeByID(unsigned ID); + Value *getFnValueByID(unsigned ID, Type *Ty) { if (Ty && Ty->isMetadataTy()) return MetadataAsValue::get(Ty->getContext(), getFnMetadataByID(ID)); return ValueList.getValueFwdRef(ID, Ty); } + Metadata *getFnMetadataByID(unsigned ID) { - return MetadataList.getMetadataFwdRef(ID); + return MDLoader->getMetadataFwdRefOrLoad(ID); } + BasicBlock *getBasicBlock(unsigned ID) const { if (ID >= FunctionBBs.size()) return nullptr; // Invalid ID return FunctionBBs[ID]; } + AttributeSet getAttributes(unsigned i) const { if (i-1 < MAttributes.size()) return MAttributes[i-1]; @@ -422,68 +595,41 @@ private: /// Converts alignment exponent (i.e. power of two (or zero)) to the /// corresponding alignment to use. If alignment is too large, returns /// a corresponding error code. - std::error_code parseAlignmentValue(uint64_t Exponent, unsigned &Alignment); - std::error_code parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind); - std::error_code parseModule(uint64_t ResumeBit, - bool ShouldLazyLoadMetadata = false); - std::error_code parseAttributeBlock(); - std::error_code parseAttributeGroupBlock(); - std::error_code parseTypeTable(); - std::error_code parseTypeTableBody(); - std::error_code parseOperandBundleTags(); - - ErrorOr<Value *> recordValue(SmallVectorImpl<uint64_t> &Record, - unsigned NameIndex, Triple &TT); - std::error_code parseValueSymbolTable(uint64_t Offset = 0); - std::error_code parseConstants(); - std::error_code rememberAndSkipFunctionBodies(); - std::error_code rememberAndSkipFunctionBody(); + Error parseAlignmentValue(uint64_t Exponent, unsigned &Alignment); + Error parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind); + Error parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata = false); + Error parseAttributeBlock(); + Error parseAttributeGroupBlock(); + Error parseTypeTable(); + Error parseTypeTableBody(); + Error parseOperandBundleTags(); + + Expected<Value *> recordValue(SmallVectorImpl<uint64_t> &Record, + unsigned NameIndex, Triple &TT); + Error parseValueSymbolTable(uint64_t Offset = 0); + Error parseConstants(); + Error rememberAndSkipFunctionBodies(); + Error rememberAndSkipFunctionBody(); /// Save the positions of the Metadata blocks and skip parsing the blocks. - std::error_code rememberAndSkipMetadata(); - std::error_code parseFunctionBody(Function *F); - std::error_code globalCleanup(); - std::error_code resolveGlobalAndIndirectSymbolInits(); - std::error_code parseMetadata(bool ModuleLevel = false); - std::error_code parseMetadataStrings(ArrayRef<uint64_t> Record, - StringRef Blob, - unsigned &NextMetadataNo); - std::error_code parseMetadataKinds(); - std::error_code parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record); - std::error_code - parseGlobalObjectAttachment(GlobalObject &GO, - ArrayRef<uint64_t> Record); - std::error_code parseMetadataAttachment(Function &F); - ErrorOr<std::string> parseModuleTriple(); - ErrorOr<bool> hasObjCCategoryInModule(); - std::error_code parseUseLists(); - std::error_code initStream(std::unique_ptr<DataStreamer> Streamer); - std::error_code initStreamFromBuffer(); - std::error_code initLazyStream(std::unique_ptr<DataStreamer> Streamer); - std::error_code findFunctionInStream( + Error rememberAndSkipMetadata(); + Error typeCheckLoadStoreInst(Type *ValType, Type *PtrType); + Error parseFunctionBody(Function *F); + Error globalCleanup(); + Error resolveGlobalAndIndirectSymbolInits(); + Error parseUseLists(); + Error findFunctionInStream( Function *F, DenseMap<Function *, uint64_t>::iterator DeferredFunctionInfoIterator); }; /// Class to manage reading and parsing function summary index bitcode /// files/sections. -class ModuleSummaryIndexBitcodeReader { - DiagnosticHandlerFunction DiagnosticHandler; - - /// Eventually points to the module index built during parsing. - ModuleSummaryIndex *TheIndex = nullptr; - - std::unique_ptr<MemoryBuffer> Buffer; - std::unique_ptr<BitstreamReader> StreamFile; - BitstreamCursor Stream; - - /// Used to indicate whether caller only wants to check for the presence - /// of the global value summary bitcode section. All blocks are skipped, - /// but the SeenGlobalValSummary boolean is set. - bool CheckGlobalValSummaryPresenceOnly = false; +class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase { + /// The module index built during parsing. + ModuleSummaryIndex &TheIndex; /// Indicates whether we have encountered a global value summary section - /// yet during parsing, used when checking if file contains global value - /// summary section. + /// yet during parsing. bool SeenGlobalValSummary = false; /// Indicates whether we have already parsed the VST, used for error checking. @@ -513,95 +659,52 @@ class ModuleSummaryIndexBitcodeReader { std::string SourceFileName; public: - std::error_code error(const Twine &Message); - ModuleSummaryIndexBitcodeReader( - MemoryBuffer *Buffer, DiagnosticHandlerFunction DiagnosticHandler, - bool CheckGlobalValSummaryPresenceOnly = false); - ~ModuleSummaryIndexBitcodeReader() { freeState(); } - - void freeState(); + BitstreamCursor Stream, ModuleSummaryIndex &TheIndex); - void releaseBuffer(); - - /// Check if the parser has encountered a summary section. - bool foundGlobalValSummary() { return SeenGlobalValSummary; } - - /// \brief Main interface to parsing a bitcode buffer. - /// \returns true if an error occurred. - std::error_code parseSummaryIndexInto(std::unique_ptr<DataStreamer> Streamer, - ModuleSummaryIndex *I); + Error parseModule(StringRef ModulePath); private: - std::error_code parseModule(); - std::error_code parseValueSymbolTable( + Error parseValueSymbolTable( uint64_t Offset, DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap); - std::error_code parseEntireSummary(); - std::error_code parseModuleStringTable(); - std::error_code initStream(std::unique_ptr<DataStreamer> Streamer); - std::error_code initStreamFromBuffer(); - std::error_code initLazyStream(std::unique_ptr<DataStreamer> Streamer); + std::vector<ValueInfo> makeRefList(ArrayRef<uint64_t> Record); + std::vector<FunctionSummary::EdgeTy> makeCallList(ArrayRef<uint64_t> Record, + bool IsOldProfileFormat, + bool HasProfile); + Error parseEntireSummary(StringRef ModulePath); + Error parseModuleStringTable(); + std::pair<GlobalValue::GUID, GlobalValue::GUID> getGUIDFromValueId(unsigned ValueId); }; -} // end anonymous namespace -BitcodeDiagnosticInfo::BitcodeDiagnosticInfo(std::error_code EC, - DiagnosticSeverity Severity, - const Twine &Msg) - : DiagnosticInfo(DK_Bitcode, Severity), Msg(Msg), EC(EC) {} - -void BitcodeDiagnosticInfo::print(DiagnosticPrinter &DP) const { DP << Msg; } - -static std::error_code error(const DiagnosticHandlerFunction &DiagnosticHandler, - std::error_code EC, const Twine &Message) { - BitcodeDiagnosticInfo DI(EC, DS_Error, Message); - DiagnosticHandler(DI); - return EC; -} - -static std::error_code error(LLVMContext &Context, std::error_code EC, - const Twine &Message) { - return error([&](const DiagnosticInfo &DI) { Context.diagnose(DI); }, EC, - Message); -} - -static std::error_code error(LLVMContext &Context, const Twine &Message) { - return error(Context, make_error_code(BitcodeError::CorruptedBitcode), - Message); -} +} // end anonymous namespace -std::error_code BitcodeReader::error(BitcodeError E, const Twine &Message) { - if (!ProducerIdentification.empty()) { - return ::error(Context, make_error_code(E), - Message + " (Producer: '" + ProducerIdentification + - "' Reader: 'LLVM " + LLVM_VERSION_STRING "')"); +std::error_code llvm::errorToErrorCodeAndEmitErrors(LLVMContext &Ctx, + Error Err) { + if (Err) { + std::error_code EC; + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + EC = EIB.convertToErrorCode(); + Ctx.emitError(EIB.message()); + }); + return EC; } - return ::error(Context, make_error_code(E), Message); + return std::error_code(); } -std::error_code BitcodeReader::error(const Twine &Message) { - if (!ProducerIdentification.empty()) { - return ::error(Context, make_error_code(BitcodeError::CorruptedBitcode), - Message + " (Producer: '" + ProducerIdentification + - "' Reader: 'LLVM " + LLVM_VERSION_STRING "')"); - } - return ::error(Context, make_error_code(BitcodeError::CorruptedBitcode), - Message); +BitcodeReader::BitcodeReader(BitstreamCursor Stream, + StringRef ProducerIdentification, + LLVMContext &Context) + : BitcodeReaderBase(std::move(Stream)), Context(Context), + ValueList(Context) { + this->ProducerIdentification = ProducerIdentification; } -BitcodeReader::BitcodeReader(MemoryBuffer *Buffer, LLVMContext &Context) - : Context(Context), Buffer(Buffer), ValueList(Context), - MetadataList(Context) {} - -BitcodeReader::BitcodeReader(LLVMContext &Context) - : Context(Context), Buffer(nullptr), ValueList(Context), - MetadataList(Context) {} - -std::error_code BitcodeReader::materializeForwardReferencedFunctions() { +Error BitcodeReader::materializeForwardReferencedFunctions() { if (WillMaterializeAllForwardRefs) - return std::error_code(); + return Error::success(); // Prevent recursion. WillMaterializeAllForwardRefs = true; @@ -622,50 +725,20 @@ std::error_code BitcodeReader::materializeForwardReferencedFunctions() { return error("Never resolved function from blockaddress"); // Try to materialize F. - if (std::error_code EC = materialize(F)) - return EC; + if (Error Err = materialize(F)) + return Err; } assert(BasicBlockFwdRefs.empty() && "Function missing from queue"); // Reset state. WillMaterializeAllForwardRefs = false; - return std::error_code(); -} - -void BitcodeReader::freeState() { - Buffer = nullptr; - std::vector<Type*>().swap(TypeList); - ValueList.clear(); - MetadataList.clear(); - std::vector<Comdat *>().swap(ComdatList); - - std::vector<AttributeSet>().swap(MAttributes); - std::vector<BasicBlock*>().swap(FunctionBBs); - std::vector<Function*>().swap(FunctionsWithBodies); - DeferredFunctionInfo.clear(); - DeferredMetadataInfo.clear(); - MDKindMap.clear(); - - assert(BasicBlockFwdRefs.empty() && "Unresolved blockaddress fwd references"); - BasicBlockFwdRefQueue.clear(); + return Error::success(); } //===----------------------------------------------------------------------===// // Helper functions to implement forward reference resolution, etc. //===----------------------------------------------------------------------===// -/// Convert a string from a record into an std::string, return true on failure. -template <typename StrTy> -static bool convertToString(ArrayRef<uint64_t> Record, unsigned Idx, - StrTy &Result) { - if (Idx > Record.size()) - return true; - - for (unsigned i = Idx, e = Record.size(); i != e; ++i) - Result += (char)Record[i]; - return false; -} - static bool hasImplicitComdat(size_t Val) { switch (Val) { default: @@ -720,7 +793,7 @@ static GlobalValue::LinkageTypes getDecodedLinkage(unsigned Val) { } } -// Decode the flags for GlobalValue in the summary +/// Decode the flags for GlobalValue in the summary. static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, uint64_t Version) { // Summary were not emitted before LLVM 3.9, we don't need to upgrade Linkage @@ -728,8 +801,12 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, // to getDecodedLinkage() will need to be taken into account here as above. auto Linkage = GlobalValue::LinkageTypes(RawFlags & 0xF); // 4 bits RawFlags = RawFlags >> 4; - auto HasSection = RawFlags & 0x1; // bool - return GlobalValueSummary::GVFlags(Linkage, HasSection); + bool NotEligibleToImport = (RawFlags & 0x1) || Version < 3; + // The LiveRoot flag wasn't introduced until version 3. For dead stripping + // to work correctly on earlier versions, we must conservatively treat all + // values as live. + bool LiveRoot = (RawFlags & 0x2) || Version < 3; + return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, LiveRoot); } static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { @@ -897,364 +974,13 @@ static FastMathFlags getDecodedFastMathFlags(unsigned Val) { return FMF; } -static void upgradeDLLImportExportLinkage(llvm::GlobalValue *GV, unsigned Val) { +static void upgradeDLLImportExportLinkage(GlobalValue *GV, unsigned Val) { switch (Val) { case 5: GV->setDLLStorageClass(GlobalValue::DLLImportStorageClass); break; case 6: GV->setDLLStorageClass(GlobalValue::DLLExportStorageClass); break; } } -namespace llvm { -namespace { -/// \brief A class for maintaining the slot number definition -/// as a placeholder for the actual definition for forward constants defs. -class ConstantPlaceHolder : public ConstantExpr { - void operator=(const ConstantPlaceHolder &) = delete; - -public: - // allocate space for exactly one operand - void *operator new(size_t s) { return User::operator new(s, 1); } - explicit ConstantPlaceHolder(Type *Ty, LLVMContext &Context) - : ConstantExpr(Ty, Instruction::UserOp1, &Op<0>(), 1) { - Op<0>() = UndefValue::get(Type::getInt32Ty(Context)); - } - - /// \brief Methods to support type inquiry through isa, cast, and dyn_cast. - static bool classof(const Value *V) { - return isa<ConstantExpr>(V) && - cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1; - } - - /// Provide fast operand accessors - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); -}; -} // end anonymous namespace - -// FIXME: can we inherit this from ConstantExpr? -template <> -struct OperandTraits<ConstantPlaceHolder> : - public FixedNumOperandTraits<ConstantPlaceHolder, 1> { -}; -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPlaceHolder, Value) -} // end namespace llvm - -void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { - if (Idx == size()) { - push_back(V); - return; - } - - if (Idx >= size()) - resize(Idx+1); - - WeakVH &OldV = ValuePtrs[Idx]; - if (!OldV) { - OldV = V; - return; - } - - // Handle constants and non-constants (e.g. instrs) differently for - // efficiency. - if (Constant *PHC = dyn_cast<Constant>(&*OldV)) { - ResolveConstants.push_back(std::make_pair(PHC, Idx)); - OldV = V; - } else { - // If there was a forward reference to this value, replace it. - Value *PrevVal = OldV; - OldV->replaceAllUsesWith(V); - delete PrevVal; - } -} - -Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx, - Type *Ty) { - if (Idx >= size()) - resize(Idx + 1); - - if (Value *V = ValuePtrs[Idx]) { - if (Ty != V->getType()) - report_fatal_error("Type mismatch in constant table!"); - return cast<Constant>(V); - } - - // Create and return a placeholder, which will later be RAUW'd. - Constant *C = new ConstantPlaceHolder(Ty, Context); - ValuePtrs[Idx] = C; - return C; -} - -Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) { - // Bail out for a clearly invalid value. This would make us call resize(0) - if (Idx == UINT_MAX) - return nullptr; - - if (Idx >= size()) - resize(Idx + 1); - - if (Value *V = ValuePtrs[Idx]) { - // If the types don't match, it's invalid. - if (Ty && Ty != V->getType()) - return nullptr; - return V; - } - - // No type specified, must be invalid reference. - if (!Ty) return nullptr; - - // Create and return a placeholder, which will later be RAUW'd. - Value *V = new Argument(Ty); - ValuePtrs[Idx] = V; - return V; -} - -/// Once all constants are read, this method bulk resolves any forward -/// references. The idea behind this is that we sometimes get constants (such -/// as large arrays) which reference *many* forward ref constants. Replacing -/// each of these causes a lot of thrashing when building/reuniquing the -/// constant. Instead of doing this, we look at all the uses and rewrite all -/// the place holders at once for any constant that uses a placeholder. -void BitcodeReaderValueList::resolveConstantForwardRefs() { - // Sort the values by-pointer so that they are efficient to look up with a - // binary search. - std::sort(ResolveConstants.begin(), ResolveConstants.end()); - - SmallVector<Constant*, 64> NewOps; - - while (!ResolveConstants.empty()) { - Value *RealVal = operator[](ResolveConstants.back().second); - Constant *Placeholder = ResolveConstants.back().first; - ResolveConstants.pop_back(); - - // Loop over all users of the placeholder, updating them to reference the - // new value. If they reference more than one placeholder, update them all - // at once. - while (!Placeholder->use_empty()) { - auto UI = Placeholder->user_begin(); - User *U = *UI; - - // If the using object isn't uniqued, just update the operands. This - // handles instructions and initializers for global variables. - if (!isa<Constant>(U) || isa<GlobalValue>(U)) { - UI.getUse().set(RealVal); - continue; - } - - // Otherwise, we have a constant that uses the placeholder. Replace that - // constant with a new constant that has *all* placeholder uses updated. - Constant *UserC = cast<Constant>(U); - for (User::op_iterator I = UserC->op_begin(), E = UserC->op_end(); - I != E; ++I) { - Value *NewOp; - if (!isa<ConstantPlaceHolder>(*I)) { - // Not a placeholder reference. - NewOp = *I; - } else if (*I == Placeholder) { - // Common case is that it just references this one placeholder. - NewOp = RealVal; - } else { - // Otherwise, look up the placeholder in ResolveConstants. - ResolveConstantsTy::iterator It = - std::lower_bound(ResolveConstants.begin(), ResolveConstants.end(), - std::pair<Constant*, unsigned>(cast<Constant>(*I), - 0)); - assert(It != ResolveConstants.end() && It->first == *I); - NewOp = operator[](It->second); - } - - NewOps.push_back(cast<Constant>(NewOp)); - } - - // Make the new constant. - Constant *NewC; - if (ConstantArray *UserCA = dyn_cast<ConstantArray>(UserC)) { - NewC = ConstantArray::get(UserCA->getType(), NewOps); - } else if (ConstantStruct *UserCS = dyn_cast<ConstantStruct>(UserC)) { - NewC = ConstantStruct::get(UserCS->getType(), NewOps); - } else if (isa<ConstantVector>(UserC)) { - NewC = ConstantVector::get(NewOps); - } else { - assert(isa<ConstantExpr>(UserC) && "Must be a ConstantExpr."); - NewC = cast<ConstantExpr>(UserC)->getWithOperands(NewOps); - } - - UserC->replaceAllUsesWith(NewC); - UserC->destroyConstant(); - NewOps.clear(); - } - - // Update all ValueHandles, they should be the only users at this point. - Placeholder->replaceAllUsesWith(RealVal); - delete Placeholder; - } -} - -void BitcodeReaderMetadataList::assignValue(Metadata *MD, unsigned Idx) { - if (Idx == size()) { - push_back(MD); - return; - } - - if (Idx >= size()) - resize(Idx+1); - - TrackingMDRef &OldMD = MetadataPtrs[Idx]; - if (!OldMD) { - OldMD.reset(MD); - return; - } - - // If there was a forward reference to this value, replace it. - TempMDTuple PrevMD(cast<MDTuple>(OldMD.get())); - PrevMD->replaceAllUsesWith(MD); - --NumFwdRefs; -} - -Metadata *BitcodeReaderMetadataList::getMetadataFwdRef(unsigned Idx) { - if (Idx >= size()) - resize(Idx + 1); - - if (Metadata *MD = MetadataPtrs[Idx]) - return MD; - - // Track forward refs to be resolved later. - if (AnyFwdRefs) { - MinFwdRef = std::min(MinFwdRef, Idx); - MaxFwdRef = std::max(MaxFwdRef, Idx); - } else { - AnyFwdRefs = true; - MinFwdRef = MaxFwdRef = Idx; - } - ++NumFwdRefs; - - // Create and return a placeholder, which will later be RAUW'd. - Metadata *MD = MDNode::getTemporary(Context, None).release(); - MetadataPtrs[Idx].reset(MD); - return MD; -} - -Metadata *BitcodeReaderMetadataList::getMetadataIfResolved(unsigned Idx) { - Metadata *MD = lookup(Idx); - if (auto *N = dyn_cast_or_null<MDNode>(MD)) - if (!N->isResolved()) - return nullptr; - return MD; -} - -MDNode *BitcodeReaderMetadataList::getMDNodeFwdRefOrNull(unsigned Idx) { - return dyn_cast_or_null<MDNode>(getMetadataFwdRef(Idx)); -} - -void BitcodeReaderMetadataList::tryToResolveCycles() { - if (NumFwdRefs) - // Still forward references... can't resolve cycles. - return; - - bool DidReplaceTypeRefs = false; - - // Give up on finding a full definition for any forward decls that remain. - for (const auto &Ref : OldTypeRefs.FwdDecls) - OldTypeRefs.Final.insert(Ref); - OldTypeRefs.FwdDecls.clear(); - - // Upgrade from old type ref arrays. In strange cases, this could add to - // OldTypeRefs.Unknown. - for (const auto &Array : OldTypeRefs.Arrays) { - DidReplaceTypeRefs = true; - Array.second->replaceAllUsesWith(resolveTypeRefArray(Array.first.get())); - } - OldTypeRefs.Arrays.clear(); - - // Replace old string-based type refs with the resolved node, if possible. - // If we haven't seen the node, leave it to the verifier to complain about - // the invalid string reference. - for (const auto &Ref : OldTypeRefs.Unknown) { - DidReplaceTypeRefs = true; - if (DICompositeType *CT = OldTypeRefs.Final.lookup(Ref.first)) - Ref.second->replaceAllUsesWith(CT); - else - Ref.second->replaceAllUsesWith(Ref.first); - } - OldTypeRefs.Unknown.clear(); - - // Make sure all the upgraded types are resolved. - if (DidReplaceTypeRefs) { - AnyFwdRefs = true; - MinFwdRef = 0; - MaxFwdRef = MetadataPtrs.size() - 1; - } - - if (!AnyFwdRefs) - // Nothing to do. - return; - - // Resolve any cycles. - for (unsigned I = MinFwdRef, E = MaxFwdRef + 1; I != E; ++I) { - auto &MD = MetadataPtrs[I]; - auto *N = dyn_cast_or_null<MDNode>(MD); - if (!N) - continue; - - assert(!N->isTemporary() && "Unexpected forward reference"); - N->resolveCycles(); - } - - // Make sure we return early again until there's another forward ref. - AnyFwdRefs = false; -} - -void BitcodeReaderMetadataList::addTypeRef(MDString &UUID, - DICompositeType &CT) { - assert(CT.getRawIdentifier() == &UUID && "Mismatched UUID"); - if (CT.isForwardDecl()) - OldTypeRefs.FwdDecls.insert(std::make_pair(&UUID, &CT)); - else - OldTypeRefs.Final.insert(std::make_pair(&UUID, &CT)); -} - -Metadata *BitcodeReaderMetadataList::upgradeTypeRef(Metadata *MaybeUUID) { - auto *UUID = dyn_cast_or_null<MDString>(MaybeUUID); - if (LLVM_LIKELY(!UUID)) - return MaybeUUID; - - if (auto *CT = OldTypeRefs.Final.lookup(UUID)) - return CT; - - auto &Ref = OldTypeRefs.Unknown[UUID]; - if (!Ref) - Ref = MDNode::getTemporary(Context, None); - return Ref.get(); -} - -Metadata *BitcodeReaderMetadataList::upgradeTypeRefArray(Metadata *MaybeTuple) { - auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple); - if (!Tuple || Tuple->isDistinct()) - return MaybeTuple; - - // Look through the array immediately if possible. - if (!Tuple->isTemporary()) - return resolveTypeRefArray(Tuple); - - // Create and return a placeholder to use for now. Eventually - // resolveTypeRefArrays() will be resolve this forward reference. - OldTypeRefs.Arrays.emplace_back( - std::piecewise_construct, std::forward_as_tuple(Tuple), - std::forward_as_tuple(MDTuple::getTemporary(Context, None))); - return OldTypeRefs.Arrays.back().second.get(); -} - -Metadata *BitcodeReaderMetadataList::resolveTypeRefArray(Metadata *MaybeTuple) { - auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple); - if (!Tuple || Tuple->isDistinct()) - return MaybeTuple; - - // Look through the DITypeRefArray, upgrading each DITypeRef. - SmallVector<Metadata *, 32> Ops; - Ops.reserve(Tuple->getNumOperands()); - for (Metadata *MD : Tuple->operands()) - Ops.push_back(upgradeTypeRef(MD)); - - return MDTuple::get(Context, Ops); -} Type *BitcodeReader::getTypeByID(unsigned ID) { // The type table size is always specified correctly. @@ -1286,6 +1012,97 @@ StructType *BitcodeReader::createIdentifiedStructType(LLVMContext &Context) { // Functions for parsing blocks from the bitcode file //===----------------------------------------------------------------------===// +static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { + switch (Val) { + case Attribute::EndAttrKinds: + llvm_unreachable("Synthetic enumerators which should never get here"); + + case Attribute::None: return 0; + case Attribute::ZExt: return 1 << 0; + case Attribute::SExt: return 1 << 1; + case Attribute::NoReturn: return 1 << 2; + case Attribute::InReg: return 1 << 3; + case Attribute::StructRet: return 1 << 4; + case Attribute::NoUnwind: return 1 << 5; + case Attribute::NoAlias: return 1 << 6; + case Attribute::ByVal: return 1 << 7; + case Attribute::Nest: return 1 << 8; + case Attribute::ReadNone: return 1 << 9; + case Attribute::ReadOnly: return 1 << 10; + case Attribute::NoInline: return 1 << 11; + case Attribute::AlwaysInline: return 1 << 12; + case Attribute::OptimizeForSize: return 1 << 13; + case Attribute::StackProtect: return 1 << 14; + case Attribute::StackProtectReq: return 1 << 15; + case Attribute::Alignment: return 31 << 16; + case Attribute::NoCapture: return 1 << 21; + case Attribute::NoRedZone: return 1 << 22; + case Attribute::NoImplicitFloat: return 1 << 23; + case Attribute::Naked: return 1 << 24; + case Attribute::InlineHint: return 1 << 25; + case Attribute::StackAlignment: return 7 << 26; + case Attribute::ReturnsTwice: return 1 << 29; + case Attribute::UWTable: return 1 << 30; + case Attribute::NonLazyBind: return 1U << 31; + case Attribute::SanitizeAddress: return 1ULL << 32; + case Attribute::MinSize: return 1ULL << 33; + case Attribute::NoDuplicate: return 1ULL << 34; + case Attribute::StackProtectStrong: return 1ULL << 35; + case Attribute::SanitizeThread: return 1ULL << 36; + case Attribute::SanitizeMemory: return 1ULL << 37; + case Attribute::NoBuiltin: return 1ULL << 38; + case Attribute::Returned: return 1ULL << 39; + case Attribute::Cold: return 1ULL << 40; + case Attribute::Builtin: return 1ULL << 41; + case Attribute::OptimizeNone: return 1ULL << 42; + case Attribute::InAlloca: return 1ULL << 43; + case Attribute::NonNull: return 1ULL << 44; + case Attribute::JumpTable: return 1ULL << 45; + case Attribute::Convergent: return 1ULL << 46; + case Attribute::SafeStack: return 1ULL << 47; + case Attribute::NoRecurse: return 1ULL << 48; + case Attribute::InaccessibleMemOnly: return 1ULL << 49; + case Attribute::InaccessibleMemOrArgMemOnly: return 1ULL << 50; + case Attribute::SwiftSelf: return 1ULL << 51; + case Attribute::SwiftError: return 1ULL << 52; + case Attribute::WriteOnly: return 1ULL << 53; + case Attribute::Dereferenceable: + llvm_unreachable("dereferenceable attribute not supported in raw format"); + break; + case Attribute::DereferenceableOrNull: + llvm_unreachable("dereferenceable_or_null attribute not supported in raw " + "format"); + break; + case Attribute::ArgMemOnly: + llvm_unreachable("argmemonly attribute not supported in raw format"); + break; + case Attribute::AllocSize: + llvm_unreachable("allocsize not supported in raw format"); + break; + } + llvm_unreachable("Unsupported attribute type"); +} + +static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) { + if (!Val) return; + + for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds; + I = Attribute::AttrKind(I + 1)) { + if (I == Attribute::Dereferenceable || + I == Attribute::DereferenceableOrNull || + I == Attribute::ArgMemOnly || + I == Attribute::AllocSize) + continue; + if (uint64_t A = (Val & getRawAttributeMask(I))) { + if (I == Attribute::Alignment) + B.addAlignmentAttr(1ULL << ((A >> 16) - 1)); + else if (I == Attribute::StackAlignment) + B.addStackAlignmentAttr(1ULL << ((A >> 26)-1)); + else + B.addAttribute(I); + } + } +} /// \brief This fills an AttrBuilder object with the LLVM attributes that have /// been decoded from the given integer. This function must stay in sync with @@ -1302,11 +1119,11 @@ static void decodeLLVMAttributesForBitcode(AttrBuilder &B, if (Alignment) B.addAlignmentAttr(Alignment); - B.addRawValue(((EncodedAttrs & (0xfffffULL << 32)) >> 11) | - (EncodedAttrs & 0xffff)); + addRawAttributeValue(B, ((EncodedAttrs & (0xfffffULL << 32)) >> 11) | + (EncodedAttrs & 0xffff)); } -std::error_code BitcodeReader::parseAttributeBlock() { +Error BitcodeReader::parseAttributeBlock() { if (Stream.EnterSubBlock(bitc::PARAMATTR_BLOCK_ID)) return error("Invalid record"); @@ -1318,7 +1135,7 @@ std::error_code BitcodeReader::parseAttributeBlock() { SmallVector<AttributeSet, 8> Attrs; // Read all the records. - while (1) { + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -1326,7 +1143,7 @@ std::error_code BitcodeReader::parseAttributeBlock() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -1476,26 +1293,24 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { } } -std::error_code BitcodeReader::parseAlignmentValue(uint64_t Exponent, - unsigned &Alignment) { +Error BitcodeReader::parseAlignmentValue(uint64_t Exponent, + unsigned &Alignment) { // Note: Alignment in bitcode files is incremented by 1, so that zero // can be used for default alignment. if (Exponent > Value::MaxAlignmentExponent + 1) return error("Invalid alignment value"); Alignment = (1 << static_cast<unsigned>(Exponent)) >> 1; - return std::error_code(); + return Error::success(); } -std::error_code BitcodeReader::parseAttrKind(uint64_t Code, - Attribute::AttrKind *Kind) { +Error BitcodeReader::parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind) { *Kind = getAttrFromCode(Code); if (*Kind == Attribute::None) - return error(BitcodeError::CorruptedBitcode, - "Unknown attribute kind (" + Twine(Code) + ")"); - return std::error_code(); + return error("Unknown attribute kind (" + Twine(Code) + ")"); + return Error::success(); } -std::error_code BitcodeReader::parseAttributeGroupBlock() { +Error BitcodeReader::parseAttributeGroupBlock() { if (Stream.EnterSubBlock(bitc::PARAMATTR_GROUP_BLOCK_ID)) return error("Invalid record"); @@ -1505,7 +1320,7 @@ std::error_code BitcodeReader::parseAttributeGroupBlock() { SmallVector<uint64_t, 64> Record; // Read all the records. - while (1) { + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -1513,7 +1328,7 @@ std::error_code BitcodeReader::parseAttributeGroupBlock() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -1535,14 +1350,14 @@ std::error_code BitcodeReader::parseAttributeGroupBlock() { for (unsigned i = 2, e = Record.size(); i != e; ++i) { if (Record[i] == 0) { // Enum attribute Attribute::AttrKind Kind; - if (std::error_code EC = parseAttrKind(Record[++i], &Kind)) - return EC; + if (Error Err = parseAttrKind(Record[++i], &Kind)) + return Err; B.addAttribute(Kind); } else if (Record[i] == 1) { // Integer attribute Attribute::AttrKind Kind; - if (std::error_code EC = parseAttrKind(Record[++i], &Kind)) - return EC; + if (Error Err = parseAttrKind(Record[++i], &Kind)) + return Err; if (Kind == Attribute::Alignment) B.addAlignmentAttr(Record[++i]); else if (Kind == Attribute::StackAlignment) @@ -1583,14 +1398,14 @@ std::error_code BitcodeReader::parseAttributeGroupBlock() { } } -std::error_code BitcodeReader::parseTypeTable() { +Error BitcodeReader::parseTypeTable() { if (Stream.EnterSubBlock(bitc::TYPE_BLOCK_ID_NEW)) return error("Invalid record"); return parseTypeTableBody(); } -std::error_code BitcodeReader::parseTypeTableBody() { +Error BitcodeReader::parseTypeTableBody() { if (!TypeList.empty()) return error("Invalid multiple blocks"); @@ -1600,7 +1415,7 @@ std::error_code BitcodeReader::parseTypeTableBody() { SmallString<64> TypeName; // Read all the records for this type table. - while (1) { + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -1610,7 +1425,7 @@ std::error_code BitcodeReader::parseTypeTableBody() { case BitstreamEntry::EndBlock: if (NumRecords != TypeList.size()) return error("Malformed block"); - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -1826,7 +1641,7 @@ std::error_code BitcodeReader::parseTypeTableBody() { } } -std::error_code BitcodeReader::parseOperandBundleTags() { +Error BitcodeReader::parseOperandBundleTags() { if (Stream.EnterSubBlock(bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID)) return error("Invalid record"); @@ -1835,7 +1650,7 @@ std::error_code BitcodeReader::parseOperandBundleTags() { SmallVector<uint64_t, 64> Record; - while (1) { + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -1843,7 +1658,7 @@ std::error_code BitcodeReader::parseOperandBundleTags() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -1863,8 +1678,8 @@ std::error_code BitcodeReader::parseOperandBundleTags() { } /// Associate a value with its name from the given index in the provided record. -ErrorOr<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record, - unsigned NameIndex, Triple &TT) { +Expected<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record, + unsigned NameIndex, Triple &TT) { SmallString<128> ValueName; if (convertToString(Record, NameIndex, ValueName)) return error("Invalid record"); @@ -1912,7 +1727,7 @@ static uint64_t jumpToValueSymbolTable(uint64_t Offset, /// Parse the value symbol table at either the current parsing location or /// at the given bit offset if provided. -std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { +Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { uint64_t CurrentBit; // Pass in the Offset to distinguish between calling for the module-level // VST (where we want to jump to the VST offset) and the function-level @@ -1943,7 +1758,8 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { // Read all the records for this value table. SmallString<128> ValueName; - while (1) { + + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -1953,7 +1769,7 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { case BitstreamEntry::EndBlock: if (Offset > 0) Stream.JumpToBit(CurrentBit); - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -1965,17 +1781,17 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { default: // Default behavior: unknown type. break; case bitc::VST_CODE_ENTRY: { // VST_CODE_ENTRY: [valueid, namechar x N] - ErrorOr<Value *> ValOrErr = recordValue(Record, 1, TT); - if (std::error_code EC = ValOrErr.getError()) - return EC; + Expected<Value *> ValOrErr = recordValue(Record, 1, TT); + if (Error Err = ValOrErr.takeError()) + return Err; ValOrErr.get(); break; } case bitc::VST_CODE_FNENTRY: { // VST_CODE_FNENTRY: [valueid, offset, namechar x N] - ErrorOr<Value *> ValOrErr = recordValue(Record, 2, TT); - if (std::error_code EC = ValOrErr.getError()) - return EC; + Expected<Value *> ValOrErr = recordValue(Record, 2, TT); + if (Error Err = ValOrErr.takeError()) + return Err; Value *V = ValOrErr.get(); auto *GO = dyn_cast<GlobalObject>(V); @@ -1988,7 +1804,10 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { assert(GO); } - uint64_t FuncWordOffset = Record[1]; + // Note that we subtract 1 here because the offset is relative to one word + // before the start of the identification or module block, which was + // historically always the start of the regular bitcode header. + uint64_t FuncWordOffset = Record[1] - 1; Function *F = dyn_cast<Function>(GO); assert(F); uint64_t FuncBitOffset = FuncWordOffset * 32; @@ -2015,748 +1834,6 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { } } -/// Parse a single METADATA_KIND record, inserting result in MDKindMap. -std::error_code -BitcodeReader::parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record) { - if (Record.size() < 2) - return error("Invalid record"); - - unsigned Kind = Record[0]; - SmallString<8> Name(Record.begin() + 1, Record.end()); - - unsigned NewKind = TheModule->getMDKindID(Name.str()); - if (!MDKindMap.insert(std::make_pair(Kind, NewKind)).second) - return error("Conflicting METADATA_KIND records"); - return std::error_code(); -} - -static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; } - -std::error_code BitcodeReader::parseMetadataStrings(ArrayRef<uint64_t> Record, - StringRef Blob, - unsigned &NextMetadataNo) { - // All the MDStrings in the block are emitted together in a single - // record. The strings are concatenated and stored in a blob along with - // their sizes. - if (Record.size() != 2) - return error("Invalid record: metadata strings layout"); - - unsigned NumStrings = Record[0]; - unsigned StringsOffset = Record[1]; - if (!NumStrings) - return error("Invalid record: metadata strings with no strings"); - if (StringsOffset > Blob.size()) - return error("Invalid record: metadata strings corrupt offset"); - - StringRef Lengths = Blob.slice(0, StringsOffset); - SimpleBitstreamCursor R(*StreamFile); - R.jumpToPointer(Lengths.begin()); - - // Ensure that Blob doesn't get invalidated, even if this is reading from - // a StreamingMemoryObject with corrupt data. - R.setArtificialByteLimit(R.getCurrentByteNo() + StringsOffset); - - StringRef Strings = Blob.drop_front(StringsOffset); - do { - if (R.AtEndOfStream()) - return error("Invalid record: metadata strings bad length"); - - unsigned Size = R.ReadVBR(6); - if (Strings.size() < Size) - return error("Invalid record: metadata strings truncated chars"); - - MetadataList.assignValue(MDString::get(Context, Strings.slice(0, Size)), - NextMetadataNo++); - Strings = Strings.drop_front(Size); - } while (--NumStrings); - - return std::error_code(); -} - -namespace { -class PlaceholderQueue { - // Placeholders would thrash around when moved, so store in a std::deque - // instead of some sort of vector. - std::deque<DistinctMDOperandPlaceholder> PHs; - -public: - DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID); - void flush(BitcodeReaderMetadataList &MetadataList); -}; -} // end namespace - -DistinctMDOperandPlaceholder &PlaceholderQueue::getPlaceholderOp(unsigned ID) { - PHs.emplace_back(ID); - return PHs.back(); -} - -void PlaceholderQueue::flush(BitcodeReaderMetadataList &MetadataList) { - while (!PHs.empty()) { - PHs.front().replaceUseWith( - MetadataList.getMetadataFwdRef(PHs.front().getID())); - PHs.pop_front(); - } -} - -/// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing -/// module level metadata. -std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { - assert((ModuleLevel || DeferredMetadataInfo.empty()) && - "Must read all module-level metadata before function-level"); - - IsMetadataMaterialized = true; - unsigned NextMetadataNo = MetadataList.size(); - - if (!ModuleLevel && MetadataList.hasFwdRefs()) - return error("Invalid metadata: fwd refs into function blocks"); - - if (Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID)) - return error("Invalid record"); - - std::vector<std::pair<DICompileUnit *, Metadata *>> CUSubprograms; - SmallVector<uint64_t, 64> Record; - - PlaceholderQueue Placeholders; - bool IsDistinct; - auto getMD = [&](unsigned ID) -> Metadata * { - if (!IsDistinct) - return MetadataList.getMetadataFwdRef(ID); - if (auto *MD = MetadataList.getMetadataIfResolved(ID)) - return MD; - return &Placeholders.getPlaceholderOp(ID); - }; - auto getMDOrNull = [&](unsigned ID) -> Metadata * { - if (ID) - return getMD(ID - 1); - return nullptr; - }; - auto getMDOrNullWithoutPlaceholders = [&](unsigned ID) -> Metadata * { - if (ID) - return MetadataList.getMetadataFwdRef(ID - 1); - return nullptr; - }; - auto getMDString = [&](unsigned ID) -> MDString *{ - // This requires that the ID is not really a forward reference. In - // particular, the MDString must already have been resolved. - return cast_or_null<MDString>(getMDOrNull(ID)); - }; - - // Support for old type refs. - auto getDITypeRefOrNull = [&](unsigned ID) { - return MetadataList.upgradeTypeRef(getMDOrNull(ID)); - }; - -#define GET_OR_DISTINCT(CLASS, ARGS) \ - (IsDistinct ? CLASS::getDistinct ARGS : CLASS::get ARGS) - - // Read all the records. - while (1) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - - switch (Entry.Kind) { - case BitstreamEntry::SubBlock: // Handled for us already. - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - // Upgrade old-style CU <-> SP pointers to point from SP to CU. - for (auto CU_SP : CUSubprograms) - if (auto *SPs = dyn_cast_or_null<MDTuple>(CU_SP.second)) - for (auto &Op : SPs->operands()) - if (auto *SP = dyn_cast_or_null<MDNode>(Op)) - SP->replaceOperandWith(7, CU_SP.first); - - MetadataList.tryToResolveCycles(); - Placeholders.flush(MetadataList); - return std::error_code(); - case BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read a record. - Record.clear(); - StringRef Blob; - unsigned Code = Stream.readRecord(Entry.ID, Record, &Blob); - IsDistinct = false; - switch (Code) { - default: // Default behavior: ignore. - break; - case bitc::METADATA_NAME: { - // Read name of the named metadata. - SmallString<8> Name(Record.begin(), Record.end()); - Record.clear(); - Code = Stream.ReadCode(); - - unsigned NextBitCode = Stream.readRecord(Code, Record); - if (NextBitCode != bitc::METADATA_NAMED_NODE) - return error("METADATA_NAME not followed by METADATA_NAMED_NODE"); - - // Read named metadata elements. - unsigned Size = Record.size(); - NamedMDNode *NMD = TheModule->getOrInsertNamedMetadata(Name); - for (unsigned i = 0; i != Size; ++i) { - MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[i]); - if (!MD) - return error("Invalid record"); - NMD->addOperand(MD); - } - break; - } - case bitc::METADATA_OLD_FN_NODE: { - // FIXME: Remove in 4.0. - // This is a LocalAsMetadata record, the only type of function-local - // metadata. - if (Record.size() % 2 == 1) - return error("Invalid record"); - - // If this isn't a LocalAsMetadata record, we're dropping it. This used - // to be legal, but there's no upgrade path. - auto dropRecord = [&] { - MetadataList.assignValue(MDNode::get(Context, None), NextMetadataNo++); - }; - if (Record.size() != 2) { - dropRecord(); - break; - } - - Type *Ty = getTypeByID(Record[0]); - if (Ty->isMetadataTy() || Ty->isVoidTy()) { - dropRecord(); - break; - } - - MetadataList.assignValue( - LocalAsMetadata::get(ValueList.getValueFwdRef(Record[1], Ty)), - NextMetadataNo++); - break; - } - case bitc::METADATA_OLD_NODE: { - // FIXME: Remove in 4.0. - if (Record.size() % 2 == 1) - return error("Invalid record"); - - unsigned Size = Record.size(); - SmallVector<Metadata *, 8> Elts; - for (unsigned i = 0; i != Size; i += 2) { - Type *Ty = getTypeByID(Record[i]); - if (!Ty) - return error("Invalid record"); - if (Ty->isMetadataTy()) - Elts.push_back(getMD(Record[i + 1])); - else if (!Ty->isVoidTy()) { - auto *MD = - ValueAsMetadata::get(ValueList.getValueFwdRef(Record[i + 1], Ty)); - assert(isa<ConstantAsMetadata>(MD) && - "Expected non-function-local metadata"); - Elts.push_back(MD); - } else - Elts.push_back(nullptr); - } - MetadataList.assignValue(MDNode::get(Context, Elts), NextMetadataNo++); - break; - } - case bitc::METADATA_VALUE: { - if (Record.size() != 2) - return error("Invalid record"); - - Type *Ty = getTypeByID(Record[0]); - if (Ty->isMetadataTy() || Ty->isVoidTy()) - return error("Invalid record"); - - MetadataList.assignValue( - ValueAsMetadata::get(ValueList.getValueFwdRef(Record[1], Ty)), - NextMetadataNo++); - break; - } - case bitc::METADATA_DISTINCT_NODE: - IsDistinct = true; - // fallthrough... - case bitc::METADATA_NODE: { - SmallVector<Metadata *, 8> Elts; - Elts.reserve(Record.size()); - for (unsigned ID : Record) - Elts.push_back(getMDOrNull(ID)); - MetadataList.assignValue(IsDistinct ? MDNode::getDistinct(Context, Elts) - : MDNode::get(Context, Elts), - NextMetadataNo++); - break; - } - case bitc::METADATA_LOCATION: { - if (Record.size() != 5) - return error("Invalid record"); - - IsDistinct = Record[0]; - unsigned Line = Record[1]; - unsigned Column = Record[2]; - Metadata *Scope = getMD(Record[3]); - Metadata *InlinedAt = getMDOrNull(Record[4]); - MetadataList.assignValue( - GET_OR_DISTINCT(DILocation, - (Context, Line, Column, Scope, InlinedAt)), - NextMetadataNo++); - break; - } - case bitc::METADATA_GENERIC_DEBUG: { - if (Record.size() < 4) - return error("Invalid record"); - - IsDistinct = Record[0]; - unsigned Tag = Record[1]; - unsigned Version = Record[2]; - - if (Tag >= 1u << 16 || Version != 0) - return error("Invalid record"); - - auto *Header = getMDString(Record[3]); - SmallVector<Metadata *, 8> DwarfOps; - for (unsigned I = 4, E = Record.size(); I != E; ++I) - DwarfOps.push_back(getMDOrNull(Record[I])); - MetadataList.assignValue( - GET_OR_DISTINCT(GenericDINode, (Context, Tag, Header, DwarfOps)), - NextMetadataNo++); - break; - } - case bitc::METADATA_SUBRANGE: { - if (Record.size() != 3) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DISubrange, - (Context, Record[1], unrotateSign(Record[2]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_ENUMERATOR: { - if (Record.size() != 3) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIEnumerator, (Context, unrotateSign(Record[1]), - getMDString(Record[2]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_BASIC_TYPE: { - if (Record.size() != 6) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIBasicType, - (Context, Record[1], getMDString(Record[2]), - Record[3], Record[4], Record[5])), - NextMetadataNo++); - break; - } - case bitc::METADATA_DERIVED_TYPE: { - if (Record.size() != 12) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT( - DIDerivedType, - (Context, Record[1], getMDString(Record[2]), - getMDOrNull(Record[3]), Record[4], getDITypeRefOrNull(Record[5]), - getDITypeRefOrNull(Record[6]), Record[7], Record[8], Record[9], - Record[10], getDITypeRefOrNull(Record[11]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_COMPOSITE_TYPE: { - if (Record.size() != 16) - return error("Invalid record"); - - // If we have a UUID and this is not a forward declaration, lookup the - // mapping. - IsDistinct = Record[0] & 0x1; - bool IsNotUsedInTypeRef = Record[0] >= 2; - unsigned Tag = Record[1]; - MDString *Name = getMDString(Record[2]); - Metadata *File = getMDOrNull(Record[3]); - unsigned Line = Record[4]; - Metadata *Scope = getDITypeRefOrNull(Record[5]); - Metadata *BaseType = getDITypeRefOrNull(Record[6]); - uint64_t SizeInBits = Record[7]; - uint64_t AlignInBits = Record[8]; - uint64_t OffsetInBits = Record[9]; - unsigned Flags = Record[10]; - Metadata *Elements = getMDOrNull(Record[11]); - unsigned RuntimeLang = Record[12]; - Metadata *VTableHolder = getDITypeRefOrNull(Record[13]); - Metadata *TemplateParams = getMDOrNull(Record[14]); - auto *Identifier = getMDString(Record[15]); - DICompositeType *CT = nullptr; - if (Identifier) - CT = DICompositeType::buildODRType( - Context, *Identifier, Tag, Name, File, Line, Scope, BaseType, - SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams); - - // Create a node if we didn't get a lazy ODR type. - if (!CT) - CT = GET_OR_DISTINCT(DICompositeType, - (Context, Tag, Name, File, Line, Scope, BaseType, - SizeInBits, AlignInBits, OffsetInBits, Flags, - Elements, RuntimeLang, VTableHolder, - TemplateParams, Identifier)); - if (!IsNotUsedInTypeRef && Identifier) - MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT)); - - MetadataList.assignValue(CT, NextMetadataNo++); - break; - } - case bitc::METADATA_SUBROUTINE_TYPE: { - if (Record.size() < 3 || Record.size() > 4) - return error("Invalid record"); - bool IsOldTypeRefArray = Record[0] < 2; - unsigned CC = (Record.size() > 3) ? Record[3] : 0; - - IsDistinct = Record[0] & 0x1; - Metadata *Types = getMDOrNull(Record[2]); - if (LLVM_UNLIKELY(IsOldTypeRefArray)) - Types = MetadataList.upgradeTypeRefArray(Types); - - MetadataList.assignValue( - GET_OR_DISTINCT(DISubroutineType, (Context, Record[1], CC, Types)), - NextMetadataNo++); - break; - } - - case bitc::METADATA_MODULE: { - if (Record.size() != 6) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIModule, - (Context, getMDOrNull(Record[1]), - getMDString(Record[2]), getMDString(Record[3]), - getMDString(Record[4]), getMDString(Record[5]))), - NextMetadataNo++); - break; - } - - case bitc::METADATA_FILE: { - if (Record.size() != 3) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIFile, (Context, getMDString(Record[1]), - getMDString(Record[2]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_COMPILE_UNIT: { - if (Record.size() < 14 || Record.size() > 16) - return error("Invalid record"); - - // Ignore Record[0], which indicates whether this compile unit is - // distinct. It's always distinct. - IsDistinct = true; - auto *CU = DICompileUnit::getDistinct( - Context, Record[1], getMDOrNull(Record[2]), getMDString(Record[3]), - Record[4], getMDString(Record[5]), Record[6], getMDString(Record[7]), - Record[8], getMDOrNull(Record[9]), getMDOrNull(Record[10]), - getMDOrNull(Record[12]), getMDOrNull(Record[13]), - Record.size() <= 15 ? nullptr : getMDOrNull(Record[15]), - Record.size() <= 14 ? 0 : Record[14]); - - MetadataList.assignValue(CU, NextMetadataNo++); - - // Move the Upgrade the list of subprograms. - if (Metadata *SPs = getMDOrNullWithoutPlaceholders(Record[11])) - CUSubprograms.push_back({CU, SPs}); - break; - } - case bitc::METADATA_SUBPROGRAM: { - if (Record.size() < 18 || Record.size() > 20) - return error("Invalid record"); - - IsDistinct = - (Record[0] & 1) || Record[8]; // All definitions should be distinct. - // Version 1 has a Function as Record[15]. - // Version 2 has removed Record[15]. - // Version 3 has the Unit as Record[15]. - // Version 4 added thisAdjustment. - bool HasUnit = Record[0] >= 2; - if (HasUnit && Record.size() < 19) - return error("Invalid record"); - Metadata *CUorFn = getMDOrNull(Record[15]); - unsigned Offset = Record.size() >= 19 ? 1 : 0; - bool HasFn = Offset && !HasUnit; - bool HasThisAdj = Record.size() >= 20; - DISubprogram *SP = GET_OR_DISTINCT( - DISubprogram, (Context, - getDITypeRefOrNull(Record[1]), // scope - getMDString(Record[2]), // name - getMDString(Record[3]), // linkageName - getMDOrNull(Record[4]), // file - Record[5], // line - getMDOrNull(Record[6]), // type - Record[7], // isLocal - Record[8], // isDefinition - Record[9], // scopeLine - getDITypeRefOrNull(Record[10]), // containingType - Record[11], // virtuality - Record[12], // virtualIndex - HasThisAdj ? Record[19] : 0, // thisAdjustment - Record[13], // flags - Record[14], // isOptimized - HasUnit ? CUorFn : nullptr, // unit - getMDOrNull(Record[15 + Offset]), // templateParams - getMDOrNull(Record[16 + Offset]), // declaration - getMDOrNull(Record[17 + Offset]) // variables - )); - MetadataList.assignValue(SP, NextMetadataNo++); - - // Upgrade sp->function mapping to function->sp mapping. - if (HasFn) { - if (auto *CMD = dyn_cast_or_null<ConstantAsMetadata>(CUorFn)) - if (auto *F = dyn_cast<Function>(CMD->getValue())) { - if (F->isMaterializable()) - // Defer until materialized; unmaterialized functions may not have - // metadata. - FunctionsWithSPs[F] = SP; - else if (!F->empty()) - F->setSubprogram(SP); - } - } - break; - } - case bitc::METADATA_LEXICAL_BLOCK: { - if (Record.size() != 5) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DILexicalBlock, - (Context, getMDOrNull(Record[1]), - getMDOrNull(Record[2]), Record[3], Record[4])), - NextMetadataNo++); - break; - } - case bitc::METADATA_LEXICAL_BLOCK_FILE: { - if (Record.size() != 4) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DILexicalBlockFile, - (Context, getMDOrNull(Record[1]), - getMDOrNull(Record[2]), Record[3])), - NextMetadataNo++); - break; - } - case bitc::METADATA_NAMESPACE: { - if (Record.size() != 5) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DINamespace, (Context, getMDOrNull(Record[1]), - getMDOrNull(Record[2]), - getMDString(Record[3]), Record[4])), - NextMetadataNo++); - break; - } - case bitc::METADATA_MACRO: { - if (Record.size() != 5) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIMacro, - (Context, Record[1], Record[2], - getMDString(Record[3]), getMDString(Record[4]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_MACRO_FILE: { - if (Record.size() != 5) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIMacroFile, - (Context, Record[1], Record[2], - getMDOrNull(Record[3]), getMDOrNull(Record[4]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_TEMPLATE_TYPE: { - if (Record.size() != 3) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue(GET_OR_DISTINCT(DITemplateTypeParameter, - (Context, getMDString(Record[1]), - getDITypeRefOrNull(Record[2]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_TEMPLATE_VALUE: { - if (Record.size() != 5) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DITemplateValueParameter, - (Context, Record[1], getMDString(Record[2]), - getDITypeRefOrNull(Record[3]), - getMDOrNull(Record[4]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_GLOBAL_VAR: { - if (Record.size() != 11) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIGlobalVariable, - (Context, getMDOrNull(Record[1]), - getMDString(Record[2]), getMDString(Record[3]), - getMDOrNull(Record[4]), Record[5], - getDITypeRefOrNull(Record[6]), Record[7], Record[8], - getMDOrNull(Record[9]), getMDOrNull(Record[10]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_LOCAL_VAR: { - // 10th field is for the obseleted 'inlinedAt:' field. - if (Record.size() < 8 || Record.size() > 10) - return error("Invalid record"); - - // 2nd field used to be an artificial tag, either DW_TAG_auto_variable or - // DW_TAG_arg_variable. - IsDistinct = Record[0]; - bool HasTag = Record.size() > 8; - MetadataList.assignValue( - GET_OR_DISTINCT(DILocalVariable, - (Context, getMDOrNull(Record[1 + HasTag]), - getMDString(Record[2 + HasTag]), - getMDOrNull(Record[3 + HasTag]), Record[4 + HasTag], - getDITypeRefOrNull(Record[5 + HasTag]), - Record[6 + HasTag], Record[7 + HasTag])), - NextMetadataNo++); - break; - } - case bitc::METADATA_EXPRESSION: { - if (Record.size() < 1) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIExpression, - (Context, makeArrayRef(Record).slice(1))), - NextMetadataNo++); - break; - } - case bitc::METADATA_OBJC_PROPERTY: { - if (Record.size() != 8) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIObjCProperty, - (Context, getMDString(Record[1]), - getMDOrNull(Record[2]), Record[3], - getMDString(Record[4]), getMDString(Record[5]), - Record[6], getDITypeRefOrNull(Record[7]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_IMPORTED_ENTITY: { - if (Record.size() != 6) - return error("Invalid record"); - - IsDistinct = Record[0]; - MetadataList.assignValue( - GET_OR_DISTINCT(DIImportedEntity, - (Context, Record[1], getMDOrNull(Record[2]), - getDITypeRefOrNull(Record[3]), Record[4], - getMDString(Record[5]))), - NextMetadataNo++); - break; - } - case bitc::METADATA_STRING_OLD: { - std::string String(Record.begin(), Record.end()); - - // Test for upgrading !llvm.loop. - HasSeenOldLoopTags |= mayBeOldLoopAttachmentTag(String); - - Metadata *MD = MDString::get(Context, String); - MetadataList.assignValue(MD, NextMetadataNo++); - break; - } - case bitc::METADATA_STRINGS: - if (std::error_code EC = - parseMetadataStrings(Record, Blob, NextMetadataNo)) - return EC; - break; - case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { - if (Record.size() % 2 == 0) - return error("Invalid record"); - unsigned ValueID = Record[0]; - if (ValueID >= ValueList.size()) - return error("Invalid record"); - if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID])) - parseGlobalObjectAttachment(*GO, ArrayRef<uint64_t>(Record).slice(1)); - break; - } - case bitc::METADATA_KIND: { - // Support older bitcode files that had METADATA_KIND records in a - // block with METADATA_BLOCK_ID. - if (std::error_code EC = parseMetadataKindRecord(Record)) - return EC; - break; - } - } - } -#undef GET_OR_DISTINCT -} - -/// Parse the metadata kinds out of the METADATA_KIND_BLOCK. -std::error_code BitcodeReader::parseMetadataKinds() { - if (Stream.EnterSubBlock(bitc::METADATA_KIND_BLOCK_ID)) - return error("Invalid record"); - - SmallVector<uint64_t, 64> Record; - - // Read all the records. - while (1) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - - switch (Entry.Kind) { - case BitstreamEntry::SubBlock: // Handled for us already. - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return std::error_code(); - case BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read a record. - Record.clear(); - unsigned Code = Stream.readRecord(Entry.ID, Record); - switch (Code) { - default: // Default behavior: ignore. - break; - case bitc::METADATA_KIND: { - if (std::error_code EC = parseMetadataKindRecord(Record)) - return EC; - break; - } - } - } -} - /// Decode a signed value stored with the sign bit in the LSB for dense VBR /// encoding. uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) { @@ -2769,7 +1846,7 @@ uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) { } /// Resolve all of the initializers for global values and aliases that we can. -std::error_code BitcodeReader::resolveGlobalAndIndirectSymbolInits() { +Error BitcodeReader::resolveGlobalAndIndirectSymbolInits() { std::vector<std::pair<GlobalVariable*, unsigned> > GlobalInitWorklist; std::vector<std::pair<GlobalIndirectSymbol*, unsigned> > IndirectSymbolInitWorklist; @@ -2852,18 +1929,18 @@ std::error_code BitcodeReader::resolveGlobalAndIndirectSymbolInits() { FunctionPersonalityFnWorklist.pop_back(); } - return std::error_code(); + return Error::success(); } static APInt readWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) { SmallVector<uint64_t, 8> Words(Vals.size()); - std::transform(Vals.begin(), Vals.end(), Words.begin(), + transform(Vals, Words.begin(), BitcodeReader::decodeSignRotatedValue); return APInt(TypeBits, Words); } -std::error_code BitcodeReader::parseConstants() { +Error BitcodeReader::parseConstants() { if (Stream.EnterSubBlock(bitc::CONSTANTS_BLOCK_ID)) return error("Invalid record"); @@ -2872,7 +1949,8 @@ std::error_code BitcodeReader::parseConstants() { // Read all the records for this value table. Type *CurTy = Type::getInt32Ty(Context); unsigned NextCstNo = ValueList.size(); - while (1) { + + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -2886,7 +1964,7 @@ std::error_code BitcodeReader::parseConstants() { // Once all the constants have been read, go through and resolve forward // references. ValueList.resolveConstantForwardRefs(); - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -2933,26 +2011,26 @@ std::error_code BitcodeReader::parseConstants() { if (Record.empty()) return error("Invalid record"); if (CurTy->isHalfTy()) - V = ConstantFP::get(Context, APFloat(APFloat::IEEEhalf, + V = ConstantFP::get(Context, APFloat(APFloat::IEEEhalf(), APInt(16, (uint16_t)Record[0]))); else if (CurTy->isFloatTy()) - V = ConstantFP::get(Context, APFloat(APFloat::IEEEsingle, + V = ConstantFP::get(Context, APFloat(APFloat::IEEEsingle(), APInt(32, (uint32_t)Record[0]))); else if (CurTy->isDoubleTy()) - V = ConstantFP::get(Context, APFloat(APFloat::IEEEdouble, + V = ConstantFP::get(Context, APFloat(APFloat::IEEEdouble(), APInt(64, Record[0]))); else if (CurTy->isX86_FP80Ty()) { // Bits are not stored the same way as a normal i80 APInt, compensate. uint64_t Rearrange[2]; Rearrange[0] = (Record[1] & 0xffffLL) | (Record[0] << 16); Rearrange[1] = Record[0] >> 48; - V = ConstantFP::get(Context, APFloat(APFloat::x87DoubleExtended, + V = ConstantFP::get(Context, APFloat(APFloat::x87DoubleExtended(), APInt(80, Rearrange))); } else if (CurTy->isFP128Ty()) - V = ConstantFP::get(Context, APFloat(APFloat::IEEEquad, + V = ConstantFP::get(Context, APFloat(APFloat::IEEEquad(), APInt(128, Record))); else if (CurTy->isPPC_FP128Ty()) - V = ConstantFP::get(Context, APFloat(APFloat::PPCDoubleDouble, + V = ConstantFP::get(Context, APFloat(APFloat::PPCDoubleDouble(), APInt(128, Record))); else V = UndefValue::get(CurTy); @@ -3095,12 +2173,25 @@ std::error_code BitcodeReader::parseConstants() { } break; } - case bitc::CST_CODE_CE_INBOUNDS_GEP: - case bitc::CST_CODE_CE_GEP: { // CE_GEP: [n x operands] + case bitc::CST_CODE_CE_INBOUNDS_GEP: // [ty, n x operands] + case bitc::CST_CODE_CE_GEP: // [ty, n x operands] + case bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX: { // [ty, flags, n x + // operands] unsigned OpNum = 0; Type *PointeeType = nullptr; - if (Record.size() % 2) + if (BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX || + Record.size() % 2) PointeeType = getTypeByID(Record[OpNum++]); + + bool InBounds = false; + Optional<unsigned> InRangeIndex; + if (BitCode == bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX) { + uint64_t Op = Record[OpNum++]; + InBounds = Op & 1; + InRangeIndex = Op >> 1; + } else if (BitCode == bitc::CST_CODE_CE_INBOUNDS_GEP) + InBounds = true; + SmallVector<Constant*, 16> Elts; while (OpNum != Record.size()) { Type *ElTy = getTypeByID(Record[OpNum++]); @@ -3111,7 +2202,7 @@ std::error_code BitcodeReader::parseConstants() { if (PointeeType && PointeeType != - cast<SequentialType>(Elts[0]->getType()->getScalarType()) + cast<PointerType>(Elts[0]->getType()->getScalarType()) ->getElementType()) return error("Explicit gep operator type does not match pointee type " "of pointer operand"); @@ -3121,8 +2212,7 @@ std::error_code BitcodeReader::parseConstants() { ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end()); V = ConstantExpr::getGetElementPtr(PointeeType, Elts[0], Indices, - BitCode == - bitc::CST_CODE_CE_INBOUNDS_GEP); + InBounds, InRangeIndex); break; } case bitc::CST_CODE_CE_SELECT: { // CE_SELECT: [opval#, opval#, opval#] @@ -3326,13 +2416,14 @@ std::error_code BitcodeReader::parseConstants() { } } -std::error_code BitcodeReader::parseUseLists() { +Error BitcodeReader::parseUseLists() { if (Stream.EnterSubBlock(bitc::USELIST_BLOCK_ID)) return error("Invalid record"); // Read all the records. SmallVector<uint64_t, 64> Record; - while (1) { + + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -3340,7 +2431,7 @@ std::error_code BitcodeReader::parseUseLists() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -3354,7 +2445,7 @@ std::error_code BitcodeReader::parseUseLists() { break; case bitc::USELIST_CODE_BB: IsBB = true; - // fallthrough + LLVM_FALLTHROUGH; case bitc::USELIST_CODE_DEFAULT: { unsigned RecordLength = Record.size(); if (RecordLength < 3) @@ -3392,7 +2483,7 @@ std::error_code BitcodeReader::parseUseLists() { /// When we see the block for metadata, remember where it is and then skip it. /// This lets us lazily deserialize the metadata. -std::error_code BitcodeReader::rememberAndSkipMetadata() { +Error BitcodeReader::rememberAndSkipMetadata() { // Save the current stream state. uint64_t CurBit = Stream.GetCurrentBitNo(); DeferredMetadataInfo.push_back(CurBit); @@ -3400,25 +2491,25 @@ std::error_code BitcodeReader::rememberAndSkipMetadata() { // Skip over the block for now. if (Stream.SkipBlock()) return error("Invalid record"); - return std::error_code(); + return Error::success(); } -std::error_code BitcodeReader::materializeMetadata() { +Error BitcodeReader::materializeMetadata() { for (uint64_t BitPos : DeferredMetadataInfo) { // Move the bit stream to the saved position. Stream.JumpToBit(BitPos); - if (std::error_code EC = parseMetadata(true)) - return EC; + if (Error Err = MDLoader->parseModuleMetadata()) + return Err; } DeferredMetadataInfo.clear(); - return std::error_code(); + return Error::success(); } void BitcodeReader::setStripDebugInfo() { StripDebugInfo = true; } /// When we see the block for a function body, remember where it is and then /// skip it. This lets us lazily deserialize the functions. -std::error_code BitcodeReader::rememberAndSkipFunctionBody() { +Error BitcodeReader::rememberAndSkipFunctionBody() { // Get the function we are talking about. if (FunctionsWithBodies.empty()) return error("Insufficient function protos"); @@ -3436,12 +2527,13 @@ std::error_code BitcodeReader::rememberAndSkipFunctionBody() { // Skip over the function block for now. if (Stream.SkipBlock()) return error("Invalid record"); - return std::error_code(); + return Error::success(); } -std::error_code BitcodeReader::globalCleanup() { +Error BitcodeReader::globalCleanup() { // Patch the initializers for globals and aliases up. - resolveGlobalAndIndirectSymbolInits(); + if (Error Err = resolveGlobalAndIndirectSymbolInits()) + return Err; if (!GlobalInits.empty() || !IndirectSymbolInits.empty()) return error("Malformed global initializer set"); @@ -3466,14 +2558,14 @@ std::error_code BitcodeReader::globalCleanup() { std::vector<std::pair<GlobalVariable*, unsigned> >().swap(GlobalInits); std::vector<std::pair<GlobalIndirectSymbol*, unsigned> >().swap( IndirectSymbolInits); - return std::error_code(); + return Error::success(); } /// Support for lazy parsing of function bodies. This is required if we /// either have an old bitcode file without a VST forward declaration record, /// or if we have an anonymous function being materialized, since anonymous /// functions do not have a name and are therefore not in the VST. -std::error_code BitcodeReader::rememberAndSkipFunctionBodies() { +Error BitcodeReader::rememberAndSkipFunctionBodies() { Stream.JumpToBit(NextUnreadBit); if (Stream.AtEndOfStream()) @@ -3488,7 +2580,7 @@ std::error_code BitcodeReader::rememberAndSkipFunctionBodies() { SmallVector<uint64_t, 64> Record; - while (1) { + while (true) { BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { default: @@ -3498,60 +2590,25 @@ std::error_code BitcodeReader::rememberAndSkipFunctionBodies() { default: return error("Expect function block"); case bitc::FUNCTION_BLOCK_ID: - if (std::error_code EC = rememberAndSkipFunctionBody()) - return EC; + if (Error Err = rememberAndSkipFunctionBody()) + return Err; NextUnreadBit = Stream.GetCurrentBitNo(); - return std::error_code(); + return Error::success(); } } } } -std::error_code BitcodeReader::parseBitcodeVersion() { - if (Stream.EnterSubBlock(bitc::IDENTIFICATION_BLOCK_ID)) - return error("Invalid record"); - - // Read all the records. - SmallVector<uint64_t, 64> Record; - while (1) { - BitstreamEntry Entry = Stream.advance(); - - switch (Entry.Kind) { - default: - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return std::error_code(); - case BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read a record. - Record.clear(); - unsigned BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { - default: // Default behavior: reject - return error("Invalid value"); - case bitc::IDENTIFICATION_CODE_STRING: { // IDENTIFICATION: [strchr x - // N] - convertToString(Record, 0, ProducerIdentification); - break; - } - case bitc::IDENTIFICATION_CODE_EPOCH: { // EPOCH: [epoch#] - unsigned epoch = (unsigned)Record[0]; - if (epoch != bitc::BITCODE_CURRENT_EPOCH) { - return error( - Twine("Incompatible epoch: Bitcode '") + Twine(epoch) + - "' vs current: '" + Twine(bitc::BITCODE_CURRENT_EPOCH) + "'"); - } - } - } - } +bool BitcodeReaderBase::readBlockInfo() { + Optional<BitstreamBlockInfo> NewBlockInfo = Stream.ReadBlockInfoBlock(); + if (!NewBlockInfo) + return true; + BlockInfo = std::move(*NewBlockInfo); + return false; } -std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, - bool ShouldLazyLoadMetadata) { +Error BitcodeReader::parseModule(uint64_t ResumeBit, + bool ShouldLazyLoadMetadata) { if (ResumeBit) Stream.JumpToBit(ResumeBit); else if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) @@ -3562,7 +2619,7 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, std::vector<std::string> GCTable; // Read all the records for this module. - while (1) { + while (true) { BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { @@ -3578,20 +2635,20 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, return error("Invalid record"); break; case bitc::BLOCKINFO_BLOCK_ID: - if (Stream.ReadBlockInfoBlock()) + if (readBlockInfo()) return error("Malformed block"); break; case bitc::PARAMATTR_BLOCK_ID: - if (std::error_code EC = parseAttributeBlock()) - return EC; + if (Error Err = parseAttributeBlock()) + return Err; break; case bitc::PARAMATTR_GROUP_BLOCK_ID: - if (std::error_code EC = parseAttributeGroupBlock()) - return EC; + if (Error Err = parseAttributeGroupBlock()) + return Err; break; case bitc::TYPE_BLOCK_ID_NEW: - if (std::error_code EC = parseTypeTable()) - return EC; + if (Error Err = parseTypeTable()) + return Err; break; case bitc::VALUE_SYMTAB_BLOCK_ID: if (!SeenValueSymbolTable) { @@ -3601,8 +2658,8 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, // normally in the stream), or there were no function blocks to // trigger an earlier parsing of the VST. assert(VSTOffset == 0 || FunctionsWithBodies.empty()); - if (std::error_code EC = parseValueSymbolTable()) - return EC; + if (Error Err = parseValueSymbolTable()) + return Err; SeenValueSymbolTable = true; } else { // We must have had a VST forward declaration record, which caused @@ -3613,32 +2670,32 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, } break; case bitc::CONSTANTS_BLOCK_ID: - if (std::error_code EC = parseConstants()) - return EC; - if (std::error_code EC = resolveGlobalAndIndirectSymbolInits()) - return EC; + if (Error Err = parseConstants()) + return Err; + if (Error Err = resolveGlobalAndIndirectSymbolInits()) + return Err; break; case bitc::METADATA_BLOCK_ID: - if (ShouldLazyLoadMetadata && !IsMetadataMaterialized) { - if (std::error_code EC = rememberAndSkipMetadata()) - return EC; + if (ShouldLazyLoadMetadata) { + if (Error Err = rememberAndSkipMetadata()) + return Err; break; } assert(DeferredMetadataInfo.empty() && "Unexpected deferred metadata"); - if (std::error_code EC = parseMetadata(true)) - return EC; + if (Error Err = MDLoader->parseModuleMetadata()) + return Err; break; case bitc::METADATA_KIND_BLOCK_ID: - if (std::error_code EC = parseMetadataKinds()) - return EC; + if (Error Err = MDLoader->parseMetadataKinds()) + return Err; break; case bitc::FUNCTION_BLOCK_ID: // If this is the first function body we've seen, reverse the // FunctionsWithBodies list. if (!SeenFirstFunctionBody) { std::reverse(FunctionsWithBodies.begin(), FunctionsWithBodies.end()); - if (std::error_code EC = globalCleanup()) - return EC; + if (Error Err = globalCleanup()) + return Err; SeenFirstFunctionBody = true; } @@ -3647,9 +2704,8 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, // parse the VST now if we haven't already. It is needed to // set up the DeferredFunctionInfo vector for lazy reading. if (!SeenValueSymbolTable) { - if (std::error_code EC = - BitcodeReader::parseValueSymbolTable(VSTOffset)) - return EC; + if (Error Err = BitcodeReader::parseValueSymbolTable(VSTOffset)) + return Err; SeenValueSymbolTable = true; // Fall through so that we record the NextUnreadBit below. // This is necessary in case we have an anonymous function that @@ -3672,8 +2728,8 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, // index in the VST, nor a VST forward declaration record, as // well as anonymous functions that do not have VST entries. // Build the DeferredFunctionInfo vector on the fly. - if (std::error_code EC = rememberAndSkipFunctionBody()) - return EC; + if (Error Err = rememberAndSkipFunctionBody()) + return Err; // Suspend parsing when we reach the function bodies. Subsequent // materialization calls will resume it when necessary. If the bitcode @@ -3681,16 +2737,18 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, // have been seen yet. In this case, just finish the parse now. if (SeenValueSymbolTable) { NextUnreadBit = Stream.GetCurrentBitNo(); - return std::error_code(); + // After the VST has been parsed, we need to make sure intrinsic name + // are auto-upgraded. + return globalCleanup(); } break; case bitc::USELIST_BLOCK_ID: - if (std::error_code EC = parseUseLists()) - return EC; + if (Error Err = parseUseLists()) + return Err; break; case bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID: - if (std::error_code EC = parseOperandBundleTags()) - return EC; + if (Error Err = parseOperandBundleTags()) + return Err; break; } continue; @@ -3803,8 +2861,8 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, uint64_t RawLinkage = Record[3]; GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); unsigned Alignment; - if (std::error_code EC = parseAlignmentValue(Record[4], Alignment)) - return EC; + if (Error Err = parseAlignmentValue(Record[4], Alignment)) + return Err; std::string Section; if (Record[5]) { if (Record[5]-1 >= SectionTable.size()) @@ -3889,8 +2947,8 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, Func->setAttributes(getAttributes(Record[4])); unsigned Alignment; - if (std::error_code EC = parseAlignmentValue(Record[5], Alignment)) - return EC; + if (Error Err = parseAlignmentValue(Record[5], Alignment)) + return Err; Func->setAlignment(Alignment); if (Record[6]) { if (Record[6]-1 >= SectionTable.size()) @@ -4011,7 +3069,10 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, case bitc::MODULE_CODE_VSTOFFSET: if (Record.size() < 1) return error("Invalid record"); - VSTOffset = Record[0]; + // Note that we subtract 1 here because the offset is relative to one word + // before the start of the identification or module block, which was + // historically always the start of the regular bitcode header. + VSTOffset = Record[0] - 1; break; /// MODULE_CODE_SOURCE_FILENAME: [namechar x N] case bitc::MODULE_CODE_SOURCE_FILENAME: @@ -4025,350 +3086,40 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, } } -/// Helper to read the header common to all bitcode files. -static bool hasValidBitcodeHeader(BitstreamCursor &Stream) { - // Sniff for the signature. - if (Stream.Read(8) != 'B' || - Stream.Read(8) != 'C' || - Stream.Read(4) != 0x0 || - Stream.Read(4) != 0xC || - Stream.Read(4) != 0xE || - Stream.Read(4) != 0xD) - return false; - return true; -} - -std::error_code -BitcodeReader::parseBitcodeInto(std::unique_ptr<DataStreamer> Streamer, - Module *M, bool ShouldLazyLoadMetadata) { +Error BitcodeReader::parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata, + bool IsImporting) { TheModule = M; - - if (std::error_code EC = initStream(std::move(Streamer))) - return EC; - - // Sniff for the signature. - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); - - // We expect a number of well-defined blocks, though we don't necessarily - // need to understand them all. - while (1) { - if (Stream.AtEndOfStream()) { - // We didn't really read a proper Module. - return error("Malformed IR file"); - } - - BitstreamEntry Entry = - Stream.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs); - - if (Entry.Kind != BitstreamEntry::SubBlock) - return error("Malformed block"); - - if (Entry.ID == bitc::IDENTIFICATION_BLOCK_ID) { - parseBitcodeVersion(); - continue; - } - - if (Entry.ID == bitc::MODULE_BLOCK_ID) - return parseModule(0, ShouldLazyLoadMetadata); - - if (Stream.SkipBlock()) - return error("Invalid record"); - } -} - -ErrorOr<std::string> BitcodeReader::parseModuleTriple() { - if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); - - SmallVector<uint64_t, 64> Record; - - std::string Triple; - // Read all the records for this module. - while (1) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - - switch (Entry.Kind) { - case BitstreamEntry::SubBlock: // Handled for us already. - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return Triple; - case BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read a record. - switch (Stream.readRecord(Entry.ID, Record)) { - default: break; // Default behavior, ignore unknown content. - case bitc::MODULE_CODE_TRIPLE: { // TRIPLE: [strchr x N] - std::string S; - if (convertToString(Record, 0, S)) - return error("Invalid record"); - Triple = S; - break; - } - } - Record.clear(); - } - llvm_unreachable("Exit infinite loop"); + MDLoader = MetadataLoader(Stream, *M, ValueList, IsImporting, + [&](unsigned ID) { return getTypeByID(ID); }); + return parseModule(0, ShouldLazyLoadMetadata); } -ErrorOr<std::string> BitcodeReader::parseTriple() { - if (std::error_code EC = initStream(nullptr)) - return EC; - // Sniff for the signature. - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); - - // We expect a number of well-defined blocks, though we don't necessarily - // need to understand them all. - while (1) { - BitstreamEntry Entry = Stream.advance(); - - switch (Entry.Kind) { - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return std::error_code(); - - case BitstreamEntry::SubBlock: - if (Entry.ID == bitc::MODULE_BLOCK_ID) - return parseModuleTriple(); - - // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); - continue; - - case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; - } - } -} - -ErrorOr<std::string> BitcodeReader::parseIdentificationBlock() { - if (std::error_code EC = initStream(nullptr)) - return EC; - - // Sniff for the signature. - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); - - // We expect a number of well-defined blocks, though we don't necessarily - // need to understand them all. - while (1) { - BitstreamEntry Entry = Stream.advance(); - switch (Entry.Kind) { - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return std::error_code(); - - case BitstreamEntry::SubBlock: - if (Entry.ID == bitc::IDENTIFICATION_BLOCK_ID) { - if (std::error_code EC = parseBitcodeVersion()) - return EC; - return ProducerIdentification; - } - // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); - continue; - case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; - } - } -} - -std::error_code BitcodeReader::parseGlobalObjectAttachment( - GlobalObject &GO, ArrayRef<uint64_t> Record) { - assert(Record.size() % 2 == 0); - for (unsigned I = 0, E = Record.size(); I != E; I += 2) { - auto K = MDKindMap.find(Record[I]); - if (K == MDKindMap.end()) - return error("Invalid ID"); - MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[I + 1]); - if (!MD) - return error("Invalid metadata attachment"); - GO.addMetadata(K->second, *MD); - } - return std::error_code(); -} - -ErrorOr<bool> BitcodeReader::hasObjCCategory() { - if (std::error_code EC = initStream(nullptr)) - return EC; - - // Sniff for the signature. - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); - - // We expect a number of well-defined blocks, though we don't necessarily - // need to understand them all. - while (1) { - BitstreamEntry Entry = Stream.advance(); - - switch (Entry.Kind) { - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return std::error_code(); - - case BitstreamEntry::SubBlock: - if (Entry.ID == bitc::MODULE_BLOCK_ID) - return hasObjCCategoryInModule(); - - // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); - continue; - - case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; - } - } -} - -ErrorOr<bool> BitcodeReader::hasObjCCategoryInModule() { - if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); - - SmallVector<uint64_t, 64> Record; - // Read all the records for this module. - while (1) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - - switch (Entry.Kind) { - case BitstreamEntry::SubBlock: // Handled for us already. - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return false; - case BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read a record. - switch (Stream.readRecord(Entry.ID, Record)) { - default: - break; // Default behavior, ignore unknown content. - case bitc::MODULE_CODE_SECTIONNAME: { // SECTIONNAME: [strchr x N] - std::string S; - if (convertToString(Record, 0, S)) - return error("Invalid record"); - // Check for the i386 and other (x86_64, ARM) conventions - if (S.find("__DATA, __objc_catlist") != std::string::npos || - S.find("__OBJC,__category") != std::string::npos) - return true; - break; - } - } - Record.clear(); - } - llvm_unreachable("Exit infinite loop"); -} - -/// Parse metadata attachments. -std::error_code BitcodeReader::parseMetadataAttachment(Function &F) { - if (Stream.EnterSubBlock(bitc::METADATA_ATTACHMENT_ID)) - return error("Invalid record"); - - SmallVector<uint64_t, 64> Record; - while (1) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - - switch (Entry.Kind) { - case BitstreamEntry::SubBlock: // Handled for us already. - case BitstreamEntry::Error: - return error("Malformed block"); - case BitstreamEntry::EndBlock: - return std::error_code(); - case BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read a metadata attachment record. - Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { - default: // Default behavior: ignore. - break; - case bitc::METADATA_ATTACHMENT: { - unsigned RecordLength = Record.size(); - if (Record.empty()) - return error("Invalid record"); - if (RecordLength % 2 == 0) { - // A function attachment. - if (std::error_code EC = parseGlobalObjectAttachment(F, Record)) - return EC; - continue; - } - - // An instruction attachment. - Instruction *Inst = InstructionList[Record[0]]; - for (unsigned i = 1; i != RecordLength; i = i+2) { - unsigned Kind = Record[i]; - DenseMap<unsigned, unsigned>::iterator I = - MDKindMap.find(Kind); - if (I == MDKindMap.end()) - return error("Invalid ID"); - Metadata *Node = MetadataList.getMetadataFwdRef(Record[i + 1]); - if (isa<LocalAsMetadata>(Node)) - // Drop the attachment. This used to be legal, but there's no - // upgrade path. - break; - MDNode *MD = dyn_cast_or_null<MDNode>(Node); - if (!MD) - return error("Invalid metadata attachment"); - - if (HasSeenOldLoopTags && I->second == LLVMContext::MD_loop) - MD = upgradeInstructionLoopAttachment(*MD); - - Inst->setMetadata(I->second, MD); - if (I->second == LLVMContext::MD_tbaa) { - InstsWithTBAATag.push_back(Inst); - continue; - } - } - break; - } - } - } -} - -static std::error_code typeCheckLoadStoreInst(Type *ValType, Type *PtrType) { - LLVMContext &Context = PtrType->getContext(); +Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) { if (!isa<PointerType>(PtrType)) - return error(Context, "Load/Store operand is not a pointer type"); + return error("Load/Store operand is not a pointer type"); Type *ElemType = cast<PointerType>(PtrType)->getElementType(); if (ValType && ValType != ElemType) - return error(Context, "Explicit load/store type does not match pointee " - "type of pointer operand"); + return error("Explicit load/store type does not match pointee " + "type of pointer operand"); if (!PointerType::isLoadableOrStorableType(ElemType)) - return error(Context, "Cannot load/store from pointer"); - return std::error_code(); + return error("Cannot load/store from pointer"); + return Error::success(); } /// Lazily parse the specified function body block. -std::error_code BitcodeReader::parseFunctionBody(Function *F) { +Error BitcodeReader::parseFunctionBody(Function *F) { if (Stream.EnterSubBlock(bitc::FUNCTION_BLOCK_ID)) return error("Invalid record"); // Unexpected unresolved metadata when parsing function. - if (MetadataList.hasFwdRefs()) + if (MDLoader->hasFwdRefs()) return error("Invalid function metadata: incoming forward references"); InstructionList.clear(); unsigned ModuleValueListSize = ValueList.size(); - unsigned ModuleMetadataListSize = MetadataList.size(); + unsigned ModuleMDLoaderSize = MDLoader->size(); // Add all the function arguments to the value table. for (Argument &I : F->args()) @@ -4392,7 +3143,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { // Read all the records. SmallVector<uint64_t, 64> Record; - while (1) { + + while (true) { BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { @@ -4408,25 +3160,27 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); break; case bitc::CONSTANTS_BLOCK_ID: - if (std::error_code EC = parseConstants()) - return EC; + if (Error Err = parseConstants()) + return Err; NextValueNo = ValueList.size(); break; case bitc::VALUE_SYMTAB_BLOCK_ID: - if (std::error_code EC = parseValueSymbolTable()) - return EC; + if (Error Err = parseValueSymbolTable()) + return Err; break; case bitc::METADATA_ATTACHMENT_ID: - if (std::error_code EC = parseMetadataAttachment(*F)) - return EC; + if (Error Err = MDLoader->parseMetadataAttachment(*F, InstructionList)) + return Err; break; case bitc::METADATA_BLOCK_ID: - if (std::error_code EC = parseMetadata()) - return EC; + assert(DeferredMetadataInfo.empty() && + "Must read all module-level metadata before function-level"); + if (Error Err = MDLoader->parseFunctionMetadata()) + return Err; break; case bitc::USELIST_BLOCK_ID: - if (std::error_code EC = parseUseLists()) - return EC; + if (Error Err = parseUseLists()) + return Err; break; } continue; @@ -4499,12 +3253,12 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { MDNode *Scope = nullptr, *IA = nullptr; if (ScopeID) { - Scope = MetadataList.getMDNodeFwdRefOrNull(ScopeID - 1); + Scope = MDLoader->getMDNodeFwdRefOrNull(ScopeID - 1); if (!Scope) return error("Invalid record"); } if (IAID) { - IA = MetadataList.getMDNodeFwdRefOrNull(IAID - 1); + IA = MDLoader->getMDNodeFwdRefOrNull(IAID - 1); if (!IA) return error("Invalid record"); } @@ -4598,10 +3352,10 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); if (!Ty) - Ty = cast<SequentialType>(BasePtr->getType()->getScalarType()) + Ty = cast<PointerType>(BasePtr->getType()->getScalarType()) ->getElementType(); else if (Ty != - cast<SequentialType>(BasePtr->getType()->getScalarType()) + cast<PointerType>(BasePtr->getType()->getScalarType()) ->getElementType()) return error( "Explicit gep type does not match pointee type of pointer operand"); @@ -5258,9 +4012,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { Type *OpTy = getTypeByID(Record[1]); Value *Size = getFnValueByID(Record[2], OpTy); unsigned Align; - if (std::error_code EC = - parseAlignmentValue(AlignRecord & ~FlagMask, Align)) { - return EC; + if (Error Err = parseAlignmentValue(AlignRecord & ~FlagMask, Align)) { + return Err; } if (!Ty || !Size) return error("Invalid record"); @@ -5281,14 +4034,14 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { Type *Ty = nullptr; if (OpNum + 3 == Record.size()) Ty = getTypeByID(Record[OpNum++]); - if (std::error_code EC = typeCheckLoadStoreInst(Ty, Op->getType())) - return EC; + if (Error Err = typeCheckLoadStoreInst(Ty, Op->getType())) + return Err; if (!Ty) Ty = cast<PointerType>(Op->getType())->getElementType(); unsigned Align; - if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align)) - return EC; + if (Error Err = parseAlignmentValue(Record[OpNum], Align)) + return Err; I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align); InstructionList.push_back(I); @@ -5305,8 +4058,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { Type *Ty = nullptr; if (OpNum + 5 == Record.size()) Ty = getTypeByID(Record[OpNum++]); - if (std::error_code EC = typeCheckLoadStoreInst(Ty, Op->getType())) - return EC; + if (Error Err = typeCheckLoadStoreInst(Ty, Op->getType())) + return Err; if (!Ty) Ty = cast<PointerType>(Op->getType())->getElementType(); @@ -5320,8 +4073,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); unsigned Align; - if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align)) - return EC; + if (Error Err = parseAlignmentValue(Record[OpNum], Align)) + return Err; I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SynchScope); InstructionList.push_back(I); @@ -5340,12 +4093,11 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { OpNum + 2 != Record.size()) return error("Invalid record"); - if (std::error_code EC = - typeCheckLoadStoreInst(Val->getType(), Ptr->getType())) - return EC; + if (Error Err = typeCheckLoadStoreInst(Val->getType(), Ptr->getType())) + return Err; unsigned Align; - if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align)) - return EC; + if (Error Err = parseAlignmentValue(Record[OpNum], Align)) + return Err; I = new StoreInst(Val, Ptr, Record[OpNum+1], Align); InstructionList.push_back(I); break; @@ -5365,9 +4117,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { OpNum + 4 != Record.size()) return error("Invalid record"); - if (std::error_code EC = - typeCheckLoadStoreInst(Val->getType(), Ptr->getType())) - return EC; + if (Error Err = typeCheckLoadStoreInst(Val->getType(), Ptr->getType())) + return Err; AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]); if (Ordering == AtomicOrdering::NotAtomic || Ordering == AtomicOrdering::Acquire || @@ -5378,8 +4129,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); unsigned Align; - if (std::error_code EC = parseAlignmentValue(Record[OpNum], Align)) - return EC; + if (Error Err = parseAlignmentValue(Record[OpNum], Align)) + return Err; I = new StoreInst(Val, Ptr, Record[OpNum+1], Align, Ordering, SynchScope); InstructionList.push_back(I); break; @@ -5405,9 +4156,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 2]); - if (std::error_code EC = - typeCheckLoadStoreInst(Cmp->getType(), Ptr->getType())) - return EC; + if (Error Err = typeCheckLoadStoreInst(Cmp->getType(), Ptr->getType())) + return Err; AtomicOrdering FailureOrdering; if (Record.size() < 7) FailureOrdering = @@ -5633,18 +4383,18 @@ OutOfRecordLoop: } // Unexpected unresolved metadata about to be dropped. - if (MetadataList.hasFwdRefs()) + if (MDLoader->hasFwdRefs()) return error("Invalid function metadata: outgoing forward refs"); // Trim the value list down to the size it was before we parsed this function. ValueList.shrinkTo(ModuleValueListSize); - MetadataList.shrinkTo(ModuleMetadataListSize); + MDLoader->shrinkTo(ModuleMDLoaderSize); std::vector<BasicBlock*>().swap(FunctionBBs); - return std::error_code(); + return Error::success(); } /// Find the function body in the bitcode stream -std::error_code BitcodeReader::findFunctionInStream( +Error BitcodeReader::findFunctionInStream( Function *F, DenseMap<Function *, uint64_t>::iterator DeferredFunctionInfoIterator) { while (DeferredFunctionInfoIterator->second == 0) { @@ -5655,41 +4405,39 @@ std::error_code BitcodeReader::findFunctionInStream( assert(VSTOffset == 0 || !F->hasName()); // Parse the next body in the stream and set its position in the // DeferredFunctionInfo map. - if (std::error_code EC = rememberAndSkipFunctionBodies()) - return EC; + if (Error Err = rememberAndSkipFunctionBodies()) + return Err; } - return std::error_code(); + return Error::success(); } //===----------------------------------------------------------------------===// // GVMaterializer implementation //===----------------------------------------------------------------------===// -void BitcodeReader::releaseBuffer() { Buffer.release(); } - -std::error_code BitcodeReader::materialize(GlobalValue *GV) { +Error BitcodeReader::materialize(GlobalValue *GV) { Function *F = dyn_cast<Function>(GV); // If it's not a function or is already material, ignore the request. if (!F || !F->isMaterializable()) - return std::error_code(); + return Error::success(); DenseMap<Function*, uint64_t>::iterator DFII = DeferredFunctionInfo.find(F); assert(DFII != DeferredFunctionInfo.end() && "Deferred function not found!"); // If its position is recorded as 0, its body is somewhere in the stream // but we haven't seen it yet. if (DFII->second == 0) - if (std::error_code EC = findFunctionInStream(F, DFII)) - return EC; + if (Error Err = findFunctionInStream(F, DFII)) + return Err; // Materialize metadata before parsing any function bodies. - if (std::error_code EC = materializeMetadata()) - return EC; + if (Error Err = materializeMetadata()) + return Err; // Move the bit stream to the saved position of the deferred function body. Stream.JumpToBit(DFII->second); - if (std::error_code EC = parseFunctionBody(F)) - return EC; + if (Error Err = parseFunctionBody(F)) + return Err; F->setIsMaterializable(false); if (StripDebugInfo) @@ -5714,17 +4462,28 @@ std::error_code BitcodeReader::materialize(GlobalValue *GV) { CallSite(*UI++).setCalledFunction(I.second); // Finish fn->subprogram upgrade for materialized functions. - if (DISubprogram *SP = FunctionsWithSPs.lookup(F)) + if (DISubprogram *SP = MDLoader->lookupSubprogramForFunction(F)) F->setSubprogram(SP); + // Check if the TBAA Metadata are valid, otherwise we will need to strip them. + if (!MDLoader->isStrippingTBAA()) { + for (auto &I : instructions(F)) { + MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa); + if (!TBAA || TBAAVerifyHelper.visitTBAAMetadata(I, TBAA)) + continue; + MDLoader->setStripTBAA(true); + stripTBAA(F->getParent()); + } + } + // Bring in any functions that this function forward-referenced via // blockaddresses. return materializeForwardReferencedFunctions(); } -std::error_code BitcodeReader::materializeModule() { - if (std::error_code EC = materializeMetadata()) - return EC; +Error BitcodeReader::materializeModule() { + if (Error Err = materializeMetadata()) + return Err; // Promise to materialize all forward references. WillMaterializeAllForwardRefs = true; @@ -5732,26 +4491,23 @@ std::error_code BitcodeReader::materializeModule() { // Iterate over the module, deserializing any functions that are still on // disk. for (Function &F : *TheModule) { - if (std::error_code EC = materialize(&F)) - return EC; + if (Error Err = materialize(&F)) + return Err; } // At this point, if there are any function bodies, parse the rest of // the bits in the module past the last function block we have recorded // through either lazy scanning or the VST. if (LastFunctionBlockBit || NextUnreadBit) - parseModule(LastFunctionBlockBit > NextUnreadBit ? LastFunctionBlockBit - : NextUnreadBit); + if (Error Err = parseModule(LastFunctionBlockBit > NextUnreadBit + ? LastFunctionBlockBit + : NextUnreadBit)) + return Err; // Check that all block address forward references got resolved (as we // promised above). if (!BasicBlockFwdRefs.empty()) return error("Never resolved function from blockaddress"); - // Upgrading intrinsic calls before TBAA can cause TBAA metadata to be lost, - // to prevent this instructions with TBAA tags should be upgraded first. - for (unsigned I = 0, E = InstsWithTBAATag.size(); I < E; I++) - UpgradeInstWithTBAATag(InstsWithTBAATag[I]); - // Upgrade any intrinsic calls that slipped through (should not happen!) and // delete the old functions to clean up. We can't do this unless the entire // module is materialized because there could always be another function body @@ -5776,80 +4532,16 @@ std::error_code BitcodeReader::materializeModule() { UpgradeDebugInfo(*TheModule); UpgradeModuleFlags(*TheModule); - return std::error_code(); + return Error::success(); } std::vector<StructType *> BitcodeReader::getIdentifiedStructTypes() const { return IdentifiedStructTypes; } -std::error_code -BitcodeReader::initStream(std::unique_ptr<DataStreamer> Streamer) { - if (Streamer) - return initLazyStream(std::move(Streamer)); - return initStreamFromBuffer(); -} - -std::error_code BitcodeReader::initStreamFromBuffer() { - const unsigned char *BufPtr = (const unsigned char*)Buffer->getBufferStart(); - const unsigned char *BufEnd = BufPtr+Buffer->getBufferSize(); - - if (Buffer->getBufferSize() & 3) - return error("Invalid bitcode signature"); - - // If we have a wrapper header, parse it and ignore the non-bc file contents. - // The magic number is 0x0B17C0DE stored in little endian. - if (isBitcodeWrapper(BufPtr, BufEnd)) - if (SkipBitcodeWrapperHeader(BufPtr, BufEnd, true)) - return error("Invalid bitcode wrapper header"); - - StreamFile.reset(new BitstreamReader(BufPtr, BufEnd)); - Stream.init(&*StreamFile); - - return std::error_code(); -} - -std::error_code -BitcodeReader::initLazyStream(std::unique_ptr<DataStreamer> Streamer) { - // Check and strip off the bitcode wrapper; BitstreamReader expects never to - // see it. - auto OwnedBytes = - llvm::make_unique<StreamingMemoryObject>(std::move(Streamer)); - StreamingMemoryObject &Bytes = *OwnedBytes; - StreamFile = llvm::make_unique<BitstreamReader>(std::move(OwnedBytes)); - Stream.init(&*StreamFile); - - unsigned char buf[16]; - if (Bytes.readBytes(buf, 16, 0) != 16) - return error("Invalid bitcode signature"); - - if (!isBitcode(buf, buf + 16)) - return error("Invalid bitcode signature"); - - if (isBitcodeWrapper(buf, buf + 4)) { - const unsigned char *bitcodeStart = buf; - const unsigned char *bitcodeEnd = buf + 16; - SkipBitcodeWrapperHeader(bitcodeStart, bitcodeEnd, false); - Bytes.dropLeadingBytes(bitcodeStart - buf); - Bytes.setKnownObjectSize(bitcodeEnd - bitcodeStart); - } - return std::error_code(); -} - -std::error_code ModuleSummaryIndexBitcodeReader::error(const Twine &Message) { - return ::error(DiagnosticHandler, - make_error_code(BitcodeError::CorruptedBitcode), Message); -} - ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader( - MemoryBuffer *Buffer, DiagnosticHandlerFunction DiagnosticHandler, - bool CheckGlobalValSummaryPresenceOnly) - : DiagnosticHandler(std::move(DiagnosticHandler)), Buffer(Buffer), - CheckGlobalValSummaryPresenceOnly(CheckGlobalValSummaryPresenceOnly) {} - -void ModuleSummaryIndexBitcodeReader::freeState() { Buffer = nullptr; } - -void ModuleSummaryIndexBitcodeReader::releaseBuffer() { Buffer.release(); } + BitstreamCursor Cursor, ModuleSummaryIndex &TheIndex) + : BitcodeReaderBase(std::move(Cursor)), TheIndex(TheIndex) {} std::pair<GlobalValue::GUID, GlobalValue::GUID> ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) { @@ -5861,7 +4553,7 @@ ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) { // Specialized value symbol table parser used when reading module index // blocks where we don't actually create global values. The parsed information // is saved in the bitcode reader for use when later parsing summaries. -std::error_code ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( +Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( uint64_t Offset, DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap) { assert(Offset > 0 && "Expected non-zero VST offset"); @@ -5874,7 +4566,8 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( // Read all the records for this value table. SmallString<128> ValueName; - while (1) { + + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -5884,7 +4577,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( case BitstreamEntry::EndBlock: // Done parsing VST, jump back to wherever we came from. Stream.JumpToBit(CurrentBit); - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -5959,7 +4652,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( // Parse just the blocks needed for building the index out of the module. // At the end of this routine the module Index is populated with a map // from global value id to GlobalValueSummary objects. -std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { +Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) return error("Invalid record"); @@ -5968,26 +4661,16 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { unsigned ValueId = 0; // Read the index for this module. - while (1) { + while (true) { BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return std::error_code(); + return Error::success(); case BitstreamEntry::SubBlock: - if (CheckGlobalValSummaryPresenceOnly) { - if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) { - SeenGlobalValSummary = true; - // No need to parse the rest since we found the summary. - return std::error_code(); - } - if (Stream.SkipBlock()) - return error("Invalid record"); - continue; - } switch (Entry.ID) { default: // Skip unknown content. if (Stream.SkipBlock()) @@ -5995,7 +4678,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { break; case bitc::BLOCKINFO_BLOCK_ID: // Need to parse these to get abbrev ids (e.g. for VST) - if (Stream.ReadBlockInfoBlock()) + if (readBlockInfo()) return error("Malformed block"); break; case bitc::VALUE_SYMTAB_BLOCK_ID: @@ -6008,20 +4691,24 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { return error("Invalid record"); break; case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: - assert(VSTOffset > 0 && "Expected non-zero VST offset"); assert(!SeenValueSymbolTable && "Already read VST when parsing summary block?"); - if (std::error_code EC = - parseValueSymbolTable(VSTOffset, ValueIdToLinkageMap)) - return EC; - SeenValueSymbolTable = true; + // We might not have a VST if there were no values in the + // summary. An empty summary block generated when we are + // performing ThinLTO compiles so we don't later invoke + // the regular LTO process on them. + if (VSTOffset > 0) { + if (Error Err = parseValueSymbolTable(VSTOffset, ValueIdToLinkageMap)) + return Err; + SeenValueSymbolTable = true; + } SeenGlobalValSummary = true; - if (std::error_code EC = parseEntireSummary()) - return EC; + if (Error Err = parseEntireSummary(ModulePath)) + return Err; break; case bitc::MODULE_STRTAB_BLOCK_ID: - if (std::error_code EC = parseModuleStringTable()) - return EC; + if (Error Err = parseModuleStringTable()) + return Err; break; } continue; @@ -6044,14 +4731,12 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { case bitc::MODULE_CODE_HASH: { if (Record.size() != 5) return error("Invalid hash length " + Twine(Record.size()).str()); - if (!TheIndex) - break; - if (TheIndex->modulePaths().empty()) - // Does not have any summary emitted. - break; - if (TheIndex->modulePaths().size() != 1) + if (TheIndex.modulePaths().empty()) + // We always seed the index with the module. + TheIndex.addModulePath(ModulePath, 0); + if (TheIndex.modulePaths().size() != 1) return error("Don't expect multiple modules defined?"); - auto &Hash = TheIndex->modulePaths().begin()->second.second; + auto &Hash = TheIndex.modulePaths().begin()->second.second; int Pos = 0; for (auto &Val : Record) { assert(!(Val >> 32) && "Unexpected high bits set"); @@ -6063,7 +4748,10 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { case bitc::MODULE_CODE_VSTOFFSET: if (Record.size() < 1) return error("Invalid record"); - VSTOffset = Record[0]; + // Note that we subtract 1 here because the offset is relative to one + // word before the start of the identification or module block, which + // was historically always the start of the regular bitcode header. + VSTOffset = Record[0] - 1; break; // GLOBALVAR: [pointer type, isconst, initid, // linkage, alignment, section, visibility, threadlocal, @@ -6105,9 +4793,37 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { } } +std::vector<ValueInfo> +ModuleSummaryIndexBitcodeReader::makeRefList(ArrayRef<uint64_t> Record) { + std::vector<ValueInfo> Ret; + Ret.reserve(Record.size()); + for (uint64_t RefValueId : Record) + Ret.push_back(getGUIDFromValueId(RefValueId).first); + return Ret; +} + +std::vector<FunctionSummary::EdgeTy> ModuleSummaryIndexBitcodeReader::makeCallList( + ArrayRef<uint64_t> Record, bool IsOldProfileFormat, bool HasProfile) { + std::vector<FunctionSummary::EdgeTy> Ret; + Ret.reserve(Record.size()); + for (unsigned I = 0, E = Record.size(); I != E; ++I) { + CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown; + GlobalValue::GUID CalleeGUID = getGUIDFromValueId(Record[I]).first; + if (IsOldProfileFormat) { + I += 1; // Skip old callsitecount field + if (HasProfile) + I += 1; // Skip old profilecount field + } else if (HasProfile) + Hotness = static_cast<CalleeInfo::HotnessType>(Record[++I]); + Ret.push_back(FunctionSummary::EdgeTy{CalleeGUID, CalleeInfo{Hotness}}); + } + return Ret; +} + // Eagerly parse the entire summary block. This populates the GlobalValueSummary // objects in the index. -std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { +Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( + StringRef ModulePath) { if (Stream.EnterSubBlock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID)) return error("Invalid record"); SmallVector<uint64_t, 64> Record; @@ -6121,15 +4837,19 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { return error("Invalid Summary Block: version expected"); } const uint64_t Version = Record[0]; - if (Version != 1) - return error("Invalid summary version " + Twine(Version) + ", 1 expected"); + const bool IsOldProfileFormat = Version == 1; + if (Version < 1 || Version > 3) + return error("Invalid summary version " + Twine(Version) + + ", 1, 2 or 3 expected"); Record.clear(); // Keep around the last seen summary to be used when we see an optional // "OriginalName" attachement. GlobalValueSummary *LastSeenSummary = nullptr; bool Combined = false; - while (1) { + std::vector<GlobalValue::GUID> PendingTypeTests; + + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -6146,8 +4866,8 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { // to clean them up (especially since that may not run for the first // module's index if we merge into that). if (!Combined) - TheIndex->removeEmptySummaryEntries(); - return std::error_code(); + TheIndex.removeEmptySummaryEntries(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -6166,10 +4886,10 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { default: // Default behavior: ignore. break; // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid, - // n x (valueid, callsitecount)] + // n x (valueid)] // FS_PERMODULE_PROFILE: [valueid, flags, instcount, numrefs, // numrefs x valueid, - // n x (valueid, callsitecount, profilecount)] + // n x (valueid, hotness)] case bitc::FS_PERMODULE: case bitc::FS_PERMODULE_PROFILE: { unsigned ValueID = Record[0]; @@ -6177,37 +4897,29 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { unsigned InstCount = Record[2]; unsigned NumRefs = Record[3]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - std::unique_ptr<FunctionSummary> FS = - llvm::make_unique<FunctionSummary>(Flags, InstCount); // The module path string ref set in the summary must be owned by the // index's module string table. Since we don't have a module path // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. - FS->setModulePath( - TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first()); static int RefListStartIndex = 4; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); - for (unsigned I = 4, E = CallGraphEdgeStartIndex; I != E; ++I) { - unsigned RefValueId = Record[I]; - GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; - FS->addRefEdge(RefGUID); - } + std::vector<ValueInfo> Refs = makeRefList( + ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs)); bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE); - for (unsigned I = CallGraphEdgeStartIndex, E = Record.size(); I != E; - ++I) { - unsigned CalleeValueId = Record[I]; - unsigned CallsiteCount = Record[++I]; - uint64_t ProfileCount = HasProfile ? Record[++I] : 0; - GlobalValue::GUID CalleeGUID = getGUIDFromValueId(CalleeValueId).first; - FS->addCallGraphEdge(CalleeGUID, - CalleeInfo(CallsiteCount, ProfileCount)); - } + std::vector<FunctionSummary::EdgeTy> Calls = makeCallList( + ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex), + IsOldProfileFormat, HasProfile); + auto FS = llvm::make_unique<FunctionSummary>( + Flags, InstCount, std::move(Refs), std::move(Calls), + std::move(PendingTypeTests)); + PendingTypeTests.clear(); auto GUID = getGUIDFromValueId(ValueID); + FS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first()); FS->setOriginalName(GUID.second); - TheIndex->addGlobalValueSummary(GUID.first, std::move(FS)); + TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); break; } // FS_ALIAS: [valueid, flags, valueid] @@ -6218,24 +4930,24 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { uint64_t RawFlags = Record[1]; unsigned AliaseeID = Record[2]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - std::unique_ptr<AliasSummary> AS = llvm::make_unique<AliasSummary>(Flags); + auto AS = + llvm::make_unique<AliasSummary>(Flags, std::vector<ValueInfo>{}); // The module path string ref set in the summary must be owned by the // index's module string table. Since we don't have a module path // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. - AS->setModulePath( - TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first()); + AS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first()); GlobalValue::GUID AliaseeGUID = getGUIDFromValueId(AliaseeID).first; - auto *AliaseeSummary = TheIndex->getGlobalValueSummary(AliaseeGUID); + auto *AliaseeSummary = TheIndex.getGlobalValueSummary(AliaseeGUID); if (!AliaseeSummary) return error("Alias expects aliasee summary to be parsed"); AS->setAliasee(AliaseeSummary); auto GUID = getGUIDFromValueId(ValueID); AS->setOriginalName(GUID.second); - TheIndex->addGlobalValueSummary(GUID.first, std::move(AS)); + TheIndex.addGlobalValueSummary(GUID.first, std::move(AS)); break; } // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, n x valueid] @@ -6243,25 +4955,19 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - std::unique_ptr<GlobalVarSummary> FS = - llvm::make_unique<GlobalVarSummary>(Flags); - FS->setModulePath( - TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first()); - for (unsigned I = 2, E = Record.size(); I != E; ++I) { - unsigned RefValueId = Record[I]; - GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; - FS->addRefEdge(RefGUID); - } + std::vector<ValueInfo> Refs = + makeRefList(ArrayRef<uint64_t>(Record).slice(2)); + auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs)); + FS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first()); auto GUID = getGUIDFromValueId(ValueID); FS->setOriginalName(GUID.second); - TheIndex->addGlobalValueSummary(GUID.first, std::move(FS)); + TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); break; } // FS_COMBINED: [valueid, modid, flags, instcount, numrefs, - // numrefs x valueid, n x (valueid, callsitecount)] + // numrefs x valueid, n x (valueid)] // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, numrefs, - // numrefs x valueid, - // n x (valueid, callsitecount, profilecount)] + // numrefs x valueid, n x (valueid, hotness)] case bitc::FS_COMBINED: case bitc::FS_COMBINED_PROFILE: { unsigned ValueID = Record[0]; @@ -6270,32 +4976,24 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { unsigned InstCount = Record[3]; unsigned NumRefs = Record[4]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - std::unique_ptr<FunctionSummary> FS = - llvm::make_unique<FunctionSummary>(Flags, InstCount); - LastSeenSummary = FS.get(); - FS->setModulePath(ModuleIdMap[ModuleId]); static int RefListStartIndex = 5; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); - for (unsigned I = RefListStartIndex, E = CallGraphEdgeStartIndex; I != E; - ++I) { - unsigned RefValueId = Record[I]; - GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; - FS->addRefEdge(RefGUID); - } + std::vector<ValueInfo> Refs = makeRefList( + ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs)); bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE); - for (unsigned I = CallGraphEdgeStartIndex, E = Record.size(); I != E; - ++I) { - unsigned CalleeValueId = Record[I]; - unsigned CallsiteCount = Record[++I]; - uint64_t ProfileCount = HasProfile ? Record[++I] : 0; - GlobalValue::GUID CalleeGUID = getGUIDFromValueId(CalleeValueId).first; - FS->addCallGraphEdge(CalleeGUID, - CalleeInfo(CallsiteCount, ProfileCount)); - } + std::vector<FunctionSummary::EdgeTy> Edges = makeCallList( + ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex), + IsOldProfileFormat, HasProfile); GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; - TheIndex->addGlobalValueSummary(GUID, std::move(FS)); + auto FS = llvm::make_unique<FunctionSummary>( + Flags, InstCount, std::move(Refs), std::move(Edges), + std::move(PendingTypeTests)); + PendingTypeTests.clear(); + LastSeenSummary = FS.get(); + FS->setModulePath(ModuleIdMap[ModuleId]); + TheIndex.addGlobalValueSummary(GUID, std::move(FS)); Combined = true; break; } @@ -6308,19 +5006,19 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { uint64_t RawFlags = Record[2]; unsigned AliaseeValueId = Record[3]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - std::unique_ptr<AliasSummary> AS = llvm::make_unique<AliasSummary>(Flags); + auto AS = llvm::make_unique<AliasSummary>(Flags, std::vector<ValueInfo>{}); LastSeenSummary = AS.get(); AS->setModulePath(ModuleIdMap[ModuleId]); auto AliaseeGUID = getGUIDFromValueId(AliaseeValueId).first; auto AliaseeInModule = - TheIndex->findSummaryInModule(AliaseeGUID, AS->modulePath()); + TheIndex.findSummaryInModule(AliaseeGUID, AS->modulePath()); if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); AS->setAliasee(AliaseeInModule); GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; - TheIndex->addGlobalValueSummary(GUID, std::move(AS)); + TheIndex.addGlobalValueSummary(GUID, std::move(AS)); Combined = true; break; } @@ -6330,17 +5028,13 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { uint64_t ModuleId = Record[1]; uint64_t RawFlags = Record[2]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - std::unique_ptr<GlobalVarSummary> FS = - llvm::make_unique<GlobalVarSummary>(Flags); + std::vector<ValueInfo> Refs = + makeRefList(ArrayRef<uint64_t>(Record).slice(3)); + auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs)); LastSeenSummary = FS.get(); FS->setModulePath(ModuleIdMap[ModuleId]); - for (unsigned I = 3, E = Record.size(); I != E; ++I) { - unsigned RefValueId = Record[I]; - GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; - FS->addRefEdge(RefGUID); - } GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; - TheIndex->addGlobalValueSummary(GUID, std::move(FS)); + TheIndex.addGlobalValueSummary(GUID, std::move(FS)); Combined = true; break; } @@ -6352,6 +5046,13 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { LastSeenSummary->setOriginalName(OriginalName); // Reset the LastSeenSummary LastSeenSummary = nullptr; + break; + } + case bitc::FS_TYPE_TESTS: { + assert(PendingTypeTests.empty()); + PendingTypeTests.insert(PendingTypeTests.end(), Record.begin(), + Record.end()); + break; } } } @@ -6360,7 +5061,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { // Parse the module string table block into the Index. // This populates the ModulePathStringTable map in the index. -std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { +Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { if (Stream.EnterSubBlock(bitc::MODULE_STRTAB_BLOCK_ID)) return error("Invalid record"); @@ -6368,7 +5069,8 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { SmallString<128> ModulePath; ModulePathStringTableTy::iterator LastSeenModulePath; - while (1) { + + while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); switch (Entry.Kind) { @@ -6376,7 +5078,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return std::error_code(); + return Error::success(); case BitstreamEntry::Record: // The interesting case. break; @@ -6393,7 +5095,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { if (convertToString(Record, 1, ModulePath)) return error("Invalid record"); - LastSeenModulePath = TheIndex->addModulePath(ModulePath, ModuleId); + LastSeenModulePath = TheIndex.addModulePath(ModulePath, ModuleId); ModuleIdMap[ModuleId] = LastSeenModulePath->first(); ModulePath.clear(); @@ -6403,7 +5105,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { case bitc::MST_CODE_HASH: { if (Record.size() != 5) return error("Invalid hash length " + Twine(Record.size()).str()); - if (LastSeenModulePath == TheIndex->modulePaths().end()) + if (LastSeenModulePath == TheIndex.modulePaths().end()) return error("Invalid hash that does not follow a module path"); int Pos = 0; for (auto &Val : Record) { @@ -6411,7 +5113,7 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { LastSeenModulePath->second.second[Pos++] = Val; } // Reset LastSeenModulePath to avoid overriding the hash unexpectedly. - LastSeenModulePath = TheIndex->modulePaths().end(); + LastSeenModulePath = TheIndex.modulePaths().end(); break; } } @@ -6419,114 +5121,25 @@ std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { llvm_unreachable("Exit infinite loop"); } -// Parse the function info index from the bitcode streamer into the given index. -std::error_code ModuleSummaryIndexBitcodeReader::parseSummaryIndexInto( - std::unique_ptr<DataStreamer> Streamer, ModuleSummaryIndex *I) { - TheIndex = I; - - if (std::error_code EC = initStream(std::move(Streamer))) - return EC; - - // Sniff for the signature. - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); - - // We expect a number of well-defined blocks, though we don't necessarily - // need to understand them all. - while (1) { - if (Stream.AtEndOfStream()) { - // We didn't really read a proper Module block. - return error("Malformed block"); - } - - BitstreamEntry Entry = - Stream.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs); - - if (Entry.Kind != BitstreamEntry::SubBlock) - return error("Malformed block"); - - // If we see a MODULE_BLOCK, parse it to find the blocks needed for - // building the function summary index. - if (Entry.ID == bitc::MODULE_BLOCK_ID) - return parseModule(); - - if (Stream.SkipBlock()) - return error("Invalid record"); - } -} - -std::error_code ModuleSummaryIndexBitcodeReader::initStream( - std::unique_ptr<DataStreamer> Streamer) { - if (Streamer) - return initLazyStream(std::move(Streamer)); - return initStreamFromBuffer(); -} - -std::error_code ModuleSummaryIndexBitcodeReader::initStreamFromBuffer() { - const unsigned char *BufPtr = (const unsigned char *)Buffer->getBufferStart(); - const unsigned char *BufEnd = BufPtr + Buffer->getBufferSize(); - - if (Buffer->getBufferSize() & 3) - return error("Invalid bitcode signature"); - - // If we have a wrapper header, parse it and ignore the non-bc file contents. - // The magic number is 0x0B17C0DE stored in little endian. - if (isBitcodeWrapper(BufPtr, BufEnd)) - if (SkipBitcodeWrapperHeader(BufPtr, BufEnd, true)) - return error("Invalid bitcode wrapper header"); - - StreamFile.reset(new BitstreamReader(BufPtr, BufEnd)); - Stream.init(&*StreamFile); - - return std::error_code(); -} - -std::error_code ModuleSummaryIndexBitcodeReader::initLazyStream( - std::unique_ptr<DataStreamer> Streamer) { - // Check and strip off the bitcode wrapper; BitstreamReader expects never to - // see it. - auto OwnedBytes = - llvm::make_unique<StreamingMemoryObject>(std::move(Streamer)); - StreamingMemoryObject &Bytes = *OwnedBytes; - StreamFile = llvm::make_unique<BitstreamReader>(std::move(OwnedBytes)); - Stream.init(&*StreamFile); - - unsigned char buf[16]; - if (Bytes.readBytes(buf, 16, 0) != 16) - return error("Invalid bitcode signature"); - - if (!isBitcode(buf, buf + 16)) - return error("Invalid bitcode signature"); - - if (isBitcodeWrapper(buf, buf + 4)) { - const unsigned char *bitcodeStart = buf; - const unsigned char *bitcodeEnd = buf + 16; - SkipBitcodeWrapperHeader(bitcodeStart, bitcodeEnd, false); - Bytes.dropLeadingBytes(bitcodeStart - buf); - Bytes.setKnownObjectSize(bitcodeEnd - bitcodeStart); - } - return std::error_code(); -} - namespace { + // FIXME: This class is only here to support the transition to llvm::Error. It // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. class BitcodeErrorCategoryType : public std::error_category { - const char *name() const LLVM_NOEXCEPT override { + const char *name() const noexcept override { return "llvm.bitcode"; } std::string message(int IE) const override { BitcodeError E = static_cast<BitcodeError>(IE); switch (E) { - case BitcodeError::InvalidBitcodeSignature: - return "Invalid bitcode signature"; case BitcodeError::CorruptedBitcode: return "Corrupted bitcode"; } llvm_unreachable("Unknown error type!"); } }; + } // end anonymous namespace static ManagedStatic<BitcodeErrorCategoryType> ErrorCategory; @@ -6539,151 +5152,251 @@ const std::error_category &llvm::BitcodeErrorCategory() { // External interface //===----------------------------------------------------------------------===// -static ErrorOr<std::unique_ptr<Module>> -getBitcodeModuleImpl(std::unique_ptr<DataStreamer> Streamer, StringRef Name, - BitcodeReader *R, LLVMContext &Context, - bool MaterializeAll, bool ShouldLazyLoadMetadata) { - std::unique_ptr<Module> M = make_unique<Module>(Name, Context); - M->setMaterializer(R); +Expected<std::vector<BitcodeModule>> +llvm::getBitcodeModuleList(MemoryBufferRef Buffer) { + Expected<BitstreamCursor> StreamOrErr = initStream(Buffer); + if (!StreamOrErr) + return StreamOrErr.takeError(); + BitstreamCursor &Stream = *StreamOrErr; - auto cleanupOnError = [&](std::error_code EC) { - R->releaseBuffer(); // Never take ownership on error. - return EC; - }; + std::vector<BitcodeModule> Modules; + while (true) { + uint64_t BCBegin = Stream.getCurrentByteNo(); - // Delay parsing Metadata if ShouldLazyLoadMetadata is true. - if (std::error_code EC = R->parseBitcodeInto(std::move(Streamer), M.get(), - ShouldLazyLoadMetadata)) - return cleanupOnError(EC); + // We may be consuming bitcode from a client that leaves garbage at the end + // of the bitcode stream (e.g. Apple's ar tool). If we are close enough to + // the end that there cannot possibly be another module, stop looking. + if (BCBegin + 8 >= Stream.getBitcodeBytes().size()) + return Modules; - if (MaterializeAll) { - // Read in the entire module, and destroy the BitcodeReader. - if (std::error_code EC = M->materializeAll()) - return cleanupOnError(EC); - } else { - // Resolve forward references from blockaddresses. - if (std::error_code EC = R->materializeForwardReferencedFunctions()) - return cleanupOnError(EC); + BitstreamEntry Entry = Stream.advance(); + switch (Entry.Kind) { + case BitstreamEntry::EndBlock: + case BitstreamEntry::Error: + return error("Malformed block"); + + case BitstreamEntry::SubBlock: { + uint64_t IdentificationBit = -1ull; + if (Entry.ID == bitc::IDENTIFICATION_BLOCK_ID) { + IdentificationBit = Stream.GetCurrentBitNo() - BCBegin * 8; + if (Stream.SkipBlock()) + return error("Malformed block"); + + Entry = Stream.advance(); + if (Entry.Kind != BitstreamEntry::SubBlock || + Entry.ID != bitc::MODULE_BLOCK_ID) + return error("Malformed block"); + } + + if (Entry.ID == bitc::MODULE_BLOCK_ID) { + uint64_t ModuleBit = Stream.GetCurrentBitNo() - BCBegin * 8; + if (Stream.SkipBlock()) + return error("Malformed block"); + + Modules.push_back({Stream.getBitcodeBytes().slice( + BCBegin, Stream.getCurrentByteNo() - BCBegin), + Buffer.getBufferIdentifier(), IdentificationBit, + ModuleBit}); + continue; + } + + if (Stream.SkipBlock()) + return error("Malformed block"); + continue; + } + case BitstreamEntry::Record: + Stream.skipRecord(Entry.ID); + continue; + } } - return std::move(M); } /// \brief Get a lazy one-at-time loading module from bitcode. /// /// This isn't always used in a lazy context. In particular, it's also used by -/// \a parseBitcodeFile(). If this is truly lazy, then we need to eagerly pull +/// \a parseModule(). If this is truly lazy, then we need to eagerly pull /// in forward-referenced functions from block address references. /// /// \param[in] MaterializeAll Set to \c true if we should materialize /// everything. -static ErrorOr<std::unique_ptr<Module>> -getLazyBitcodeModuleImpl(std::unique_ptr<MemoryBuffer> &&Buffer, - LLVMContext &Context, bool MaterializeAll, - bool ShouldLazyLoadMetadata = false) { - BitcodeReader *R = new BitcodeReader(Buffer.get(), Context); - - ErrorOr<std::unique_ptr<Module>> Ret = - getBitcodeModuleImpl(nullptr, Buffer->getBufferIdentifier(), R, Context, - MaterializeAll, ShouldLazyLoadMetadata); - if (!Ret) - return Ret; - - Buffer.release(); // The BitcodeReader owns it now. - return Ret; +Expected<std::unique_ptr<Module>> +BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, + bool ShouldLazyLoadMetadata, bool IsImporting) { + BitstreamCursor Stream(Buffer); + + std::string ProducerIdentification; + if (IdentificationBit != -1ull) { + Stream.JumpToBit(IdentificationBit); + Expected<std::string> ProducerIdentificationOrErr = + readIdentificationBlock(Stream); + if (!ProducerIdentificationOrErr) + return ProducerIdentificationOrErr.takeError(); + + ProducerIdentification = *ProducerIdentificationOrErr; + } + + Stream.JumpToBit(ModuleBit); + auto *R = + new BitcodeReader(std::move(Stream), ProducerIdentification, Context); + + std::unique_ptr<Module> M = + llvm::make_unique<Module>(ModuleIdentifier, Context); + M->setMaterializer(R); + + // Delay parsing Metadata if ShouldLazyLoadMetadata is true. + if (Error Err = + R->parseBitcodeInto(M.get(), ShouldLazyLoadMetadata, IsImporting)) + return std::move(Err); + + if (MaterializeAll) { + // Read in the entire module, and destroy the BitcodeReader. + if (Error Err = M->materializeAll()) + return std::move(Err); + } else { + // Resolve forward references from blockaddresses. + if (Error Err = R->materializeForwardReferencedFunctions()) + return std::move(Err); + } + return std::move(M); } -ErrorOr<std::unique_ptr<Module>> -llvm::getLazyBitcodeModule(std::unique_ptr<MemoryBuffer> &&Buffer, - LLVMContext &Context, bool ShouldLazyLoadMetadata) { - return getLazyBitcodeModuleImpl(std::move(Buffer), Context, false, - ShouldLazyLoadMetadata); +Expected<std::unique_ptr<Module>> +BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, + bool IsImporting) { + return getModuleImpl(Context, false, ShouldLazyLoadMetadata, IsImporting); } -ErrorOr<std::unique_ptr<Module>> -llvm::getStreamedBitcodeModule(StringRef Name, - std::unique_ptr<DataStreamer> Streamer, - LLVMContext &Context) { - std::unique_ptr<Module> M = make_unique<Module>(Name, Context); - BitcodeReader *R = new BitcodeReader(Context); +// Parse the specified bitcode buffer, returning the function info index. +Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() { + BitstreamCursor Stream(Buffer); + Stream.JumpToBit(ModuleBit); - return getBitcodeModuleImpl(std::move(Streamer), Name, R, Context, false, - false); + auto Index = llvm::make_unique<ModuleSummaryIndex>(); + ModuleSummaryIndexBitcodeReader R(std::move(Stream), *Index); + + if (Error Err = R.parseModule(ModuleIdentifier)) + return std::move(Err); + + return std::move(Index); } -ErrorOr<std::unique_ptr<Module>> llvm::parseBitcodeFile(MemoryBufferRef Buffer, - LLVMContext &Context) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - return getLazyBitcodeModuleImpl(std::move(Buf), Context, true); - // TODO: Restore the use-lists to the in-memory state when the bitcode was - // written. We must defer until the Module has been fully materialized. +// Check if the given bitcode buffer contains a global value summary block. +Expected<bool> BitcodeModule::hasSummary() { + BitstreamCursor Stream(Buffer); + Stream.JumpToBit(ModuleBit); + + if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return error("Invalid record"); + + while (true) { + BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return false; + + case BitstreamEntry::SubBlock: + if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) + return true; + + // Ignore other sub-blocks. + if (Stream.SkipBlock()) + return error("Malformed block"); + continue; + + case BitstreamEntry::Record: + Stream.skipRecord(Entry.ID); + continue; + } + } } -std::string llvm::getBitcodeTargetTriple(MemoryBufferRef Buffer, - LLVMContext &Context) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - auto R = llvm::make_unique<BitcodeReader>(Buf.release(), Context); - ErrorOr<std::string> Triple = R->parseTriple(); - if (Triple.getError()) - return ""; - return Triple.get(); +static Expected<BitcodeModule> getSingleModule(MemoryBufferRef Buffer) { + Expected<std::vector<BitcodeModule>> MsOrErr = getBitcodeModuleList(Buffer); + if (!MsOrErr) + return MsOrErr.takeError(); + + if (MsOrErr->size() != 1) + return error("Expected a single module"); + + return (*MsOrErr)[0]; } -bool llvm::isBitcodeContainingObjCCategory(MemoryBufferRef Buffer, - LLVMContext &Context) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - auto R = llvm::make_unique<BitcodeReader>(Buf.release(), Context); - ErrorOr<bool> hasObjCCategory = R->hasObjCCategory(); - if (hasObjCCategory.getError()) - return false; - return hasObjCCategory.get(); +Expected<std::unique_ptr<Module>> +llvm::getLazyBitcodeModule(MemoryBufferRef Buffer, LLVMContext &Context, + bool ShouldLazyLoadMetadata, bool IsImporting) { + Expected<BitcodeModule> BM = getSingleModule(Buffer); + if (!BM) + return BM.takeError(); + + return BM->getLazyModule(Context, ShouldLazyLoadMetadata, IsImporting); } -std::string llvm::getBitcodeProducerString(MemoryBufferRef Buffer, - LLVMContext &Context) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - BitcodeReader R(Buf.release(), Context); - ErrorOr<std::string> ProducerString = R.parseIdentificationBlock(); - if (ProducerString.getError()) - return ""; - return ProducerString.get(); +Expected<std::unique_ptr<Module>> llvm::getOwningLazyBitcodeModule( + std::unique_ptr<MemoryBuffer> &&Buffer, LLVMContext &Context, + bool ShouldLazyLoadMetadata, bool IsImporting) { + auto MOrErr = getLazyBitcodeModule(*Buffer, Context, ShouldLazyLoadMetadata, + IsImporting); + if (MOrErr) + (*MOrErr)->setOwnedMemoryBuffer(std::move(Buffer)); + return MOrErr; } -// Parse the specified bitcode buffer, returning the function info index. -ErrorOr<std::unique_ptr<ModuleSummaryIndex>> llvm::getModuleSummaryIndex( - MemoryBufferRef Buffer, - const DiagnosticHandlerFunction &DiagnosticHandler) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - ModuleSummaryIndexBitcodeReader R(Buf.get(), DiagnosticHandler); +Expected<std::unique_ptr<Module>> +BitcodeModule::parseModule(LLVMContext &Context) { + return getModuleImpl(Context, true, false, false); + // TODO: Restore the use-lists to the in-memory state when the bitcode was + // written. We must defer until the Module has been fully materialized. +} - auto Index = llvm::make_unique<ModuleSummaryIndex>(); +Expected<std::unique_ptr<Module>> llvm::parseBitcodeFile(MemoryBufferRef Buffer, + LLVMContext &Context) { + Expected<BitcodeModule> BM = getSingleModule(Buffer); + if (!BM) + return BM.takeError(); - auto cleanupOnError = [&](std::error_code EC) { - R.releaseBuffer(); // Never take ownership on error. - return EC; - }; + return BM->parseModule(Context); +} - if (std::error_code EC = R.parseSummaryIndexInto(nullptr, Index.get())) - return cleanupOnError(EC); +Expected<std::string> llvm::getBitcodeTargetTriple(MemoryBufferRef Buffer) { + Expected<BitstreamCursor> StreamOrErr = initStream(Buffer); + if (!StreamOrErr) + return StreamOrErr.takeError(); - Buf.release(); // The ModuleSummaryIndexBitcodeReader owns it now. - return std::move(Index); + return readTriple(*StreamOrErr); } -// Check if the given bitcode buffer contains a global value summary block. -bool llvm::hasGlobalValueSummary( - MemoryBufferRef Buffer, - const DiagnosticHandlerFunction &DiagnosticHandler) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - ModuleSummaryIndexBitcodeReader R(Buf.get(), DiagnosticHandler, true); - - auto cleanupOnError = [&](std::error_code EC) { - R.releaseBuffer(); // Never take ownership on error. - return false; - }; +Expected<bool> llvm::isBitcodeContainingObjCCategory(MemoryBufferRef Buffer) { + Expected<BitstreamCursor> StreamOrErr = initStream(Buffer); + if (!StreamOrErr) + return StreamOrErr.takeError(); + + return hasObjCCategory(*StreamOrErr); +} + +Expected<std::string> llvm::getBitcodeProducerString(MemoryBufferRef Buffer) { + Expected<BitstreamCursor> StreamOrErr = initStream(Buffer); + if (!StreamOrErr) + return StreamOrErr.takeError(); + + return readIdentificationCode(*StreamOrErr); +} + +Expected<std::unique_ptr<ModuleSummaryIndex>> +llvm::getModuleSummaryIndex(MemoryBufferRef Buffer) { + Expected<BitcodeModule> BM = getSingleModule(Buffer); + if (!BM) + return BM.takeError(); + + return BM->getSummary(); +} - if (std::error_code EC = R.parseSummaryIndexInto(nullptr, nullptr)) - return cleanupOnError(EC); +Expected<bool> llvm::hasGlobalValueSummary(MemoryBufferRef Buffer) { + Expected<BitcodeModule> BM = getSingleModule(Buffer); + if (!BM) + return BM.takeError(); - Buf.release(); // The ModuleSummaryIndexBitcodeReader owns it now. - return R.foundGlobalValSummary(); + return BM->hasSummary(); } diff --git a/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp index 60360d2..771cf3d 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp @@ -8,6 +8,9 @@ //===----------------------------------------------------------------------===// #include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/ADT/StringRef.h" +#include <cassert> +#include <string> using namespace llvm; @@ -15,14 +18,6 @@ using namespace llvm; // BitstreamCursor implementation //===----------------------------------------------------------------------===// -void BitstreamCursor::freeState() { - // Free all the Abbrevs. - CurAbbrevs.clear(); - - // Free all the Abbrevs in the block scope. - BlockScope.clear(); -} - /// EnterSubBlock - Having read the ENTER_SUBBLOCK abbrevid, enter /// the block, and return true if the block has an error. bool BitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) { @@ -31,10 +26,12 @@ bool BitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) { BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); // Add the abbrevs specific to this block to the CurAbbrevs list. - if (const BitstreamReader::BlockInfo *Info = - getBitStreamReader()->getBlockInfo(BlockID)) { - CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), - Info->Abbrevs.end()); + if (BlockInfo) { + if (const BitstreamBlockInfo::BlockInfo *Info = + BlockInfo->getBlockInfo(BlockID)) { + CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), + Info->Abbrevs.end()); + } } // Get the codesize of this block. @@ -95,23 +92,30 @@ static void skipAbbreviatedField(BitstreamCursor &Cursor, } } - - /// skipRecord - Read the current record and discard it. -void BitstreamCursor::skipRecord(unsigned AbbrevID) { +unsigned BitstreamCursor::skipRecord(unsigned AbbrevID) { // Skip unabbreviated records by reading past their entries. if (AbbrevID == bitc::UNABBREV_RECORD) { unsigned Code = ReadVBR(6); - (void)Code; unsigned NumElts = ReadVBR(6); for (unsigned i = 0; i != NumElts; ++i) (void)ReadVBR64(6); - return; + return Code; } const BitCodeAbbrev *Abbv = getAbbrev(AbbrevID); + const BitCodeAbbrevOp &CodeOp = Abbv->getOperandInfo(0); + unsigned Code; + if (CodeOp.isLiteral()) + Code = CodeOp.getLiteralValue(); + else { + if (CodeOp.getEncoding() == BitCodeAbbrevOp::Array || + CodeOp.getEncoding() == BitCodeAbbrevOp::Blob) + report_fatal_error("Abbreviation starts with an Array or a Blob"); + Code = readAbbreviatedField(*this, CodeOp); + } - for (unsigned i = 0, e = Abbv->getNumOperandInfos(); i != e; ++i) { + for (unsigned i = 1, e = Abbv->getNumOperandInfos(); i < e; ++i) { const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); if (Op.isLiteral()) continue; @@ -136,18 +140,16 @@ void BitstreamCursor::skipRecord(unsigned AbbrevID) { default: report_fatal_error("Array element type can't be an Array or a Blob"); case BitCodeAbbrevOp::Fixed: - assert((unsigned)Op.getEncodingData() <= MaxChunkSize); - for (; NumElts; --NumElts) - Read((unsigned)EltEnc.getEncodingData()); + assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize); + JumpToBit(GetCurrentBitNo() + NumElts * EltEnc.getEncodingData()); break; case BitCodeAbbrevOp::VBR: - assert((unsigned)Op.getEncodingData() <= MaxChunkSize); + assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize); for (; NumElts; --NumElts) ReadVBR64((unsigned)EltEnc.getEncodingData()); break; case BitCodeAbbrevOp::Char6: - for (; NumElts; --NumElts) - Read(6); + JumpToBit(GetCurrentBitNo() + NumElts * 6); break; } continue; @@ -171,6 +173,7 @@ void BitstreamCursor::skipRecord(unsigned AbbrevID) { // Skip over the blob. JumpToBit(NewEnd); } + return Code; } unsigned BitstreamCursor::readRecord(unsigned AbbrevID, @@ -279,9 +282,8 @@ unsigned BitstreamCursor::readRecord(unsigned AbbrevID, return Code; } - void BitstreamCursor::ReadAbbrevRecord() { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); unsigned NumOpInfo = ReadVBR(5); for (unsigned i = 0; i != NumOpInfo; ++i) { bool IsLiteral = Read(1); @@ -315,29 +317,28 @@ void BitstreamCursor::ReadAbbrevRecord() { if (Abbv->getNumOperandInfos() == 0) report_fatal_error("Abbrev record with no operands"); - CurAbbrevs.push_back(Abbv); + CurAbbrevs.push_back(std::move(Abbv)); } -bool BitstreamCursor::ReadBlockInfoBlock() { - // If this is the second stream to get to the block info block, skip it. - if (getBitStreamReader()->hasBlockInfoRecords()) - return SkipBlock(); +Optional<BitstreamBlockInfo> +BitstreamCursor::ReadBlockInfoBlock(bool ReadBlockInfoNames) { + if (EnterSubBlock(bitc::BLOCKINFO_BLOCK_ID)) return None; - if (EnterSubBlock(bitc::BLOCKINFO_BLOCK_ID)) return true; + BitstreamBlockInfo NewBlockInfo; SmallVector<uint64_t, 64> Record; - BitstreamReader::BlockInfo *CurBlockInfo = nullptr; + BitstreamBlockInfo::BlockInfo *CurBlockInfo = nullptr; // Read all the records for this module. - while (1) { + while (true) { BitstreamEntry Entry = advanceSkippingSubblocks(AF_DontAutoprocessAbbrevs); switch (Entry.Kind) { case llvm::BitstreamEntry::SubBlock: // Handled for us already. case llvm::BitstreamEntry::Error: - return true; + return None; case llvm::BitstreamEntry::EndBlock: - return false; + return std::move(NewBlockInfo); case llvm::BitstreamEntry::Record: // The interesting case. break; @@ -345,7 +346,7 @@ bool BitstreamCursor::ReadBlockInfoBlock() { // Read abbrev records, associate them with CurBID. if (Entry.ID == bitc::DEFINE_ABBREV) { - if (!CurBlockInfo) return true; + if (!CurBlockInfo) return None; ReadAbbrevRecord(); // ReadAbbrevRecord installs the abbrev in CurAbbrevs. Move it to the @@ -360,13 +361,12 @@ bool BitstreamCursor::ReadBlockInfoBlock() { switch (readRecord(Entry.ID, Record)) { default: break; // Default behavior, ignore unknown content. case bitc::BLOCKINFO_CODE_SETBID: - if (Record.size() < 1) return true; - CurBlockInfo = - &getBitStreamReader()->getOrCreateBlockInfo((unsigned)Record[0]); + if (Record.size() < 1) return None; + CurBlockInfo = &NewBlockInfo.getOrCreateBlockInfo((unsigned)Record[0]); break; case bitc::BLOCKINFO_CODE_BLOCKNAME: { - if (!CurBlockInfo) return true; - if (getBitStreamReader()->isIgnoringBlockInfoNames()) + if (!CurBlockInfo) return None; + if (!ReadBlockInfoNames) break; // Ignore name. std::string Name; for (unsigned i = 0, e = Record.size(); i != e; ++i) @@ -375,8 +375,8 @@ bool BitstreamCursor::ReadBlockInfoBlock() { break; } case bitc::BLOCKINFO_CODE_SETRECORDNAME: { - if (!CurBlockInfo) return true; - if (getBitStreamReader()->isIgnoringBlockInfoNames()) + if (!CurBlockInfo) return None; + if (!ReadBlockInfoNames) break; // Ignore name. std::string Name; for (unsigned i = 1, e = Record.size(); i != e; ++i) @@ -388,4 +388,3 @@ bool BitstreamCursor::ReadBlockInfoBlock() { } } } - diff --git a/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp new file mode 100644 index 0000000..b89f5be --- /dev/null +++ b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp @@ -0,0 +1,1850 @@ +//===- MetadataLoader.cpp - Internal BitcodeReader implementation ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "MetadataLoader.h" +#include "ValueList.h" + +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/CallingConv.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GVMaterializer.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" +#include "llvm/IR/GlobalIndirectSymbol.h" +#include "llvm/IR/GlobalObject.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/IR/OperandTraits.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/TrackingMDRef.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <deque> +#include <limits> +#include <map> +#include <memory> +#include <string> +#include <system_error> +#include <tuple> +#include <utility> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "bitcode-reader" + +STATISTIC(NumMDStringLoaded, "Number of MDStrings loaded"); +STATISTIC(NumMDNodeTemporary, "Number of MDNode::Temporary created"); +STATISTIC(NumMDRecordLoaded, "Number of Metadata records loaded"); + +/// Flag whether we need to import full type definitions for ThinLTO. +/// Currently needed for Darwin and LLDB. +static cl::opt<bool> ImportFullTypeDefinitions( + "import-full-type-definitions", cl::init(false), cl::Hidden, + cl::desc("Import full type definitions for ThinLTO.")); + +static cl::opt<bool> DisableLazyLoading( + "disable-ondemand-mds-loading", cl::init(false), cl::Hidden, + cl::desc("Force disable the lazy-loading on-demand of metadata when " + "loading bitcode for importing.")); + +namespace { + +static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; } + +class BitcodeReaderMetadataList { + /// Array of metadata references. + /// + /// Don't use std::vector here. Some versions of libc++ copy (instead of + /// move) on resize, and TrackingMDRef is very expensive to copy. + SmallVector<TrackingMDRef, 1> MetadataPtrs; + + /// The set of indices in MetadataPtrs above of forward references that were + /// generated. + SmallDenseSet<unsigned, 1> ForwardReference; + + /// The set of indices in MetadataPtrs above of Metadata that need to be + /// resolved. + SmallDenseSet<unsigned, 1> UnresolvedNodes; + + /// Structures for resolving old type refs. + struct { + SmallDenseMap<MDString *, TempMDTuple, 1> Unknown; + SmallDenseMap<MDString *, DICompositeType *, 1> Final; + SmallDenseMap<MDString *, DICompositeType *, 1> FwdDecls; + SmallVector<std::pair<TrackingMDRef, TempMDTuple>, 1> Arrays; + } OldTypeRefs; + + LLVMContext &Context; + +public: + BitcodeReaderMetadataList(LLVMContext &C) : Context(C) {} + + // vector compatibility methods + unsigned size() const { return MetadataPtrs.size(); } + void resize(unsigned N) { MetadataPtrs.resize(N); } + void push_back(Metadata *MD) { MetadataPtrs.emplace_back(MD); } + void clear() { MetadataPtrs.clear(); } + Metadata *back() const { return MetadataPtrs.back(); } + void pop_back() { MetadataPtrs.pop_back(); } + bool empty() const { return MetadataPtrs.empty(); } + + Metadata *operator[](unsigned i) const { + assert(i < MetadataPtrs.size()); + return MetadataPtrs[i]; + } + + Metadata *lookup(unsigned I) const { + if (I < MetadataPtrs.size()) + return MetadataPtrs[I]; + return nullptr; + } + + void shrinkTo(unsigned N) { + assert(N <= size() && "Invalid shrinkTo request!"); + assert(ForwardReference.empty() && "Unexpected forward refs"); + assert(UnresolvedNodes.empty() && "Unexpected unresolved node"); + MetadataPtrs.resize(N); + } + + /// Return the given metadata, creating a replaceable forward reference if + /// necessary. + Metadata *getMetadataFwdRef(unsigned Idx); + + /// Return the the given metadata only if it is fully resolved. + /// + /// Gives the same result as \a lookup(), unless \a MDNode::isResolved() + /// would give \c false. + Metadata *getMetadataIfResolved(unsigned Idx); + + MDNode *getMDNodeFwdRefOrNull(unsigned Idx); + void assignValue(Metadata *MD, unsigned Idx); + void tryToResolveCycles(); + bool hasFwdRefs() const { return !ForwardReference.empty(); } + int getNextFwdRef() { + assert(hasFwdRefs()); + return *ForwardReference.begin(); + } + + /// Upgrade a type that had an MDString reference. + void addTypeRef(MDString &UUID, DICompositeType &CT); + + /// Upgrade a type that had an MDString reference. + Metadata *upgradeTypeRef(Metadata *MaybeUUID); + + /// Upgrade a type ref array that may have MDString references. + Metadata *upgradeTypeRefArray(Metadata *MaybeTuple); + +private: + Metadata *resolveTypeRefArray(Metadata *MaybeTuple); +}; + +void BitcodeReaderMetadataList::assignValue(Metadata *MD, unsigned Idx) { + if (auto *MDN = dyn_cast<MDNode>(MD)) + if (!MDN->isResolved()) + UnresolvedNodes.insert(Idx); + + if (Idx == size()) { + push_back(MD); + return; + } + + if (Idx >= size()) + resize(Idx + 1); + + TrackingMDRef &OldMD = MetadataPtrs[Idx]; + if (!OldMD) { + OldMD.reset(MD); + return; + } + + // If there was a forward reference to this value, replace it. + TempMDTuple PrevMD(cast<MDTuple>(OldMD.get())); + PrevMD->replaceAllUsesWith(MD); + ForwardReference.erase(Idx); +} + +Metadata *BitcodeReaderMetadataList::getMetadataFwdRef(unsigned Idx) { + if (Idx >= size()) + resize(Idx + 1); + + if (Metadata *MD = MetadataPtrs[Idx]) + return MD; + + // Track forward refs to be resolved later. + ForwardReference.insert(Idx); + + // Create and return a placeholder, which will later be RAUW'd. + ++NumMDNodeTemporary; + Metadata *MD = MDNode::getTemporary(Context, None).release(); + MetadataPtrs[Idx].reset(MD); + return MD; +} + +Metadata *BitcodeReaderMetadataList::getMetadataIfResolved(unsigned Idx) { + Metadata *MD = lookup(Idx); + if (auto *N = dyn_cast_or_null<MDNode>(MD)) + if (!N->isResolved()) + return nullptr; + return MD; +} + +MDNode *BitcodeReaderMetadataList::getMDNodeFwdRefOrNull(unsigned Idx) { + return dyn_cast_or_null<MDNode>(getMetadataFwdRef(Idx)); +} + +void BitcodeReaderMetadataList::tryToResolveCycles() { + if (!ForwardReference.empty()) + // Still forward references... can't resolve cycles. + return; + + // Give up on finding a full definition for any forward decls that remain. + for (const auto &Ref : OldTypeRefs.FwdDecls) + OldTypeRefs.Final.insert(Ref); + OldTypeRefs.FwdDecls.clear(); + + // Upgrade from old type ref arrays. In strange cases, this could add to + // OldTypeRefs.Unknown. + for (const auto &Array : OldTypeRefs.Arrays) + Array.second->replaceAllUsesWith(resolveTypeRefArray(Array.first.get())); + OldTypeRefs.Arrays.clear(); + + // Replace old string-based type refs with the resolved node, if possible. + // If we haven't seen the node, leave it to the verifier to complain about + // the invalid string reference. + for (const auto &Ref : OldTypeRefs.Unknown) { + if (DICompositeType *CT = OldTypeRefs.Final.lookup(Ref.first)) + Ref.second->replaceAllUsesWith(CT); + else + Ref.second->replaceAllUsesWith(Ref.first); + } + OldTypeRefs.Unknown.clear(); + + if (UnresolvedNodes.empty()) + // Nothing to do. + return; + + // Resolve any cycles. + for (unsigned I : UnresolvedNodes) { + auto &MD = MetadataPtrs[I]; + auto *N = dyn_cast_or_null<MDNode>(MD); + if (!N) + continue; + + assert(!N->isTemporary() && "Unexpected forward reference"); + N->resolveCycles(); + } + + // Make sure we return early again until there's another unresolved ref. + UnresolvedNodes.clear(); +} + +void BitcodeReaderMetadataList::addTypeRef(MDString &UUID, + DICompositeType &CT) { + assert(CT.getRawIdentifier() == &UUID && "Mismatched UUID"); + if (CT.isForwardDecl()) + OldTypeRefs.FwdDecls.insert(std::make_pair(&UUID, &CT)); + else + OldTypeRefs.Final.insert(std::make_pair(&UUID, &CT)); +} + +Metadata *BitcodeReaderMetadataList::upgradeTypeRef(Metadata *MaybeUUID) { + auto *UUID = dyn_cast_or_null<MDString>(MaybeUUID); + if (LLVM_LIKELY(!UUID)) + return MaybeUUID; + + if (auto *CT = OldTypeRefs.Final.lookup(UUID)) + return CT; + + auto &Ref = OldTypeRefs.Unknown[UUID]; + if (!Ref) + Ref = MDNode::getTemporary(Context, None); + return Ref.get(); +} + +Metadata *BitcodeReaderMetadataList::upgradeTypeRefArray(Metadata *MaybeTuple) { + auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple); + if (!Tuple || Tuple->isDistinct()) + return MaybeTuple; + + // Look through the array immediately if possible. + if (!Tuple->isTemporary()) + return resolveTypeRefArray(Tuple); + + // Create and return a placeholder to use for now. Eventually + // resolveTypeRefArrays() will be resolve this forward reference. + OldTypeRefs.Arrays.emplace_back( + std::piecewise_construct, std::forward_as_tuple(Tuple), + std::forward_as_tuple(MDTuple::getTemporary(Context, None))); + return OldTypeRefs.Arrays.back().second.get(); +} + +Metadata *BitcodeReaderMetadataList::resolveTypeRefArray(Metadata *MaybeTuple) { + auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple); + if (!Tuple || Tuple->isDistinct()) + return MaybeTuple; + + // Look through the DITypeRefArray, upgrading each DITypeRef. + SmallVector<Metadata *, 32> Ops; + Ops.reserve(Tuple->getNumOperands()); + for (Metadata *MD : Tuple->operands()) + Ops.push_back(upgradeTypeRef(MD)); + + return MDTuple::get(Context, Ops); +} + +namespace { + +class PlaceholderQueue { + // Placeholders would thrash around when moved, so store in a std::deque + // instead of some sort of vector. + std::deque<DistinctMDOperandPlaceholder> PHs; + +public: + bool empty() { return PHs.empty(); } + DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID); + void flush(BitcodeReaderMetadataList &MetadataList); + + /// Return the list of temporaries nodes in the queue, these need to be + /// loaded before we can flush the queue. + void getTemporaries(BitcodeReaderMetadataList &MetadataList, + DenseSet<unsigned> &Temporaries) { + for (auto &PH : PHs) { + auto ID = PH.getID(); + auto *MD = MetadataList.lookup(ID); + if (!MD) { + Temporaries.insert(ID); + continue; + } + auto *N = dyn_cast_or_null<MDNode>(MD); + if (N && N->isTemporary()) + Temporaries.insert(ID); + } + } +}; + +} // end anonymous namespace + +DistinctMDOperandPlaceholder &PlaceholderQueue::getPlaceholderOp(unsigned ID) { + PHs.emplace_back(ID); + return PHs.back(); +} + +void PlaceholderQueue::flush(BitcodeReaderMetadataList &MetadataList) { + while (!PHs.empty()) { + auto *MD = MetadataList.lookup(PHs.front().getID()); + assert(MD && "Flushing placeholder on unassigned MD"); +#ifndef NDEBUG + if (auto *MDN = dyn_cast<MDNode>(MD)) + assert(MDN->isResolved() && + "Flushing Placeholder while cycles aren't resolved"); +#endif + PHs.front().replaceUseWith(MD); + PHs.pop_front(); + } +} + +} // anonynous namespace + +class MetadataLoader::MetadataLoaderImpl { + BitcodeReaderMetadataList MetadataList; + BitcodeReaderValueList &ValueList; + BitstreamCursor &Stream; + LLVMContext &Context; + Module &TheModule; + std::function<Type *(unsigned)> getTypeByID; + + /// Cursor associated with the lazy-loading of Metadata. This is the easy way + /// to keep around the right "context" (Abbrev list) to be able to jump in + /// the middle of the metadata block and load any record. + BitstreamCursor IndexCursor; + + /// Index that keeps track of MDString values. + std::vector<StringRef> MDStringRef; + + /// On-demand loading of a single MDString. Requires the index above to be + /// populated. + MDString *lazyLoadOneMDString(unsigned Idx); + + /// Index that keeps track of where to find a metadata record in the stream. + std::vector<uint64_t> GlobalMetadataBitPosIndex; + + /// Populate the index above to enable lazily loading of metadata, and load + /// the named metadata as well as the transitively referenced global + /// Metadata. + Expected<bool> lazyLoadModuleMetadataBlock(); + + /// On-demand loading of a single metadata. Requires the index above to be + /// populated. + void lazyLoadOneMetadata(unsigned Idx, PlaceholderQueue &Placeholders); + + // Keep mapping of seens pair of old-style CU <-> SP, and update pointers to + // point from SP to CU after a block is completly parsed. + std::vector<std::pair<DICompileUnit *, Metadata *>> CUSubprograms; + + /// Functions that need to be matched with subprograms when upgrading old + /// metadata. + SmallDenseMap<Function *, DISubprogram *, 16> FunctionsWithSPs; + + // Map the bitcode's custom MDKind ID to the Module's MDKind ID. + DenseMap<unsigned, unsigned> MDKindMap; + + bool StripTBAA = false; + bool HasSeenOldLoopTags = false; + bool NeedUpgradeToDIGlobalVariableExpression = false; + + /// True if metadata is being parsed for a module being ThinLTO imported. + bool IsImporting = false; + + Error parseOneMetadata(SmallVectorImpl<uint64_t> &Record, unsigned Code, + PlaceholderQueue &Placeholders, StringRef Blob, + unsigned &NextMetadataNo); + Error parseMetadataStrings(ArrayRef<uint64_t> Record, StringRef Blob, + std::function<void(StringRef)> CallBack); + Error parseGlobalObjectAttachment(GlobalObject &GO, + ArrayRef<uint64_t> Record); + Error parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record); + + void resolveForwardRefsAndPlaceholders(PlaceholderQueue &Placeholders); + + /// Upgrade old-style CU <-> SP pointers to point from SP to CU. + void upgradeCUSubprograms() { + for (auto CU_SP : CUSubprograms) + if (auto *SPs = dyn_cast_or_null<MDTuple>(CU_SP.second)) + for (auto &Op : SPs->operands()) + if (auto *SP = dyn_cast_or_null<MDNode>(Op)) + SP->replaceOperandWith(7, CU_SP.first); + CUSubprograms.clear(); + } + + /// Upgrade old-style bare DIGlobalVariables to DIGlobalVariableExpressions. + void upgradeCUVariables() { + if (!NeedUpgradeToDIGlobalVariableExpression) + return; + + // Upgrade list of variables attached to the CUs. + if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) + for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) { + auto *CU = cast<DICompileUnit>(CUNodes->getOperand(I)); + if (auto *GVs = dyn_cast_or_null<MDTuple>(CU->getRawGlobalVariables())) + for (unsigned I = 0; I < GVs->getNumOperands(); I++) + if (auto *GV = + dyn_cast_or_null<DIGlobalVariable>(GVs->getOperand(I))) { + auto *DGVE = + DIGlobalVariableExpression::getDistinct(Context, GV, nullptr); + GVs->replaceOperandWith(I, DGVE); + } + } + + // Upgrade variables attached to globals. + for (auto &GV : TheModule.globals()) { + SmallVector<MDNode *, 1> MDs, NewMDs; + GV.getMetadata(LLVMContext::MD_dbg, MDs); + GV.eraseMetadata(LLVMContext::MD_dbg); + for (auto *MD : MDs) + if (auto *DGV = dyn_cast_or_null<DIGlobalVariable>(MD)) { + auto *DGVE = + DIGlobalVariableExpression::getDistinct(Context, DGV, nullptr); + GV.addMetadata(LLVMContext::MD_dbg, *DGVE); + } else + GV.addMetadata(LLVMContext::MD_dbg, *MD); + } + } + + void upgradeDebugInfo() { + upgradeCUSubprograms(); + upgradeCUVariables(); + } + +public: + MetadataLoaderImpl(BitstreamCursor &Stream, Module &TheModule, + BitcodeReaderValueList &ValueList, + std::function<Type *(unsigned)> getTypeByID, + bool IsImporting) + : MetadataList(TheModule.getContext()), ValueList(ValueList), + Stream(Stream), Context(TheModule.getContext()), TheModule(TheModule), + getTypeByID(getTypeByID), IsImporting(IsImporting) {} + + Error parseMetadata(bool ModuleLevel); + + bool hasFwdRefs() const { return MetadataList.hasFwdRefs(); } + + Metadata *getMetadataFwdRefOrLoad(unsigned ID) { + if (ID < MDStringRef.size()) + return lazyLoadOneMDString(ID); + if (auto *MD = MetadataList.lookup(ID)) + return MD; + // If lazy-loading is enabled, we try recursively to load the operand + // instead of creating a temporary. + if (ID < (MDStringRef.size() + GlobalMetadataBitPosIndex.size())) { + PlaceholderQueue Placeholders; + lazyLoadOneMetadata(ID, Placeholders); + resolveForwardRefsAndPlaceholders(Placeholders); + return MetadataList.lookup(ID); + } + return MetadataList.getMetadataFwdRef(ID); + } + + MDNode *getMDNodeFwdRefOrNull(unsigned Idx) { + return MetadataList.getMDNodeFwdRefOrNull(Idx); + } + + DISubprogram *lookupSubprogramForFunction(Function *F) { + return FunctionsWithSPs.lookup(F); + } + + bool hasSeenOldLoopTags() { return HasSeenOldLoopTags; } + + Error parseMetadataAttachment( + Function &F, const SmallVectorImpl<Instruction *> &InstructionList); + + Error parseMetadataKinds(); + + void setStripTBAA(bool Value) { StripTBAA = Value; } + bool isStrippingTBAA() { return StripTBAA; } + + unsigned size() const { return MetadataList.size(); } + void shrinkTo(unsigned N) { MetadataList.shrinkTo(N); } +}; + +Error error(const Twine &Message) { + return make_error<StringError>( + Message, make_error_code(BitcodeError::CorruptedBitcode)); +} + +Expected<bool> +MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { + IndexCursor = Stream; + SmallVector<uint64_t, 64> Record; + // Get the abbrevs, and preload record positions to make them lazy-loadable. + while (true) { + BitstreamEntry Entry = IndexCursor.advanceSkippingSubblocks( + BitstreamCursor::AF_DontPopBlockAtEnd); + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: { + return true; + } + case BitstreamEntry::Record: { + // The interesting case. + ++NumMDRecordLoaded; + uint64_t CurrentPos = IndexCursor.GetCurrentBitNo(); + auto Code = IndexCursor.skipRecord(Entry.ID); + switch (Code) { + case bitc::METADATA_STRINGS: { + // Rewind and parse the strings. + IndexCursor.JumpToBit(CurrentPos); + StringRef Blob; + Record.clear(); + IndexCursor.readRecord(Entry.ID, Record, &Blob); + unsigned NumStrings = Record[0]; + MDStringRef.reserve(NumStrings); + auto IndexNextMDString = [&](StringRef Str) { + MDStringRef.push_back(Str); + }; + if (auto Err = parseMetadataStrings(Record, Blob, IndexNextMDString)) + return std::move(Err); + break; + } + case bitc::METADATA_INDEX_OFFSET: { + // This is the offset to the index, when we see this we skip all the + // records and load only an index to these. + IndexCursor.JumpToBit(CurrentPos); + Record.clear(); + IndexCursor.readRecord(Entry.ID, Record); + if (Record.size() != 2) + return error("Invalid record"); + auto Offset = Record[0] + (Record[1] << 32); + auto BeginPos = IndexCursor.GetCurrentBitNo(); + IndexCursor.JumpToBit(BeginPos + Offset); + Entry = IndexCursor.advanceSkippingSubblocks( + BitstreamCursor::AF_DontPopBlockAtEnd); + assert(Entry.Kind == BitstreamEntry::Record && + "Corrupted bitcode: Expected `Record` when trying to find the " + "Metadata index"); + Record.clear(); + auto Code = IndexCursor.readRecord(Entry.ID, Record); + (void)Code; + assert(Code == bitc::METADATA_INDEX && "Corrupted bitcode: Expected " + "`METADATA_INDEX` when trying " + "to find the Metadata index"); + + // Delta unpack + auto CurrentValue = BeginPos; + GlobalMetadataBitPosIndex.reserve(Record.size()); + for (auto &Elt : Record) { + CurrentValue += Elt; + GlobalMetadataBitPosIndex.push_back(CurrentValue); + } + break; + } + case bitc::METADATA_INDEX: + // We don't expect to get there, the Index is loaded when we encounter + // the offset. + return error("Corrupted Metadata block"); + case bitc::METADATA_NAME: { + // Named metadata need to be materialized now and aren't deferred. + IndexCursor.JumpToBit(CurrentPos); + Record.clear(); + unsigned Code = IndexCursor.readRecord(Entry.ID, Record); + assert(Code == bitc::METADATA_NAME); + + // Read name of the named metadata. + SmallString<8> Name(Record.begin(), Record.end()); + Code = IndexCursor.ReadCode(); + + // Named Metadata comes in two parts, we expect the name to be followed + // by the node + Record.clear(); + unsigned NextBitCode = IndexCursor.readRecord(Code, Record); + assert(NextBitCode == bitc::METADATA_NAMED_NODE); + (void)NextBitCode; + + // Read named metadata elements. + unsigned Size = Record.size(); + NamedMDNode *NMD = TheModule.getOrInsertNamedMetadata(Name); + for (unsigned i = 0; i != Size; ++i) { + // FIXME: We could use a placeholder here, however NamedMDNode are + // taking MDNode as operand and not using the Metadata infrastructure. + // It is acknowledged by 'TODO: Inherit from Metadata' in the + // NamedMDNode class definition. + MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[i]); + assert(MD && "Invalid record"); + NMD->addOperand(MD); + } + break; + } + case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { + // FIXME: we need to do this early because we don't materialize global + // value explicitly. + IndexCursor.JumpToBit(CurrentPos); + Record.clear(); + IndexCursor.readRecord(Entry.ID, Record); + if (Record.size() % 2 == 0) + return error("Invalid record"); + unsigned ValueID = Record[0]; + if (ValueID >= ValueList.size()) + return error("Invalid record"); + if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID])) + if (Error Err = parseGlobalObjectAttachment( + *GO, ArrayRef<uint64_t>(Record).slice(1))) + return std::move(Err); + break; + } + case bitc::METADATA_KIND: + case bitc::METADATA_STRING_OLD: + case bitc::METADATA_OLD_FN_NODE: + case bitc::METADATA_OLD_NODE: + case bitc::METADATA_VALUE: + case bitc::METADATA_DISTINCT_NODE: + case bitc::METADATA_NODE: + case bitc::METADATA_LOCATION: + case bitc::METADATA_GENERIC_DEBUG: + case bitc::METADATA_SUBRANGE: + case bitc::METADATA_ENUMERATOR: + case bitc::METADATA_BASIC_TYPE: + case bitc::METADATA_DERIVED_TYPE: + case bitc::METADATA_COMPOSITE_TYPE: + case bitc::METADATA_SUBROUTINE_TYPE: + case bitc::METADATA_MODULE: + case bitc::METADATA_FILE: + case bitc::METADATA_COMPILE_UNIT: + case bitc::METADATA_SUBPROGRAM: + case bitc::METADATA_LEXICAL_BLOCK: + case bitc::METADATA_LEXICAL_BLOCK_FILE: + case bitc::METADATA_NAMESPACE: + case bitc::METADATA_MACRO: + case bitc::METADATA_MACRO_FILE: + case bitc::METADATA_TEMPLATE_TYPE: + case bitc::METADATA_TEMPLATE_VALUE: + case bitc::METADATA_GLOBAL_VAR: + case bitc::METADATA_LOCAL_VAR: + case bitc::METADATA_EXPRESSION: + case bitc::METADATA_OBJC_PROPERTY: + case bitc::METADATA_IMPORTED_ENTITY: + case bitc::METADATA_GLOBAL_VAR_EXPR: + // We don't expect to see any of these, if we see one, give up on + // lazy-loading and fallback. + MDStringRef.clear(); + GlobalMetadataBitPosIndex.clear(); + return false; + } + break; + } + } + } +} + +/// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing +/// module level metadata. +Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { + if (!ModuleLevel && MetadataList.hasFwdRefs()) + return error("Invalid metadata: fwd refs into function blocks"); + + // Record the entry position so that we can jump back here and efficiently + // skip the whole block in case we lazy-load. + auto EntryPos = Stream.GetCurrentBitNo(); + + if (Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID)) + return error("Invalid record"); + + SmallVector<uint64_t, 64> Record; + PlaceholderQueue Placeholders; + + // We lazy-load module-level metadata: we build an index for each record, and + // then load individual record as needed, starting with the named metadata. + if (ModuleLevel && IsImporting && MetadataList.empty() && + !DisableLazyLoading) { + auto SuccessOrErr = lazyLoadModuleMetadataBlock(); + if (!SuccessOrErr) + return SuccessOrErr.takeError(); + if (SuccessOrErr.get()) { + // An index was successfully created and we will be able to load metadata + // on-demand. + MetadataList.resize(MDStringRef.size() + + GlobalMetadataBitPosIndex.size()); + + // Reading the named metadata created forward references and/or + // placeholders, that we flush here. + resolveForwardRefsAndPlaceholders(Placeholders); + upgradeDebugInfo(); + // Return at the beginning of the block, since it is easy to skip it + // entirely from there. + Stream.ReadBlockEnd(); // Pop the abbrev block context. + Stream.JumpToBit(EntryPos); + if (Stream.SkipBlock()) + return error("Invalid record"); + return Error::success(); + } + // Couldn't load an index, fallback to loading all the block "old-style". + } + + unsigned NextMetadataNo = MetadataList.size(); + + // Read all the records. + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + resolveForwardRefsAndPlaceholders(Placeholders); + upgradeDebugInfo(); + return Error::success(); + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read a record. + Record.clear(); + StringRef Blob; + ++NumMDRecordLoaded; + unsigned Code = Stream.readRecord(Entry.ID, Record, &Blob); + if (Error Err = + parseOneMetadata(Record, Code, Placeholders, Blob, NextMetadataNo)) + return Err; + } +} + +MDString *MetadataLoader::MetadataLoaderImpl::lazyLoadOneMDString(unsigned ID) { + ++NumMDStringLoaded; + if (Metadata *MD = MetadataList.lookup(ID)) + return cast<MDString>(MD); + auto MDS = MDString::get(Context, MDStringRef[ID]); + MetadataList.assignValue(MDS, ID); + return MDS; +} + +void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata( + unsigned ID, PlaceholderQueue &Placeholders) { + assert(ID < (MDStringRef.size()) + GlobalMetadataBitPosIndex.size()); + assert(ID >= MDStringRef.size() && "Unexpected lazy-loading of MDString"); + // Lookup first if the metadata hasn't already been loaded. + if (auto *MD = MetadataList.lookup(ID)) { + auto *N = dyn_cast_or_null<MDNode>(MD); + if (!N->isTemporary()) + return; + } + SmallVector<uint64_t, 64> Record; + StringRef Blob; + IndexCursor.JumpToBit(GlobalMetadataBitPosIndex[ID - MDStringRef.size()]); + auto Entry = IndexCursor.advanceSkippingSubblocks(); + ++NumMDRecordLoaded; + unsigned Code = IndexCursor.readRecord(Entry.ID, Record, &Blob); + if (Error Err = parseOneMetadata(Record, Code, Placeholders, Blob, ID)) + report_fatal_error("Can't lazyload MD"); +} + +/// Ensure that all forward-references and placeholders are resolved. +/// Iteratively lazy-loading metadata on-demand if needed. +void MetadataLoader::MetadataLoaderImpl::resolveForwardRefsAndPlaceholders( + PlaceholderQueue &Placeholders) { + DenseSet<unsigned> Temporaries; + while (1) { + // Populate Temporaries with the placeholders that haven't been loaded yet. + Placeholders.getTemporaries(MetadataList, Temporaries); + + // If we don't have any temporary, or FwdReference, we're done! + if (Temporaries.empty() && !MetadataList.hasFwdRefs()) + break; + + // First, load all the temporaries. This can add new placeholders or + // forward references. + for (auto ID : Temporaries) + lazyLoadOneMetadata(ID, Placeholders); + Temporaries.clear(); + + // Second, load the forward-references. This can also add new placeholders + // or forward references. + while (MetadataList.hasFwdRefs()) + lazyLoadOneMetadata(MetadataList.getNextFwdRef(), Placeholders); + } + // At this point we don't have any forward reference remaining, or temporary + // that haven't been loaded. We can safely drop RAUW support and mark cycles + // as resolved. + MetadataList.tryToResolveCycles(); + + // Finally, everything is in place, we can replace the placeholders operands + // with the final node they refer to. + Placeholders.flush(MetadataList); +} + +Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( + SmallVectorImpl<uint64_t> &Record, unsigned Code, + PlaceholderQueue &Placeholders, StringRef Blob, unsigned &NextMetadataNo) { + + bool IsDistinct = false; + auto getMD = [&](unsigned ID) -> Metadata * { + if (ID < MDStringRef.size()) + return lazyLoadOneMDString(ID); + if (!IsDistinct) { + if (auto *MD = MetadataList.lookup(ID)) + return MD; + // If lazy-loading is enabled, we try recursively to load the operand + // instead of creating a temporary. + if (ID < (MDStringRef.size() + GlobalMetadataBitPosIndex.size())) { + // Create a temporary for the node that is referencing the operand we + // will lazy-load. It is needed before recursing in case there are + // uniquing cycles. + MetadataList.getMetadataFwdRef(NextMetadataNo); + lazyLoadOneMetadata(ID, Placeholders); + return MetadataList.lookup(ID); + } + // Return a temporary. + return MetadataList.getMetadataFwdRef(ID); + } + if (auto *MD = MetadataList.getMetadataIfResolved(ID)) + return MD; + return &Placeholders.getPlaceholderOp(ID); + }; + auto getMDOrNull = [&](unsigned ID) -> Metadata * { + if (ID) + return getMD(ID - 1); + return nullptr; + }; + auto getMDOrNullWithoutPlaceholders = [&](unsigned ID) -> Metadata * { + if (ID) + return MetadataList.getMetadataFwdRef(ID - 1); + return nullptr; + }; + auto getMDString = [&](unsigned ID) -> MDString * { + // This requires that the ID is not really a forward reference. In + // particular, the MDString must already have been resolved. + auto MDS = getMDOrNull(ID); + return cast_or_null<MDString>(MDS); + }; + + // Support for old type refs. + auto getDITypeRefOrNull = [&](unsigned ID) { + return MetadataList.upgradeTypeRef(getMDOrNull(ID)); + }; + +#define GET_OR_DISTINCT(CLASS, ARGS) \ + (IsDistinct ? CLASS::getDistinct ARGS : CLASS::get ARGS) + + switch (Code) { + default: // Default behavior: ignore. + break; + case bitc::METADATA_NAME: { + // Read name of the named metadata. + SmallString<8> Name(Record.begin(), Record.end()); + Record.clear(); + Code = Stream.ReadCode(); + + ++NumMDRecordLoaded; + unsigned NextBitCode = Stream.readRecord(Code, Record); + if (NextBitCode != bitc::METADATA_NAMED_NODE) + return error("METADATA_NAME not followed by METADATA_NAMED_NODE"); + + // Read named metadata elements. + unsigned Size = Record.size(); + NamedMDNode *NMD = TheModule.getOrInsertNamedMetadata(Name); + for (unsigned i = 0; i != Size; ++i) { + MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[i]); + if (!MD) + return error("Invalid record"); + NMD->addOperand(MD); + } + break; + } + case bitc::METADATA_OLD_FN_NODE: { + // FIXME: Remove in 4.0. + // This is a LocalAsMetadata record, the only type of function-local + // metadata. + if (Record.size() % 2 == 1) + return error("Invalid record"); + + // If this isn't a LocalAsMetadata record, we're dropping it. This used + // to be legal, but there's no upgrade path. + auto dropRecord = [&] { + MetadataList.assignValue(MDNode::get(Context, None), NextMetadataNo); + NextMetadataNo++; + }; + if (Record.size() != 2) { + dropRecord(); + break; + } + + Type *Ty = getTypeByID(Record[0]); + if (Ty->isMetadataTy() || Ty->isVoidTy()) { + dropRecord(); + break; + } + + MetadataList.assignValue( + LocalAsMetadata::get(ValueList.getValueFwdRef(Record[1], Ty)), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_OLD_NODE: { + // FIXME: Remove in 4.0. + if (Record.size() % 2 == 1) + return error("Invalid record"); + + unsigned Size = Record.size(); + SmallVector<Metadata *, 8> Elts; + for (unsigned i = 0; i != Size; i += 2) { + Type *Ty = getTypeByID(Record[i]); + if (!Ty) + return error("Invalid record"); + if (Ty->isMetadataTy()) + Elts.push_back(getMD(Record[i + 1])); + else if (!Ty->isVoidTy()) { + auto *MD = + ValueAsMetadata::get(ValueList.getValueFwdRef(Record[i + 1], Ty)); + assert(isa<ConstantAsMetadata>(MD) && + "Expected non-function-local metadata"); + Elts.push_back(MD); + } else + Elts.push_back(nullptr); + } + MetadataList.assignValue(MDNode::get(Context, Elts), NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_VALUE: { + if (Record.size() != 2) + return error("Invalid record"); + + Type *Ty = getTypeByID(Record[0]); + if (Ty->isMetadataTy() || Ty->isVoidTy()) + return error("Invalid record"); + + MetadataList.assignValue( + ValueAsMetadata::get(ValueList.getValueFwdRef(Record[1], Ty)), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_DISTINCT_NODE: + IsDistinct = true; + LLVM_FALLTHROUGH; + case bitc::METADATA_NODE: { + SmallVector<Metadata *, 8> Elts; + Elts.reserve(Record.size()); + for (unsigned ID : Record) + Elts.push_back(getMDOrNull(ID)); + MetadataList.assignValue(IsDistinct ? MDNode::getDistinct(Context, Elts) + : MDNode::get(Context, Elts), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_LOCATION: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0]; + unsigned Line = Record[1]; + unsigned Column = Record[2]; + Metadata *Scope = getMD(Record[3]); + Metadata *InlinedAt = getMDOrNull(Record[4]); + MetadataList.assignValue( + GET_OR_DISTINCT(DILocation, (Context, Line, Column, Scope, InlinedAt)), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_GENERIC_DEBUG: { + if (Record.size() < 4) + return error("Invalid record"); + + IsDistinct = Record[0]; + unsigned Tag = Record[1]; + unsigned Version = Record[2]; + + if (Tag >= 1u << 16 || Version != 0) + return error("Invalid record"); + + auto *Header = getMDString(Record[3]); + SmallVector<Metadata *, 8> DwarfOps; + for (unsigned I = 4, E = Record.size(); I != E; ++I) + DwarfOps.push_back(getMDOrNull(Record[I])); + MetadataList.assignValue( + GET_OR_DISTINCT(GenericDINode, (Context, Tag, Header, DwarfOps)), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_SUBRANGE: { + if (Record.size() != 3) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DISubrange, + (Context, Record[1], unrotateSign(Record[2]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_ENUMERATOR: { + if (Record.size() != 3) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIEnumerator, (Context, unrotateSign(Record[1]), + getMDString(Record[2]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_BASIC_TYPE: { + if (Record.size() != 6) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIBasicType, + (Context, Record[1], getMDString(Record[2]), Record[3], + Record[4], Record[5])), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_DERIVED_TYPE: { + if (Record.size() != 12) + return error("Invalid record"); + + IsDistinct = Record[0]; + DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]); + MetadataList.assignValue( + GET_OR_DISTINCT(DIDerivedType, + (Context, Record[1], getMDString(Record[2]), + getMDOrNull(Record[3]), Record[4], + getDITypeRefOrNull(Record[5]), + getDITypeRefOrNull(Record[6]), Record[7], Record[8], + Record[9], Flags, getDITypeRefOrNull(Record[11]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_COMPOSITE_TYPE: { + if (Record.size() != 16) + return error("Invalid record"); + + // If we have a UUID and this is not a forward declaration, lookup the + // mapping. + IsDistinct = Record[0] & 0x1; + bool IsNotUsedInTypeRef = Record[0] >= 2; + unsigned Tag = Record[1]; + MDString *Name = getMDString(Record[2]); + Metadata *File = getMDOrNull(Record[3]); + unsigned Line = Record[4]; + Metadata *Scope = getDITypeRefOrNull(Record[5]); + Metadata *BaseType = nullptr; + uint64_t SizeInBits = Record[7]; + if (Record[8] > (uint64_t)std::numeric_limits<uint32_t>::max()) + return error("Alignment value is too large"); + uint32_t AlignInBits = Record[8]; + uint64_t OffsetInBits = 0; + DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]); + Metadata *Elements = nullptr; + unsigned RuntimeLang = Record[12]; + Metadata *VTableHolder = nullptr; + Metadata *TemplateParams = nullptr; + auto *Identifier = getMDString(Record[15]); + // If this module is being parsed so that it can be ThinLTO imported + // into another module, composite types only need to be imported + // as type declarations (unless full type definitions requested). + // Create type declarations up front to save memory. Also, buildODRType + // handles the case where this is type ODRed with a definition needed + // by the importing module, in which case the existing definition is + // used. + if (IsImporting && !ImportFullTypeDefinitions && Identifier && + (Tag == dwarf::DW_TAG_enumeration_type || + Tag == dwarf::DW_TAG_class_type || + Tag == dwarf::DW_TAG_structure_type || + Tag == dwarf::DW_TAG_union_type)) { + Flags = Flags | DINode::FlagFwdDecl; + } else { + BaseType = getDITypeRefOrNull(Record[6]); + OffsetInBits = Record[9]; + Elements = getMDOrNull(Record[11]); + VTableHolder = getDITypeRefOrNull(Record[13]); + TemplateParams = getMDOrNull(Record[14]); + } + DICompositeType *CT = nullptr; + if (Identifier) + CT = DICompositeType::buildODRType( + Context, *Identifier, Tag, Name, File, Line, Scope, BaseType, + SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, + VTableHolder, TemplateParams); + + // Create a node if we didn't get a lazy ODR type. + if (!CT) + CT = GET_OR_DISTINCT(DICompositeType, + (Context, Tag, Name, File, Line, Scope, BaseType, + SizeInBits, AlignInBits, OffsetInBits, Flags, + Elements, RuntimeLang, VTableHolder, TemplateParams, + Identifier)); + if (!IsNotUsedInTypeRef && Identifier) + MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT)); + + MetadataList.assignValue(CT, NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_SUBROUTINE_TYPE: { + if (Record.size() < 3 || Record.size() > 4) + return error("Invalid record"); + bool IsOldTypeRefArray = Record[0] < 2; + unsigned CC = (Record.size() > 3) ? Record[3] : 0; + + IsDistinct = Record[0] & 0x1; + DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[1]); + Metadata *Types = getMDOrNull(Record[2]); + if (LLVM_UNLIKELY(IsOldTypeRefArray)) + Types = MetadataList.upgradeTypeRefArray(Types); + + MetadataList.assignValue( + GET_OR_DISTINCT(DISubroutineType, (Context, Flags, CC, Types)), + NextMetadataNo); + NextMetadataNo++; + break; + } + + case bitc::METADATA_MODULE: { + if (Record.size() != 6) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIModule, + (Context, getMDOrNull(Record[1]), + getMDString(Record[2]), getMDString(Record[3]), + getMDString(Record[4]), getMDString(Record[5]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + + case bitc::METADATA_FILE: { + if (Record.size() != 3 && Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT( + DIFile, + (Context, getMDString(Record[1]), getMDString(Record[2]), + Record.size() == 3 ? DIFile::CSK_None + : static_cast<DIFile::ChecksumKind>(Record[3]), + Record.size() == 3 ? nullptr : getMDString(Record[4]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_COMPILE_UNIT: { + if (Record.size() < 14 || Record.size() > 17) + return error("Invalid record"); + + // Ignore Record[0], which indicates whether this compile unit is + // distinct. It's always distinct. + IsDistinct = true; + auto *CU = DICompileUnit::getDistinct( + Context, Record[1], getMDOrNull(Record[2]), getMDString(Record[3]), + Record[4], getMDString(Record[5]), Record[6], getMDString(Record[7]), + Record[8], getMDOrNull(Record[9]), getMDOrNull(Record[10]), + getMDOrNull(Record[12]), getMDOrNull(Record[13]), + Record.size() <= 15 ? nullptr : getMDOrNull(Record[15]), + Record.size() <= 14 ? 0 : Record[14], + Record.size() <= 16 ? true : Record[16]); + + MetadataList.assignValue(CU, NextMetadataNo); + NextMetadataNo++; + + // Move the Upgrade the list of subprograms. + if (Metadata *SPs = getMDOrNullWithoutPlaceholders(Record[11])) + CUSubprograms.push_back({CU, SPs}); + break; + } + case bitc::METADATA_SUBPROGRAM: { + if (Record.size() < 18 || Record.size() > 20) + return error("Invalid record"); + + IsDistinct = + (Record[0] & 1) || Record[8]; // All definitions should be distinct. + // Version 1 has a Function as Record[15]. + // Version 2 has removed Record[15]. + // Version 3 has the Unit as Record[15]. + // Version 4 added thisAdjustment. + bool HasUnit = Record[0] >= 2; + if (HasUnit && Record.size() < 19) + return error("Invalid record"); + Metadata *CUorFn = getMDOrNull(Record[15]); + unsigned Offset = Record.size() >= 19 ? 1 : 0; + bool HasFn = Offset && !HasUnit; + bool HasThisAdj = Record.size() >= 20; + DISubprogram *SP = GET_OR_DISTINCT( + DISubprogram, (Context, + getDITypeRefOrNull(Record[1]), // scope + getMDString(Record[2]), // name + getMDString(Record[3]), // linkageName + getMDOrNull(Record[4]), // file + Record[5], // line + getMDOrNull(Record[6]), // type + Record[7], // isLocal + Record[8], // isDefinition + Record[9], // scopeLine + getDITypeRefOrNull(Record[10]), // containingType + Record[11], // virtuality + Record[12], // virtualIndex + HasThisAdj ? Record[19] : 0, // thisAdjustment + static_cast<DINode::DIFlags>(Record[13] // flags + ), + Record[14], // isOptimized + HasUnit ? CUorFn : nullptr, // unit + getMDOrNull(Record[15 + Offset]), // templateParams + getMDOrNull(Record[16 + Offset]), // declaration + getMDOrNull(Record[17 + Offset]) // variables + )); + MetadataList.assignValue(SP, NextMetadataNo); + NextMetadataNo++; + + // Upgrade sp->function mapping to function->sp mapping. + if (HasFn) { + if (auto *CMD = dyn_cast_or_null<ConstantAsMetadata>(CUorFn)) + if (auto *F = dyn_cast<Function>(CMD->getValue())) { + if (F->isMaterializable()) + // Defer until materialized; unmaterialized functions may not have + // metadata. + FunctionsWithSPs[F] = SP; + else if (!F->empty()) + F->setSubprogram(SP); + } + } + break; + } + case bitc::METADATA_LEXICAL_BLOCK: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DILexicalBlock, + (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]), Record[3], Record[4])), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_LEXICAL_BLOCK_FILE: { + if (Record.size() != 4) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DILexicalBlockFile, + (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]), Record[3])), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_NAMESPACE: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0] & 1; + bool ExportSymbols = Record[0] & 2; + MetadataList.assignValue( + GET_OR_DISTINCT(DINamespace, + (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]), getMDString(Record[3]), + Record[4], ExportSymbols)), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_MACRO: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIMacro, + (Context, Record[1], Record[2], getMDString(Record[3]), + getMDString(Record[4]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_MACRO_FILE: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIMacroFile, + (Context, Record[1], Record[2], getMDOrNull(Record[3]), + getMDOrNull(Record[4]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_TEMPLATE_TYPE: { + if (Record.size() != 3) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue(GET_OR_DISTINCT(DITemplateTypeParameter, + (Context, getMDString(Record[1]), + getDITypeRefOrNull(Record[2]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_TEMPLATE_VALUE: { + if (Record.size() != 5) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DITemplateValueParameter, + (Context, Record[1], getMDString(Record[2]), + getDITypeRefOrNull(Record[3]), + getMDOrNull(Record[4]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_GLOBAL_VAR: { + if (Record.size() < 11 || Record.size() > 12) + return error("Invalid record"); + + IsDistinct = Record[0] & 1; + unsigned Version = Record[0] >> 1; + + if (Version == 1) { + MetadataList.assignValue( + GET_OR_DISTINCT(DIGlobalVariable, + (Context, getMDOrNull(Record[1]), + getMDString(Record[2]), getMDString(Record[3]), + getMDOrNull(Record[4]), Record[5], + getDITypeRefOrNull(Record[6]), Record[7], Record[8], + getMDOrNull(Record[10]), Record[11])), + NextMetadataNo); + NextMetadataNo++; + } else if (Version == 0) { + // Upgrade old metadata, which stored a global variable reference or a + // ConstantInt here. + Metadata *Expr = getMDOrNull(Record[9]); + uint32_t AlignInBits = 0; + if (Record.size() > 11) { + if (Record[11] > (uint64_t)std::numeric_limits<uint32_t>::max()) + return error("Alignment value is too large"); + AlignInBits = Record[11]; + } + GlobalVariable *Attach = nullptr; + if (auto *CMD = dyn_cast_or_null<ConstantAsMetadata>(Expr)) { + if (auto *GV = dyn_cast<GlobalVariable>(CMD->getValue())) { + Attach = GV; + Expr = nullptr; + } else if (auto *CI = dyn_cast<ConstantInt>(CMD->getValue())) { + Expr = DIExpression::get(Context, + {dwarf::DW_OP_constu, CI->getZExtValue(), + dwarf::DW_OP_stack_value}); + } else { + Expr = nullptr; + } + } + DIGlobalVariable *DGV = GET_OR_DISTINCT( + DIGlobalVariable, + (Context, getMDOrNull(Record[1]), getMDString(Record[2]), + getMDString(Record[3]), getMDOrNull(Record[4]), Record[5], + getDITypeRefOrNull(Record[6]), Record[7], Record[8], + getMDOrNull(Record[10]), AlignInBits)); + + DIGlobalVariableExpression *DGVE = nullptr; + if (Attach || Expr) + DGVE = DIGlobalVariableExpression::getDistinct(Context, DGV, Expr); + else + NeedUpgradeToDIGlobalVariableExpression = true; + if (Attach) + Attach->addDebugInfo(DGVE); + + auto *MDNode = Expr ? cast<Metadata>(DGVE) : cast<Metadata>(DGV); + MetadataList.assignValue(MDNode, NextMetadataNo); + NextMetadataNo++; + } else + return error("Invalid record"); + + break; + } + case bitc::METADATA_LOCAL_VAR: { + // 10th field is for the obseleted 'inlinedAt:' field. + if (Record.size() < 8 || Record.size() > 10) + return error("Invalid record"); + + IsDistinct = Record[0] & 1; + bool HasAlignment = Record[0] & 2; + // 2nd field used to be an artificial tag, either DW_TAG_auto_variable or + // DW_TAG_arg_variable, if we have alignment flag encoded it means, that + // this is newer version of record which doesn't have artifical tag. + bool HasTag = !HasAlignment && Record.size() > 8; + DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[7 + HasTag]); + uint32_t AlignInBits = 0; + if (HasAlignment) { + if (Record[8 + HasTag] > (uint64_t)std::numeric_limits<uint32_t>::max()) + return error("Alignment value is too large"); + AlignInBits = Record[8 + HasTag]; + } + MetadataList.assignValue( + GET_OR_DISTINCT(DILocalVariable, + (Context, getMDOrNull(Record[1 + HasTag]), + getMDString(Record[2 + HasTag]), + getMDOrNull(Record[3 + HasTag]), Record[4 + HasTag], + getDITypeRefOrNull(Record[5 + HasTag]), + Record[6 + HasTag], Flags, AlignInBits)), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_EXPRESSION: { + if (Record.size() < 1) + return error("Invalid record"); + + IsDistinct = Record[0] & 1; + bool HasOpFragment = Record[0] & 2; + auto Elts = MutableArrayRef<uint64_t>(Record).slice(1); + if (!HasOpFragment) + if (unsigned N = Elts.size()) + if (N >= 3 && Elts[N - 3] == dwarf::DW_OP_bit_piece) + Elts[N - 3] = dwarf::DW_OP_LLVM_fragment; + + MetadataList.assignValue( + GET_OR_DISTINCT(DIExpression, (Context, makeArrayRef(Record).slice(1))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_GLOBAL_VAR_EXPR: { + if (Record.size() != 3) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue(GET_OR_DISTINCT(DIGlobalVariableExpression, + (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_OBJC_PROPERTY: { + if (Record.size() != 8) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIObjCProperty, + (Context, getMDString(Record[1]), + getMDOrNull(Record[2]), Record[3], + getMDString(Record[4]), getMDString(Record[5]), + Record[6], getDITypeRefOrNull(Record[7]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_IMPORTED_ENTITY: { + if (Record.size() != 6) + return error("Invalid record"); + + IsDistinct = Record[0]; + MetadataList.assignValue( + GET_OR_DISTINCT(DIImportedEntity, + (Context, Record[1], getMDOrNull(Record[2]), + getDITypeRefOrNull(Record[3]), Record[4], + getMDString(Record[5]))), + NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_STRING_OLD: { + std::string String(Record.begin(), Record.end()); + + // Test for upgrading !llvm.loop. + HasSeenOldLoopTags |= mayBeOldLoopAttachmentTag(String); + ++NumMDStringLoaded; + Metadata *MD = MDString::get(Context, String); + MetadataList.assignValue(MD, NextMetadataNo); + NextMetadataNo++; + break; + } + case bitc::METADATA_STRINGS: { + auto CreateNextMDString = [&](StringRef Str) { + ++NumMDStringLoaded; + MetadataList.assignValue(MDString::get(Context, Str), NextMetadataNo); + NextMetadataNo++; + }; + if (Error Err = parseMetadataStrings(Record, Blob, CreateNextMDString)) + return Err; + break; + } + case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { + if (Record.size() % 2 == 0) + return error("Invalid record"); + unsigned ValueID = Record[0]; + if (ValueID >= ValueList.size()) + return error("Invalid record"); + if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID])) + if (Error Err = parseGlobalObjectAttachment( + *GO, ArrayRef<uint64_t>(Record).slice(1))) + return Err; + break; + } + case bitc::METADATA_KIND: { + // Support older bitcode files that had METADATA_KIND records in a + // block with METADATA_BLOCK_ID. + if (Error Err = parseMetadataKindRecord(Record)) + return Err; + break; + } + } + return Error::success(); +#undef GET_OR_DISTINCT +} + +Error MetadataLoader::MetadataLoaderImpl::parseMetadataStrings( + ArrayRef<uint64_t> Record, StringRef Blob, + std::function<void(StringRef)> CallBack) { + // All the MDStrings in the block are emitted together in a single + // record. The strings are concatenated and stored in a blob along with + // their sizes. + if (Record.size() != 2) + return error("Invalid record: metadata strings layout"); + + unsigned NumStrings = Record[0]; + unsigned StringsOffset = Record[1]; + if (!NumStrings) + return error("Invalid record: metadata strings with no strings"); + if (StringsOffset > Blob.size()) + return error("Invalid record: metadata strings corrupt offset"); + + StringRef Lengths = Blob.slice(0, StringsOffset); + SimpleBitstreamCursor R(Lengths); + + StringRef Strings = Blob.drop_front(StringsOffset); + do { + if (R.AtEndOfStream()) + return error("Invalid record: metadata strings bad length"); + + unsigned Size = R.ReadVBR(6); + if (Strings.size() < Size) + return error("Invalid record: metadata strings truncated chars"); + + CallBack(Strings.slice(0, Size)); + Strings = Strings.drop_front(Size); + } while (--NumStrings); + + return Error::success(); +} + +Error MetadataLoader::MetadataLoaderImpl::parseGlobalObjectAttachment( + GlobalObject &GO, ArrayRef<uint64_t> Record) { + assert(Record.size() % 2 == 0); + for (unsigned I = 0, E = Record.size(); I != E; I += 2) { + auto K = MDKindMap.find(Record[I]); + if (K == MDKindMap.end()) + return error("Invalid ID"); + MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[I + 1]); + if (!MD) + return error("Invalid metadata attachment"); + GO.addMetadata(K->second, *MD); + } + return Error::success(); +} + +/// Parse metadata attachments. +Error MetadataLoader::MetadataLoaderImpl::parseMetadataAttachment( + Function &F, const SmallVectorImpl<Instruction *> &InstructionList) { + if (Stream.EnterSubBlock(bitc::METADATA_ATTACHMENT_ID)) + return error("Invalid record"); + + SmallVector<uint64_t, 64> Record; + PlaceholderQueue Placeholders; + + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + resolveForwardRefsAndPlaceholders(Placeholders); + return Error::success(); + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read a metadata attachment record. + Record.clear(); + ++NumMDRecordLoaded; + switch (Stream.readRecord(Entry.ID, Record)) { + default: // Default behavior: ignore. + break; + case bitc::METADATA_ATTACHMENT: { + unsigned RecordLength = Record.size(); + if (Record.empty()) + return error("Invalid record"); + if (RecordLength % 2 == 0) { + // A function attachment. + if (Error Err = parseGlobalObjectAttachment(F, Record)) + return Err; + continue; + } + + // An instruction attachment. + Instruction *Inst = InstructionList[Record[0]]; + for (unsigned i = 1; i != RecordLength; i = i + 2) { + unsigned Kind = Record[i]; + DenseMap<unsigned, unsigned>::iterator I = MDKindMap.find(Kind); + if (I == MDKindMap.end()) + return error("Invalid ID"); + if (I->second == LLVMContext::MD_tbaa && StripTBAA) + continue; + + auto Idx = Record[i + 1]; + if (Idx < (MDStringRef.size() + GlobalMetadataBitPosIndex.size()) && + !MetadataList.lookup(Idx)) { + // Load the attachment if it is in the lazy-loadable range and hasn't + // been loaded yet. + lazyLoadOneMetadata(Idx, Placeholders); + resolveForwardRefsAndPlaceholders(Placeholders); + } + + Metadata *Node = MetadataList.getMetadataFwdRef(Idx); + if (isa<LocalAsMetadata>(Node)) + // Drop the attachment. This used to be legal, but there's no + // upgrade path. + break; + MDNode *MD = dyn_cast_or_null<MDNode>(Node); + if (!MD) + return error("Invalid metadata attachment"); + + if (HasSeenOldLoopTags && I->second == LLVMContext::MD_loop) + MD = upgradeInstructionLoopAttachment(*MD); + + if (I->second == LLVMContext::MD_tbaa) { + assert(!MD->isTemporary() && "should load MDs before attachments"); + MD = UpgradeTBAANode(*MD); + } + Inst->setMetadata(I->second, MD); + } + break; + } + } + } +} + +/// Parse a single METADATA_KIND record, inserting result in MDKindMap. +Error MetadataLoader::MetadataLoaderImpl::parseMetadataKindRecord( + SmallVectorImpl<uint64_t> &Record) { + if (Record.size() < 2) + return error("Invalid record"); + + unsigned Kind = Record[0]; + SmallString<8> Name(Record.begin() + 1, Record.end()); + + unsigned NewKind = TheModule.getMDKindID(Name.str()); + if (!MDKindMap.insert(std::make_pair(Kind, NewKind)).second) + return error("Conflicting METADATA_KIND records"); + return Error::success(); +} + +/// Parse the metadata kinds out of the METADATA_KIND_BLOCK. +Error MetadataLoader::MetadataLoaderImpl::parseMetadataKinds() { + if (Stream.EnterSubBlock(bitc::METADATA_KIND_BLOCK_ID)) + return error("Invalid record"); + + SmallVector<uint64_t, 64> Record; + + // Read all the records. + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return Error::success(); + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read a record. + Record.clear(); + ++NumMDRecordLoaded; + unsigned Code = Stream.readRecord(Entry.ID, Record); + switch (Code) { + default: // Default behavior: ignore. + break; + case bitc::METADATA_KIND: { + if (Error Err = parseMetadataKindRecord(Record)) + return Err; + break; + } + } + } +} + +MetadataLoader &MetadataLoader::operator=(MetadataLoader &&RHS) { + Pimpl = std::move(RHS.Pimpl); + return *this; +} +MetadataLoader::MetadataLoader(MetadataLoader &&RHS) + : Pimpl(std::move(RHS.Pimpl)) {} + +MetadataLoader::~MetadataLoader() = default; +MetadataLoader::MetadataLoader(BitstreamCursor &Stream, Module &TheModule, + BitcodeReaderValueList &ValueList, + bool IsImporting, + std::function<Type *(unsigned)> getTypeByID) + : Pimpl(llvm::make_unique<MetadataLoaderImpl>(Stream, TheModule, ValueList, + getTypeByID, IsImporting)) {} + +Error MetadataLoader::parseMetadata(bool ModuleLevel) { + return Pimpl->parseMetadata(ModuleLevel); +} + +bool MetadataLoader::hasFwdRefs() const { return Pimpl->hasFwdRefs(); } + +/// Return the given metadata, creating a replaceable forward reference if +/// necessary. +Metadata *MetadataLoader::getMetadataFwdRefOrLoad(unsigned Idx) { + return Pimpl->getMetadataFwdRefOrLoad(Idx); +} + +MDNode *MetadataLoader::getMDNodeFwdRefOrNull(unsigned Idx) { + return Pimpl->getMDNodeFwdRefOrNull(Idx); +} + +DISubprogram *MetadataLoader::lookupSubprogramForFunction(Function *F) { + return Pimpl->lookupSubprogramForFunction(F); +} + +Error MetadataLoader::parseMetadataAttachment( + Function &F, const SmallVectorImpl<Instruction *> &InstructionList) { + return Pimpl->parseMetadataAttachment(F, InstructionList); +} + +Error MetadataLoader::parseMetadataKinds() { + return Pimpl->parseMetadataKinds(); +} + +void MetadataLoader::setStripTBAA(bool StripTBAA) { + return Pimpl->setStripTBAA(StripTBAA); +} + +bool MetadataLoader::isStrippingTBAA() { return Pimpl->isStrippingTBAA(); } + +unsigned MetadataLoader::size() const { return Pimpl->size(); } +void MetadataLoader::shrinkTo(unsigned N) { return Pimpl->shrinkTo(N); } diff --git a/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h new file mode 100644 index 0000000..442dfc9 --- /dev/null +++ b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h @@ -0,0 +1,85 @@ +//===-- Bitcode/Reader/MetadataLoader.h - Load Metadatas -------*- C++ -*-====// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This class handles loading Metadatas. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_BITCODE_READER_METADATALOADER_H +#define LLVM_LIB_BITCODE_READER_METADATALOADER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Error.h" + +#include <functional> +#include <memory> + +namespace llvm { +class BitcodeReaderValueList; +class BitstreamCursor; +class DISubprogram; +class Error; +class Function; +class Instruction; +class Metadata; +class MDNode; +class Module; +class Type; + +/// Helper class that handles loading Metadatas and keeping them available. +class MetadataLoader { + class MetadataLoaderImpl; + std::unique_ptr<MetadataLoaderImpl> Pimpl; + Error parseMetadata(bool ModuleLevel); + +public: + ~MetadataLoader(); + MetadataLoader(BitstreamCursor &Stream, Module &TheModule, + BitcodeReaderValueList &ValueList, bool IsImporting, + std::function<Type *(unsigned)> getTypeByID); + MetadataLoader &operator=(MetadataLoader &&); + MetadataLoader(MetadataLoader &&); + + // Parse a module metadata block + Error parseModuleMetadata() { return parseMetadata(true); } + + // Parse a function metadata block + Error parseFunctionMetadata() { return parseMetadata(false); } + + /// Set the mode to strip TBAA metadata on load. + void setStripTBAA(bool StripTBAA = true); + + /// Return true if the Loader is stripping TBAA metadata. + bool isStrippingTBAA(); + + // Return true there are remaining unresolved forward references. + bool hasFwdRefs() const; + + /// Return the given metadata, creating a replaceable forward reference if + /// necessary. + Metadata *getMetadataFwdRefOrLoad(unsigned Idx); + + MDNode *getMDNodeFwdRefOrNull(unsigned Idx); + + /// Return the DISubprogra metadata for a Function if any, null otherwise. + DISubprogram *lookupSubprogramForFunction(Function *F); + + /// Parse a `METADATA_ATTACHMENT` block for a function. + Error parseMetadataAttachment( + Function &F, const SmallVectorImpl<Instruction *> &InstructionList); + + /// Parse a `METADATA_KIND` block for the current module. + Error parseMetadataKinds(); + + unsigned size() const; + void shrinkTo(unsigned N); +}; +} + +#endif // LLVM_LIB_BITCODE_READER_METADATALOADER_H diff --git a/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp b/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp new file mode 100644 index 0000000..7152a51 --- /dev/null +++ b/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp @@ -0,0 +1,199 @@ +//===----- ValueList.cpp - Internal BitcodeReader implementation ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ValueList.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Instructions.h" + +using namespace llvm; + +namespace llvm { +namespace { + +/// \brief A class for maintaining the slot number definition +/// as a placeholder for the actual definition for forward constants defs. +class ConstantPlaceHolder : public ConstantExpr { + void operator=(const ConstantPlaceHolder &) = delete; + +public: + // allocate space for exactly one operand + void *operator new(size_t s) { return User::operator new(s, 1); } + explicit ConstantPlaceHolder(Type *Ty, LLVMContext &Context) + : ConstantExpr(Ty, Instruction::UserOp1, &Op<0>(), 1) { + Op<0>() = UndefValue::get(Type::getInt32Ty(Context)); + } + + /// \brief Methods to support type inquiry through isa, cast, and dyn_cast. + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && + cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1; + } + + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); +}; + +} // end anonymous namespace + +// FIXME: can we inherit this from ConstantExpr? +template <> +struct OperandTraits<ConstantPlaceHolder> + : public FixedNumOperandTraits<ConstantPlaceHolder, 1> {}; +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPlaceHolder, Value) + +} // end namespace llvm + +void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { + if (Idx == size()) { + push_back(V); + return; + } + + if (Idx >= size()) + resize(Idx + 1); + + WeakVH &OldV = ValuePtrs[Idx]; + if (!OldV) { + OldV = V; + return; + } + + // Handle constants and non-constants (e.g. instrs) differently for + // efficiency. + if (Constant *PHC = dyn_cast<Constant>(&*OldV)) { + ResolveConstants.push_back(std::make_pair(PHC, Idx)); + OldV = V; + } else { + // If there was a forward reference to this value, replace it. + Value *PrevVal = OldV; + OldV->replaceAllUsesWith(V); + delete PrevVal; + } +} + +Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx, Type *Ty) { + if (Idx >= size()) + resize(Idx + 1); + + if (Value *V = ValuePtrs[Idx]) { + if (Ty != V->getType()) + report_fatal_error("Type mismatch in constant table!"); + return cast<Constant>(V); + } + + // Create and return a placeholder, which will later be RAUW'd. + Constant *C = new ConstantPlaceHolder(Ty, Context); + ValuePtrs[Idx] = C; + return C; +} + +Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) { + // Bail out for a clearly invalid value. This would make us call resize(0) + if (Idx == std::numeric_limits<unsigned>::max()) + return nullptr; + + if (Idx >= size()) + resize(Idx + 1); + + if (Value *V = ValuePtrs[Idx]) { + // If the types don't match, it's invalid. + if (Ty && Ty != V->getType()) + return nullptr; + return V; + } + + // No type specified, must be invalid reference. + if (!Ty) + return nullptr; + + // Create and return a placeholder, which will later be RAUW'd. + Value *V = new Argument(Ty); + ValuePtrs[Idx] = V; + return V; +} + +/// Once all constants are read, this method bulk resolves any forward +/// references. The idea behind this is that we sometimes get constants (such +/// as large arrays) which reference *many* forward ref constants. Replacing +/// each of these causes a lot of thrashing when building/reuniquing the +/// constant. Instead of doing this, we look at all the uses and rewrite all +/// the place holders at once for any constant that uses a placeholder. +void BitcodeReaderValueList::resolveConstantForwardRefs() { + // Sort the values by-pointer so that they are efficient to look up with a + // binary search. + std::sort(ResolveConstants.begin(), ResolveConstants.end()); + + SmallVector<Constant *, 64> NewOps; + + while (!ResolveConstants.empty()) { + Value *RealVal = operator[](ResolveConstants.back().second); + Constant *Placeholder = ResolveConstants.back().first; + ResolveConstants.pop_back(); + + // Loop over all users of the placeholder, updating them to reference the + // new value. If they reference more than one placeholder, update them all + // at once. + while (!Placeholder->use_empty()) { + auto UI = Placeholder->user_begin(); + User *U = *UI; + + // If the using object isn't uniqued, just update the operands. This + // handles instructions and initializers for global variables. + if (!isa<Constant>(U) || isa<GlobalValue>(U)) { + UI.getUse().set(RealVal); + continue; + } + + // Otherwise, we have a constant that uses the placeholder. Replace that + // constant with a new constant that has *all* placeholder uses updated. + Constant *UserC = cast<Constant>(U); + for (User::op_iterator I = UserC->op_begin(), E = UserC->op_end(); I != E; + ++I) { + Value *NewOp; + if (!isa<ConstantPlaceHolder>(*I)) { + // Not a placeholder reference. + NewOp = *I; + } else if (*I == Placeholder) { + // Common case is that it just references this one placeholder. + NewOp = RealVal; + } else { + // Otherwise, look up the placeholder in ResolveConstants. + ResolveConstantsTy::iterator It = std::lower_bound( + ResolveConstants.begin(), ResolveConstants.end(), + std::pair<Constant *, unsigned>(cast<Constant>(*I), 0)); + assert(It != ResolveConstants.end() && It->first == *I); + NewOp = operator[](It->second); + } + + NewOps.push_back(cast<Constant>(NewOp)); + } + + // Make the new constant. + Constant *NewC; + if (ConstantArray *UserCA = dyn_cast<ConstantArray>(UserC)) { + NewC = ConstantArray::get(UserCA->getType(), NewOps); + } else if (ConstantStruct *UserCS = dyn_cast<ConstantStruct>(UserC)) { + NewC = ConstantStruct::get(UserCS->getType(), NewOps); + } else if (isa<ConstantVector>(UserC)) { + NewC = ConstantVector::get(NewOps); + } else { + assert(isa<ConstantExpr>(UserC) && "Must be a ConstantExpr."); + NewC = cast<ConstantExpr>(UserC)->getWithOperands(NewOps); + } + + UserC->replaceAllUsesWith(NewC); + UserC->destroyConstant(); + NewOps.clear(); + } + + // Update all ValueHandles, they should be the only users at this point. + Placeholder->replaceAllUsesWith(RealVal); + delete Placeholder; + } +} diff --git a/contrib/llvm/lib/Bitcode/Reader/ValueList.h b/contrib/llvm/lib/Bitcode/Reader/ValueList.h new file mode 100644 index 0000000..3119d77 --- /dev/null +++ b/contrib/llvm/lib/Bitcode/Reader/ValueList.h @@ -0,0 +1,76 @@ +//===-- Bitcode/Reader/ValueEnumerator.h - Number values --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This class gives values and types Unique ID's. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/ValueHandle.h" + +#include <vector> + +namespace llvm { +class Constant; + +class BitcodeReaderValueList { + std::vector<WeakVH> ValuePtrs; + + /// As we resolve forward-referenced constants, we add information about them + /// to this vector. This allows us to resolve them in bulk instead of + /// resolving each reference at a time. See the code in + /// ResolveConstantForwardRefs for more information about this. + /// + /// The key of this vector is the placeholder constant, the value is the slot + /// number that holds the resolved value. + typedef std::vector<std::pair<Constant *, unsigned>> ResolveConstantsTy; + ResolveConstantsTy ResolveConstants; + LLVMContext &Context; + +public: + BitcodeReaderValueList(LLVMContext &C) : Context(C) {} + ~BitcodeReaderValueList() { + assert(ResolveConstants.empty() && "Constants not resolved?"); + } + + // vector compatibility methods + unsigned size() const { return ValuePtrs.size(); } + void resize(unsigned N) { ValuePtrs.resize(N); } + void push_back(Value *V) { ValuePtrs.emplace_back(V); } + + void clear() { + assert(ResolveConstants.empty() && "Constants not resolved?"); + ValuePtrs.clear(); + } + + Value *operator[](unsigned i) const { + assert(i < ValuePtrs.size()); + return ValuePtrs[i]; + } + + Value *back() const { return ValuePtrs.back(); } + void pop_back() { ValuePtrs.pop_back(); } + bool empty() const { return ValuePtrs.empty(); } + + void shrinkTo(unsigned N) { + assert(N <= size() && "Invalid shrinkTo request!"); + ValuePtrs.resize(N); + } + + Constant *getConstantFwdRef(unsigned Idx, Type *Ty); + Value *getValueFwdRef(unsigned Idx, Type *Ty); + + void assignValue(Value *V, unsigned Idx); + + /// Once all constants are read, this method bulk resolves any forward + /// references. + void resolveConstantForwardRefs(); +}; + +} // namespace llvm diff --git a/contrib/llvm/lib/Bitcode/Writer/BitWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitWriter.cpp index 7218ea0..e038841 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitWriter.cpp @@ -8,9 +8,10 @@ //===----------------------------------------------------------------------===// #include "llvm-c/BitWriter.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/Module.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index dcb8b58..ebb2022 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Bitcode/BitcodeWriter.h" #include "ValueEnumerator.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" -#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -38,6 +38,11 @@ using namespace llvm; namespace { + +cl::opt<unsigned> + IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(25), + cl::desc("Number of metadatas above which we emit an index " + "to enable lazy-loading")); /// These are manifest constants used by the bitcode writer. They do not need to /// be kept in sync with the reader, but need to be consistent within this file. enum { @@ -65,36 +70,20 @@ enum { }; /// Abstract class to manage the bitcode writing, subclassed for each bitcode -/// file type. Owns the BitstreamWriter, and includes the main entry point for -/// writing. -class BitcodeWriter { +/// file type. +class BitcodeWriterBase { protected: - /// Pointer to the buffer allocated by caller for bitcode writing. - const SmallVectorImpl<char> &Buffer; - - /// The stream created and owned by the BitodeWriter. - BitstreamWriter Stream; + /// The stream created and owned by the client. + BitstreamWriter &Stream; /// Saves the offset of the VSTOffset record that must eventually be /// backpatched with the offset of the actual VST. uint64_t VSTOffsetPlaceholder = 0; public: - /// Constructs a BitcodeWriter object, and initializes a BitstreamRecord, - /// writing to the provided \p Buffer. - BitcodeWriter(SmallVectorImpl<char> &Buffer) - : Buffer(Buffer), Stream(Buffer) {} - - virtual ~BitcodeWriter() = default; - - /// Main entry point to write the bitcode file, which writes the bitcode - /// header and will then invoke the virtual writeBlocks() method. - void write(); - -private: - /// Derived classes must implement this to write the corresponding blocks for - /// that bitcode file type. - virtual void writeBlocks() = 0; + /// Constructs a BitcodeWriterBase object that writes to the provided + /// \p Stream. + BitcodeWriterBase(BitstreamWriter &Stream) : Stream(Stream) {} protected: bool hasVSTOffsetPlaceholder() { return VSTOffsetPlaceholder != 0; } @@ -103,7 +92,10 @@ protected: }; /// Class to manage the bitcode writing for a module. -class ModuleBitcodeWriter : public BitcodeWriter { +class ModuleBitcodeWriter : public BitcodeWriterBase { + /// Pointer to the buffer allocated by caller for bitcode writing. + const SmallVectorImpl<char> &Buffer; + /// The Module to write to bitcode. const Module &M; @@ -116,8 +108,8 @@ class ModuleBitcodeWriter : public BitcodeWriter { /// True if a module hash record should be written. bool GenerateHash; - /// The start bit of the module block, for use in generating a module hash - uint64_t BitcodeStartBit = 0; + /// The start bit of the identification block. + uint64_t BitcodeStartBit; /// Map that holds the correspondence between GUIDs in the summary index, /// that came from indirect call profiles, and a value id generated by this @@ -131,51 +123,38 @@ public: /// Constructs a ModuleBitcodeWriter object for the given Module, /// writing to the provided \p Buffer. ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, - bool ShouldPreserveUseListOrder, + BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, bool GenerateHash) - : BitcodeWriter(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), - Index(Index), GenerateHash(GenerateHash) { - // Save the start bit of the actual bitcode, in case there is space - // saved at the start for the darwin header above. The reader stream - // will start at the bitcode, and we need the offset of the VST - // to line up. - BitcodeStartBit = Stream.GetCurrentBitNo(); - + : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M), + VE(*M, ShouldPreserveUseListOrder), Index(Index), + GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). // The starting ValueId is just after the number of values in the // ValueEnumerator, so that they can be emitted in the VST. GlobalValueId = VE.getValues().size(); - if (Index) - for (const auto &GUIDSummaryLists : *Index) - // Examine all summaries for this GUID. - for (auto &Summary : GUIDSummaryLists.second) - if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) - // For each call in the function summary, see if the call - // is to a GUID (which means it is for an indirect call, - // otherwise we would have a Value for it). If so, synthesize - // a value id. - for (auto &CallEdge : FS->calls()) - if (CallEdge.first.isGUID()) - assignValueId(CallEdge.first.getGUID()); + if (!Index) + return; + for (const auto &GUIDSummaryLists : *Index) + // Examine all summaries for this GUID. + for (auto &Summary : GUIDSummaryLists.second) + if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) + // For each call in the function summary, see if the call + // is to a GUID (which means it is for an indirect call, + // otherwise we would have a Value for it). If so, synthesize + // a value id. + for (auto &CallEdge : FS->calls()) + if (CallEdge.first.isGUID()) + assignValueId(CallEdge.first.getGUID()); } -private: - /// Main entry point for writing a module to bitcode, invoked by - /// BitcodeWriter::write() after it writes the header. - void writeBlocks() override; - - /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the - /// current llvm version, and a record for the epoch number. - void writeIdentificationBlock(); - /// Emit the current module to the bitstream. - void writeModule(); + void write(); +private: uint64_t bitcodeStartBit() { return BitcodeStartBit; } - void writeStringRecord(unsigned Code, StringRef Str, unsigned AbbrevToUse); void writeAttributeGroupTable(); void writeAttributeTable(); void writeTypeTable(); @@ -236,6 +215,9 @@ private: SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); void writeDIExpression(const DIExpression *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); void writeDIObjCProperty(const DIObjCProperty *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); void writeDIImportedEntity(const DIImportedEntity *N, @@ -247,7 +229,9 @@ private: void writeMetadataStrings(ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record); void writeMetadataRecords(ArrayRef<const Metadata *> MDs, - SmallVectorImpl<uint64_t> &Record); + SmallVectorImpl<uint64_t> &Record, + std::vector<unsigned> *MDAbbrevs = nullptr, + std::vector<uint64_t> *IndexPos = nullptr); void writeModuleMetadata(); void writeFunctionMetadata(const Function &F); void writeFunctionMetadataAttachment(const Function &F); @@ -293,7 +277,10 @@ private: } unsigned getValueId(GlobalValue::GUID ValGUID) { const auto &VMI = GUIDToValueIdMap.find(ValGUID); - assert(VMI != GUIDToValueIdMap.end()); + // Expect that any GUID value had a value Id assigned by an + // earlier call to assignValueId. + assert(VMI != GUIDToValueIdMap.end() && + "GUID does not have assigned value Id"); return VMI->second; } // Helper to get the valueId for the type of value recorded in VI. @@ -306,13 +293,13 @@ private: }; /// Class to manage the bitcode writing for a combined index. -class IndexBitcodeWriter : public BitcodeWriter { +class IndexBitcodeWriter : public BitcodeWriterBase { /// The combined index to write to bitcode. const ModuleSummaryIndex &Index; /// When writing a subset of the index for distributed backends, client /// provides a map of modules to the corresponding GUIDs/summaries to write. - std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex; + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex; /// Map that holds the correspondence between the GUID used in the combined /// index and a value id generated by this class to use in references. @@ -325,11 +312,10 @@ public: /// Constructs a IndexBitcodeWriter object for the given combined index, /// writing to the provided \p Buffer. When writing a subset of the index /// for a distributed backend, provide a \p ModuleToSummariesForIndex map. - IndexBitcodeWriter(SmallVectorImpl<char> &Buffer, - const ModuleSummaryIndex &Index, - std::map<std::string, GVSummaryMapTy> + IndexBitcodeWriter(BitstreamWriter &Stream, const ModuleSummaryIndex &Index, + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex = nullptr) - : BitcodeWriter(Buffer), Index(Index), + : BitcodeWriterBase(Stream), Index(Index), ModuleToSummariesForIndex(ModuleToSummariesForIndex) { // Assign unique value ids to all summaries to be written, for use // in writing out the call graph edges. Save the mapping from GUID @@ -355,11 +341,11 @@ public: // ModuleToSummariesForIndex map: /// Points to the last element in outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesBack; + std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesBack; /// Iterator on outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesIter; + std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesIter; /// Iterator on an inner global variable summary map. - GVSummaryMapTy::iterator ModuleGVSummariesIter; + GVSummaryMapTy::const_iterator ModuleGVSummariesIter; // Iterators used when writing all summaries in the index: @@ -476,11 +462,10 @@ public: /// Obtain the end iterator over the summaries to be written. iterator end() { return iterator(*this, /*IsAtEnd=*/true); } -private: - /// Main entry point for writing a combined index to bitcode, invoked by - /// BitcodeWriter::write() after it writes the header. - void writeBlocks() override; + /// Main entry point for writing a combined index to bitcode. + void write(); +private: void writeIndex(); void writeModStrings(); void writeCombinedValueSymbolTable(); @@ -593,8 +578,8 @@ static unsigned getEncodedSynchScope(SynchronizationScope SynchScope) { llvm_unreachable("Invalid synch scope"); } -void ModuleBitcodeWriter::writeStringRecord(unsigned Code, StringRef Str, - unsigned AbbrevToUse) { +static void writeStringRecord(BitstreamWriter &Stream, unsigned Code, + StringRef Str, unsigned AbbrevToUse) { SmallVector<unsigned, 64> Vals; // Code: [strchar x N] @@ -799,53 +784,53 @@ void ModuleBitcodeWriter::writeTypeTable() { uint64_t NumBits = VE.computeBitsRequiredForTypeIndicies(); // Abbrev for TYPE_CODE_POINTER. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_POINTER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); Abbv->Add(BitCodeAbbrevOp(0)); // Addrspace = 0 - unsigned PtrAbbrev = Stream.EmitAbbrev(Abbv); + unsigned PtrAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_FUNCTION. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FunctionAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_ANON. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv); + unsigned StructAnonAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_NAME. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv); + unsigned StructNameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_NAMED. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv); + unsigned StructNamedAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_ARRAY. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv); + unsigned ArrayAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Emit an entry count so the reader can reserve space. TypeVals.push_back(TypeList.size()); @@ -918,7 +903,7 @@ void ModuleBitcodeWriter::writeTypeTable() { // Emit the name if it is present. if (!ST->getName().empty()) - writeStringRecord(bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), + writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), StructNameAbbrev); } break; @@ -986,8 +971,8 @@ static unsigned getEncodedLinkage(const GlobalValue &GV) { static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { uint64_t RawFlags = 0; - RawFlags |= Flags.HasSection; // bool - + RawFlags |= Flags.NotEligibleToImport; // bool + RawFlags |= (Flags.LiveRoot << 1); // Linkage don't need to be remapped at that time for the summary. Any future // change to the getEncodedLinkage() function will need to be taken into // account here as well. @@ -1068,18 +1053,18 @@ void ModuleBitcodeWriter::writeComdats() { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched /// after the real VST is written. Saves the bit offset to backpatch. -void BitcodeWriter::writeValueSymbolTableForwardDecl() { +void BitcodeWriterBase::writeValueSymbolTableForwardDecl() { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be // updated when the real VST is written. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET)); // Blocks are 32-bit aligned, so we can use a 32-bit word offset to // hold the real VST offset. Must use fixed instead of VBR as we don't // know how many VBR chunks to reserve ahead of time. Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); - unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv); + unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Emit the placeholder uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0}; @@ -1115,13 +1100,13 @@ static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { void ModuleBitcodeWriter::writeModuleInfo() { // Emit various pieces of data attached to a module. if (!M.getTargetTriple().empty()) - writeStringRecord(bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(), + writeStringRecord(Stream, bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(), 0 /*TODO*/); const std::string &DL = M.getDataLayoutStr(); if (!DL.empty()) - writeStringRecord(bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/); + writeStringRecord(Stream, bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/); if (!M.getModuleInlineAsm().empty()) - writeStringRecord(bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(), + writeStringRecord(Stream, bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(), 0 /*TODO*/); // Emit information about sections and GC, computing how many there are. Also @@ -1137,7 +1122,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Give section names unique ID's. unsigned &Entry = SectionMap[GV.getSection()]; if (!Entry) { - writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), + writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), 0 /*TODO*/); Entry = SectionMap.size(); } @@ -1149,7 +1134,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Give section names unique ID's. unsigned &Entry = SectionMap[F.getSection()]; if (!Entry) { - writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, F.getSection(), + writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, F.getSection(), 0 /*TODO*/); Entry = SectionMap.size(); } @@ -1158,7 +1143,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Same for GC names. unsigned &Entry = GCMap[F.getGC()]; if (!Entry) { - writeStringRecord(bitc::MODULE_CODE_GCNAME, F.getGC(), 0 /*TODO*/); + writeStringRecord(Stream, bitc::MODULE_CODE_GCNAME, F.getGC(), + 0 /*TODO*/); Entry = GCMap.size(); } } @@ -1168,7 +1154,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { unsigned SimpleGVarAbbrev = 0; if (!M.global_empty()) { // Add an abbrev for common globals with no visibility or thread localness. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(MaxGlobalType+1))); @@ -1190,7 +1176,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(SectionMap.size()+1))); // Don't bother emitting vis + thread local. - SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv); + SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } // Emit the global variable information. @@ -1298,11 +1284,11 @@ void ModuleBitcodeWriter::writeModuleInfo() { AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); // MODULE_CODE_SOURCE_FILENAME: [namechar x N] - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(AbbrevOpToUse); - unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); for (const auto P : M.getSourceFileName()) Vals.push_back((unsigned char)P); @@ -1373,14 +1359,14 @@ void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N, unsigned ModuleBitcodeWriter::createDILocationAbbrev() { // Assume the column is usually under 128, and always output the inlined-at // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } void ModuleBitcodeWriter::writeDILocation(const DILocation *N, @@ -1402,7 +1388,7 @@ void ModuleBitcodeWriter::writeDILocation(const DILocation *N, unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() { // Assume the column is usually under 128, and always output the inlined-at // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); @@ -1410,7 +1396,7 @@ unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N, @@ -1535,6 +1521,8 @@ void ModuleBitcodeWriter::writeDIFile(const DIFile *N, Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawFilename())); Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory())); + Record.push_back(N->getChecksumKind()); + Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum())); Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev); Record.clear(); @@ -1560,6 +1548,7 @@ void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get())); Record.push_back(N->getDWOId()); Record.push_back(VE.getMetadataOrNullID(N->getMacros().get())); + Record.push_back(N->getSplitDebugInlining()); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -1622,7 +1611,7 @@ void ModuleBitcodeWriter::writeDILexicalBlockFile( void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + Record.push_back(N->isDistinct() | N->getExportSymbols() << 1); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1696,7 +1685,8 @@ void ModuleBitcodeWriter::writeDITemplateValueParameter( void ModuleBitcodeWriter::writeDIGlobalVariable( const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + const uint64_t Version = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | Version); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName())); @@ -1705,8 +1695,9 @@ void ModuleBitcodeWriter::writeDIGlobalVariable( Record.push_back(VE.getMetadataOrNullID(N->getType())); Record.push_back(N->isLocalToUnit()); Record.push_back(N->isDefinition()); - Record.push_back(VE.getMetadataOrNullID(N->getRawVariable())); + Record.push_back(/* expr */ 0); Record.push_back(VE.getMetadataOrNullID(N->getStaticDataMemberDeclaration())); + Record.push_back(N->getAlignInBits()); Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR, Record, Abbrev); Record.clear(); @@ -1715,7 +1706,21 @@ void ModuleBitcodeWriter::writeDIGlobalVariable( void ModuleBitcodeWriter::writeDILocalVariable( const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + // In order to support all possible bitcode formats in BitcodeReader we need + // to distinguish the following cases: + // 1) Record has no artificial tag (Record[1]), + // has no obsolete inlinedAt field (Record[9]). + // In this case Record size will be 8, HasAlignment flag is false. + // 2) Record has artificial tag (Record[1]), + // has no obsolete inlignedAt field (Record[9]). + // In this case Record size will be 9, HasAlignment flag is false. + // 3) Record has both artificial tag (Record[1]) and + // obsolete inlignedAt field (Record[9]). + // In this case Record size will be 10, HasAlignment flag is false. + // 4) Record has neither artificial tag, nor inlignedAt field, but + // HasAlignment flag is true and Record[8] contains alignment value. + const uint64_t HasAlignmentFlag = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | HasAlignmentFlag); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1723,6 +1728,7 @@ void ModuleBitcodeWriter::writeDILocalVariable( Record.push_back(VE.getMetadataOrNullID(N->getType())); Record.push_back(N->getArg()); Record.push_back(N->getFlags()); + Record.push_back(N->getAlignInBits()); Stream.EmitRecord(bitc::METADATA_LOCAL_VAR, Record, Abbrev); Record.clear(); @@ -1733,13 +1739,25 @@ void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, unsigned Abbrev) { Record.reserve(N->getElements().size() + 1); - Record.push_back(N->isDistinct()); + const uint64_t HasOpFragmentFlag = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | HasOpFragmentFlag); Record.append(N->elements_begin(), N->elements_end()); Stream.EmitRecord(bitc::METADATA_EXPRESSION, Record, Abbrev); Record.clear(); } +void ModuleBitcodeWriter::writeDIGlobalVariableExpression( + const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + Record.push_back(N->isDistinct()); + Record.push_back(VE.getMetadataOrNullID(N->getVariable())); + Record.push_back(VE.getMetadataOrNullID(N->getExpression())); + + Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR_EXPR, Record, Abbrev); + Record.clear(); +} + void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { @@ -1771,11 +1789,11 @@ void ModuleBitcodeWriter::writeDIImportedEntity( } unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } void ModuleBitcodeWriter::writeNamedMetadata( @@ -1800,12 +1818,12 @@ void ModuleBitcodeWriter::writeNamedMetadata( } unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRINGS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } /// Write out a record for MDString. @@ -1842,8 +1860,16 @@ void ModuleBitcodeWriter::writeMetadataStrings( Record.clear(); } +// Generates an enum to use as an index in the Abbrev array of Metadata record. +enum MetadataAbbrev : unsigned { +#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID, +#include "llvm/IR/Metadata.def" + LastPlusOne +}; + void ModuleBitcodeWriter::writeMetadataRecords( - ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record) { + ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record, + std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) { if (MDs.empty()) return; @@ -1852,6 +1878,8 @@ void ModuleBitcodeWriter::writeMetadataRecords( #include "llvm/IR/Metadata.def" for (const Metadata *MD : MDs) { + if (IndexPos) + IndexPos->push_back(Stream.GetCurrentBitNo()); if (const MDNode *N = dyn_cast<MDNode>(MD)) { assert(N->isResolved() && "Expected forward references to be resolved"); @@ -1860,7 +1888,11 @@ void ModuleBitcodeWriter::writeMetadataRecords( llvm_unreachable("Invalid MDNode subclass"); #define HANDLE_MDNODE_LEAF(CLASS) \ case Metadata::CLASS##Kind: \ - write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \ + if (MDAbbrevs) \ + write##CLASS(cast<CLASS>(N), Record, \ + (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \ + else \ + write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \ continue; #include "llvm/IR/Metadata.def" } @@ -1873,10 +1905,77 @@ void ModuleBitcodeWriter::writeModuleMetadata() { if (!VE.hasMDs() && M.named_metadata_empty()) return; - Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); + Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 4); SmallVector<uint64_t, 64> Record; + + // Emit all abbrevs upfront, so that the reader can jump in the middle of the + // block and load any metadata. + std::vector<unsigned> MDAbbrevs; + + MDAbbrevs.resize(MetadataAbbrev::LastPlusOne); + MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev(); + MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] = + createGenericDINodeAbbrev(); + + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + unsigned OffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + unsigned IndexAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + // Emit MDStrings together upfront. writeMetadataStrings(VE.getMDStrings(), Record); - writeMetadataRecords(VE.getNonMDStrings(), Record); + + // We only emit an index for the metadata record if we have more than a given + // (naive) threshold of metadatas, otherwise it is not worth it. + if (VE.getNonMDStrings().size() > IndexThreshold) { + // Write a placeholder value in for the offset of the metadata index, + // which is written after the records, so that it can include + // the offset of each entry. The placeholder offset will be + // updated after all records are emitted. + uint64_t Vals[] = {0, 0}; + Stream.EmitRecord(bitc::METADATA_INDEX_OFFSET, Vals, OffsetAbbrev); + } + + // Compute and save the bit offset to the current position, which will be + // patched when we emit the index later. We can simply subtract the 64-bit + // fixed size from the current bit number to get the location to backpatch. + uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo(); + + // This index will contain the bitpos for each individual record. + std::vector<uint64_t> IndexPos; + IndexPos.reserve(VE.getNonMDStrings().size()); + + // Write all the records + writeMetadataRecords(VE.getNonMDStrings(), Record, &MDAbbrevs, &IndexPos); + + if (VE.getNonMDStrings().size() > IndexThreshold) { + // Now that we have emitted all the records we will emit the index. But + // first + // backpatch the forward reference so that the reader can skip the records + // efficiently. + Stream.BackpatchWord64(IndexOffsetRecordBitPos - 64, + Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos); + + // Delta encode the index. + uint64_t PreviousValue = IndexOffsetRecordBitPos; + for (auto &Elt : IndexPos) { + auto EltDelta = Elt - PreviousValue; + PreviousValue = Elt; + Elt = EltDelta; + } + // Emit the index record. + Stream.EmitRecord(bitc::METADATA_INDEX, IndexPos, IndexAbbrev); + IndexPos.clear(); + } + + // Write the named metadata now. writeNamedMetadata(Record); auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) { @@ -2025,30 +2124,30 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, // If this is a constant pool for the module, emit module-specific abbrevs. if (isGlobal) { // Abbrev for CST_CODE_AGGREGATE. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(LastVal+1))); - AggregateAbbrev = Stream.EmitAbbrev(Abbv); + AggregateAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_STRING. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_STRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - String8Abbrev = Stream.EmitAbbrev(Abbv); + String8Abbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_CSTRING. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - CString7Abbrev = Stream.EmitAbbrev(Abbv); + CString7Abbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_CSTRING. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - CString6Abbrev = Stream.EmitAbbrev(Abbv); + CString6Abbrev = Stream.EmitAbbrev(std::move(Abbv)); } SmallVector<uint64_t, 64> Record; @@ -2196,9 +2295,12 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, case Instruction::GetElementPtr: { Code = bitc::CST_CODE_CE_GEP; const auto *GO = cast<GEPOperator>(C); - if (GO->isInBounds()) - Code = bitc::CST_CODE_CE_INBOUNDS_GEP; Record.push_back(VE.getTypeID(GO->getSourceElementType())); + if (Optional<unsigned> Idx = GO->getInRangeIndex()) { + Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX; + Record.push_back((*Idx << 1) | GO->isInBounds()); + } else if (GO->isInBounds()) + Code = bitc::CST_CODE_CE_INBOUNDS_GEP; for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) { Record.push_back(VE.getTypeID(C->getOperand(i)->getType())); Record.push_back(VE.getValueID(C->getOperand(i))); @@ -2495,7 +2597,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, // Emit type/value pairs for varargs params. if (FTy->isVarArg()) { - for (unsigned i = FTy->getNumParams(), e = I.getNumOperands()-3; + for (unsigned i = FTy->getNumParams(), e = II->getNumArgOperands(); i != e; ++i) pushValueAndType(I.getOperand(i), InstID, Vals); // vararg } @@ -2736,11 +2838,13 @@ void ModuleBitcodeWriter::writeValueSymbolTable( // Get the offset of the VST we are writing, and backpatch it into // the VST forward declaration record. uint64_t VSTOffset = Stream.GetCurrentBitNo(); - // The BitcodeStartBit was the stream offset of the actual bitcode - // (e.g. excluding any initial darwin header). + // The BitcodeStartBit was the stream offset of the identification block. VSTOffset -= bitcodeStartBit(); assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); - Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); + // Note that we add 1 here because the offset is relative to one word + // before the start of the identification block, which was historically + // always the start of the regular bitcode header. + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1); } Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); @@ -2753,39 +2857,39 @@ void ModuleBitcodeWriter::writeValueSymbolTable( unsigned GUIDEntryAbbrev; if (IsModuleLevel && hasVSTOffsetPlaceholder()) { // 8-bit fixed-width VST_CODE_FNENTRY function strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - FnEntry8BitAbbrev = Stream.EmitAbbrev(Abbv); + FnEntry8BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // 7-bit fixed width VST_CODE_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - FnEntry7BitAbbrev = Stream.EmitAbbrev(Abbv); + FnEntry7BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // 6-bit char6 VST_CODE_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - FnEntry6BitAbbrev = Stream.EmitAbbrev(Abbv); + FnEntry6BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // FIXME: Change the name of this record as it is now used by // the per-module index as well. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - GUIDEntryAbbrev = Stream.EmitAbbrev(Abbv); + GUIDEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } // FIXME: Set up the abbrev, we know how many values there are! @@ -2828,7 +2932,10 @@ void ModuleBitcodeWriter::writeValueSymbolTable( // actual bitcode written to the stream). uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - bitcodeStartBit(); assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); - NameVals.push_back(BitcodeIndex / 32); + // Note that we add 1 here because the offset is relative to one word + // before the start of the identification block, which was historically + // always the start of the regular bitcode header. + NameVals.push_back(BitcodeIndex / 32 + 1); Code = bitc::VST_CODE_FNENTRY; AbbrevToUse = FnEntry8BitAbbrev; @@ -2876,11 +2983,11 @@ void IndexBitcodeWriter::writeCombinedValueSymbolTable() { Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv); + unsigned EntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<uint64_t, 64> NameVals; for (const auto &GVI : valueIds()) { @@ -2994,7 +3101,8 @@ void ModuleBitcodeWriter::writeFunction( } // Emit names for all the instructions etc. - writeValueSymbolTable(F.getValueSymbolTable()); + if (auto *Symtab = F.getValueSymbolTable()) + writeValueSymbolTable(*Symtab); if (NeedsMetadataAttachment) writeFunctionMetadataAttachment(F); @@ -3009,10 +3117,10 @@ void ModuleBitcodeWriter::writeBlockInfo() { // We only want to emit block info records for blocks that have multiple // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK. // Other blocks can define their abbrevs inline. - Stream.EnterBlockInfoBlock(2); + Stream.EnterBlockInfoBlock(); { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3023,7 +3131,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // 7-bit fixed width VST_CODE_ENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3033,7 +3141,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // 6-bit char6 VST_CODE_ENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3043,7 +3151,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // 6-bit char6 VST_CODE_BBENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3056,7 +3164,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { { // SETTYPE abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, VE.computeBitsRequiredForTypeIndicies())); @@ -3066,7 +3174,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // INTEGER abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_INTEGER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != @@ -3075,7 +3183,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // CE_CAST abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid @@ -3087,7 +3195,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // NULL abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_NULL)); if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != CONSTANTS_NULL_Abbrev) @@ -3097,7 +3205,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { // FIXME: This should only use space for first class types! { // INST_LOAD abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty @@ -3109,7 +3217,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS @@ -3119,7 +3227,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS @@ -3130,7 +3238,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_CAST abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty @@ -3142,14 +3250,14 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // INST_RET abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_RET_VOID_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_RET abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != @@ -3157,14 +3265,14 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE)); if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_UNREACHABLE_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty @@ -3187,38 +3295,38 @@ void IndexBitcodeWriter::writeModStrings() { // TODO: See which abbrev sizes we actually need to emit // 8-bit fixed-width MST_ENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv); + unsigned Abbrev8Bit = Stream.EmitAbbrev(std::move(Abbv)); // 7-bit fixed width MST_ENTRY strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv); + unsigned Abbrev7Bit = Stream.EmitAbbrev(std::move(Abbv)); // 6-bit char6 MST_ENTRY strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv); + unsigned Abbrev6Bit = Stream.EmitAbbrev(std::move(Abbv)); // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); - unsigned AbbrevHash = Stream.EmitAbbrev(Abbv); + unsigned AbbrevHash = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<unsigned, 64> Vals; for (const auto &MPSE : Index.modulePaths()) { @@ -3267,30 +3375,21 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( NameVals.push_back(ValueID); FunctionSummary *FS = cast<FunctionSummary>(Summary); + if (!FS->type_tests().empty()) + Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); NameVals.push_back(FS->refs().size()); - unsigned SizeBeforeRefs = NameVals.size(); for (auto &RI : FS->refs()) NameVals.push_back(VE.getValueID(RI.getValue())); - // Sort the refs for determinism output, the vector returned by FS->refs() has - // been initialized from a DenseSet. - std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); - std::vector<FunctionSummary::EdgeTy> Calls = FS->calls(); - std::sort(Calls.begin(), Calls.end(), - [this](const FunctionSummary::EdgeTy &L, - const FunctionSummary::EdgeTy &R) { - return getValueId(L.first) < getValueId(R.first); - }); bool HasProfileData = F.getEntryCount().hasValue(); - for (auto &ECI : Calls) { + for (auto &ECI : FS->calls()) { NameVals.push_back(getValueId(ECI.first)); - assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite"); - NameVals.push_back(ECI.second.CallsiteCount); if (HasProfileData) - NameVals.push_back(ECI.second.ProfileCount); + NameVals.push_back(static_cast<uint8_t>(ECI.second.Hotness)); } unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); @@ -3307,13 +3406,18 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( void ModuleBitcodeWriter::writeModuleLevelReferences( const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev) { - // Only interested in recording variable defs in the summary. - if (V.isDeclaration()) + auto Summaries = + Index->findGlobalValueSummaryList(GlobalValue::getGUID(V.getName())); + if (Summaries == Index->end()) { + // Only declarations should not have a summary (a declaration might however + // have a summary if the def was in module level asm). + assert(V.isDeclaration()); return; + } + auto *Summary = Summaries->second.front().get(); NameVals.push_back(VE.getValueID(&V)); - NameVals.push_back(getEncodedGVSummaryFlags(V)); - auto *Summary = Index->getGlobalValueSummary(V); GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary); + NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); unsigned SizeBeforeRefs = NameVals.size(); for (auto &RI : VS->refs()) @@ -3330,71 +3434,79 @@ void ModuleBitcodeWriter::writeModuleLevelReferences( // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 1; +static const uint64_t INDEX_VERSION = 3; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { - if (Index->begin() == Index->end()) - return; - Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + if (Index->begin() == Index->end()) { + Stream.ExitBlock(); + return; + } + // Abbrev for FS_PERMODULE. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount) + // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_PERMODULE_PROFILE. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount, profilecount) + // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_ALIAS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<uint64_t, 64> NameVals; // Iterate over the list of functions instead of the Index to // ensure the ordering is stable. for (const Function &F : M) { - if (F.isDeclaration()) - continue; // Summary emission does not support anonymous functions, they have to // renamed using the anonymous function renaming pass. if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - auto *Summary = Index->getGlobalValueSummary(F); + auto Summaries = + Index->findGlobalValueSummaryList(GlobalValue::getGUID(F.getName())); + if (Summaries == Index->end()) { + // Only declarations should not have a summary (a declaration might + // however have a summary if the def was in module level asm). + assert(F.isDeclaration()); + continue; + } + auto *Summary = Summaries->second.front().get(); writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), FSCallsAbbrev, FSCallsProfileAbbrev, F); } @@ -3412,7 +3524,9 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { auto AliasId = VE.getValueID(&A); auto AliaseeId = VE.getValueID(Aliasee); NameVals.push_back(AliasId); - NameVals.push_back(getEncodedGVSummaryFlags(A)); + auto *Summary = Index->getGlobalValueSummary(A); + AliasSummary *AS = cast<AliasSummary>(Summary); + NameVals.push_back(getEncodedGVSummaryFlags(AS->flags())); NameVals.push_back(AliaseeId); Stream.EmitRecord(bitc::FS_ALIAS, NameVals, FSAliasAbbrev); NameVals.clear(); @@ -3427,49 +3541,49 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); // Abbrev for FS_COMBINED. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount) + // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_PROFILE. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount, profilecount) + // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_ALIAS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // The aliases are emitted as a post-pass, and will point to the value // id of the aliasee. Save them in a vector for post-processing. @@ -3522,6 +3636,9 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { } auto *FS = cast<FunctionSummary>(S); + if (!FS->type_tests().empty()) + Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + NameVals.push_back(ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); @@ -3534,7 +3651,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { bool HasProfileData = false; for (auto &EI : FS->calls()) { - HasProfileData |= EI.second.ProfileCount != 0; + HasProfileData |= EI.second.Hotness != CalleeInfo::HotnessType::Unknown; if (HasProfileData) break; } @@ -3545,10 +3662,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { if (!hasValueId(EI.first.getGUID())) continue; NameVals.push_back(getValueId(EI.first.getGUID())); - assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); - NameVals.push_back(EI.second.CallsiteCount); if (HasProfileData) - NameVals.push_back(EI.second.ProfileCount); + NameVals.push_back(static_cast<uint8_t>(EI.second.Hotness)); } unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); @@ -3580,23 +3695,25 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.ExitBlock(); } -void ModuleBitcodeWriter::writeIdentificationBlock() { +/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the +/// current llvm version, and a record for the epoch number. +void writeIdentificationBlock(BitstreamWriter &Stream) { Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5); // Write the "user readable" string identifying the bitcode producer - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - auto StringAbbrev = Stream.EmitAbbrev(Abbv); - writeStringRecord(bitc::IDENTIFICATION_CODE_STRING, + auto StringAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + writeStringRecord(Stream, bitc::IDENTIFICATION_CODE_STRING, "LLVM" LLVM_VERSION_STRING, StringAbbrev); // Write the epoch version - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - auto EpochAbbrev = Stream.EmitAbbrev(Abbv); + auto EpochAbbrev = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<unsigned, 1> Vals = {bitc::BITCODE_CURRENT_EPOCH}; Stream.EmitRecord(bitc::IDENTIFICATION_CODE_EPOCH, Vals, EpochAbbrev); Stream.ExitBlock(); @@ -3608,39 +3725,19 @@ void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { SHA1 Hasher; Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], Buffer.size() - BlockStartPos)); - auto Hash = Hasher.result(); - SmallVector<uint64_t, 20> Vals; - auto LShift = [&](unsigned char Val, unsigned Amount) - -> uint64_t { return ((uint64_t)Val) << Amount; }; + StringRef Hash = Hasher.result(); + uint32_t Vals[5]; for (int Pos = 0; Pos < 20; Pos += 4) { - uint32_t SubHash = LShift(Hash[Pos + 0], 24); - SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) | - (unsigned)(unsigned char)Hash[Pos + 3]; - Vals.push_back(SubHash); + Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); } // Emit the finished record. Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); } -void BitcodeWriter::write() { - // Emit the file header first. - writeBitcodeHeader(); - - writeBlocks(); -} - -void ModuleBitcodeWriter::writeBlocks() { - writeIdentificationBlock(); - writeModule(); -} - -void IndexBitcodeWriter::writeBlocks() { - // Index contains only a single outer (module) block. - writeIndex(); -} +void ModuleBitcodeWriter::write() { + writeIdentificationBlock(Stream); -void ModuleBitcodeWriter::writeModule() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); size_t BlockStartPos = Buffer.size(); @@ -3769,7 +3866,7 @@ static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, } /// Helper to write the header common to all bitcode files. -void BitcodeWriter::writeBitcodeHeader() { +static void writeBitcodeHeader(BitstreamWriter &Stream) { // Emit the file header. Stream.Emit((unsigned)'B', 8); Stream.Emit((unsigned)'C', 8); @@ -3779,6 +3876,22 @@ void BitcodeWriter::writeBitcodeHeader() { Stream.Emit(0xD, 4); } +BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer) + : Buffer(Buffer), Stream(new BitstreamWriter(Buffer)) { + writeBitcodeHeader(*Stream); +} + +BitcodeWriter::~BitcodeWriter() = default; + +void BitcodeWriter::writeModule(const Module *M, + bool ShouldPreserveUseListOrder, + const ModuleSummaryIndex *Index, + bool GenerateHash) { + ModuleBitcodeWriter ModuleWriter( + M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash); + ModuleWriter.write(); +} + /// WriteBitcodeToFile - Write the specified module to the specified output /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, @@ -3794,10 +3907,8 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); - // Emit the module into the buffer. - ModuleBitcodeWriter ModuleWriter(M, Buffer, ShouldPreserveUseListOrder, Index, - GenerateHash); - ModuleWriter.write(); + BitcodeWriter Writer(Buffer); + Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) emitDarwinBCHeaderAndTrailer(Buffer, TT); @@ -3806,7 +3917,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, Out.write((char*)&Buffer.front(), Buffer.size()); } -void IndexBitcodeWriter::writeIndex() { +void IndexBitcodeWriter::write() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); SmallVector<unsigned, 1> Vals; @@ -3836,11 +3947,14 @@ void IndexBitcodeWriter::writeIndex() { // index for a distributed backend, provide a \p ModuleToSummariesForIndex map. void llvm::WriteIndexToFile( const ModuleSummaryIndex &Index, raw_ostream &Out, - std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { SmallVector<char, 0> Buffer; Buffer.reserve(256 * 1024); - IndexBitcodeWriter IndexWriter(Buffer, Index, ModuleToSummariesForIndex); + BitstreamWriter Stream(Buffer); + writeBitcodeHeader(Stream); + + IndexBitcodeWriter IndexWriter(Stream, Index, ModuleToSummariesForIndex); IndexWriter.write(); Out.write((char *)&Buffer.front(), Buffer.size()); diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 3e89ade..80cab76 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -13,18 +13,17 @@ #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" using namespace llvm; -PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &) { - std::unique_ptr<ModuleSummaryIndex> Index; - if (EmitSummaryIndex) - Index = ModuleSummaryIndexBuilder(&M).takeIndex(); - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index.get(), - EmitModuleHash); +PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { + const ModuleSummaryIndex *Index = + EmitSummaryIndex ? &(AM.getResult<ModuleSummaryIndexAnalysis>(M)) + : nullptr; + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash); return PreservedAnalyses::all(); } @@ -49,7 +48,7 @@ namespace { initializeWriteBitcodePassPass(*PassRegistry::getPassRegistry()); } - const char *getPassName() const override { return "Bitcode Writer"; } + StringRef getPassName() const override { return "Bitcode Writer"; } bool runOnModule(Module &M) override { const ModuleSummaryIndex *Index = |
