diff options
Diffstat (limited to 'contrib/llvm/lib/Bitcode')
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitReader.cpp | 5 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 1786 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp | 68 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2225 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp | 44 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 321 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h | 121 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/module.modulemap | 1 |
8 files changed, 3173 insertions, 1398 deletions
diff --git a/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp index 385c18a..9ac3cb9 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitReader.cpp @@ -25,14 +25,13 @@ using namespace llvm; Optionally returns a human-readable error message via OutMessage. */ LLVMBool LLVMParseBitcode(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutModule, char **OutMessage) { - return LLVMParseBitcodeInContext(wrap(&getGlobalContext()), MemBuf, OutModule, + return LLVMParseBitcodeInContext(LLVMGetGlobalContext(), MemBuf, OutModule, OutMessage); } LLVMBool LLVMParseBitcode2(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutModule) { - return LLVMParseBitcodeInContext2(wrap(&getGlobalContext()), MemBuf, - OutModule); + return LLVMParseBitcodeInContext2(LLVMGetGlobalContext(), MemBuf, OutModule); } static void diagnosticHandler(const DiagnosticInfo &DI, void *C) { diff --git a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 2ad4b32..73a30c6 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -7,14 +7,15 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" #include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -25,18 +26,27 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/OperandTraits.h" #include "llvm/IR/Operator.h" -#include "llvm/IR/FunctionInfo.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/DataStream.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" #include <deque> +#include <utility> + using namespace llvm; +static cl::opt<bool> PrintSummaryGUIDs( + "print-summary-global-ids", cl::init(false), cl::Hidden, + cl::desc( + "Print the global id for each value when reading the module summary")); + namespace { enum { SWITCH_INST_MAGIC = 0x4B5 // May 2012 => 1205 => Hex @@ -77,7 +87,7 @@ public: } Value *back() const { return ValuePtrs.back(); } - void pop_back() { ValuePtrs.pop_back(); } + void pop_back() { ValuePtrs.pop_back(); } bool empty() const { return ValuePtrs.empty(); } void shrinkTo(unsigned N) { assert(N <= size() && "Invalid shrinkTo request!"); @@ -99,7 +109,20 @@ class BitcodeReaderMetadataList { bool AnyFwdRefs; unsigned MinFwdRef; unsigned MaxFwdRef; - std::vector<TrackingMDRef> MetadataPtrs; + + /// Array of metadata references. + /// + /// Don't use std::vector here. Some versions of libc++ copy (instead of + /// move) on resize, and TrackingMDRef is very expensive to copy. + SmallVector<TrackingMDRef, 1> MetadataPtrs; + + /// Structures for resolving old type refs. + struct { + SmallDenseMap<MDString *, TempMDTuple, 1> Unknown; + SmallDenseMap<MDString *, DICompositeType *, 1> Final; + SmallDenseMap<MDString *, DICompositeType *, 1> FwdDecls; + SmallVector<std::pair<TrackingMDRef, TempMDTuple>, 1> Arrays; + } OldTypeRefs; LLVMContext &Context; public: @@ -120,14 +143,44 @@ public: return MetadataPtrs[i]; } + Metadata *lookup(unsigned I) const { + if (I < MetadataPtrs.size()) + return MetadataPtrs[I]; + return nullptr; + } + void shrinkTo(unsigned N) { assert(N <= size() && "Invalid shrinkTo request!"); + assert(!AnyFwdRefs && "Unexpected forward refs"); MetadataPtrs.resize(N); } - Metadata *getValueFwdRef(unsigned Idx); + /// Return the given metadata, creating a replaceable forward reference if + /// necessary. + Metadata *getMetadataFwdRef(unsigned Idx); + + /// Return the the given metadata only if it is fully resolved. + /// + /// Gives the same result as \a lookup(), unless \a MDNode::isResolved() + /// would give \c false. + Metadata *getMetadataIfResolved(unsigned Idx); + + MDNode *getMDNodeFwdRefOrNull(unsigned Idx); void assignValue(Metadata *MD, unsigned Idx); void tryToResolveCycles(); + bool hasFwdRefs() const { return AnyFwdRefs; } + + /// Upgrade a type that had an MDString reference. + void addTypeRef(MDString &UUID, DICompositeType &CT); + + /// Upgrade a type that had an MDString reference. + Metadata *upgradeTypeRef(Metadata *MaybeUUID); + + /// Upgrade a type ref array that may have MDString references. + Metadata *upgradeTypeRefArray(Metadata *MaybeTuple); + +private: + Metadata *resolveTypeRefArray(Metadata *MaybeTuple); }; class BitcodeReader : public GVMaterializer { @@ -144,11 +197,6 @@ class BitcodeReader : public GVMaterializer { uint64_t VSTOffset = 0; // Contains an arbitrary and optional string identifying the bitcode producer std::string ProducerIdentification; - // Number of module level metadata records specified by the - // MODULE_CODE_METADATA_VALUES record. - unsigned NumModuleMDs = 0; - // Support older bitcode without the MODULE_CODE_METADATA_VALUES record. - bool SeenModuleValuesRecord = false; std::vector<Type*> TypeList; BitcodeReaderValueList ValueList; @@ -157,13 +205,15 @@ class BitcodeReader : public GVMaterializer { SmallVector<Instruction *, 64> InstructionList; std::vector<std::pair<GlobalVariable*, unsigned> > GlobalInits; - std::vector<std::pair<GlobalAlias*, unsigned> > AliasInits; + std::vector<std::pair<GlobalIndirectSymbol*, unsigned> > IndirectSymbolInits; std::vector<std::pair<Function*, unsigned> > FunctionPrefixes; std::vector<std::pair<Function*, unsigned> > FunctionPrologues; std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFns; SmallVector<Instruction*, 64> InstsWithTBAATag; + bool HasSeenOldLoopTags = false; + /// The set of attributes by index. Index zero in the file is for null, and /// is thus not represented here. As such all indices are off by one. std::vector<AttributeSet> MAttributes; @@ -181,8 +231,10 @@ class BitcodeReader : public GVMaterializer { // When intrinsic functions are encountered which require upgrading they are // stored here with their replacement function. - typedef DenseMap<Function*, Function*> UpgradedIntrinsicMap; - UpgradedIntrinsicMap UpgradedIntrinsics; + typedef DenseMap<Function*, Function*> UpdatedIntrinsicMap; + UpdatedIntrinsicMap UpgradedIntrinsics; + // Intrinsics which were remangled because of types rename + UpdatedIntrinsicMap RemangledIntrinsics; // Map the bitcode's custom MDKind ID to the Module's MDKind ID. DenseMap<unsigned, unsigned> MDKindMap; @@ -232,7 +284,6 @@ class BitcodeReader : public GVMaterializer { public: std::error_code error(BitcodeError E, const Twine &Message); - std::error_code error(BitcodeError E); std::error_code error(const Twine &Message); BitcodeReader(MemoryBuffer *Buffer, LLVMContext &Context); @@ -262,6 +313,10 @@ public: /// Cheap mechanism to just extract the identification block out of bitcode. ErrorOr<std::string> parseIdentificationBlock(); + /// Peak at the module content and return true if any ObjC category or class + /// is found. + ErrorOr<bool> hasObjCCategory(); + static uint64_t decodeSignRotatedValue(uint64_t V); /// Materialize any deferred Metadata block. @@ -269,14 +324,6 @@ public: void setStripDebugInfo() override; - /// Save the mapping between the metadata values and the corresponding - /// value id that were recorded in the MetadataList during parsing. If - /// OnlyTempMD is true, then only record those entries that are still - /// temporary metadata. This interface is used when metadata linking is - /// performed as a postpass, such as during function importing. - void saveMetadataList(DenseMap<const Metadata *, unsigned> &MetadataToIDs, - bool OnlyTempMD) override; - private: /// Parse the "IDENTIFICATION_BLOCK_ID" block, populate the // ProducerIdentification data member, and do some basic enforcement on the @@ -294,7 +341,7 @@ private: return ValueList.getValueFwdRef(ID, Ty); } Metadata *getFnMetadataByID(unsigned ID) { - return MetadataList.getValueFwdRef(ID); + return MetadataList.getMetadataFwdRef(ID); } BasicBlock *getBasicBlock(unsigned ID) const { if (ID >= FunctionBBs.size()) return nullptr; // Invalid ID @@ -395,12 +442,19 @@ private: std::error_code rememberAndSkipMetadata(); std::error_code parseFunctionBody(Function *F); std::error_code globalCleanup(); - std::error_code resolveGlobalAndAliasInits(); + std::error_code resolveGlobalAndIndirectSymbolInits(); std::error_code parseMetadata(bool ModuleLevel = false); + std::error_code parseMetadataStrings(ArrayRef<uint64_t> Record, + StringRef Blob, + unsigned &NextMetadataNo); std::error_code parseMetadataKinds(); std::error_code parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record); + std::error_code + parseGlobalObjectAttachment(GlobalObject &GO, + ArrayRef<uint64_t> Record); std::error_code parseMetadataAttachment(Function &F); ErrorOr<std::string> parseModuleTriple(); + ErrorOr<bool> hasObjCCategoryInModule(); std::error_code parseUseLists(); std::error_code initStream(std::unique_ptr<DataStreamer> Streamer); std::error_code initStreamFromBuffer(); @@ -412,92 +466,86 @@ private: /// Class to manage reading and parsing function summary index bitcode /// files/sections. -class FunctionIndexBitcodeReader { +class ModuleSummaryIndexBitcodeReader { DiagnosticHandlerFunction DiagnosticHandler; - /// Eventually points to the function index built during parsing. - FunctionInfoIndex *TheIndex = nullptr; + /// Eventually points to the module index built during parsing. + ModuleSummaryIndex *TheIndex = nullptr; std::unique_ptr<MemoryBuffer> Buffer; std::unique_ptr<BitstreamReader> StreamFile; BitstreamCursor Stream; - /// \brief Used to indicate whether we are doing lazy parsing of summary data. - /// - /// If false, the summary section is fully parsed into the index during - /// the initial parse. Otherwise, if true, the caller is expected to - /// invoke \a readFunctionSummary for each summary needed, and the summary - /// section is thus parsed lazily. - bool IsLazy = false; - /// Used to indicate whether caller only wants to check for the presence - /// of the function summary bitcode section. All blocks are skipped, - /// but the SeenFuncSummary boolean is set. - bool CheckFuncSummaryPresenceOnly = false; + /// of the global value summary bitcode section. All blocks are skipped, + /// but the SeenGlobalValSummary boolean is set. + bool CheckGlobalValSummaryPresenceOnly = false; - /// Indicates whether we have encountered a function summary section - /// yet during parsing, used when checking if file contains function + /// Indicates whether we have encountered a global value summary section + /// yet during parsing, used when checking if file contains global value /// summary section. - bool SeenFuncSummary = false; + bool SeenGlobalValSummary = false; - /// \brief Map populated during function summary section parsing, and - /// consumed during ValueSymbolTable parsing. - /// - /// Used to correlate summary records with VST entries. For the per-module - /// index this maps the ValueID to the parsed function summary, and - /// for the combined index this maps the summary record's bitcode - /// offset to the function summary (since in the combined index the - /// VST records do not hold value IDs but rather hold the function - /// summary record offset). - DenseMap<uint64_t, std::unique_ptr<FunctionSummary>> SummaryMap; + /// Indicates whether we have already parsed the VST, used for error checking. + bool SeenValueSymbolTable = false; + + /// Set to the offset of the VST recorded in the MODULE_CODE_VSTOFFSET record. + /// Used to enable on-demand parsing of the VST. + uint64_t VSTOffset = 0; + + // Map to save ValueId to GUID association that was recorded in the + // ValueSymbolTable. It is used after the VST is parsed to convert + // call graph edges read from the function summary from referencing + // callees by their ValueId to using the GUID instead, which is how + // they are recorded in the summary index being built. + // We save a second GUID which is the same as the first one, but ignoring the + // linkage, i.e. for value other than local linkage they are identical. + DenseMap<unsigned, std::pair<GlobalValue::GUID, GlobalValue::GUID>> + ValueIdToCallGraphGUIDMap; /// Map populated during module path string table parsing, from the /// module ID to a string reference owned by the index's module - /// path string table, used to correlate with combined index function + /// path string table, used to correlate with combined index /// summary records. DenseMap<uint64_t, StringRef> ModuleIdMap; + /// Original source file name recorded in a bitcode record. + std::string SourceFileName; + public: - std::error_code error(BitcodeError E, const Twine &Message); - std::error_code error(BitcodeError E); std::error_code error(const Twine &Message); - FunctionIndexBitcodeReader(MemoryBuffer *Buffer, - DiagnosticHandlerFunction DiagnosticHandler, - bool IsLazy = false, - bool CheckFuncSummaryPresenceOnly = false); - FunctionIndexBitcodeReader(DiagnosticHandlerFunction DiagnosticHandler, - bool IsLazy = false, - bool CheckFuncSummaryPresenceOnly = false); - ~FunctionIndexBitcodeReader() { freeState(); } + ModuleSummaryIndexBitcodeReader( + MemoryBuffer *Buffer, DiagnosticHandlerFunction DiagnosticHandler, + bool CheckGlobalValSummaryPresenceOnly = false); + ~ModuleSummaryIndexBitcodeReader() { freeState(); } void freeState(); void releaseBuffer(); - /// Check if the parser has encountered a function summary section. - bool foundFuncSummary() { return SeenFuncSummary; } + /// Check if the parser has encountered a summary section. + bool foundGlobalValSummary() { return SeenGlobalValSummary; } /// \brief Main interface to parsing a bitcode buffer. /// \returns true if an error occurred. std::error_code parseSummaryIndexInto(std::unique_ptr<DataStreamer> Streamer, - FunctionInfoIndex *I); - - /// \brief Interface for parsing a function summary lazily. - std::error_code parseFunctionSummary(std::unique_ptr<DataStreamer> Streamer, - FunctionInfoIndex *I, - size_t FunctionSummaryOffset); + ModuleSummaryIndex *I); private: std::error_code parseModule(); - std::error_code parseValueSymbolTable(); + std::error_code parseValueSymbolTable( + uint64_t Offset, + DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap); std::error_code parseEntireSummary(); std::error_code parseModuleStringTable(); std::error_code initStream(std::unique_ptr<DataStreamer> Streamer); std::error_code initStreamFromBuffer(); std::error_code initLazyStream(std::unique_ptr<DataStreamer> Streamer); + std::pair<GlobalValue::GUID, GlobalValue::GUID> + getGUIDFromValueId(unsigned ValueId); }; -} // namespace +} // end anonymous namespace BitcodeDiagnosticInfo::BitcodeDiagnosticInfo(std::error_code EC, DiagnosticSeverity Severity, @@ -506,28 +554,19 @@ BitcodeDiagnosticInfo::BitcodeDiagnosticInfo(std::error_code EC, void BitcodeDiagnosticInfo::print(DiagnosticPrinter &DP) const { DP << Msg; } -static std::error_code error(DiagnosticHandlerFunction DiagnosticHandler, +static std::error_code error(const DiagnosticHandlerFunction &DiagnosticHandler, std::error_code EC, const Twine &Message) { BitcodeDiagnosticInfo DI(EC, DS_Error, Message); DiagnosticHandler(DI); return EC; } -static std::error_code error(DiagnosticHandlerFunction DiagnosticHandler, - std::error_code EC) { - return error(DiagnosticHandler, EC, EC.message()); -} - static std::error_code error(LLVMContext &Context, std::error_code EC, const Twine &Message) { return error([&](const DiagnosticInfo &DI) { Context.diagnose(DI); }, EC, Message); } -static std::error_code error(LLVMContext &Context, std::error_code EC) { - return error(Context, EC, EC.message()); -} - static std::error_code error(LLVMContext &Context, const Twine &Message) { return error(Context, make_error_code(BitcodeError::CorruptedBitcode), Message); @@ -552,10 +591,6 @@ std::error_code BitcodeReader::error(const Twine &Message) { Message); } -std::error_code BitcodeReader::error(BitcodeError E) { - return ::error(Context, make_error_code(E)); -} - BitcodeReader::BitcodeReader(MemoryBuffer *Buffer, LLVMContext &Context) : Context(Context), Buffer(Buffer), ValueList(Context), MetadataList(Context) {} @@ -685,6 +720,18 @@ static GlobalValue::LinkageTypes getDecodedLinkage(unsigned Val) { } } +// Decode the flags for GlobalValue in the summary +static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, + uint64_t Version) { + // Summary were not emitted before LLVM 3.9, we don't need to upgrade Linkage + // like getDecodedLinkage() above. Any future change to the linkage enum and + // to getDecodedLinkage() will need to be taken into account here as above. + auto Linkage = GlobalValue::LinkageTypes(RawFlags & 0xF); // 4 bits + RawFlags = RawFlags >> 4; + auto HasSection = RawFlags & 0x1; // bool + return GlobalValueSummary::GVFlags(Linkage, HasSection); +} + static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { switch (Val) { default: // Map unknown visibilities to default. @@ -715,6 +762,15 @@ static GlobalVariable::ThreadLocalMode getDecodedThreadLocalMode(unsigned Val) { } } +static GlobalVariable::UnnamedAddr getDecodedUnnamedAddrType(unsigned Val) { + switch (Val) { + default: // Map unknown to UnnamedAddr::None. + case 0: return GlobalVariable::UnnamedAddr::None; + case 1: return GlobalVariable::UnnamedAddr::Global; + case 2: return GlobalVariable::UnnamedAddr::Local; + } +} + static int getDecodedCastOpcode(unsigned Val) { switch (Val) { default: return -1; @@ -791,14 +847,14 @@ static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) { static AtomicOrdering getDecodedOrdering(unsigned Val) { switch (Val) { - case bitc::ORDERING_NOTATOMIC: return NotAtomic; - case bitc::ORDERING_UNORDERED: return Unordered; - case bitc::ORDERING_MONOTONIC: return Monotonic; - case bitc::ORDERING_ACQUIRE: return Acquire; - case bitc::ORDERING_RELEASE: return Release; - case bitc::ORDERING_ACQREL: return AcquireRelease; + case bitc::ORDERING_NOTATOMIC: return AtomicOrdering::NotAtomic; + case bitc::ORDERING_UNORDERED: return AtomicOrdering::Unordered; + case bitc::ORDERING_MONOTONIC: return AtomicOrdering::Monotonic; + case bitc::ORDERING_ACQUIRE: return AtomicOrdering::Acquire; + case bitc::ORDERING_RELEASE: return AtomicOrdering::Release; + case bitc::ORDERING_ACQREL: return AtomicOrdering::AcquireRelease; default: // Map unknown orderings to sequentially-consistent. - case bitc::ORDERING_SEQCST: return SequentiallyConsistent; + case bitc::ORDERING_SEQCST: return AtomicOrdering::SequentiallyConsistent; } } @@ -872,7 +928,7 @@ public: /// Provide fast operand accessors DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; -} +} // end anonymous namespace // FIXME: can we inherit this from ConstantExpr? template <> @@ -880,7 +936,7 @@ struct OperandTraits<ConstantPlaceHolder> : public FixedNumOperandTraits<ConstantPlaceHolder, 1> { }; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPlaceHolder, Value) -} +} // end namespace llvm void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { if (Idx == size()) { @@ -908,11 +964,8 @@ void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { OldV->replaceAllUsesWith(V); delete PrevVal; } - - return; } - Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx, Type *Ty) { if (Idx >= size()) @@ -1056,7 +1109,7 @@ void BitcodeReaderMetadataList::assignValue(Metadata *MD, unsigned Idx) { --NumFwdRefs; } -Metadata *BitcodeReaderMetadataList::getValueFwdRef(unsigned Idx) { +Metadata *BitcodeReaderMetadataList::getMetadataFwdRef(unsigned Idx) { if (Idx >= size()) resize(Idx + 1); @@ -1079,15 +1132,61 @@ Metadata *BitcodeReaderMetadataList::getValueFwdRef(unsigned Idx) { return MD; } -void BitcodeReaderMetadataList::tryToResolveCycles() { - if (!AnyFwdRefs) - // Nothing to do. - return; +Metadata *BitcodeReaderMetadataList::getMetadataIfResolved(unsigned Idx) { + Metadata *MD = lookup(Idx); + if (auto *N = dyn_cast_or_null<MDNode>(MD)) + if (!N->isResolved()) + return nullptr; + return MD; +} +MDNode *BitcodeReaderMetadataList::getMDNodeFwdRefOrNull(unsigned Idx) { + return dyn_cast_or_null<MDNode>(getMetadataFwdRef(Idx)); +} + +void BitcodeReaderMetadataList::tryToResolveCycles() { if (NumFwdRefs) // Still forward references... can't resolve cycles. return; + bool DidReplaceTypeRefs = false; + + // Give up on finding a full definition for any forward decls that remain. + for (const auto &Ref : OldTypeRefs.FwdDecls) + OldTypeRefs.Final.insert(Ref); + OldTypeRefs.FwdDecls.clear(); + + // Upgrade from old type ref arrays. In strange cases, this could add to + // OldTypeRefs.Unknown. + for (const auto &Array : OldTypeRefs.Arrays) { + DidReplaceTypeRefs = true; + Array.second->replaceAllUsesWith(resolveTypeRefArray(Array.first.get())); + } + OldTypeRefs.Arrays.clear(); + + // Replace old string-based type refs with the resolved node, if possible. + // If we haven't seen the node, leave it to the verifier to complain about + // the invalid string reference. + for (const auto &Ref : OldTypeRefs.Unknown) { + DidReplaceTypeRefs = true; + if (DICompositeType *CT = OldTypeRefs.Final.lookup(Ref.first)) + Ref.second->replaceAllUsesWith(CT); + else + Ref.second->replaceAllUsesWith(Ref.first); + } + OldTypeRefs.Unknown.clear(); + + // Make sure all the upgraded types are resolved. + if (DidReplaceTypeRefs) { + AnyFwdRefs = true; + MinFwdRef = 0; + MaxFwdRef = MetadataPtrs.size() - 1; + } + + if (!AnyFwdRefs) + // Nothing to do. + return; + // Resolve any cycles. for (unsigned I = MinFwdRef, E = MaxFwdRef + 1; I != E; ++I) { auto &MD = MetadataPtrs[I]; @@ -1103,6 +1202,60 @@ void BitcodeReaderMetadataList::tryToResolveCycles() { AnyFwdRefs = false; } +void BitcodeReaderMetadataList::addTypeRef(MDString &UUID, + DICompositeType &CT) { + assert(CT.getRawIdentifier() == &UUID && "Mismatched UUID"); + if (CT.isForwardDecl()) + OldTypeRefs.FwdDecls.insert(std::make_pair(&UUID, &CT)); + else + OldTypeRefs.Final.insert(std::make_pair(&UUID, &CT)); +} + +Metadata *BitcodeReaderMetadataList::upgradeTypeRef(Metadata *MaybeUUID) { + auto *UUID = dyn_cast_or_null<MDString>(MaybeUUID); + if (LLVM_LIKELY(!UUID)) + return MaybeUUID; + + if (auto *CT = OldTypeRefs.Final.lookup(UUID)) + return CT; + + auto &Ref = OldTypeRefs.Unknown[UUID]; + if (!Ref) + Ref = MDNode::getTemporary(Context, None); + return Ref.get(); +} + +Metadata *BitcodeReaderMetadataList::upgradeTypeRefArray(Metadata *MaybeTuple) { + auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple); + if (!Tuple || Tuple->isDistinct()) + return MaybeTuple; + + // Look through the array immediately if possible. + if (!Tuple->isTemporary()) + return resolveTypeRefArray(Tuple); + + // Create and return a placeholder to use for now. Eventually + // resolveTypeRefArrays() will be resolve this forward reference. + OldTypeRefs.Arrays.emplace_back( + std::piecewise_construct, std::forward_as_tuple(Tuple), + std::forward_as_tuple(MDTuple::getTemporary(Context, None))); + return OldTypeRefs.Arrays.back().second.get(); +} + +Metadata *BitcodeReaderMetadataList::resolveTypeRefArray(Metadata *MaybeTuple) { + auto *Tuple = dyn_cast_or_null<MDTuple>(MaybeTuple); + if (!Tuple || Tuple->isDistinct()) + return MaybeTuple; + + // Look through the DITypeRefArray, upgrading each DITypeRef. + SmallVector<Metadata *, 32> Ops; + Ops.reserve(Tuple->getNumOperands()); + for (Metadata *MD : Tuple->operands()) + Ops.push_back(upgradeTypeRef(MD)); + + return MDTuple::get(Context, Ops); +} + Type *BitcodeReader::getTypeByID(unsigned ID) { // The type table size is always specified correctly. if (ID >= TypeList.size()) @@ -1129,7 +1282,6 @@ StructType *BitcodeReader::createIdentifiedStructType(LLVMContext &Context) { return Ret; } - //===----------------------------------------------------------------------===// // Functions for parsing blocks from the bitcode file //===----------------------------------------------------------------------===// @@ -1271,6 +1423,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::Dereferenceable; case bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL: return Attribute::DereferenceableOrNull; + case bitc::ATTR_KIND_ALLOC_SIZE: + return Attribute::AllocSize; case bitc::ATTR_KIND_NO_RED_ZONE: return Attribute::NoRedZone; case bitc::ATTR_KIND_NO_RETURN: @@ -1309,8 +1463,14 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::SanitizeThread; case bitc::ATTR_KIND_SANITIZE_MEMORY: return Attribute::SanitizeMemory; + case bitc::ATTR_KIND_SWIFT_ERROR: + return Attribute::SwiftError; + case bitc::ATTR_KIND_SWIFT_SELF: + return Attribute::SwiftSelf; case bitc::ATTR_KIND_UW_TABLE: return Attribute::UWTable; + case bitc::ATTR_KIND_WRITEONLY: + return Attribute::WriteOnly; case bitc::ATTR_KIND_Z_EXT: return Attribute::ZExt; } @@ -1391,6 +1551,8 @@ std::error_code BitcodeReader::parseAttributeGroupBlock() { B.addDereferenceableAttr(Record[++i]); else if (Kind == Attribute::DereferenceableOrNull) B.addDereferenceableOrNullAttr(Record[++i]); + else if (Kind == Attribute::AllocSize) + B.addAllocSizeAttrFromRawRepr(Record[++i]); } else { // String attribute assert((Record[i] == 3 || Record[i] == 4) && "Invalid attribute group entry"); @@ -1727,6 +1889,27 @@ ErrorOr<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record, return V; } +/// Helper to note and return the current location, and jump to the given +/// offset. +static uint64_t jumpToValueSymbolTable(uint64_t Offset, + BitstreamCursor &Stream) { + // Save the current parsing location so we can jump back at the end + // of the VST read. + uint64_t CurrentBit = Stream.GetCurrentBitNo(); + Stream.JumpToBit(Offset * 32); +#ifndef NDEBUG + // Do some checking if we are in debug mode. + BitstreamEntry Entry = Stream.advance(); + assert(Entry.Kind == BitstreamEntry::SubBlock); + assert(Entry.ID == bitc::VALUE_SYMTAB_BLOCK_ID); +#else + // In NDEBUG mode ignore the output so we don't get an unused variable + // warning. + Stream.advance(); +#endif + return CurrentBit; +} + /// Parse the value symbol table at either the current parsing location or /// at the given bit offset if provided. std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { @@ -1734,22 +1917,8 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { // Pass in the Offset to distinguish between calling for the module-level // VST (where we want to jump to the VST offset) and the function-level // VST (where we don't). - if (Offset > 0) { - // Save the current parsing location so we can jump back at the end - // of the VST read. - CurrentBit = Stream.GetCurrentBitNo(); - Stream.JumpToBit(Offset * 32); -#ifndef NDEBUG - // Do some checking if we are in debug mode. - BitstreamEntry Entry = Stream.advance(); - assert(Entry.Kind == BitstreamEntry::SubBlock); - assert(Entry.ID == bitc::VALUE_SYMTAB_BLOCK_ID); -#else - // In NDEBUG mode ignore the output so we don't get an unused variable - // warning. - Stream.advance(); -#endif - } + if (Offset > 0) + CurrentBit = jumpToValueSymbolTable(Offset, Stream); // Compute the delta between the bitcode indices in the VST (the word offset // to the word-aligned ENTER_SUBBLOCK for the function block, and that @@ -1795,7 +1964,7 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { switch (Stream.readRecord(Entry.ID, Record)) { default: // Default behavior: unknown type. break; - case bitc::VST_CODE_ENTRY: { // VST_ENTRY: [valueid, namechar x N] + case bitc::VST_CODE_ENTRY: { // VST_CODE_ENTRY: [valueid, namechar x N] ErrorOr<Value *> ValOrErr = recordValue(Record, 1, TT); if (std::error_code EC = ValOrErr.getError()) return EC; @@ -1803,7 +1972,7 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) { break; } case bitc::VST_CODE_FNENTRY: { - // VST_FNENTRY: [valueid, offset, namechar x N] + // VST_CODE_FNENTRY: [valueid, offset, namechar x N] ErrorOr<Value *> ValOrErr = recordValue(Record, 2, TT); if (std::error_code EC = ValOrErr.getError()) return EC; @@ -1863,47 +2032,122 @@ BitcodeReader::parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record) { static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; } +std::error_code BitcodeReader::parseMetadataStrings(ArrayRef<uint64_t> Record, + StringRef Blob, + unsigned &NextMetadataNo) { + // All the MDStrings in the block are emitted together in a single + // record. The strings are concatenated and stored in a blob along with + // their sizes. + if (Record.size() != 2) + return error("Invalid record: metadata strings layout"); + + unsigned NumStrings = Record[0]; + unsigned StringsOffset = Record[1]; + if (!NumStrings) + return error("Invalid record: metadata strings with no strings"); + if (StringsOffset > Blob.size()) + return error("Invalid record: metadata strings corrupt offset"); + + StringRef Lengths = Blob.slice(0, StringsOffset); + SimpleBitstreamCursor R(*StreamFile); + R.jumpToPointer(Lengths.begin()); + + // Ensure that Blob doesn't get invalidated, even if this is reading from + // a StreamingMemoryObject with corrupt data. + R.setArtificialByteLimit(R.getCurrentByteNo() + StringsOffset); + + StringRef Strings = Blob.drop_front(StringsOffset); + do { + if (R.AtEndOfStream()) + return error("Invalid record: metadata strings bad length"); + + unsigned Size = R.ReadVBR(6); + if (Strings.size() < Size) + return error("Invalid record: metadata strings truncated chars"); + + MetadataList.assignValue(MDString::get(Context, Strings.slice(0, Size)), + NextMetadataNo++); + Strings = Strings.drop_front(Size); + } while (--NumStrings); + + return std::error_code(); +} + +namespace { +class PlaceholderQueue { + // Placeholders would thrash around when moved, so store in a std::deque + // instead of some sort of vector. + std::deque<DistinctMDOperandPlaceholder> PHs; + +public: + DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID); + void flush(BitcodeReaderMetadataList &MetadataList); +}; +} // end namespace + +DistinctMDOperandPlaceholder &PlaceholderQueue::getPlaceholderOp(unsigned ID) { + PHs.emplace_back(ID); + return PHs.back(); +} + +void PlaceholderQueue::flush(BitcodeReaderMetadataList &MetadataList) { + while (!PHs.empty()) { + PHs.front().replaceUseWith( + MetadataList.getMetadataFwdRef(PHs.front().getID())); + PHs.pop_front(); + } +} + /// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing /// module level metadata. std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { + assert((ModuleLevel || DeferredMetadataInfo.empty()) && + "Must read all module-level metadata before function-level"); + IsMetadataMaterialized = true; unsigned NextMetadataNo = MetadataList.size(); - if (ModuleLevel && SeenModuleValuesRecord) { - // Now that we are parsing the module level metadata, we want to restart - // the numbering of the MD values, and replace temp MD created earlier - // with their real values. If we saw a METADATA_VALUE record then we - // would have set the MetadataList size to the number specified in that - // record, to support parsing function-level metadata first, and we need - // to reset back to 0 to fill the MetadataList in with the parsed module - // The function-level metadata parsing should have reset the MetadataList - // size back to the value reported by the METADATA_VALUE record, saved in - // NumModuleMDs. - assert(NumModuleMDs == MetadataList.size() && - "Expected MetadataList to only contain module level values"); - NextMetadataNo = 0; - } + + if (!ModuleLevel && MetadataList.hasFwdRefs()) + return error("Invalid metadata: fwd refs into function blocks"); if (Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID)) return error("Invalid record"); + std::vector<std::pair<DICompileUnit *, Metadata *>> CUSubprograms; SmallVector<uint64_t, 64> Record; + PlaceholderQueue Placeholders; + bool IsDistinct; auto getMD = [&](unsigned ID) -> Metadata * { - return MetadataList.getValueFwdRef(ID); + if (!IsDistinct) + return MetadataList.getMetadataFwdRef(ID); + if (auto *MD = MetadataList.getMetadataIfResolved(ID)) + return MD; + return &Placeholders.getPlaceholderOp(ID); }; - auto getMDOrNull = [&](unsigned ID) -> Metadata *{ + auto getMDOrNull = [&](unsigned ID) -> Metadata * { if (ID) return getMD(ID - 1); return nullptr; }; + auto getMDOrNullWithoutPlaceholders = [&](unsigned ID) -> Metadata * { + if (ID) + return MetadataList.getMetadataFwdRef(ID - 1); + return nullptr; + }; auto getMDString = [&](unsigned ID) -> MDString *{ // This requires that the ID is not really a forward reference. In // particular, the MDString must already have been resolved. return cast_or_null<MDString>(getMDOrNull(ID)); }; -#define GET_OR_DISTINCT(CLASS, DISTINCT, ARGS) \ - (DISTINCT ? CLASS::getDistinct ARGS : CLASS::get ARGS) + // Support for old type refs. + auto getDITypeRefOrNull = [&](unsigned ID) { + return MetadataList.upgradeTypeRef(getMDOrNull(ID)); + }; + +#define GET_OR_DISTINCT(CLASS, ARGS) \ + (IsDistinct ? CLASS::getDistinct ARGS : CLASS::get ARGS) // Read all the records. while (1) { @@ -1914,10 +2158,15 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: + // Upgrade old-style CU <-> SP pointers to point from SP to CU. + for (auto CU_SP : CUSubprograms) + if (auto *SPs = dyn_cast_or_null<MDTuple>(CU_SP.second)) + for (auto &Op : SPs->operands()) + if (auto *SP = dyn_cast_or_null<MDNode>(Op)) + SP->replaceOperandWith(7, CU_SP.first); + MetadataList.tryToResolveCycles(); - assert((!(ModuleLevel && SeenModuleValuesRecord) || - NumModuleMDs == MetadataList.size()) && - "Inconsistent bitcode: METADATA_VALUES mismatch"); + Placeholders.flush(MetadataList); return std::error_code(); case BitstreamEntry::Record: // The interesting case. @@ -1926,8 +2175,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { // Read a record. Record.clear(); - unsigned Code = Stream.readRecord(Entry.ID, Record); - bool IsDistinct = false; + StringRef Blob; + unsigned Code = Stream.readRecord(Entry.ID, Record, &Blob); + IsDistinct = false; switch (Code) { default: // Default behavior: ignore. break; @@ -1945,8 +2195,7 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { unsigned Size = Record.size(); NamedMDNode *NMD = TheModule->getOrInsertNamedMetadata(Name); for (unsigned i = 0; i != Size; ++i) { - MDNode *MD = - dyn_cast_or_null<MDNode>(MetadataList.getValueFwdRef(Record[i])); + MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[i]); if (!MD) return error("Invalid record"); NMD->addOperand(MD); @@ -1993,7 +2242,7 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (!Ty) return error("Invalid record"); if (Ty->isMetadataTy()) - Elts.push_back(MetadataList.getValueFwdRef(Record[i + 1])); + Elts.push_back(getMD(Record[i + 1])); else if (!Ty->isVoidTy()) { auto *MD = ValueAsMetadata::get(ValueList.getValueFwdRef(Record[i + 1], Ty)); @@ -2026,7 +2275,7 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { SmallVector<Metadata *, 8> Elts; Elts.reserve(Record.size()); for (unsigned ID : Record) - Elts.push_back(ID ? MetadataList.getValueFwdRef(ID - 1) : nullptr); + Elts.push_back(getMDOrNull(ID)); MetadataList.assignValue(IsDistinct ? MDNode::getDistinct(Context, Elts) : MDNode::get(Context, Elts), NextMetadataNo++); @@ -2036,13 +2285,13 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 5) return error("Invalid record"); + IsDistinct = Record[0]; unsigned Line = Record[1]; unsigned Column = Record[2]; - MDNode *Scope = cast<MDNode>(MetadataList.getValueFwdRef(Record[3])); - Metadata *InlinedAt = - Record[4] ? MetadataList.getValueFwdRef(Record[4] - 1) : nullptr; + Metadata *Scope = getMD(Record[3]); + Metadata *InlinedAt = getMDOrNull(Record[4]); MetadataList.assignValue( - GET_OR_DISTINCT(DILocation, Record[0], + GET_OR_DISTINCT(DILocation, (Context, Line, Column, Scope, InlinedAt)), NextMetadataNo++); break; @@ -2051,6 +2300,7 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() < 4) return error("Invalid record"); + IsDistinct = Record[0]; unsigned Tag = Record[1]; unsigned Version = Record[2]; @@ -2060,11 +2310,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { auto *Header = getMDString(Record[3]); SmallVector<Metadata *, 8> DwarfOps; for (unsigned I = 4, E = Record.size(); I != E; ++I) - DwarfOps.push_back( - Record[I] ? MetadataList.getValueFwdRef(Record[I] - 1) : nullptr); + DwarfOps.push_back(getMDOrNull(Record[I])); MetadataList.assignValue( - GET_OR_DISTINCT(GenericDINode, Record[0], - (Context, Tag, Header, DwarfOps)), + GET_OR_DISTINCT(GenericDINode, (Context, Tag, Header, DwarfOps)), NextMetadataNo++); break; } @@ -2072,8 +2320,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 3) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DISubrange, Record[0], + GET_OR_DISTINCT(DISubrange, (Context, Record[1], unrotateSign(Record[2]))), NextMetadataNo++); break; @@ -2082,10 +2331,10 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 3) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT( - DIEnumerator, Record[0], - (Context, unrotateSign(Record[1]), getMDString(Record[2]))), + GET_OR_DISTINCT(DIEnumerator, (Context, unrotateSign(Record[1]), + getMDString(Record[2]))), NextMetadataNo++); break; } @@ -2093,8 +2342,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 6) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIBasicType, Record[0], + GET_OR_DISTINCT(DIBasicType, (Context, Record[1], getMDString(Record[2]), Record[3], Record[4], Record[5])), NextMetadataNo++); @@ -2104,13 +2354,14 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 12) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIDerivedType, Record[0], - (Context, Record[1], getMDString(Record[2]), - getMDOrNull(Record[3]), Record[4], - getMDOrNull(Record[5]), getMDOrNull(Record[6]), - Record[7], Record[8], Record[9], Record[10], - getMDOrNull(Record[11]))), + GET_OR_DISTINCT( + DIDerivedType, + (Context, Record[1], getMDString(Record[2]), + getMDOrNull(Record[3]), Record[4], getDITypeRefOrNull(Record[5]), + getDITypeRefOrNull(Record[6]), Record[7], Record[8], Record[9], + Record[10], getDITypeRefOrNull(Record[11]))), NextMetadataNo++); break; } @@ -2118,25 +2369,58 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 16) return error("Invalid record"); - MetadataList.assignValue( - GET_OR_DISTINCT(DICompositeType, Record[0], - (Context, Record[1], getMDString(Record[2]), - getMDOrNull(Record[3]), Record[4], - getMDOrNull(Record[5]), getMDOrNull(Record[6]), - Record[7], Record[8], Record[9], Record[10], - getMDOrNull(Record[11]), Record[12], - getMDOrNull(Record[13]), getMDOrNull(Record[14]), - getMDString(Record[15]))), - NextMetadataNo++); + // If we have a UUID and this is not a forward declaration, lookup the + // mapping. + IsDistinct = Record[0] & 0x1; + bool IsNotUsedInTypeRef = Record[0] >= 2; + unsigned Tag = Record[1]; + MDString *Name = getMDString(Record[2]); + Metadata *File = getMDOrNull(Record[3]); + unsigned Line = Record[4]; + Metadata *Scope = getDITypeRefOrNull(Record[5]); + Metadata *BaseType = getDITypeRefOrNull(Record[6]); + uint64_t SizeInBits = Record[7]; + uint64_t AlignInBits = Record[8]; + uint64_t OffsetInBits = Record[9]; + unsigned Flags = Record[10]; + Metadata *Elements = getMDOrNull(Record[11]); + unsigned RuntimeLang = Record[12]; + Metadata *VTableHolder = getDITypeRefOrNull(Record[13]); + Metadata *TemplateParams = getMDOrNull(Record[14]); + auto *Identifier = getMDString(Record[15]); + DICompositeType *CT = nullptr; + if (Identifier) + CT = DICompositeType::buildODRType( + Context, *Identifier, Tag, Name, File, Line, Scope, BaseType, + SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, + VTableHolder, TemplateParams); + + // Create a node if we didn't get a lazy ODR type. + if (!CT) + CT = GET_OR_DISTINCT(DICompositeType, + (Context, Tag, Name, File, Line, Scope, BaseType, + SizeInBits, AlignInBits, OffsetInBits, Flags, + Elements, RuntimeLang, VTableHolder, + TemplateParams, Identifier)); + if (!IsNotUsedInTypeRef && Identifier) + MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT)); + + MetadataList.assignValue(CT, NextMetadataNo++); break; } case bitc::METADATA_SUBROUTINE_TYPE: { - if (Record.size() != 3) + if (Record.size() < 3 || Record.size() > 4) return error("Invalid record"); + bool IsOldTypeRefArray = Record[0] < 2; + unsigned CC = (Record.size() > 3) ? Record[3] : 0; + + IsDistinct = Record[0] & 0x1; + Metadata *Types = getMDOrNull(Record[2]); + if (LLVM_UNLIKELY(IsOldTypeRefArray)) + Types = MetadataList.upgradeTypeRefArray(Types); MetadataList.assignValue( - GET_OR_DISTINCT(DISubroutineType, Record[0], - (Context, Record[1], getMDOrNull(Record[2]))), + GET_OR_DISTINCT(DISubroutineType, (Context, Record[1], CC, Types)), NextMetadataNo++); break; } @@ -2145,8 +2429,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 6) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIModule, Record[0], + GET_OR_DISTINCT(DIModule, (Context, getMDOrNull(Record[1]), getMDString(Record[2]), getMDString(Record[3]), getMDString(Record[4]), getMDString(Record[5]))), @@ -2158,9 +2443,10 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 3) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIFile, Record[0], (Context, getMDString(Record[1]), - getMDString(Record[2]))), + GET_OR_DISTINCT(DIFile, (Context, getMDString(Record[1]), + getMDString(Record[2]))), NextMetadataNo++); break; } @@ -2170,38 +2456,66 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { // Ignore Record[0], which indicates whether this compile unit is // distinct. It's always distinct. - MetadataList.assignValue( - DICompileUnit::getDistinct( - Context, Record[1], getMDOrNull(Record[2]), - getMDString(Record[3]), Record[4], getMDString(Record[5]), - Record[6], getMDString(Record[7]), Record[8], - getMDOrNull(Record[9]), getMDOrNull(Record[10]), - getMDOrNull(Record[11]), getMDOrNull(Record[12]), - getMDOrNull(Record[13]), - Record.size() <= 15 ? 0 : getMDOrNull(Record[15]), - Record.size() <= 14 ? 0 : Record[14]), - NextMetadataNo++); + IsDistinct = true; + auto *CU = DICompileUnit::getDistinct( + Context, Record[1], getMDOrNull(Record[2]), getMDString(Record[3]), + Record[4], getMDString(Record[5]), Record[6], getMDString(Record[7]), + Record[8], getMDOrNull(Record[9]), getMDOrNull(Record[10]), + getMDOrNull(Record[12]), getMDOrNull(Record[13]), + Record.size() <= 15 ? nullptr : getMDOrNull(Record[15]), + Record.size() <= 14 ? 0 : Record[14]); + + MetadataList.assignValue(CU, NextMetadataNo++); + + // Move the Upgrade the list of subprograms. + if (Metadata *SPs = getMDOrNullWithoutPlaceholders(Record[11])) + CUSubprograms.push_back({CU, SPs}); break; } case bitc::METADATA_SUBPROGRAM: { - if (Record.size() != 18 && Record.size() != 19) - return error("Invalid record"); - - bool HasFn = Record.size() == 19; + if (Record.size() < 18 || Record.size() > 20) + return error("Invalid record"); + + IsDistinct = + (Record[0] & 1) || Record[8]; // All definitions should be distinct. + // Version 1 has a Function as Record[15]. + // Version 2 has removed Record[15]. + // Version 3 has the Unit as Record[15]. + // Version 4 added thisAdjustment. + bool HasUnit = Record[0] >= 2; + if (HasUnit && Record.size() < 19) + return error("Invalid record"); + Metadata *CUorFn = getMDOrNull(Record[15]); + unsigned Offset = Record.size() >= 19 ? 1 : 0; + bool HasFn = Offset && !HasUnit; + bool HasThisAdj = Record.size() >= 20; DISubprogram *SP = GET_OR_DISTINCT( - DISubprogram, - Record[0] || Record[8], // All definitions should be distinct. - (Context, getMDOrNull(Record[1]), getMDString(Record[2]), - getMDString(Record[3]), getMDOrNull(Record[4]), Record[5], - getMDOrNull(Record[6]), Record[7], Record[8], Record[9], - getMDOrNull(Record[10]), Record[11], Record[12], Record[13], - Record[14], getMDOrNull(Record[15 + HasFn]), - getMDOrNull(Record[16 + HasFn]), getMDOrNull(Record[17 + HasFn]))); + DISubprogram, (Context, + getDITypeRefOrNull(Record[1]), // scope + getMDString(Record[2]), // name + getMDString(Record[3]), // linkageName + getMDOrNull(Record[4]), // file + Record[5], // line + getMDOrNull(Record[6]), // type + Record[7], // isLocal + Record[8], // isDefinition + Record[9], // scopeLine + getDITypeRefOrNull(Record[10]), // containingType + Record[11], // virtuality + Record[12], // virtualIndex + HasThisAdj ? Record[19] : 0, // thisAdjustment + Record[13], // flags + Record[14], // isOptimized + HasUnit ? CUorFn : nullptr, // unit + getMDOrNull(Record[15 + Offset]), // templateParams + getMDOrNull(Record[16 + Offset]), // declaration + getMDOrNull(Record[17 + Offset]) // variables + )); MetadataList.assignValue(SP, NextMetadataNo++); // Upgrade sp->function mapping to function->sp mapping. - if (HasFn && Record[15]) { - if (auto *CMD = dyn_cast<ConstantAsMetadata>(getMDOrNull(Record[15]))) + if (HasFn) { + if (auto *CMD = dyn_cast_or_null<ConstantAsMetadata>(CUorFn)) if (auto *F = dyn_cast<Function>(CMD->getValue())) { if (F->isMaterializable()) // Defer until materialized; unmaterialized functions may not have @@ -2217,8 +2531,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 5) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DILexicalBlock, Record[0], + GET_OR_DISTINCT(DILexicalBlock, (Context, getMDOrNull(Record[1]), getMDOrNull(Record[2]), Record[3], Record[4])), NextMetadataNo++); @@ -2228,8 +2543,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 4) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DILexicalBlockFile, Record[0], + GET_OR_DISTINCT(DILexicalBlockFile, (Context, getMDOrNull(Record[1]), getMDOrNull(Record[2]), Record[3])), NextMetadataNo++); @@ -2239,11 +2555,11 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 5) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DINamespace, Record[0], - (Context, getMDOrNull(Record[1]), - getMDOrNull(Record[2]), getMDString(Record[3]), - Record[4])), + GET_OR_DISTINCT(DINamespace, (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]), + getMDString(Record[3]), Record[4])), NextMetadataNo++); break; } @@ -2251,8 +2567,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 5) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIMacro, Record[0], + GET_OR_DISTINCT(DIMacro, (Context, Record[1], Record[2], getMDString(Record[3]), getMDString(Record[4]))), NextMetadataNo++); @@ -2262,8 +2579,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 5) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIMacroFile, Record[0], + GET_OR_DISTINCT(DIMacroFile, (Context, Record[1], Record[2], getMDOrNull(Record[3]), getMDOrNull(Record[4]))), NextMetadataNo++); @@ -2273,10 +2591,10 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 3) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue(GET_OR_DISTINCT(DITemplateTypeParameter, - Record[0], (Context, getMDString(Record[1]), - getMDOrNull(Record[2]))), + getDITypeRefOrNull(Record[2]))), NextMetadataNo++); break; } @@ -2284,10 +2602,12 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 5) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DITemplateValueParameter, Record[0], + GET_OR_DISTINCT(DITemplateValueParameter, (Context, Record[1], getMDString(Record[2]), - getMDOrNull(Record[3]), getMDOrNull(Record[4]))), + getDITypeRefOrNull(Record[3]), + getMDOrNull(Record[4]))), NextMetadataNo++); break; } @@ -2295,12 +2615,13 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 11) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIGlobalVariable, Record[0], + GET_OR_DISTINCT(DIGlobalVariable, (Context, getMDOrNull(Record[1]), getMDString(Record[2]), getMDString(Record[3]), getMDOrNull(Record[4]), Record[5], - getMDOrNull(Record[6]), Record[7], Record[8], + getDITypeRefOrNull(Record[6]), Record[7], Record[8], getMDOrNull(Record[9]), getMDOrNull(Record[10]))), NextMetadataNo++); break; @@ -2312,14 +2633,15 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { // 2nd field used to be an artificial tag, either DW_TAG_auto_variable or // DW_TAG_arg_variable. + IsDistinct = Record[0]; bool HasTag = Record.size() > 8; MetadataList.assignValue( - GET_OR_DISTINCT(DILocalVariable, Record[0], + GET_OR_DISTINCT(DILocalVariable, (Context, getMDOrNull(Record[1 + HasTag]), getMDString(Record[2 + HasTag]), getMDOrNull(Record[3 + HasTag]), Record[4 + HasTag], - getMDOrNull(Record[5 + HasTag]), Record[6 + HasTag], - Record[7 + HasTag])), + getDITypeRefOrNull(Record[5 + HasTag]), + Record[6 + HasTag], Record[7 + HasTag])), NextMetadataNo++); break; } @@ -2327,8 +2649,9 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() < 1) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIExpression, Record[0], + GET_OR_DISTINCT(DIExpression, (Context, makeArrayRef(Record).slice(1))), NextMetadataNo++); break; @@ -2337,12 +2660,13 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 8) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIObjCProperty, Record[0], + GET_OR_DISTINCT(DIObjCProperty, (Context, getMDString(Record[1]), getMDOrNull(Record[2]), Record[3], getMDString(Record[4]), getMDString(Record[5]), - Record[6], getMDOrNull(Record[7]))), + Record[6], getDITypeRefOrNull(Record[7]))), NextMetadataNo++); break; } @@ -2350,21 +2674,40 @@ std::error_code BitcodeReader::parseMetadata(bool ModuleLevel) { if (Record.size() != 6) return error("Invalid record"); + IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIImportedEntity, Record[0], + GET_OR_DISTINCT(DIImportedEntity, (Context, Record[1], getMDOrNull(Record[2]), - getMDOrNull(Record[3]), Record[4], + getDITypeRefOrNull(Record[3]), Record[4], getMDString(Record[5]))), NextMetadataNo++); break; } - case bitc::METADATA_STRING: { + case bitc::METADATA_STRING_OLD: { std::string String(Record.begin(), Record.end()); - llvm::UpgradeMDStringConstant(String); + + // Test for upgrading !llvm.loop. + HasSeenOldLoopTags |= mayBeOldLoopAttachmentTag(String); + Metadata *MD = MDString::get(Context, String); MetadataList.assignValue(MD, NextMetadataNo++); break; } + case bitc::METADATA_STRINGS: + if (std::error_code EC = + parseMetadataStrings(Record, Blob, NextMetadataNo)) + return EC; + break; + case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { + if (Record.size() % 2 == 0) + return error("Invalid record"); + unsigned ValueID = Record[0]; + if (ValueID >= ValueList.size()) + return error("Invalid record"); + if (auto *GO = dyn_cast<GlobalObject>(ValueList[ValueID])) + parseGlobalObjectAttachment(*GO, ArrayRef<uint64_t>(Record).slice(1)); + break; + } case bitc::METADATA_KIND: { // Support older bitcode files that had METADATA_KIND records in a // block with METADATA_BLOCK_ID. @@ -2426,15 +2769,16 @@ uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) { } /// Resolve all of the initializers for global values and aliases that we can. -std::error_code BitcodeReader::resolveGlobalAndAliasInits() { +std::error_code BitcodeReader::resolveGlobalAndIndirectSymbolInits() { std::vector<std::pair<GlobalVariable*, unsigned> > GlobalInitWorklist; - std::vector<std::pair<GlobalAlias*, unsigned> > AliasInitWorklist; + std::vector<std::pair<GlobalIndirectSymbol*, unsigned> > + IndirectSymbolInitWorklist; std::vector<std::pair<Function*, unsigned> > FunctionPrefixWorklist; std::vector<std::pair<Function*, unsigned> > FunctionPrologueWorklist; std::vector<std::pair<Function*, unsigned> > FunctionPersonalityFnWorklist; GlobalInitWorklist.swap(GlobalInits); - AliasInitWorklist.swap(AliasInits); + IndirectSymbolInitWorklist.swap(IndirectSymbolInits); FunctionPrefixWorklist.swap(FunctionPrefixes); FunctionPrologueWorklist.swap(FunctionPrologues); FunctionPersonalityFnWorklist.swap(FunctionPersonalityFns); @@ -2453,20 +2797,20 @@ std::error_code BitcodeReader::resolveGlobalAndAliasInits() { GlobalInitWorklist.pop_back(); } - while (!AliasInitWorklist.empty()) { - unsigned ValID = AliasInitWorklist.back().second; + while (!IndirectSymbolInitWorklist.empty()) { + unsigned ValID = IndirectSymbolInitWorklist.back().second; if (ValID >= ValueList.size()) { - AliasInits.push_back(AliasInitWorklist.back()); + IndirectSymbolInits.push_back(IndirectSymbolInitWorklist.back()); } else { Constant *C = dyn_cast_or_null<Constant>(ValueList[ValID]); if (!C) return error("Expected a constant"); - GlobalAlias *Alias = AliasInitWorklist.back().first; - if (C->getType() != Alias->getType()) + GlobalIndirectSymbol *GIS = IndirectSymbolInitWorklist.back().first; + if (isa<GlobalAlias>(GIS) && C->getType() != GIS->getType()) return error("Alias and aliasee types don't match"); - Alias->setAliasee(C); + GIS->setIndirectSymbol(C); } - AliasInitWorklist.pop_back(); + IndirectSymbolInitWorklist.pop_back(); } while (!FunctionPrefixWorklist.empty()) { @@ -2537,7 +2881,7 @@ std::error_code BitcodeReader::parseConstants() { return error("Malformed block"); case BitstreamEntry::EndBlock: if (NextCstNo != ValueList.size()) - return error("Invalid ronstant reference"); + return error("Invalid constant reference"); // Once all the constants have been read, go through and resolve forward // references. @@ -2550,6 +2894,7 @@ std::error_code BitcodeReader::parseConstants() { // Read a record. Record.clear(); + Type *VoidType = Type::getVoidTy(Context); Value *V = nullptr; unsigned BitCode = Stream.readRecord(Entry.ID, Record); switch (BitCode) { @@ -2562,6 +2907,8 @@ std::error_code BitcodeReader::parseConstants() { return error("Invalid record"); if (Record[0] >= TypeList.size() || !TypeList[Record[0]]) return error("Invalid record"); + if (TypeList[Record[0]] == VoidType) + return error("Invalid constant type"); CurTy = TypeList[Record[0]]; continue; // Skip the ValueList manipulation. case bitc::CST_CODE_NULL: // NULL @@ -2701,7 +3048,6 @@ std::error_code BitcodeReader::parseConstants() { } break; } - case bitc::CST_CODE_CE_BINOP: { // CE_BINOP: [opcode, opval, opval] if (Record.size() < 3) return error("Invalid record"); @@ -2770,6 +3116,9 @@ std::error_code BitcodeReader::parseConstants() { return error("Explicit gep operator type does not match pointee type " "of pointer operand"); + if (Elts.size() < 1) + return error("Invalid gep with no operands"); + ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end()); V = ConstantExpr::getGetElementPtr(PointeeType, Elts[0], Indices, BitCode == @@ -3067,35 +3416,6 @@ std::error_code BitcodeReader::materializeMetadata() { void BitcodeReader::setStripDebugInfo() { StripDebugInfo = true; } -void BitcodeReader::saveMetadataList( - DenseMap<const Metadata *, unsigned> &MetadataToIDs, bool OnlyTempMD) { - for (unsigned ID = 0; ID < MetadataList.size(); ++ID) { - Metadata *MD = MetadataList[ID]; - auto *N = dyn_cast_or_null<MDNode>(MD); - assert((!N || (N->isResolved() || N->isTemporary())) && - "Found non-resolved non-temp MDNode while saving metadata"); - // Save all values if !OnlyTempMD, otherwise just the temporary metadata. - // Note that in the !OnlyTempMD case we need to save all Metadata, not - // just MDNode, as we may have references to other types of module-level - // metadata (e.g. ValueAsMetadata) from instructions. - if (!OnlyTempMD || (N && N->isTemporary())) { - // Will call this after materializing each function, in order to - // handle remapping of the function's instructions/metadata. - // See if we already have an entry in that case. - if (OnlyTempMD && MetadataToIDs.count(MD)) { - assert(MetadataToIDs[MD] == ID && "Inconsistent metadata value id"); - continue; - } - if (N && N->isTemporary()) - // Ensure that we assert if someone tries to RAUW this temporary - // metadata while it is the key of a map. The flag will be set back - // to true when the saved metadata list is destroyed. - N->setCanReplace(false); - MetadataToIDs[MD] = ID; - } - } -} - /// When we see the block for a function body, remember where it is and then /// skip it. This lets us lazily deserialize the functions. std::error_code BitcodeReader::rememberAndSkipFunctionBody() { @@ -3121,8 +3441,8 @@ std::error_code BitcodeReader::rememberAndSkipFunctionBody() { std::error_code BitcodeReader::globalCleanup() { // Patch the initializers for globals and aliases up. - resolveGlobalAndAliasInits(); - if (!GlobalInits.empty() || !AliasInits.empty()) + resolveGlobalAndIndirectSymbolInits(); + if (!GlobalInits.empty() || !IndirectSymbolInits.empty()) return error("Malformed global initializer set"); // Look for intrinsic functions which need to be upgraded at some point @@ -3130,6 +3450,11 @@ std::error_code BitcodeReader::globalCleanup() { Function *NewFn; if (UpgradeIntrinsicFunction(&F, NewFn)) UpgradedIntrinsics[&F] = NewFn; + else if (auto Remangled = Intrinsic::remangleIntrinsicFunction(&F)) + // Some types could be renamed during loading if several modules are + // loaded in the same LLVMContext (LTO scenario). In this case we should + // remangle intrinsics names as well. + RemangledIntrinsics[&F] = Remangled.getValue(); } // Look for global variables which need to be renamed. @@ -3139,7 +3464,8 @@ std::error_code BitcodeReader::globalCleanup() { // Force deallocation of memory for these vectors to favor the client that // want lazy deserialization. std::vector<std::pair<GlobalVariable*, unsigned> >().swap(GlobalInits); - std::vector<std::pair<GlobalAlias*, unsigned> >().swap(AliasInits); + std::vector<std::pair<GlobalIndirectSymbol*, unsigned> >().swap( + IndirectSymbolInits); return std::error_code(); } @@ -3289,7 +3615,7 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, case bitc::CONSTANTS_BLOCK_ID: if (std::error_code EC = parseConstants()) return EC; - if (std::error_code EC = resolveGlobalAndAliasInits()) + if (std::error_code EC = resolveGlobalAndIndirectSymbolInits()) return EC; break; case bitc::METADATA_BLOCK_ID: @@ -3374,7 +3700,6 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, break; } - // Read a record. auto BitCode = Stream.readRecord(Entry.ID, Record); switch (BitCode) { @@ -3496,9 +3821,9 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, if (Record.size() > 7) TLM = getDecodedThreadLocalMode(Record[7]); - bool UnnamedAddr = false; + GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; if (Record.size() > 8) - UnnamedAddr = Record[8]; + UnnamedAddr = getDecodedUnnamedAddrType(Record[8]); bool ExternallyInitialized = false; if (Record.size() > 9) @@ -3533,6 +3858,7 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, } else if (hasImplicitComdat(RawLinkage)) { NewGV->setComdat(reinterpret_cast<Comdat *>(1)); } + break; } // FUNCTION: [type, callingconv, isproto, linkage, paramattr, @@ -3578,11 +3904,11 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, if (Record.size() > 8 && Record[8]) { if (Record[8]-1 >= GCTable.size()) return error("Invalid ID"); - Func->setGC(GCTable[Record[8]-1].c_str()); + Func->setGC(GCTable[Record[8] - 1]); } - bool UnnamedAddr = false; + GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; if (Record.size() > 9) - UnnamedAddr = Record[9]; + UnnamedAddr = getDecodedUnnamedAddrType(Record[9]); Func->setUnnamedAddr(UnnamedAddr); if (Record.size() > 10 && Record[10] != 0) FunctionPrologues.push_back(std::make_pair(Func, Record[10]-1)); @@ -3621,9 +3947,11 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, } // ALIAS: [alias type, addrspace, aliasee val#, linkage] // ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility, dllstorageclass] + // IFUNC: [alias type, addrspace, aliasee val#, linkage, visibility, dllstorageclass] + case bitc::MODULE_CODE_IFUNC: case bitc::MODULE_CODE_ALIAS: case bitc::MODULE_CODE_ALIAS_OLD: { - bool NewRecord = BitCode == bitc::MODULE_CODE_ALIAS; + bool NewRecord = BitCode != bitc::MODULE_CODE_ALIAS_OLD; if (Record.size() < (3 + (unsigned)NewRecord)) return error("Invalid record"); unsigned OpNum = 0; @@ -3644,8 +3972,14 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, auto Val = Record[OpNum++]; auto Linkage = Record[OpNum++]; - auto *NewGA = GlobalAlias::create( - Ty, AddrSpace, getDecodedLinkage(Linkage), "", TheModule); + GlobalIndirectSymbol *NewGA; + if (BitCode == bitc::MODULE_CODE_ALIAS || + BitCode == bitc::MODULE_CODE_ALIAS_OLD) + NewGA = GlobalAlias::create(Ty, AddrSpace, getDecodedLinkage(Linkage), + "", TheModule); + else + NewGA = GlobalIFunc::create(Ty, AddrSpace, getDecodedLinkage(Linkage), + "", nullptr, TheModule); // Old bitcode files didn't have visibility field. // Local linkage must have default visibility. if (OpNum != Record.size()) { @@ -3661,9 +3995,9 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, if (OpNum != Record.size()) NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++])); if (OpNum != Record.size()) - NewGA->setUnnamedAddr(Record[OpNum++]); + NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++])); ValueList.push_back(NewGA); - AliasInits.push_back(std::make_pair(NewGA, Val)); + IndirectSymbolInits.push_back(std::make_pair(NewGA, Val)); break; } /// MODULE_CODE_PURGEVALS: [numvals] @@ -3679,27 +4013,12 @@ std::error_code BitcodeReader::parseModule(uint64_t ResumeBit, return error("Invalid record"); VSTOffset = Record[0]; break; - /// MODULE_CODE_METADATA_VALUES: [numvals] - case bitc::MODULE_CODE_METADATA_VALUES: - if (Record.size() < 1) + /// MODULE_CODE_SOURCE_FILENAME: [namechar x N] + case bitc::MODULE_CODE_SOURCE_FILENAME: + SmallString<128> ValueName; + if (convertToString(Record, 0, ValueName)) return error("Invalid record"); - assert(!IsMetadataMaterialized); - // This record contains the number of metadata values in the module-level - // METADATA_BLOCK. It is used to support lazy parsing of metadata as - // a postpass, where we will parse function-level metadata first. - // This is needed because the ids of metadata are assigned implicitly - // based on their ordering in the bitcode, with the function-level - // metadata ids starting after the module-level metadata ids. Otherwise, - // we would have to parse the module-level metadata block to prime the - // MetadataList when we are lazy loading metadata during function - // importing. Initialize the MetadataList size here based on the - // record value, regardless of whether we are doing lazy metadata - // loading, so that we have consistent handling and assertion - // checking in parseMetadata for module-level metadata. - NumModuleMDs = Record[0]; - SeenModuleValuesRecord = true; - assert(MetadataList.size() == 0); - MetadataList.resize(NumModuleMDs); + TheModule->setSourceFileName(ValueName); break; } Record.clear(); @@ -3866,6 +4185,96 @@ ErrorOr<std::string> BitcodeReader::parseIdentificationBlock() { } } +std::error_code BitcodeReader::parseGlobalObjectAttachment( + GlobalObject &GO, ArrayRef<uint64_t> Record) { + assert(Record.size() % 2 == 0); + for (unsigned I = 0, E = Record.size(); I != E; I += 2) { + auto K = MDKindMap.find(Record[I]); + if (K == MDKindMap.end()) + return error("Invalid ID"); + MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[I + 1]); + if (!MD) + return error("Invalid metadata attachment"); + GO.addMetadata(K->second, *MD); + } + return std::error_code(); +} + +ErrorOr<bool> BitcodeReader::hasObjCCategory() { + if (std::error_code EC = initStream(nullptr)) + return EC; + + // Sniff for the signature. + if (!hasValidBitcodeHeader(Stream)) + return error("Invalid bitcode signature"); + + // We expect a number of well-defined blocks, though we don't necessarily + // need to understand them all. + while (1) { + BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return std::error_code(); + + case BitstreamEntry::SubBlock: + if (Entry.ID == bitc::MODULE_BLOCK_ID) + return hasObjCCategoryInModule(); + + // Ignore other sub-blocks. + if (Stream.SkipBlock()) + return error("Malformed block"); + continue; + + case BitstreamEntry::Record: + Stream.skipRecord(Entry.ID); + continue; + } + } +} + +ErrorOr<bool> BitcodeReader::hasObjCCategoryInModule() { + if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return error("Invalid record"); + + SmallVector<uint64_t, 64> Record; + // Read all the records for this module. + while (1) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return false; + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read a record. + switch (Stream.readRecord(Entry.ID, Record)) { + default: + break; // Default behavior, ignore unknown content. + case bitc::MODULE_CODE_SECTIONNAME: { // SECTIONNAME: [strchr x N] + std::string S; + if (convertToString(Record, 0, S)) + return error("Invalid record"); + // Check for the i386 and other (x86_64, ARM) conventions + if (S.find("__DATA, __objc_catlist") != std::string::npos || + S.find("__OBJC,__category") != std::string::npos) + return true; + break; + } + } + Record.clear(); + } + llvm_unreachable("Exit infinite loop"); +} + /// Parse metadata attachments. std::error_code BitcodeReader::parseMetadataAttachment(Function &F) { if (Stream.EnterSubBlock(bitc::METADATA_ATTACHMENT_ID)) @@ -3897,13 +4306,8 @@ std::error_code BitcodeReader::parseMetadataAttachment(Function &F) { return error("Invalid record"); if (RecordLength % 2 == 0) { // A function attachment. - for (unsigned I = 0; I != RecordLength; I += 2) { - auto K = MDKindMap.find(Record[I]); - if (K == MDKindMap.end()) - return error("Invalid ID"); - Metadata *MD = MetadataList.getValueFwdRef(Record[I + 1]); - F.setMetadata(K->second, cast<MDNode>(MD)); - } + if (std::error_code EC = parseGlobalObjectAttachment(F, Record)) + return EC; continue; } @@ -3915,14 +4319,23 @@ std::error_code BitcodeReader::parseMetadataAttachment(Function &F) { MDKindMap.find(Kind); if (I == MDKindMap.end()) return error("Invalid ID"); - Metadata *Node = MetadataList.getValueFwdRef(Record[i + 1]); + Metadata *Node = MetadataList.getMetadataFwdRef(Record[i + 1]); if (isa<LocalAsMetadata>(Node)) // Drop the attachment. This used to be legal, but there's no // upgrade path. break; - Inst->setMetadata(I->second, cast<MDNode>(Node)); - if (I->second == LLVMContext::MD_tbaa) + MDNode *MD = dyn_cast_or_null<MDNode>(Node); + if (!MD) + return error("Invalid metadata attachment"); + + if (HasSeenOldLoopTags && I->second == LLVMContext::MD_loop) + MD = upgradeInstructionLoopAttachment(*MD); + + Inst->setMetadata(I->second, MD); + if (I->second == LLVMContext::MD_tbaa) { InstsWithTBAATag.push_back(Inst); + continue; + } } break; } @@ -3949,6 +4362,10 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { if (Stream.EnterSubBlock(bitc::FUNCTION_BLOCK_ID)) return error("Invalid record"); + // Unexpected unresolved metadata when parsing function. + if (MetadataList.hasFwdRefs()) + return error("Invalid function metadata: incoming forward references"); + InstructionList.clear(); unsigned ModuleValueListSize = ValueList.size(); unsigned ModuleMetadataListSize = MetadataList.size(); @@ -4081,10 +4498,16 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { unsigned ScopeID = Record[2], IAID = Record[3]; MDNode *Scope = nullptr, *IA = nullptr; - if (ScopeID) - Scope = cast<MDNode>(MetadataList.getValueFwdRef(ScopeID - 1)); - if (IAID) - IA = cast<MDNode>(MetadataList.getValueFwdRef(IAID - 1)); + if (ScopeID) { + Scope = MetadataList.getMDNodeFwdRefOrNull(ScopeID - 1); + if (!Scope) + return error("Invalid record"); + } + if (IAID) { + IA = MetadataList.getMDNodeFwdRefOrNull(IAID - 1); + if (!IA) + return error("Invalid record"); + } LastLoc = DebugLoc::get(Line, Col, Scope, IA); I->setDebugLoc(LastLoc); I = nullptr; @@ -4820,10 +5243,11 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { uint64_t AlignRecord = Record[3]; const uint64_t InAllocaMask = uint64_t(1) << 5; const uint64_t ExplicitTypeMask = uint64_t(1) << 6; - // Reserve bit 7 for SwiftError flag. - // const uint64_t SwiftErrorMask = uint64_t(1) << 7; - const uint64_t FlagMask = InAllocaMask | ExplicitTypeMask; + const uint64_t SwiftErrorMask = uint64_t(1) << 7; + const uint64_t FlagMask = InAllocaMask | ExplicitTypeMask | + SwiftErrorMask; bool InAlloca = AlignRecord & InAllocaMask; + bool SwiftError = AlignRecord & SwiftErrorMask; Type *Ty = getTypeByID(Record[0]); if ((AlignRecord & ExplicitTypeMask) == 0) { auto *PTy = dyn_cast_or_null<PointerType>(Ty); @@ -4842,6 +5266,7 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); AllocaInst *AI = new AllocaInst(Ty, Size, Align); AI->setUsedWithInAlloca(InAlloca); + AI->setSwiftError(SwiftError); I = AI; InstructionList.push_back(I); break; @@ -4886,10 +5311,11 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { Ty = cast<PointerType>(Op->getType())->getElementType(); AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]); - if (Ordering == NotAtomic || Ordering == Release || - Ordering == AcquireRelease) + if (Ordering == AtomicOrdering::NotAtomic || + Ordering == AtomicOrdering::Release || + Ordering == AtomicOrdering::AcquireRelease) return error("Invalid record"); - if (Ordering != NotAtomic && Record[OpNum] == 0) + if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0) return error("Invalid record"); SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); @@ -4930,6 +5356,7 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { unsigned OpNum = 0; Value *Val, *Ptr; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || + !isa<PointerType>(Ptr->getType()) || (BitCode == bitc::FUNC_CODE_INST_STOREATOMIC ? getValueTypePair(Record, OpNum, NextValueNo, Val) : popValue(Record, OpNum, NextValueNo, @@ -4942,11 +5369,12 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { typeCheckLoadStoreInst(Val->getType(), Ptr->getType())) return EC; AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]); - if (Ordering == NotAtomic || Ordering == Acquire || - Ordering == AcquireRelease) + if (Ordering == AtomicOrdering::NotAtomic || + Ordering == AtomicOrdering::Acquire || + Ordering == AtomicOrdering::AcquireRelease) return error("Invalid record"); SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); - if (Ordering != NotAtomic && Record[OpNum] == 0) + if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0) return error("Invalid record"); unsigned Align; @@ -4972,7 +5400,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { Record.size() < OpNum + 3 || Record.size() > OpNum + 5) return error("Invalid record"); AtomicOrdering SuccessOrdering = getDecodedOrdering(Record[OpNum + 1]); - if (SuccessOrdering == NotAtomic || SuccessOrdering == Unordered) + if (SuccessOrdering == AtomicOrdering::NotAtomic || + SuccessOrdering == AtomicOrdering::Unordered) return error("Invalid record"); SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 2]); @@ -5008,6 +5437,7 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { unsigned OpNum = 0; Value *Ptr, *Val; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || + !isa<PointerType>(Ptr->getType()) || popValue(Record, OpNum, NextValueNo, cast<PointerType>(Ptr->getType())->getElementType(), Val) || OpNum+4 != Record.size()) @@ -5017,7 +5447,8 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { Operation > AtomicRMWInst::LAST_BINOP) return error("Invalid record"); AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]); - if (Ordering == NotAtomic || Ordering == Unordered) + if (Ordering == AtomicOrdering::NotAtomic || + Ordering == AtomicOrdering::Unordered) return error("Invalid record"); SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SynchScope); @@ -5029,8 +5460,9 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { if (2 != Record.size()) return error("Invalid record"); AtomicOrdering Ordering = getDecodedOrdering(Record[0]); - if (Ordering == NotAtomic || Ordering == Unordered || - Ordering == Monotonic) + if (Ordering == AtomicOrdering::NotAtomic || + Ordering == AtomicOrdering::Unordered || + Ordering == AtomicOrdering::Monotonic) return error("Invalid record"); SynchronizationScope SynchScope = getDecodedSynchScope(Record[1]); I = new FenceInst(Context, Ordering, SynchScope); @@ -5200,8 +5632,9 @@ OutOfRecordLoop: } } - // FIXME: Check for unresolved forward-declared metadata references - // and clean up leaks. + // Unexpected unresolved metadata about to be dropped. + if (MetadataList.hasFwdRefs()) + return error("Invalid function metadata: outgoing forward refs"); // Trim the value list down to the size it was before we parsed this function. ValueList.shrinkTo(ModuleValueListSize); @@ -5235,13 +5668,6 @@ std::error_code BitcodeReader::findFunctionInStream( void BitcodeReader::releaseBuffer() { Buffer.release(); } std::error_code BitcodeReader::materialize(GlobalValue *GV) { - // In older bitcode we must materialize the metadata before parsing - // any functions, in order to set up the MetadataList properly. - if (!SeenModuleValuesRecord) { - if (std::error_code EC = materializeMetadata()) - return EC; - } - Function *F = dyn_cast<Function>(GV); // If it's not a function or is already material, ignore the request. if (!F || !F->isMaterializable()) @@ -5255,6 +5681,10 @@ std::error_code BitcodeReader::materialize(GlobalValue *GV) { if (std::error_code EC = findFunctionInStream(F, DFII)) return EC; + // Materialize metadata before parsing any function bodies. + if (std::error_code EC = materializeMetadata()) + return EC; + // Move the bit stream to the saved position of the deferred function body. Stream.JumpToBit(DFII->second); @@ -5276,6 +5706,13 @@ std::error_code BitcodeReader::materialize(GlobalValue *GV) { } } + // Update calls to the remangled intrinsics + for (auto &I : RemangledIntrinsics) + for (auto UI = I.first->materialized_user_begin(), UE = I.first->user_end(); + UI != UE;) + // Don't expect any other users than call sites + CallSite(*UI++).setCalledFunction(I.second); + // Finish fn->subprogram upgrade for materialized functions. if (DISubprogram *SP = FunctionsWithSPs.lookup(F)) F->setSubprogram(SP); @@ -5310,6 +5747,11 @@ std::error_code BitcodeReader::materializeModule() { if (!BasicBlockFwdRefs.empty()) return error("Never resolved function from blockaddress"); + // Upgrading intrinsic calls before TBAA can cause TBAA metadata to be lost, + // to prevent this instructions with TBAA tags should be upgraded first. + for (unsigned I = 0, E = InstsWithTBAATag.size(); I < E; I++) + UpgradeInstWithTBAATag(InstsWithTBAATag[I]); + // Upgrade any intrinsic calls that slipped through (should not happen!) and // delete the old functions to clean up. We can't do this unless the entire // module is materialized because there could always be another function body @@ -5324,11 +5766,16 @@ std::error_code BitcodeReader::materializeModule() { I.first->eraseFromParent(); } UpgradedIntrinsics.clear(); - - for (unsigned I = 0, E = InstsWithTBAATag.size(); I < E; I++) - UpgradeInstWithTBAATag(InstsWithTBAATag[I]); + // Do the same for remangled intrinsics + for (auto &I : RemangledIntrinsics) { + I.first->replaceAllUsesWith(I.second); + I.first->eraseFromParent(); + } + RemangledIntrinsics.clear(); UpgradeDebugInfo(*TheModule); + + UpgradeModuleFlags(*TheModule); return std::error_code(); } @@ -5389,43 +5836,37 @@ BitcodeReader::initLazyStream(std::unique_ptr<DataStreamer> Streamer) { return std::error_code(); } -std::error_code FunctionIndexBitcodeReader::error(BitcodeError E, - const Twine &Message) { - return ::error(DiagnosticHandler, make_error_code(E), Message); -} - -std::error_code FunctionIndexBitcodeReader::error(const Twine &Message) { +std::error_code ModuleSummaryIndexBitcodeReader::error(const Twine &Message) { return ::error(DiagnosticHandler, make_error_code(BitcodeError::CorruptedBitcode), Message); } -std::error_code FunctionIndexBitcodeReader::error(BitcodeError E) { - return ::error(DiagnosticHandler, make_error_code(E)); +ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader( + MemoryBuffer *Buffer, DiagnosticHandlerFunction DiagnosticHandler, + bool CheckGlobalValSummaryPresenceOnly) + : DiagnosticHandler(std::move(DiagnosticHandler)), Buffer(Buffer), + CheckGlobalValSummaryPresenceOnly(CheckGlobalValSummaryPresenceOnly) {} + +void ModuleSummaryIndexBitcodeReader::freeState() { Buffer = nullptr; } + +void ModuleSummaryIndexBitcodeReader::releaseBuffer() { Buffer.release(); } + +std::pair<GlobalValue::GUID, GlobalValue::GUID> +ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) { + auto VGI = ValueIdToCallGraphGUIDMap.find(ValueId); + assert(VGI != ValueIdToCallGraphGUIDMap.end()); + return VGI->second; } -FunctionIndexBitcodeReader::FunctionIndexBitcodeReader( - MemoryBuffer *Buffer, DiagnosticHandlerFunction DiagnosticHandler, - bool IsLazy, bool CheckFuncSummaryPresenceOnly) - : DiagnosticHandler(DiagnosticHandler), Buffer(Buffer), IsLazy(IsLazy), - CheckFuncSummaryPresenceOnly(CheckFuncSummaryPresenceOnly) {} - -FunctionIndexBitcodeReader::FunctionIndexBitcodeReader( - DiagnosticHandlerFunction DiagnosticHandler, bool IsLazy, - bool CheckFuncSummaryPresenceOnly) - : DiagnosticHandler(DiagnosticHandler), Buffer(nullptr), IsLazy(IsLazy), - CheckFuncSummaryPresenceOnly(CheckFuncSummaryPresenceOnly) {} - -void FunctionIndexBitcodeReader::freeState() { Buffer = nullptr; } - -void FunctionIndexBitcodeReader::releaseBuffer() { Buffer.release(); } - -// Specialized value symbol table parser used when reading function index -// blocks where we don't actually create global values. -// At the end of this routine the function index is populated with a map -// from function name to FunctionInfo. The function info contains -// the function block's bitcode offset as well as the offset into the -// function summary section. -std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() { +// Specialized value symbol table parser used when reading module index +// blocks where we don't actually create global values. The parsed information +// is saved in the bitcode reader for use when later parsing summaries. +std::error_code ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( + uint64_t Offset, + DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap) { + assert(Offset > 0 && "Expected non-zero VST offset"); + uint64_t CurrentBit = jumpToValueSymbolTable(Offset, Stream); + if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) return error("Invalid record"); @@ -5441,6 +5882,8 @@ std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: + // Done parsing VST, jump back to wherever we came from. + Stream.JumpToBit(CurrentBit); return std::error_code(); case BitstreamEntry::Record: // The interesting case. @@ -5452,58 +5895,79 @@ std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() { switch (Stream.readRecord(Entry.ID, Record)) { default: // Default behavior: ignore (e.g. VST_CODE_BBENTRY records). break; - case bitc::VST_CODE_FNENTRY: { - // VST_FNENTRY: [valueid, offset, namechar x N] - if (convertToString(Record, 2, ValueName)) + case bitc::VST_CODE_ENTRY: { // VST_CODE_ENTRY: [valueid, namechar x N] + if (convertToString(Record, 1, ValueName)) return error("Invalid record"); unsigned ValueID = Record[0]; - uint64_t FuncOffset = Record[1]; - std::unique_ptr<FunctionInfo> FuncInfo = - llvm::make_unique<FunctionInfo>(FuncOffset); - if (foundFuncSummary() && !IsLazy) { - DenseMap<uint64_t, std::unique_ptr<FunctionSummary>>::iterator SMI = - SummaryMap.find(ValueID); - assert(SMI != SummaryMap.end() && "Summary info not found"); - FuncInfo->setFunctionSummary(std::move(SMI->second)); - } - TheIndex->addFunctionInfo(ValueName, std::move(FuncInfo)); - + assert(!SourceFileName.empty()); + auto VLI = ValueIdToLinkageMap.find(ValueID); + assert(VLI != ValueIdToLinkageMap.end() && + "No linkage found for VST entry?"); + auto Linkage = VLI->second; + std::string GlobalId = + GlobalValue::getGlobalIdentifier(ValueName, Linkage, SourceFileName); + auto ValueGUID = GlobalValue::getGUID(GlobalId); + auto OriginalNameID = ValueGUID; + if (GlobalValue::isLocalLinkage(Linkage)) + OriginalNameID = GlobalValue::getGUID(ValueName); + if (PrintSummaryGUIDs) + dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is " + << ValueName << "\n"; + ValueIdToCallGraphGUIDMap[ValueID] = + std::make_pair(ValueGUID, OriginalNameID); ValueName.clear(); break; } - case bitc::VST_CODE_COMBINED_FNENTRY: { - // VST_FNENTRY: [offset, namechar x N] - if (convertToString(Record, 1, ValueName)) + case bitc::VST_CODE_FNENTRY: { + // VST_CODE_FNENTRY: [valueid, offset, namechar x N] + if (convertToString(Record, 2, ValueName)) return error("Invalid record"); - uint64_t FuncSummaryOffset = Record[0]; - std::unique_ptr<FunctionInfo> FuncInfo = - llvm::make_unique<FunctionInfo>(FuncSummaryOffset); - if (foundFuncSummary() && !IsLazy) { - DenseMap<uint64_t, std::unique_ptr<FunctionSummary>>::iterator SMI = - SummaryMap.find(FuncSummaryOffset); - assert(SMI != SummaryMap.end() && "Summary info not found"); - FuncInfo->setFunctionSummary(std::move(SMI->second)); - } - TheIndex->addFunctionInfo(ValueName, std::move(FuncInfo)); + unsigned ValueID = Record[0]; + assert(!SourceFileName.empty()); + auto VLI = ValueIdToLinkageMap.find(ValueID); + assert(VLI != ValueIdToLinkageMap.end() && + "No linkage found for VST entry?"); + auto Linkage = VLI->second; + std::string FunctionGlobalId = GlobalValue::getGlobalIdentifier( + ValueName, VLI->second, SourceFileName); + auto FunctionGUID = GlobalValue::getGUID(FunctionGlobalId); + auto OriginalNameID = FunctionGUID; + if (GlobalValue::isLocalLinkage(Linkage)) + OriginalNameID = GlobalValue::getGUID(ValueName); + if (PrintSummaryGUIDs) + dbgs() << "GUID " << FunctionGUID << "(" << OriginalNameID << ") is " + << ValueName << "\n"; + ValueIdToCallGraphGUIDMap[ValueID] = + std::make_pair(FunctionGUID, OriginalNameID); ValueName.clear(); break; } + case bitc::VST_CODE_COMBINED_ENTRY: { + // VST_CODE_COMBINED_ENTRY: [valueid, refguid] + unsigned ValueID = Record[0]; + GlobalValue::GUID RefGUID = Record[1]; + // The "original name", which is the second value of the pair will be + // overriden later by a FS_COMBINED_ORIGINAL_NAME in the combined index. + ValueIdToCallGraphGUIDMap[ValueID] = std::make_pair(RefGUID, RefGUID); + break; + } } } } -// Parse just the blocks needed for function index building out of the module. -// At the end of this routine the function Index is populated with a map -// from function name to FunctionInfo. The function info contains -// either the parsed function summary information (when parsing summaries -// eagerly), or just to the function summary record's offset -// if parsing lazily (IsLazy). -std::error_code FunctionIndexBitcodeReader::parseModule() { +// Parse just the blocks needed for building the index out of the module. +// At the end of this routine the module Index is populated with a map +// from global value id to GlobalValueSummary objects. +std::error_code ModuleSummaryIndexBitcodeReader::parseModule() { if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) return error("Invalid record"); - // Read the function index for this module. + SmallVector<uint64_t, 64> Record; + DenseMap<unsigned, GlobalValue::LinkageTypes> ValueIdToLinkageMap; + unsigned ValueId = 0; + + // Read the index for this module. while (1) { BitstreamEntry Entry = Stream.advance(); @@ -5514,9 +5978,9 @@ std::error_code FunctionIndexBitcodeReader::parseModule() { return std::error_code(); case BitstreamEntry::SubBlock: - if (CheckFuncSummaryPresenceOnly) { - if (Entry.ID == bitc::FUNCTION_SUMMARY_BLOCK_ID) { - SeenFuncSummary = true; + if (CheckGlobalValSummaryPresenceOnly) { + if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) { + SeenGlobalValSummary = true; // No need to parse the rest since we found the summary. return std::error_code(); } @@ -5535,16 +5999,24 @@ std::error_code FunctionIndexBitcodeReader::parseModule() { return error("Malformed block"); break; case bitc::VALUE_SYMTAB_BLOCK_ID: - if (std::error_code EC = parseValueSymbolTable()) - return EC; + // Should have been parsed earlier via VSTOffset, unless there + // is no summary section. + assert(((SeenValueSymbolTable && VSTOffset > 0) || + !SeenGlobalValSummary) && + "Expected early VST parse via VSTOffset record"); + if (Stream.SkipBlock()) + return error("Invalid record"); break; - case bitc::FUNCTION_SUMMARY_BLOCK_ID: - SeenFuncSummary = true; - if (IsLazy) { - // Lazy parsing of summary info, skip it. - if (Stream.SkipBlock()) - return error("Invalid record"); - } else if (std::error_code EC = parseEntireSummary()) + case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: + assert(VSTOffset > 0 && "Expected non-zero VST offset"); + assert(!SeenValueSymbolTable && + "Already read VST when parsing summary block?"); + if (std::error_code EC = + parseValueSymbolTable(VSTOffset, ValueIdToLinkageMap)) + return EC; + SeenValueSymbolTable = true; + SeenGlobalValSummary = true; + if (std::error_code EC = parseEntireSummary()) return EC; break; case bitc::MODULE_STRTAB_BLOCK_ID: @@ -5554,22 +6026,109 @@ std::error_code FunctionIndexBitcodeReader::parseModule() { } continue; - case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); + case BitstreamEntry::Record: { + Record.clear(); + auto BitCode = Stream.readRecord(Entry.ID, Record); + switch (BitCode) { + default: + break; // Default behavior, ignore unknown content. + /// MODULE_CODE_SOURCE_FILENAME: [namechar x N] + case bitc::MODULE_CODE_SOURCE_FILENAME: { + SmallString<128> ValueName; + if (convertToString(Record, 0, ValueName)) + return error("Invalid record"); + SourceFileName = ValueName.c_str(); + break; + } + /// MODULE_CODE_HASH: [5*i32] + case bitc::MODULE_CODE_HASH: { + if (Record.size() != 5) + return error("Invalid hash length " + Twine(Record.size()).str()); + if (!TheIndex) + break; + if (TheIndex->modulePaths().empty()) + // Does not have any summary emitted. + break; + if (TheIndex->modulePaths().size() != 1) + return error("Don't expect multiple modules defined?"); + auto &Hash = TheIndex->modulePaths().begin()->second.second; + int Pos = 0; + for (auto &Val : Record) { + assert(!(Val >> 32) && "Unexpected high bits set"); + Hash[Pos++] = Val; + } + break; + } + /// MODULE_CODE_VSTOFFSET: [offset] + case bitc::MODULE_CODE_VSTOFFSET: + if (Record.size() < 1) + return error("Invalid record"); + VSTOffset = Record[0]; + break; + // GLOBALVAR: [pointer type, isconst, initid, + // linkage, alignment, section, visibility, threadlocal, + // unnamed_addr, externally_initialized, dllstorageclass, + // comdat] + case bitc::MODULE_CODE_GLOBALVAR: { + if (Record.size() < 6) + return error("Invalid record"); + uint64_t RawLinkage = Record[3]; + GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); + ValueIdToLinkageMap[ValueId++] = Linkage; + break; + } + // FUNCTION: [type, callingconv, isproto, linkage, paramattr, + // alignment, section, visibility, gc, unnamed_addr, + // prologuedata, dllstorageclass, comdat, prefixdata] + case bitc::MODULE_CODE_FUNCTION: { + if (Record.size() < 8) + return error("Invalid record"); + uint64_t RawLinkage = Record[3]; + GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); + ValueIdToLinkageMap[ValueId++] = Linkage; + break; + } + // ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility, + // dllstorageclass] + case bitc::MODULE_CODE_ALIAS: { + if (Record.size() < 6) + return error("Invalid record"); + uint64_t RawLinkage = Record[3]; + GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); + ValueIdToLinkageMap[ValueId++] = Linkage; + break; + } + } + } continue; } } } -// Eagerly parse the entire function summary block (i.e. for all functions -// in the index). This populates the FunctionSummary objects in -// the index. -std::error_code FunctionIndexBitcodeReader::parseEntireSummary() { - if (Stream.EnterSubBlock(bitc::FUNCTION_SUMMARY_BLOCK_ID)) +// Eagerly parse the entire summary block. This populates the GlobalValueSummary +// objects in the index. +std::error_code ModuleSummaryIndexBitcodeReader::parseEntireSummary() { + if (Stream.EnterSubBlock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID)) return error("Invalid record"); - SmallVector<uint64_t, 64> Record; + // Parse version + { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + if (Entry.Kind != BitstreamEntry::Record) + return error("Invalid Summary Block: record for version expected"); + if (Stream.readRecord(Entry.ID, Record) != bitc::FS_VERSION) + return error("Invalid Summary Block: version expected"); + } + const uint64_t Version = Record[0]; + if (Version != 1) + return error("Invalid summary version " + Twine(Version) + ", 1 expected"); + Record.clear(); + + // Keep around the last seen summary to be used when we see an optional + // "OriginalName" attachement. + GlobalValueSummary *LastSeenSummary = nullptr; + bool Combined = false; while (1) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); @@ -5578,6 +6137,16 @@ std::error_code FunctionIndexBitcodeReader::parseEntireSummary() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: + // For a per-module index, remove any entries that still have empty + // summaries. The VST parsing creates entries eagerly for all symbols, + // but not all have associated summaries (e.g. it doesn't know how to + // distinguish between VST_CODE_ENTRY for function declarations vs global + // variables with initializers that end up with a summary). Remove those + // entries now so that we don't need to rely on the combined index merger + // to clean them up (especially since that may not run for the first + // module's index if we merge into that). + if (!Combined) + TheIndex->removeEmptySummaryEntries(); return std::error_code(); case BitstreamEntry::Record: // The interesting case. @@ -5592,35 +6161,197 @@ std::error_code FunctionIndexBitcodeReader::parseEntireSummary() { // in the combined index VST entries). The records also contain // information used for ThinLTO renaming and importing. Record.clear(); - uint64_t CurRecordBit = Stream.GetCurrentBitNo(); - switch (Stream.readRecord(Entry.ID, Record)) { + auto BitCode = Stream.readRecord(Entry.ID, Record); + switch (BitCode) { default: // Default behavior: ignore. break; - // FS_PERMODULE_ENTRY: [valueid, islocal, instcount] - case bitc::FS_CODE_PERMODULE_ENTRY: { + // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid, + // n x (valueid, callsitecount)] + // FS_PERMODULE_PROFILE: [valueid, flags, instcount, numrefs, + // numrefs x valueid, + // n x (valueid, callsitecount, profilecount)] + case bitc::FS_PERMODULE: + case bitc::FS_PERMODULE_PROFILE: { unsigned ValueID = Record[0]; - bool IsLocal = Record[1]; + uint64_t RawFlags = Record[1]; unsigned InstCount = Record[2]; + unsigned NumRefs = Record[3]; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); std::unique_ptr<FunctionSummary> FS = - llvm::make_unique<FunctionSummary>(InstCount); - FS->setLocalFunction(IsLocal); + llvm::make_unique<FunctionSummary>(Flags, InstCount); // The module path string ref set in the summary must be owned by the // index's module string table. Since we don't have a module path // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. FS->setModulePath( - TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)); - SummaryMap[ValueID] = std::move(FS); + TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first()); + static int RefListStartIndex = 4; + int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; + assert(Record.size() >= RefListStartIndex + NumRefs && + "Record size inconsistent with number of references"); + for (unsigned I = 4, E = CallGraphEdgeStartIndex; I != E; ++I) { + unsigned RefValueId = Record[I]; + GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; + FS->addRefEdge(RefGUID); + } + bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE); + for (unsigned I = CallGraphEdgeStartIndex, E = Record.size(); I != E; + ++I) { + unsigned CalleeValueId = Record[I]; + unsigned CallsiteCount = Record[++I]; + uint64_t ProfileCount = HasProfile ? Record[++I] : 0; + GlobalValue::GUID CalleeGUID = getGUIDFromValueId(CalleeValueId).first; + FS->addCallGraphEdge(CalleeGUID, + CalleeInfo(CallsiteCount, ProfileCount)); + } + auto GUID = getGUIDFromValueId(ValueID); + FS->setOriginalName(GUID.second); + TheIndex->addGlobalValueSummary(GUID.first, std::move(FS)); + break; } - // FS_COMBINED_ENTRY: [modid, instcount] - case bitc::FS_CODE_COMBINED_ENTRY: { - uint64_t ModuleId = Record[0]; - unsigned InstCount = Record[1]; + // FS_ALIAS: [valueid, flags, valueid] + // Aliases must be emitted (and parsed) after all FS_PERMODULE entries, as + // they expect all aliasee summaries to be available. + case bitc::FS_ALIAS: { + unsigned ValueID = Record[0]; + uint64_t RawFlags = Record[1]; + unsigned AliaseeID = Record[2]; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + std::unique_ptr<AliasSummary> AS = llvm::make_unique<AliasSummary>(Flags); + // The module path string ref set in the summary must be owned by the + // index's module string table. Since we don't have a module path + // string table section in the per-module index, we create a single + // module path string table entry with an empty (0) ID to take + // ownership. + AS->setModulePath( + TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first()); + + GlobalValue::GUID AliaseeGUID = getGUIDFromValueId(AliaseeID).first; + auto *AliaseeSummary = TheIndex->getGlobalValueSummary(AliaseeGUID); + if (!AliaseeSummary) + return error("Alias expects aliasee summary to be parsed"); + AS->setAliasee(AliaseeSummary); + + auto GUID = getGUIDFromValueId(ValueID); + AS->setOriginalName(GUID.second); + TheIndex->addGlobalValueSummary(GUID.first, std::move(AS)); + break; + } + // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, n x valueid] + case bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS: { + unsigned ValueID = Record[0]; + uint64_t RawFlags = Record[1]; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + std::unique_ptr<GlobalVarSummary> FS = + llvm::make_unique<GlobalVarSummary>(Flags); + FS->setModulePath( + TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first()); + for (unsigned I = 2, E = Record.size(); I != E; ++I) { + unsigned RefValueId = Record[I]; + GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; + FS->addRefEdge(RefGUID); + } + auto GUID = getGUIDFromValueId(ValueID); + FS->setOriginalName(GUID.second); + TheIndex->addGlobalValueSummary(GUID.first, std::move(FS)); + break; + } + // FS_COMBINED: [valueid, modid, flags, instcount, numrefs, + // numrefs x valueid, n x (valueid, callsitecount)] + // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, numrefs, + // numrefs x valueid, + // n x (valueid, callsitecount, profilecount)] + case bitc::FS_COMBINED: + case bitc::FS_COMBINED_PROFILE: { + unsigned ValueID = Record[0]; + uint64_t ModuleId = Record[1]; + uint64_t RawFlags = Record[2]; + unsigned InstCount = Record[3]; + unsigned NumRefs = Record[4]; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); std::unique_ptr<FunctionSummary> FS = - llvm::make_unique<FunctionSummary>(InstCount); + llvm::make_unique<FunctionSummary>(Flags, InstCount); + LastSeenSummary = FS.get(); + FS->setModulePath(ModuleIdMap[ModuleId]); + static int RefListStartIndex = 5; + int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; + assert(Record.size() >= RefListStartIndex + NumRefs && + "Record size inconsistent with number of references"); + for (unsigned I = RefListStartIndex, E = CallGraphEdgeStartIndex; I != E; + ++I) { + unsigned RefValueId = Record[I]; + GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; + FS->addRefEdge(RefGUID); + } + bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE); + for (unsigned I = CallGraphEdgeStartIndex, E = Record.size(); I != E; + ++I) { + unsigned CalleeValueId = Record[I]; + unsigned CallsiteCount = Record[++I]; + uint64_t ProfileCount = HasProfile ? Record[++I] : 0; + GlobalValue::GUID CalleeGUID = getGUIDFromValueId(CalleeValueId).first; + FS->addCallGraphEdge(CalleeGUID, + CalleeInfo(CallsiteCount, ProfileCount)); + } + GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; + TheIndex->addGlobalValueSummary(GUID, std::move(FS)); + Combined = true; + break; + } + // FS_COMBINED_ALIAS: [valueid, modid, flags, valueid] + // Aliases must be emitted (and parsed) after all FS_COMBINED entries, as + // they expect all aliasee summaries to be available. + case bitc::FS_COMBINED_ALIAS: { + unsigned ValueID = Record[0]; + uint64_t ModuleId = Record[1]; + uint64_t RawFlags = Record[2]; + unsigned AliaseeValueId = Record[3]; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + std::unique_ptr<AliasSummary> AS = llvm::make_unique<AliasSummary>(Flags); + LastSeenSummary = AS.get(); + AS->setModulePath(ModuleIdMap[ModuleId]); + + auto AliaseeGUID = getGUIDFromValueId(AliaseeValueId).first; + auto AliaseeInModule = + TheIndex->findSummaryInModule(AliaseeGUID, AS->modulePath()); + if (!AliaseeInModule) + return error("Alias expects aliasee summary to be parsed"); + AS->setAliasee(AliaseeInModule); + + GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; + TheIndex->addGlobalValueSummary(GUID, std::move(AS)); + Combined = true; + break; + } + // FS_COMBINED_GLOBALVAR_INIT_REFS: [valueid, modid, flags, n x valueid] + case bitc::FS_COMBINED_GLOBALVAR_INIT_REFS: { + unsigned ValueID = Record[0]; + uint64_t ModuleId = Record[1]; + uint64_t RawFlags = Record[2]; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + std::unique_ptr<GlobalVarSummary> FS = + llvm::make_unique<GlobalVarSummary>(Flags); + LastSeenSummary = FS.get(); FS->setModulePath(ModuleIdMap[ModuleId]); - SummaryMap[CurRecordBit] = std::move(FS); + for (unsigned I = 3, E = Record.size(); I != E; ++I) { + unsigned RefValueId = Record[I]; + GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; + FS->addRefEdge(RefGUID); + } + GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; + TheIndex->addGlobalValueSummary(GUID, std::move(FS)); + Combined = true; + break; + } + // FS_COMBINED_ORIGINAL_NAME: [original_name] + case bitc::FS_COMBINED_ORIGINAL_NAME: { + uint64_t OriginalName = Record[0]; + if (!LastSeenSummary) + return error("Name attachment that does not follow a combined record"); + LastSeenSummary->setOriginalName(OriginalName); + // Reset the LastSeenSummary + LastSeenSummary = nullptr; } } } @@ -5629,13 +6360,14 @@ std::error_code FunctionIndexBitcodeReader::parseEntireSummary() { // Parse the module string table block into the Index. // This populates the ModulePathStringTable map in the index. -std::error_code FunctionIndexBitcodeReader::parseModuleStringTable() { +std::error_code ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { if (Stream.EnterSubBlock(bitc::MODULE_STRTAB_BLOCK_ID)) return error("Invalid record"); SmallVector<uint64_t, 64> Record; SmallString<128> ModulePath; + ModulePathStringTableTy::iterator LastSeenModulePath; while (1) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); @@ -5656,22 +6388,40 @@ std::error_code FunctionIndexBitcodeReader::parseModuleStringTable() { break; case bitc::MST_CODE_ENTRY: { // MST_ENTRY: [modid, namechar x N] + uint64_t ModuleId = Record[0]; + if (convertToString(Record, 1, ModulePath)) return error("Invalid record"); - uint64_t ModuleId = Record[0]; - StringRef ModulePathInMap = TheIndex->addModulePath(ModulePath, ModuleId); - ModuleIdMap[ModuleId] = ModulePathInMap; + + LastSeenModulePath = TheIndex->addModulePath(ModulePath, ModuleId); + ModuleIdMap[ModuleId] = LastSeenModulePath->first(); + ModulePath.clear(); break; } + /// MST_CODE_HASH: [5*i32] + case bitc::MST_CODE_HASH: { + if (Record.size() != 5) + return error("Invalid hash length " + Twine(Record.size()).str()); + if (LastSeenModulePath == TheIndex->modulePaths().end()) + return error("Invalid hash that does not follow a module path"); + int Pos = 0; + for (auto &Val : Record) { + assert(!(Val >> 32) && "Unexpected high bits set"); + LastSeenModulePath->second.second[Pos++] = Val; + } + // Reset LastSeenModulePath to avoid overriding the hash unexpectedly. + LastSeenModulePath = TheIndex->modulePaths().end(); + break; + } } } llvm_unreachable("Exit infinite loop"); } // Parse the function info index from the bitcode streamer into the given index. -std::error_code FunctionIndexBitcodeReader::parseSummaryIndexInto( - std::unique_ptr<DataStreamer> Streamer, FunctionInfoIndex *I) { +std::error_code ModuleSummaryIndexBitcodeReader::parseSummaryIndexInto( + std::unique_ptr<DataStreamer> Streamer, ModuleSummaryIndex *I) { TheIndex = I; if (std::error_code EC = initStream(std::move(Streamer))) @@ -5705,55 +6455,14 @@ std::error_code FunctionIndexBitcodeReader::parseSummaryIndexInto( } } -// Parse the function information at the given offset in the buffer into -// the index. Used to support lazy parsing of function summaries from the -// combined index during importing. -// TODO: This function is not yet complete as it won't have a consumer -// until ThinLTO function importing is added. -std::error_code FunctionIndexBitcodeReader::parseFunctionSummary( - std::unique_ptr<DataStreamer> Streamer, FunctionInfoIndex *I, - size_t FunctionSummaryOffset) { - TheIndex = I; - - if (std::error_code EC = initStream(std::move(Streamer))) - return EC; - - // Sniff for the signature. - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); - - Stream.JumpToBit(FunctionSummaryOffset); - - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); - - switch (Entry.Kind) { - default: - return error("Malformed block"); - case BitstreamEntry::Record: - // The expected case. - break; - } - - // TODO: Read a record. This interface will be completed when ThinLTO - // importing is added so that it can be tested. - SmallVector<uint64_t, 64> Record; - switch (Stream.readRecord(Entry.ID, Record)) { - case bitc::FS_CODE_COMBINED_ENTRY: - default: - return error("Invalid record"); - } - - return std::error_code(); -} - -std::error_code -FunctionIndexBitcodeReader::initStream(std::unique_ptr<DataStreamer> Streamer) { +std::error_code ModuleSummaryIndexBitcodeReader::initStream( + std::unique_ptr<DataStreamer> Streamer) { if (Streamer) return initLazyStream(std::move(Streamer)); return initStreamFromBuffer(); } -std::error_code FunctionIndexBitcodeReader::initStreamFromBuffer() { +std::error_code ModuleSummaryIndexBitcodeReader::initStreamFromBuffer() { const unsigned char *BufPtr = (const unsigned char *)Buffer->getBufferStart(); const unsigned char *BufEnd = BufPtr + Buffer->getBufferSize(); @@ -5772,7 +6481,7 @@ std::error_code FunctionIndexBitcodeReader::initStreamFromBuffer() { return std::error_code(); } -std::error_code FunctionIndexBitcodeReader::initLazyStream( +std::error_code ModuleSummaryIndexBitcodeReader::initLazyStream( std::unique_ptr<DataStreamer> Streamer) { // Check and strip off the bitcode wrapper; BitstreamReader expects never to // see it. @@ -5800,6 +6509,9 @@ std::error_code FunctionIndexBitcodeReader::initLazyStream( } namespace { +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. class BitcodeErrorCategoryType : public std::error_category { const char *name() const LLVM_NOEXCEPT override { return "llvm.bitcode"; @@ -5815,7 +6527,7 @@ class BitcodeErrorCategoryType : public std::error_category { llvm_unreachable("Unknown error type!"); } }; -} +} // end anonymous namespace static ManagedStatic<BitcodeErrorCategoryType> ErrorCategory; @@ -5916,6 +6628,16 @@ std::string llvm::getBitcodeTargetTriple(MemoryBufferRef Buffer, return Triple.get(); } +bool llvm::isBitcodeContainingObjCCategory(MemoryBufferRef Buffer, + LLVMContext &Context) { + std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); + auto R = llvm::make_unique<BitcodeReader>(Buf.release(), Context); + ErrorOr<bool> hasObjCCategory = R->hasObjCCategory(); + if (hasObjCCategory.getError()) + return false; + return hasObjCCategory.get(); +} + std::string llvm::getBitcodeProducerString(MemoryBufferRef Buffer, LLVMContext &Context) { std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); @@ -5927,18 +6649,13 @@ std::string llvm::getBitcodeProducerString(MemoryBufferRef Buffer, } // Parse the specified bitcode buffer, returning the function info index. -// If IsLazy is false, parse the entire function summary into -// the index. Otherwise skip the function summary section, and only create -// an index object with a map from function name to function summary offset. -// The index is used to perform lazy function summary reading later. -ErrorOr<std::unique_ptr<FunctionInfoIndex>> -llvm::getFunctionInfoIndex(MemoryBufferRef Buffer, - DiagnosticHandlerFunction DiagnosticHandler, - bool IsLazy) { +ErrorOr<std::unique_ptr<ModuleSummaryIndex>> llvm::getModuleSummaryIndex( + MemoryBufferRef Buffer, + const DiagnosticHandlerFunction &DiagnosticHandler) { std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - FunctionIndexBitcodeReader R(Buf.get(), DiagnosticHandler, IsLazy); + ModuleSummaryIndexBitcodeReader R(Buf.get(), DiagnosticHandler); - auto Index = llvm::make_unique<FunctionInfoIndex>(); + auto Index = llvm::make_unique<ModuleSummaryIndex>(); auto cleanupOnError = [&](std::error_code EC) { R.releaseBuffer(); // Never take ownership on error. @@ -5948,15 +6665,16 @@ llvm::getFunctionInfoIndex(MemoryBufferRef Buffer, if (std::error_code EC = R.parseSummaryIndexInto(nullptr, Index.get())) return cleanupOnError(EC); - Buf.release(); // The FunctionIndexBitcodeReader owns it now. + Buf.release(); // The ModuleSummaryIndexBitcodeReader owns it now. return std::move(Index); } -// Check if the given bitcode buffer contains a function summary block. -bool llvm::hasFunctionSummary(MemoryBufferRef Buffer, - DiagnosticHandlerFunction DiagnosticHandler) { +// Check if the given bitcode buffer contains a global value summary block. +bool llvm::hasGlobalValueSummary( + MemoryBufferRef Buffer, + const DiagnosticHandlerFunction &DiagnosticHandler) { std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - FunctionIndexBitcodeReader R(Buf.get(), DiagnosticHandler, false, true); + ModuleSummaryIndexBitcodeReader R(Buf.get(), DiagnosticHandler, true); auto cleanupOnError = [&](std::error_code EC) { R.releaseBuffer(); // Never take ownership on error. @@ -5966,38 +6684,6 @@ bool llvm::hasFunctionSummary(MemoryBufferRef Buffer, if (std::error_code EC = R.parseSummaryIndexInto(nullptr, nullptr)) return cleanupOnError(EC); - Buf.release(); // The FunctionIndexBitcodeReader owns it now. - return R.foundFuncSummary(); -} - -// This method supports lazy reading of function summary data from the combined -// index during ThinLTO function importing. When reading the combined index -// file, getFunctionInfoIndex is first invoked with IsLazy=true. -// Then this method is called for each function considered for importing, -// to parse the summary information for the given function name into -// the index. -std::error_code llvm::readFunctionSummary( - MemoryBufferRef Buffer, DiagnosticHandlerFunction DiagnosticHandler, - StringRef FunctionName, std::unique_ptr<FunctionInfoIndex> Index) { - std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false); - FunctionIndexBitcodeReader R(Buf.get(), DiagnosticHandler); - - auto cleanupOnError = [&](std::error_code EC) { - R.releaseBuffer(); // Never take ownership on error. - return EC; - }; - - // Lookup the given function name in the FunctionMap, which may - // contain a list of function infos in the case of a COMDAT. Walk through - // and parse each function summary info at the function summary offset - // recorded when parsing the value symbol table. - for (const auto &FI : Index->getFunctionInfoList(FunctionName)) { - size_t FunctionSummaryOffset = FI->bitcodeIndex(); - if (std::error_code EC = - R.parseFunctionSummary(nullptr, Index.get(), FunctionSummaryOffset)) - return cleanupOnError(EC); - } - - Buf.release(); // The FunctionIndexBitcodeReader owns it now. - return std::error_code(); + Buf.release(); // The ModuleSummaryIndexBitcodeReader owns it now. + return R.foundGlobalValSummary(); } diff --git a/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp index a103fbd..60360d2 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitstreamReader.cpp @@ -32,7 +32,7 @@ bool BitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) { // Add the abbrevs specific to this block to the CurAbbrevs list. if (const BitstreamReader::BlockInfo *Info = - BitStream->getBlockInfo(BlockID)) { + getBitStreamReader()->getBlockInfo(BlockID)) { CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), Info->Abbrevs.end()); } @@ -131,8 +131,25 @@ void BitstreamCursor::skipRecord(unsigned AbbrevID) { const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); // Read all the elements. - for (; NumElts; --NumElts) - skipAbbreviatedField(*this, EltEnc); + // Decode the value as we are commanded. + switch (EltEnc.getEncoding()) { + default: + report_fatal_error("Array element type can't be an Array or a Blob"); + case BitCodeAbbrevOp::Fixed: + assert((unsigned)Op.getEncodingData() <= MaxChunkSize); + for (; NumElts; --NumElts) + Read((unsigned)EltEnc.getEncodingData()); + break; + case BitCodeAbbrevOp::VBR: + assert((unsigned)Op.getEncodingData() <= MaxChunkSize); + for (; NumElts; --NumElts) + ReadVBR64((unsigned)EltEnc.getEncodingData()); + break; + case BitCodeAbbrevOp::Char6: + for (; NumElts; --NumElts) + Read(6); + break; + } continue; } @@ -147,7 +164,7 @@ void BitstreamCursor::skipRecord(unsigned AbbrevID) { // If this would read off the end of the bitcode file, just set the // record to empty and return. if (!canSkipToPos(NewEnd/8)) { - NextChar = BitStream->getBitcodeBytes().getExtent(); + skipToEnd(); break; } @@ -206,13 +223,23 @@ unsigned BitstreamCursor::readRecord(unsigned AbbrevID, if (!EltEnc.isEncoding()) report_fatal_error( "Array element type has to be an encoding of a type"); - if (EltEnc.getEncoding() == BitCodeAbbrevOp::Array || - EltEnc.getEncoding() == BitCodeAbbrevOp::Blob) - report_fatal_error("Array element type can't be an Array or a Blob"); // Read all the elements. - for (; NumElts; --NumElts) - Vals.push_back(readAbbreviatedField(*this, EltEnc)); + switch (EltEnc.getEncoding()) { + default: + report_fatal_error("Array element type can't be an Array or a Blob"); + case BitCodeAbbrevOp::Fixed: + for (; NumElts; --NumElts) + Vals.push_back(Read((unsigned)EltEnc.getEncodingData())); + break; + case BitCodeAbbrevOp::VBR: + for (; NumElts; --NumElts) + Vals.push_back(ReadVBR64((unsigned)EltEnc.getEncodingData())); + break; + case BitCodeAbbrevOp::Char6: + for (; NumElts; --NumElts) + Vals.push_back(BitCodeAbbrevOp::DecodeChar6(Read(6))); + } continue; } @@ -229,13 +256,15 @@ unsigned BitstreamCursor::readRecord(unsigned AbbrevID, // record to empty and return. if (!canSkipToPos(NewEnd/8)) { Vals.append(NumElts, 0); - NextChar = BitStream->getBitcodeBytes().getExtent(); + skipToEnd(); break; } - // Otherwise, inform the streamer that we need these bytes in memory. - const char *Ptr = (const char*) - BitStream->getBitcodeBytes().getPointer(CurBitPos/8, NumElts); + // Otherwise, inform the streamer that we need these bytes in memory. Skip + // over tail padding first, in case jumping to NewEnd invalidates the Blob + // pointer. + JumpToBit(NewEnd); + const char *Ptr = (const char *)getPointerToBit(CurBitPos, NumElts); // If we can return a reference to the data, do so to avoid copying it. if (Blob) { @@ -245,8 +274,6 @@ unsigned BitstreamCursor::readRecord(unsigned AbbrevID, for (; NumElts; --NumElts) Vals.push_back((unsigned char)*Ptr++); } - // Skip over tail padding. - JumpToBit(NewEnd); } return Code; @@ -293,7 +320,7 @@ void BitstreamCursor::ReadAbbrevRecord() { bool BitstreamCursor::ReadBlockInfoBlock() { // If this is the second stream to get to the block info block, skip it. - if (BitStream->hasBlockInfoRecords()) + if (getBitStreamReader()->hasBlockInfoRecords()) return SkipBlock(); if (EnterSubBlock(bitc::BLOCKINFO_BLOCK_ID)) return true; @@ -334,11 +361,13 @@ bool BitstreamCursor::ReadBlockInfoBlock() { default: break; // Default behavior, ignore unknown content. case bitc::BLOCKINFO_CODE_SETBID: if (Record.size() < 1) return true; - CurBlockInfo = &BitStream->getOrCreateBlockInfo((unsigned)Record[0]); + CurBlockInfo = + &getBitStreamReader()->getOrCreateBlockInfo((unsigned)Record[0]); break; case bitc::BLOCKINFO_CODE_BLOCKNAME: { if (!CurBlockInfo) return true; - if (BitStream->isIgnoringBlockInfoNames()) break; // Ignore name. + if (getBitStreamReader()->isIgnoringBlockInfoNames()) + break; // Ignore name. std::string Name; for (unsigned i = 0, e = Record.size(); i != e; ++i) Name += (char)Record[i]; @@ -347,7 +376,8 @@ bool BitstreamCursor::ReadBlockInfoBlock() { } case bitc::BLOCKINFO_CODE_SETRECORDNAME: { if (!CurBlockInfo) return true; - if (BitStream->isIgnoringBlockInfoNames()) break; // Ignore name. + if (getBitStreamReader()->isIgnoringBlockInfoNames()) + break; // Ignore name. std::string Name; for (unsigned i = 1, e = Record.size(); i != e; ++i) Name += (char)Record[i]; diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index a899a0c..dcb8b58 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Bitcode/ReaderWriter.h" #include "ValueEnumerator.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -24,20 +24,20 @@ #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/UseListOrder.h" #include "llvm/IR/ValueSymbolTable.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Program.h" +#include "llvm/Support/SHA1.h" #include "llvm/Support/raw_ostream.h" #include <cctype> #include <map> using namespace llvm; +namespace { /// These are manifest constants used by the bitcode writer. They do not need to /// be kept in sync with the reader, but need to be consistent within this file. enum { @@ -64,7 +64,455 @@ enum { FUNCTION_INST_GEP_ABBREV, }; -static unsigned GetEncodedCastOpcode(unsigned Opcode) { +/// Abstract class to manage the bitcode writing, subclassed for each bitcode +/// file type. Owns the BitstreamWriter, and includes the main entry point for +/// writing. +class BitcodeWriter { +protected: + /// Pointer to the buffer allocated by caller for bitcode writing. + const SmallVectorImpl<char> &Buffer; + + /// The stream created and owned by the BitodeWriter. + BitstreamWriter Stream; + + /// Saves the offset of the VSTOffset record that must eventually be + /// backpatched with the offset of the actual VST. + uint64_t VSTOffsetPlaceholder = 0; + +public: + /// Constructs a BitcodeWriter object, and initializes a BitstreamRecord, + /// writing to the provided \p Buffer. + BitcodeWriter(SmallVectorImpl<char> &Buffer) + : Buffer(Buffer), Stream(Buffer) {} + + virtual ~BitcodeWriter() = default; + + /// Main entry point to write the bitcode file, which writes the bitcode + /// header and will then invoke the virtual writeBlocks() method. + void write(); + +private: + /// Derived classes must implement this to write the corresponding blocks for + /// that bitcode file type. + virtual void writeBlocks() = 0; + +protected: + bool hasVSTOffsetPlaceholder() { return VSTOffsetPlaceholder != 0; } + void writeValueSymbolTableForwardDecl(); + void writeBitcodeHeader(); +}; + +/// Class to manage the bitcode writing for a module. +class ModuleBitcodeWriter : public BitcodeWriter { + /// The Module to write to bitcode. + const Module &M; + + /// Enumerates ids for all values in the module. + ValueEnumerator VE; + + /// Optional per-module index to write for ThinLTO. + const ModuleSummaryIndex *Index; + + /// True if a module hash record should be written. + bool GenerateHash; + + /// The start bit of the module block, for use in generating a module hash + uint64_t BitcodeStartBit = 0; + + /// Map that holds the correspondence between GUIDs in the summary index, + /// that came from indirect call profiles, and a value id generated by this + /// class to use in the VST and summary block records. + std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap; + + /// Tracks the last value id recorded in the GUIDToValueMap. + unsigned GlobalValueId; + +public: + /// Constructs a ModuleBitcodeWriter object for the given Module, + /// writing to the provided \p Buffer. + ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, + bool ShouldPreserveUseListOrder, + const ModuleSummaryIndex *Index, bool GenerateHash) + : BitcodeWriter(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), + Index(Index), GenerateHash(GenerateHash) { + // Save the start bit of the actual bitcode, in case there is space + // saved at the start for the darwin header above. The reader stream + // will start at the bitcode, and we need the offset of the VST + // to line up. + BitcodeStartBit = Stream.GetCurrentBitNo(); + + // Assign ValueIds to any callee values in the index that came from + // indirect call profiles and were recorded as a GUID not a Value* + // (which would have been assigned an ID by the ValueEnumerator). + // The starting ValueId is just after the number of values in the + // ValueEnumerator, so that they can be emitted in the VST. + GlobalValueId = VE.getValues().size(); + if (Index) + for (const auto &GUIDSummaryLists : *Index) + // Examine all summaries for this GUID. + for (auto &Summary : GUIDSummaryLists.second) + if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) + // For each call in the function summary, see if the call + // is to a GUID (which means it is for an indirect call, + // otherwise we would have a Value for it). If so, synthesize + // a value id. + for (auto &CallEdge : FS->calls()) + if (CallEdge.first.isGUID()) + assignValueId(CallEdge.first.getGUID()); + } + +private: + /// Main entry point for writing a module to bitcode, invoked by + /// BitcodeWriter::write() after it writes the header. + void writeBlocks() override; + + /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the + /// current llvm version, and a record for the epoch number. + void writeIdentificationBlock(); + + /// Emit the current module to the bitstream. + void writeModule(); + + uint64_t bitcodeStartBit() { return BitcodeStartBit; } + + void writeStringRecord(unsigned Code, StringRef Str, unsigned AbbrevToUse); + void writeAttributeGroupTable(); + void writeAttributeTable(); + void writeTypeTable(); + void writeComdats(); + void writeModuleInfo(); + void writeValueAsMetadata(const ValueAsMetadata *MD, + SmallVectorImpl<uint64_t> &Record); + void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + unsigned createDILocationAbbrev(); + void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record, + unsigned &Abbrev); + unsigned createGenericDINodeAbbrev(); + void writeGenericDINode(const GenericDINode *N, + SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev); + void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIEnumerator(const DIEnumerator *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIDerivedType(const DIDerivedType *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDICompositeType(const DICompositeType *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDISubroutineType(const DISubroutineType *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDICompileUnit(const DICompileUnit *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDISubprogram(const DISubprogram *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDILexicalBlock(const DILexicalBlock *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDILexicalBlockFile(const DILexicalBlockFile *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDITemplateTypeParameter(const DITemplateTypeParameter *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDITemplateValueParameter(const DITemplateValueParameter *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIGlobalVariable(const DIGlobalVariable *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDILocalVariable(const DILocalVariable *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIExpression(const DIExpression *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIObjCProperty(const DIObjCProperty *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIImportedEntity(const DIImportedEntity *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + unsigned createNamedMetadataAbbrev(); + void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record); + unsigned createMetadataStringsAbbrev(); + void writeMetadataStrings(ArrayRef<const Metadata *> Strings, + SmallVectorImpl<uint64_t> &Record); + void writeMetadataRecords(ArrayRef<const Metadata *> MDs, + SmallVectorImpl<uint64_t> &Record); + void writeModuleMetadata(); + void writeFunctionMetadata(const Function &F); + void writeFunctionMetadataAttachment(const Function &F); + void writeGlobalVariableMetadataAttachment(const GlobalVariable &GV); + void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record, + const GlobalObject &GO); + void writeModuleMetadataKinds(); + void writeOperandBundleTags(); + void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal); + void writeModuleConstants(); + bool pushValueAndType(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals); + void writeOperandBundles(ImmutableCallSite CS, unsigned InstID); + void pushValue(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals); + void pushValueSigned(const Value *V, unsigned InstID, + SmallVectorImpl<uint64_t> &Vals); + void writeInstruction(const Instruction &I, unsigned InstID, + SmallVectorImpl<unsigned> &Vals); + void writeValueSymbolTable( + const ValueSymbolTable &VST, bool IsModuleLevel = false, + DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex = nullptr); + void writeUseList(UseListOrder &&Order); + void writeUseListBlock(const Function *F); + void + writeFunction(const Function &F, + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex); + void writeBlockInfo(); + void writePerModuleFunctionSummaryRecord(SmallVector<uint64_t, 64> &NameVals, + GlobalValueSummary *Summary, + unsigned ValueID, + unsigned FSCallsAbbrev, + unsigned FSCallsProfileAbbrev, + const Function &F); + void writeModuleLevelReferences(const GlobalVariable &V, + SmallVector<uint64_t, 64> &NameVals, + unsigned FSModRefsAbbrev); + void writePerModuleGlobalValueSummary(); + void writeModuleHash(size_t BlockStartPos); + + void assignValueId(GlobalValue::GUID ValGUID) { + GUIDToValueIdMap[ValGUID] = ++GlobalValueId; + } + unsigned getValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + assert(VMI != GUIDToValueIdMap.end()); + return VMI->second; + } + // Helper to get the valueId for the type of value recorded in VI. + unsigned getValueId(ValueInfo VI) { + if (VI.isGUID()) + return getValueId(VI.getGUID()); + return VE.getValueID(VI.getValue()); + } + std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } +}; + +/// Class to manage the bitcode writing for a combined index. +class IndexBitcodeWriter : public BitcodeWriter { + /// The combined index to write to bitcode. + const ModuleSummaryIndex &Index; + + /// When writing a subset of the index for distributed backends, client + /// provides a map of modules to the corresponding GUIDs/summaries to write. + std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex; + + /// Map that holds the correspondence between the GUID used in the combined + /// index and a value id generated by this class to use in references. + std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap; + + /// Tracks the last value id recorded in the GUIDToValueMap. + unsigned GlobalValueId = 0; + +public: + /// Constructs a IndexBitcodeWriter object for the given combined index, + /// writing to the provided \p Buffer. When writing a subset of the index + /// for a distributed backend, provide a \p ModuleToSummariesForIndex map. + IndexBitcodeWriter(SmallVectorImpl<char> &Buffer, + const ModuleSummaryIndex &Index, + std::map<std::string, GVSummaryMapTy> + *ModuleToSummariesForIndex = nullptr) + : BitcodeWriter(Buffer), Index(Index), + ModuleToSummariesForIndex(ModuleToSummariesForIndex) { + // Assign unique value ids to all summaries to be written, for use + // in writing out the call graph edges. Save the mapping from GUID + // to the new global value id to use when writing those edges, which + // are currently saved in the index in terms of GUID. + for (const auto &I : *this) + GUIDToValueIdMap[I.first] = ++GlobalValueId; + } + + /// The below iterator returns the GUID and associated summary. + typedef std::pair<GlobalValue::GUID, GlobalValueSummary *> GVInfo; + + /// Iterator over the value GUID and summaries to be written to bitcode, + /// hides the details of whether they are being pulled from the entire + /// index or just those in a provided ModuleToSummariesForIndex map. + class iterator + : public llvm::iterator_facade_base<iterator, std::forward_iterator_tag, + GVInfo> { + /// Enables access to parent class. + const IndexBitcodeWriter &Writer; + + // Iterators used when writing only those summaries in a provided + // ModuleToSummariesForIndex map: + + /// Points to the last element in outer ModuleToSummariesForIndex map. + std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesBack; + /// Iterator on outer ModuleToSummariesForIndex map. + std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesIter; + /// Iterator on an inner global variable summary map. + GVSummaryMapTy::iterator ModuleGVSummariesIter; + + // Iterators used when writing all summaries in the index: + + /// Points to the last element in the Index outer GlobalValueMap. + const_gvsummary_iterator IndexSummariesBack; + /// Iterator on outer GlobalValueMap. + const_gvsummary_iterator IndexSummariesIter; + /// Iterator on an inner GlobalValueSummaryList. + GlobalValueSummaryList::const_iterator IndexGVSummariesIter; + + public: + /// Construct iterator from parent \p Writer and indicate if we are + /// constructing the end iterator. + iterator(const IndexBitcodeWriter &Writer, bool IsAtEnd) : Writer(Writer) { + // Set up the appropriate set of iterators given whether we are writing + // the full index or just a subset. + // Can't setup the Back or inner iterators if the corresponding map + // is empty. This will be handled specially in operator== as well. + if (Writer.ModuleToSummariesForIndex && + !Writer.ModuleToSummariesForIndex->empty()) { + for (ModuleSummariesBack = Writer.ModuleToSummariesForIndex->begin(); + std::next(ModuleSummariesBack) != + Writer.ModuleToSummariesForIndex->end(); + ModuleSummariesBack++) + ; + ModuleSummariesIter = !IsAtEnd + ? Writer.ModuleToSummariesForIndex->begin() + : ModuleSummariesBack; + ModuleGVSummariesIter = !IsAtEnd ? ModuleSummariesIter->second.begin() + : ModuleSummariesBack->second.end(); + } else if (!Writer.ModuleToSummariesForIndex && + Writer.Index.begin() != Writer.Index.end()) { + for (IndexSummariesBack = Writer.Index.begin(); + std::next(IndexSummariesBack) != Writer.Index.end(); + IndexSummariesBack++) + ; + IndexSummariesIter = + !IsAtEnd ? Writer.Index.begin() : IndexSummariesBack; + IndexGVSummariesIter = !IsAtEnd ? IndexSummariesIter->second.begin() + : IndexSummariesBack->second.end(); + } + } + + /// Increment the appropriate set of iterators. + iterator &operator++() { + // First the inner iterator is incremented, then if it is at the end + // and there are more outer iterations to go, the inner is reset to + // the start of the next inner list. + if (Writer.ModuleToSummariesForIndex) { + ++ModuleGVSummariesIter; + if (ModuleGVSummariesIter == ModuleSummariesIter->second.end() && + ModuleSummariesIter != ModuleSummariesBack) { + ++ModuleSummariesIter; + ModuleGVSummariesIter = ModuleSummariesIter->second.begin(); + } + } else { + ++IndexGVSummariesIter; + if (IndexGVSummariesIter == IndexSummariesIter->second.end() && + IndexSummariesIter != IndexSummariesBack) { + ++IndexSummariesIter; + IndexGVSummariesIter = IndexSummariesIter->second.begin(); + } + } + return *this; + } + + /// Access the <GUID,GlobalValueSummary*> pair corresponding to the current + /// outer and inner iterator positions. + GVInfo operator*() { + if (Writer.ModuleToSummariesForIndex) + return std::make_pair(ModuleGVSummariesIter->first, + ModuleGVSummariesIter->second); + return std::make_pair(IndexSummariesIter->first, + IndexGVSummariesIter->get()); + } + + /// Checks if the iterators are equal, with special handling for empty + /// indexes. + bool operator==(const iterator &RHS) const { + if (Writer.ModuleToSummariesForIndex) { + // First ensure that both are writing the same subset. + if (Writer.ModuleToSummariesForIndex != + RHS.Writer.ModuleToSummariesForIndex) + return false; + // Already determined above that maps are the same, so if one is + // empty, they both are. + if (Writer.ModuleToSummariesForIndex->empty()) + return true; + // Ensure the ModuleGVSummariesIter are iterating over the same + // container before checking them below. + if (ModuleSummariesIter != RHS.ModuleSummariesIter) + return false; + return ModuleGVSummariesIter == RHS.ModuleGVSummariesIter; + } + // First ensure RHS also writing the full index, and that both are + // writing the same full index. + if (RHS.Writer.ModuleToSummariesForIndex || + &Writer.Index != &RHS.Writer.Index) + return false; + // Already determined above that maps are the same, so if one is + // empty, they both are. + if (Writer.Index.begin() == Writer.Index.end()) + return true; + // Ensure the IndexGVSummariesIter are iterating over the same + // container before checking them below. + if (IndexSummariesIter != RHS.IndexSummariesIter) + return false; + return IndexGVSummariesIter == RHS.IndexGVSummariesIter; + } + }; + + /// Obtain the start iterator over the summaries to be written. + iterator begin() { return iterator(*this, /*IsAtEnd=*/false); } + /// Obtain the end iterator over the summaries to be written. + iterator end() { return iterator(*this, /*IsAtEnd=*/true); } + +private: + /// Main entry point for writing a combined index to bitcode, invoked by + /// BitcodeWriter::write() after it writes the header. + void writeBlocks() override; + + void writeIndex(); + void writeModStrings(); + void writeCombinedValueSymbolTable(); + void writeCombinedGlobalValueSummary(); + + /// Indicates whether the provided \p ModulePath should be written into + /// the module string table, e.g. if full index written or if it is in + /// the provided subset. + bool doIncludeModule(StringRef ModulePath) { + return !ModuleToSummariesForIndex || + ModuleToSummariesForIndex->count(ModulePath); + } + + bool hasValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + return VMI != GUIDToValueIdMap.end(); + } + unsigned getValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + // If this GUID doesn't have an entry, assign one. + if (VMI == GUIDToValueIdMap.end()) { + GUIDToValueIdMap[ValGUID] = ++GlobalValueId; + return GlobalValueId; + } else { + return VMI->second; + } + } + std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } +}; +} // end anonymous namespace + +static unsigned getEncodedCastOpcode(unsigned Opcode) { switch (Opcode) { default: llvm_unreachable("Unknown cast instruction!"); case Instruction::Trunc : return bitc::CAST_TRUNC; @@ -83,7 +531,7 @@ static unsigned GetEncodedCastOpcode(unsigned Opcode) { } } -static unsigned GetEncodedBinaryOpcode(unsigned Opcode) { +static unsigned getEncodedBinaryOpcode(unsigned Opcode) { switch (Opcode) { default: llvm_unreachable("Unknown binary instruction!"); case Instruction::Add: @@ -107,7 +555,7 @@ static unsigned GetEncodedBinaryOpcode(unsigned Opcode) { } } -static unsigned GetEncodedRMWOperation(AtomicRMWInst::BinOp Op) { +static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) { switch (Op) { default: llvm_unreachable("Unknown RMW operation!"); case AtomicRMWInst::Xchg: return bitc::RMW_XCHG; @@ -124,20 +572,20 @@ static unsigned GetEncodedRMWOperation(AtomicRMWInst::BinOp Op) { } } -static unsigned GetEncodedOrdering(AtomicOrdering Ordering) { +static unsigned getEncodedOrdering(AtomicOrdering Ordering) { switch (Ordering) { - case NotAtomic: return bitc::ORDERING_NOTATOMIC; - case Unordered: return bitc::ORDERING_UNORDERED; - case Monotonic: return bitc::ORDERING_MONOTONIC; - case Acquire: return bitc::ORDERING_ACQUIRE; - case Release: return bitc::ORDERING_RELEASE; - case AcquireRelease: return bitc::ORDERING_ACQREL; - case SequentiallyConsistent: return bitc::ORDERING_SEQCST; + case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC; + case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED; + case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC; + case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE; + case AtomicOrdering::Release: return bitc::ORDERING_RELEASE; + case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL; + case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST; } llvm_unreachable("Invalid ordering"); } -static unsigned GetEncodedSynchScope(SynchronizationScope SynchScope) { +static unsigned getEncodedSynchScope(SynchronizationScope SynchScope) { switch (SynchScope) { case SingleThread: return bitc::SYNCHSCOPE_SINGLETHREAD; case CrossThread: return bitc::SYNCHSCOPE_CROSSTHREAD; @@ -145,8 +593,8 @@ static unsigned GetEncodedSynchScope(SynchronizationScope SynchScope) { llvm_unreachable("Invalid synch scope"); } -static void WriteStringRecord(unsigned Code, StringRef Str, - unsigned AbbrevToUse, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeStringRecord(unsigned Code, StringRef Str, + unsigned AbbrevToUse) { SmallVector<unsigned, 64> Vals; // Code: [strchar x N] @@ -164,6 +612,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { switch (Kind) { case Attribute::Alignment: return bitc::ATTR_KIND_ALIGNMENT; + case Attribute::AllocSize: + return bitc::ATTR_KIND_ALLOC_SIZE; case Attribute::AlwaysInline: return bitc::ATTR_KIND_ALWAYS_INLINE; case Attribute::ArgMemOnly: @@ -254,8 +704,14 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_SANITIZE_THREAD; case Attribute::SanitizeMemory: return bitc::ATTR_KIND_SANITIZE_MEMORY; + case Attribute::SwiftError: + return bitc::ATTR_KIND_SWIFT_ERROR; + case Attribute::SwiftSelf: + return bitc::ATTR_KIND_SWIFT_SELF; case Attribute::UWTable: return bitc::ATTR_KIND_UW_TABLE; + case Attribute::WriteOnly: + return bitc::ATTR_KIND_WRITEONLY; case Attribute::ZExt: return bitc::ATTR_KIND_Z_EXT; case Attribute::EndAttrKinds: @@ -267,8 +723,7 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { llvm_unreachable("Trying to encode unknown attribute"); } -static void WriteAttributeGroupTable(const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeAttributeGroupTable() { const std::vector<AttributeSet> &AttrGrps = VE.getAttributeGroups(); if (AttrGrps.empty()) return; @@ -315,8 +770,7 @@ static void WriteAttributeGroupTable(const ValueEnumerator &VE, Stream.ExitBlock(); } -static void WriteAttributeTable(const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeAttributeTable() { const std::vector<AttributeSet> &Attrs = VE.getAttributes(); if (Attrs.empty()) return; @@ -336,7 +790,7 @@ static void WriteAttributeTable(const ValueEnumerator &VE, } /// WriteTypeTable - Write out the type table for a module. -static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeTypeTable() { const ValueEnumerator::TypeList &TypeList = VE.getTypes(); Stream.EnterSubblock(bitc::TYPE_BLOCK_ID_NEW, 4 /*count from # abbrevs */); @@ -464,8 +918,8 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) { // Emit the name if it is present. if (!ST->getName().empty()) - WriteStringRecord(bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), - StructNameAbbrev, Stream); + writeStringRecord(bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), + StructNameAbbrev); } break; } @@ -496,8 +950,8 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) { Stream.ExitBlock(); } -static unsigned getEncodedLinkage(const GlobalValue &GV) { - switch (GV.getLinkage()) { +static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) { + switch (Linkage) { case GlobalValue::ExternalLinkage: return 0; case GlobalValue::WeakAnyLinkage: @@ -524,6 +978,24 @@ static unsigned getEncodedLinkage(const GlobalValue &GV) { llvm_unreachable("Invalid linkage"); } +static unsigned getEncodedLinkage(const GlobalValue &GV) { + return getEncodedLinkage(GV.getLinkage()); +} + +// Decode the flags for GlobalValue in the summary +static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { + uint64_t RawFlags = 0; + + RawFlags |= Flags.HasSection; // bool + + // Linkage don't need to be remapped at that time for the summary. Any future + // change to the getEncodedLinkage() function will need to be taken into + // account here as well. + RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits + + return RawFlags; +} + static unsigned getEncodedVisibility(const GlobalValue &GV) { switch (GV.getVisibility()) { case GlobalValue::DefaultVisibility: return 0; @@ -569,13 +1041,22 @@ static unsigned getEncodedComdatSelectionKind(const Comdat &C) { llvm_unreachable("Invalid selection kind"); } -static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) { - SmallVector<uint16_t, 64> Vals; +static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) { + switch (GV.getUnnamedAddr()) { + case GlobalValue::UnnamedAddr::None: return 0; + case GlobalValue::UnnamedAddr::Local: return 2; + case GlobalValue::UnnamedAddr::Global: return 1; + } + llvm_unreachable("Invalid unnamed_addr"); +} + +void ModuleBitcodeWriter::writeComdats() { + SmallVector<unsigned, 64> Vals; for (const Comdat *C : VE.getComdats()) { // COMDAT: [selection_kind, name] Vals.push_back(getEncodedComdatSelectionKind(*C)); size_t Size = C->getName().size(); - assert(isUInt<16>(Size)); + assert(isUInt<32>(Size)); Vals.push_back(Size); for (char Chr : C->getName()) Vals.push_back((unsigned char)Chr); @@ -586,12 +1067,8 @@ static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched -/// after the real VST is written. Returns the bit offset to backpatch. -static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST, - BitstreamWriter &Stream) { - if (VST.empty()) - return 0; - +/// after the real VST is written. Saves the bit offset to backpatch. +void BitcodeWriter::writeValueSymbolTableForwardDecl() { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be @@ -608,27 +1085,44 @@ static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST, uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0}; Stream.EmitRecordWithAbbrev(VSTOffsetAbbrev, Vals); - // Compute and return the bit offset to the placeholder, which will be + // Compute and save the bit offset to the placeholder, which will be // patched when the real VST is written. We can simply subtract the 32-bit // fixed size from the current bit number to get the location to backpatch. - return Stream.GetCurrentBitNo() - 32; + VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32; +} + +enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 }; + +/// Determine the encoding to use for the given string name and length. +static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { + bool isChar6 = true; + for (const char *C = Str, *E = C + StrLen; C != E; ++C) { + if (isChar6) + isChar6 = BitCodeAbbrevOp::isChar6(*C); + if ((unsigned char)*C & 128) + // don't bother scanning the rest. + return SE_Fixed8; + } + if (isChar6) + return SE_Char6; + else + return SE_Fixed7; } /// Emit top-level description of module, including target triple, inline asm, /// descriptors for global variables, and function prototype info. /// Returns the bit offset to backpatch with the location of the real VST. -static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeModuleInfo() { // Emit various pieces of data attached to a module. - if (!M->getTargetTriple().empty()) - WriteStringRecord(bitc::MODULE_CODE_TRIPLE, M->getTargetTriple(), - 0/*TODO*/, Stream); - const std::string &DL = M->getDataLayoutStr(); + if (!M.getTargetTriple().empty()) + writeStringRecord(bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(), + 0 /*TODO*/); + const std::string &DL = M.getDataLayoutStr(); if (!DL.empty()) - WriteStringRecord(bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/, Stream); - if (!M->getModuleInlineAsm().empty()) - WriteStringRecord(bitc::MODULE_CODE_ASM, M->getModuleInlineAsm(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/); + if (!M.getModuleInlineAsm().empty()) + writeStringRecord(bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(), + 0 /*TODO*/); // Emit information about sections and GC, computing how many there are. Also // compute the maximum alignment value. @@ -636,27 +1130,27 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, std::map<std::string, unsigned> GCMap; unsigned MaxAlignment = 0; unsigned MaxGlobalType = 0; - for (const GlobalValue &GV : M->globals()) { + for (const GlobalValue &GV : M.globals()) { MaxAlignment = std::max(MaxAlignment, GV.getAlignment()); MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType())); if (GV.hasSection()) { // Give section names unique ID's. unsigned &Entry = SectionMap[GV.getSection()]; if (!Entry) { - WriteStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), + 0 /*TODO*/); Entry = SectionMap.size(); } } } - for (const Function &F : *M) { + for (const Function &F : M) { MaxAlignment = std::max(MaxAlignment, F.getAlignment()); if (F.hasSection()) { // Give section names unique ID's. unsigned &Entry = SectionMap[F.getSection()]; if (!Entry) { - WriteStringRecord(bitc::MODULE_CODE_SECTIONNAME, F.getSection(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, F.getSection(), + 0 /*TODO*/); Entry = SectionMap.size(); } } @@ -664,8 +1158,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, // Same for GC names. unsigned &Entry = GCMap[F.getGC()]; if (!Entry) { - WriteStringRecord(bitc::MODULE_CODE_GCNAME, F.getGC(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_GCNAME, F.getGC(), 0 /*TODO*/); Entry = GCMap.size(); } } @@ -673,7 +1166,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, // Emit abbrev for globals, now that we know # sections and max alignment. unsigned SimpleGVarAbbrev = 0; - if (!M->global_empty()) { + if (!M.global_empty()) { // Add an abbrev for common globals with no visibility or thread localness. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR)); @@ -702,7 +1195,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, // Emit the global variable information. SmallVector<unsigned, 64> Vals; - for (const GlobalVariable &GV : M->globals()) { + for (const GlobalVariable &GV : M.globals()) { unsigned AbbrevToUse = 0; // GLOBALVAR: [type, isconst, initid, @@ -718,12 +1211,13 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.push_back(GV.hasSection() ? SectionMap[GV.getSection()] : 0); if (GV.isThreadLocal() || GV.getVisibility() != GlobalValue::DefaultVisibility || - GV.hasUnnamedAddr() || GV.isExternallyInitialized() || + GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None || + GV.isExternallyInitialized() || GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass || GV.hasComdat()) { Vals.push_back(getEncodedVisibility(GV)); Vals.push_back(getEncodedThreadLocalMode(GV)); - Vals.push_back(GV.hasUnnamedAddr()); + Vals.push_back(getEncodedUnnamedAddr(GV)); Vals.push_back(GV.isExternallyInitialized()); Vals.push_back(getEncodedDLLStorageClass(GV)); Vals.push_back(GV.hasComdat() ? VE.getComdatID(GV.getComdat()) : 0); @@ -736,7 +1230,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, } // Emit the function proto information. - for (const Function &F : *M) { + for (const Function &F : M) { // FUNCTION: [type, callingconv, isproto, linkage, paramattrs, alignment, // section, visibility, gc, unnamed_addr, prologuedata, // dllstorageclass, comdat, prefixdata, personalityfn] @@ -749,7 +1243,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.push_back(F.hasSection() ? SectionMap[F.getSection()] : 0); Vals.push_back(getEncodedVisibility(F)); Vals.push_back(F.hasGC() ? GCMap[F.getGC()] : 0); - Vals.push_back(F.hasUnnamedAddr()); + Vals.push_back(getEncodedUnnamedAddr(F)); Vals.push_back(F.hasPrologueData() ? (VE.getValueID(F.getPrologueData()) + 1) : 0); Vals.push_back(getEncodedDLLStorageClass(F)); @@ -765,8 +1259,9 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, } // Emit the alias information. - for (const GlobalAlias &A : M->aliases()) { - // ALIAS: [alias type, aliasee val#, linkage, visibility] + for (const GlobalAlias &A : M.aliases()) { + // ALIAS: [alias type, aliasee val#, linkage, visibility, dllstorageclass, + // threadlocal, unnamed_addr] Vals.push_back(VE.getTypeID(A.getValueType())); Vals.push_back(A.getType()->getAddressSpace()); Vals.push_back(VE.getValueID(A.getAliasee())); @@ -774,33 +1269,56 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.push_back(getEncodedVisibility(A)); Vals.push_back(getEncodedDLLStorageClass(A)); Vals.push_back(getEncodedThreadLocalMode(A)); - Vals.push_back(A.hasUnnamedAddr()); + Vals.push_back(getEncodedUnnamedAddr(A)); unsigned AbbrevToUse = 0; Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals, AbbrevToUse); Vals.clear(); } - // Write a record indicating the number of module-level metadata IDs - // This is needed because the ids of metadata are assigned implicitly - // based on their ordering in the bitcode, with the function-level - // metadata ids starting after the module-level metadata ids. For - // function importing where we lazy load the metadata as a postpass, - // we want to avoid parsing the module-level metadata before parsing - // the imported functions. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_METADATA_VALUES)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - unsigned MDValsAbbrev = Stream.EmitAbbrev(Abbv); - Vals.push_back(VE.numMDs()); - Stream.EmitRecord(bitc::MODULE_CODE_METADATA_VALUES, Vals, MDValsAbbrev); - Vals.clear(); + // Emit the ifunc information. + for (const GlobalIFunc &I : M.ifuncs()) { + // IFUNC: [ifunc type, address space, resolver val#, linkage, visibility] + Vals.push_back(VE.getTypeID(I.getValueType())); + Vals.push_back(I.getType()->getAddressSpace()); + Vals.push_back(VE.getValueID(I.getResolver())); + Vals.push_back(getEncodedLinkage(I)); + Vals.push_back(getEncodedVisibility(I)); + Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals); + Vals.clear(); + } + + // Emit the module's source file name. + { + StringEncoding Bits = getStringEncoding(M.getSourceFileName().data(), + M.getSourceFileName().size()); + BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); + if (Bits == SE_Char6) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); + else if (Bits == SE_Fixed7) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); + + // MODULE_CODE_SOURCE_FILENAME: [namechar x N] + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(AbbrevOpToUse); + unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv); + + for (const auto P : M.getSourceFileName()) + Vals.push_back((unsigned char)P); - uint64_t VSTOffsetPlaceholder = - WriteValueSymbolTableForwardDecl(M->getValueSymbolTable(), Stream); - return VSTOffsetPlaceholder; + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); + Vals.clear(); + } + + // If we have a VST, write the VSTOFFSET record placeholder. + if (M.getValueSymbolTable().empty()) + return; + writeValueSymbolTableForwardDecl(); } -static uint64_t GetOptimizationFlags(const Value *V) { +static uint64_t getOptimizationFlags(const Value *V) { uint64_t Flags = 0; if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(V)) { @@ -827,10 +1345,8 @@ static uint64_t GetOptimizationFlags(const Value *V) { return Flags; } -static void WriteValueAsMetadata(const ValueAsMetadata *MD, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record) { +void ModuleBitcodeWriter::writeValueAsMetadata( + const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) { // Mimic an MDNode with a value as one operand. Value *V = MD->getValue(); Record.push_back(VE.getTypeID(V->getType())); @@ -839,9 +1355,9 @@ static void WriteValueAsMetadata(const ValueAsMetadata *MD, Record.clear(); } -static void WriteMDTuple(const MDTuple *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { Metadata *MD = N->getOperand(i); assert(!(MD && isa<LocalAsMetadata>(MD)) && @@ -854,10 +1370,25 @@ static void WriteMDTuple(const MDTuple *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDILocation(const DILocation *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +unsigned ModuleBitcodeWriter::createDILocationAbbrev() { + // Assume the column is usually under 128, and always output the inlined-at + // location (it's never more expensive than building an array size 1). + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + return Stream.EmitAbbrev(Abbv); +} + +void ModuleBitcodeWriter::writeDILocation(const DILocation *N, + SmallVectorImpl<uint64_t> &Record, + unsigned &Abbrev) { + if (!Abbrev) + Abbrev = createDILocationAbbrev(); + Record.push_back(N->isDistinct()); Record.push_back(N->getLine()); Record.push_back(N->getColumn()); @@ -868,11 +1399,26 @@ static void WriteDILocation(const DILocation *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteGenericDINode(const GenericDINode *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() { + // Assume the column is usually under 128, and always output the inlined-at + // location (it's never more expensive than building an array size 1). + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + return Stream.EmitAbbrev(Abbv); +} + +void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N, + SmallVectorImpl<uint64_t> &Record, + unsigned &Abbrev) { + if (!Abbrev) + Abbrev = createGenericDINodeAbbrev(); + Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(0); // Per-tag version field; unused for now. @@ -889,10 +1435,9 @@ static uint64_t rotateSign(int64_t I) { return I < 0 ? ~(U << 1) : U << 1; } -static void WriteDISubrange(const DISubrange *N, const ValueEnumerator &, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getCount()); Record.push_back(rotateSign(N->getLowerBound())); @@ -901,10 +1446,9 @@ static void WriteDISubrange(const DISubrange *N, const ValueEnumerator &, Record.clear(); } -static void WriteDIEnumerator(const DIEnumerator *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(rotateSign(N->getValue())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -913,10 +1457,9 @@ static void WriteDIEnumerator(const DIEnumerator *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIBasicType(const DIBasicType *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -928,11 +1471,9 @@ static void WriteDIBasicType(const DIBasicType *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIDerivedType(const DIDerivedType *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -950,12 +1491,11 @@ static void WriteDIDerivedType(const DIDerivedType *N, Record.clear(); } -static void WriteDICompositeType(const DICompositeType *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { - Record.push_back(N->isDistinct()); +void ModuleBitcodeWriter::writeDICompositeType( + const DICompositeType *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + const unsigned IsNotUsedInOldTypeRef = 0x2; + Record.push_back(IsNotUsedInOldTypeRef | (unsigned)N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -976,22 +1516,22 @@ static void WriteDICompositeType(const DICompositeType *N, Record.clear(); } -static void WriteDISubroutineType(const DISubroutineType *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { - Record.push_back(N->isDistinct()); +void ModuleBitcodeWriter::writeDISubroutineType( + const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + const unsigned HasNoOldTypeRefs = 0x2; + Record.push_back(HasNoOldTypeRefs | (unsigned)N->isDistinct()); Record.push_back(N->getFlags()); Record.push_back(VE.getMetadataOrNullID(N->getTypeArray().get())); + Record.push_back(N->getCC()); Stream.EmitRecord(bitc::METADATA_SUBROUTINE_TYPE, Record, Abbrev); Record.clear(); } -static void WriteDIFile(const DIFile *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIFile(const DIFile *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawFilename())); Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory())); @@ -1000,11 +1540,9 @@ static void WriteDIFile(const DIFile *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDICompileUnit(const DICompileUnit *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { assert(N->isDistinct() && "Expected distinct compile units"); Record.push_back(/* IsDistinct */ true); Record.push_back(N->getSourceLanguage()); @@ -1017,7 +1555,7 @@ static void WriteDICompileUnit(const DICompileUnit *N, Record.push_back(N->getEmissionKind()); Record.push_back(VE.getMetadataOrNullID(N->getEnumTypes().get())); Record.push_back(VE.getMetadataOrNullID(N->getRetainedTypes().get())); - Record.push_back(VE.getMetadataOrNullID(N->getSubprograms().get())); + Record.push_back(/* subprograms */ 0); Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables().get())); Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get())); Record.push_back(N->getDWOId()); @@ -1027,11 +1565,11 @@ static void WriteDICompileUnit(const DICompileUnit *N, Record.clear(); } -static void WriteDISubprogram(const DISubprogram *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { - Record.push_back(N->isDistinct()); +void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + uint64_t HasUnitFlag = 1 << 1; + Record.push_back(N->isDistinct() | HasUnitFlag); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName())); @@ -1046,19 +1584,19 @@ static void WriteDISubprogram(const DISubprogram *N, const ValueEnumerator &VE, Record.push_back(N->getVirtualIndex()); Record.push_back(N->getFlags()); Record.push_back(N->isOptimized()); + Record.push_back(VE.getMetadataOrNullID(N->getRawUnit())); Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getDeclaration())); Record.push_back(VE.getMetadataOrNullID(N->getVariables().get())); + Record.push_back(N->getThisAdjustment()); Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev); Record.clear(); } -static void WriteDILexicalBlock(const DILexicalBlock *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1069,11 +1607,9 @@ static void WriteDILexicalBlock(const DILexicalBlock *N, Record.clear(); } -static void WriteDILexicalBlockFile(const DILexicalBlockFile *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILexicalBlockFile( + const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1083,10 +1619,9 @@ static void WriteDILexicalBlockFile(const DILexicalBlockFile *N, Record.clear(); } -static void WriteDINamespace(const DINamespace *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1097,9 +1632,9 @@ static void WriteDINamespace(const DINamespace *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIMacro(const DIMacro *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getMacinfoType()); Record.push_back(N->getLine()); @@ -1110,10 +1645,9 @@ static void WriteDIMacro(const DIMacro *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIMacroFile(const DIMacroFile *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getMacinfoType()); Record.push_back(N->getLine()); @@ -1124,9 +1658,9 @@ static void WriteDIMacroFile(const DIMacroFile *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIModule(const DIModule *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIModule(const DIModule *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); for (auto &I : N->operands()) Record.push_back(VE.getMetadataOrNullID(I)); @@ -1135,11 +1669,9 @@ static void WriteDIModule(const DIModule *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDITemplateTypeParameter(const DITemplateTypeParameter *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDITemplateTypeParameter( + const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getType())); @@ -1148,11 +1680,9 @@ static void WriteDITemplateTypeParameter(const DITemplateTypeParameter *N, Record.clear(); } -static void WriteDITemplateValueParameter(const DITemplateValueParameter *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDITemplateValueParameter( + const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1163,11 +1693,9 @@ static void WriteDITemplateValueParameter(const DITemplateValueParameter *N, Record.clear(); } -static void WriteDIGlobalVariable(const DIGlobalVariable *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIGlobalVariable( + const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1184,11 +1712,9 @@ static void WriteDIGlobalVariable(const DIGlobalVariable *N, Record.clear(); } -static void WriteDILocalVariable(const DILocalVariable *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILocalVariable( + const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1202,10 +1728,9 @@ static void WriteDILocalVariable(const DILocalVariable *N, Record.clear(); } -static void WriteDIExpression(const DIExpression *N, const ValueEnumerator &, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.reserve(N->getElements().size() + 1); Record.push_back(N->isDistinct()); @@ -1215,11 +1740,9 @@ static void WriteDIExpression(const DIExpression *N, const ValueEnumerator &, Record.clear(); } -static void WriteDIObjCProperty(const DIObjCProperty *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1233,11 +1756,9 @@ static void WriteDIObjCProperty(const DIObjCProperty *N, Record.clear(); } -static void WriteDIImportedEntity(const DIImportedEntity *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIImportedEntity( + const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); @@ -1249,71 +1770,87 @@ static void WriteDIImportedEntity(const DIImportedEntity *N, Record.clear(); } -static void WriteModuleMetadata(const Module *M, - const ValueEnumerator &VE, - BitstreamWriter &Stream) { - const auto &MDs = VE.getMDs(); - if (MDs.empty() && M->named_metadata_empty()) +unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() { + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); + return Stream.EmitAbbrev(Abbv); +} + +void ModuleBitcodeWriter::writeNamedMetadata( + SmallVectorImpl<uint64_t> &Record) { + if (M.named_metadata_empty()) return; - Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); + unsigned Abbrev = createNamedMetadataAbbrev(); + for (const NamedMDNode &NMD : M.named_metadata()) { + // Write name. + StringRef Str = NMD.getName(); + Record.append(Str.bytes_begin(), Str.bytes_end()); + Stream.EmitRecord(bitc::METADATA_NAME, Record, Abbrev); + Record.clear(); - unsigned MDSAbbrev = 0; - if (VE.hasMDString()) { - // Abbrev for METADATA_STRING. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRING)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - MDSAbbrev = Stream.EmitAbbrev(Abbv); + // Write named metadata operands. + for (const MDNode *N : NMD.operands()) + Record.push_back(VE.getMetadataID(N)); + Stream.EmitRecord(bitc::METADATA_NAMED_NODE, Record, 0); + Record.clear(); } +} - // Initialize MDNode abbreviations. -#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0; -#include "llvm/IR/Metadata.def" +unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() { + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRINGS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + return Stream.EmitAbbrev(Abbv); +} - if (VE.hasDILocation()) { - // Abbrev for METADATA_LOCATION. - // - // Assume the column is usually under 128, and always output the inlined-at - // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - DILocationAbbrev = Stream.EmitAbbrev(Abbv); - } +/// Write out a record for MDString. +/// +/// All the metadata strings in a metadata block are emitted in a single +/// record. The sizes and strings themselves are shoved into a blob. +void ModuleBitcodeWriter::writeMetadataStrings( + ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) { + if (Strings.empty()) + return; - if (VE.hasGenericDINode()) { - // Abbrev for METADATA_GENERIC_DEBUG. - // - // Assume the column is usually under 128, and always output the inlined-at - // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - GenericDINodeAbbrev = Stream.EmitAbbrev(Abbv); - } + // Start the record with the number of strings. + Record.push_back(bitc::METADATA_STRINGS); + Record.push_back(Strings.size()); - unsigned NameAbbrev = 0; - if (!M->named_metadata_empty()) { - // Abbrev for METADATA_NAME. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - NameAbbrev = Stream.EmitAbbrev(Abbv); + // Emit the sizes of the strings in the blob. + SmallString<256> Blob; + { + BitstreamWriter W(Blob); + for (const Metadata *MD : Strings) + W.EmitVBR(cast<MDString>(MD)->getLength(), 6); + W.FlushToWord(); } - SmallVector<uint64_t, 64> Record; + // Add the offset to the strings to the record. + Record.push_back(Blob.size()); + + // Add the strings to the blob. + for (const Metadata *MD : Strings) + Blob.append(cast<MDString>(MD)->getString()); + + // Emit the final record. + Stream.EmitRecordWithBlob(createMetadataStringsAbbrev(), Record, Blob); + Record.clear(); +} + +void ModuleBitcodeWriter::writeMetadataRecords( + ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record) { + if (MDs.empty()) + return; + + // Initialize MDNode abbreviations. +#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0; +#include "llvm/IR/Metadata.def" + for (const Metadata *MD : MDs) { if (const MDNode *N = dyn_cast<MDNode>(MD)) { assert(N->isResolved() && "Expected forward references to be resolved"); @@ -1323,82 +1860,79 @@ static void WriteModuleMetadata(const Module *M, llvm_unreachable("Invalid MDNode subclass"); #define HANDLE_MDNODE_LEAF(CLASS) \ case Metadata::CLASS##Kind: \ - Write##CLASS(cast<CLASS>(N), VE, Stream, Record, CLASS##Abbrev); \ + write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \ continue; #include "llvm/IR/Metadata.def" } } - if (const auto *MDC = dyn_cast<ConstantAsMetadata>(MD)) { - WriteValueAsMetadata(MDC, VE, Stream, Record); - continue; - } - const MDString *MDS = cast<MDString>(MD); - // Code: [strchar x N] - Record.append(MDS->bytes_begin(), MDS->bytes_end()); - - // Emit the finished record. - Stream.EmitRecord(bitc::METADATA_STRING, Record, MDSAbbrev); - Record.clear(); + writeValueAsMetadata(cast<ValueAsMetadata>(MD), Record); } +} - // Write named metadata. - for (const NamedMDNode &NMD : M->named_metadata()) { - // Write name. - StringRef Str = NMD.getName(); - Record.append(Str.bytes_begin(), Str.bytes_end()); - Stream.EmitRecord(bitc::METADATA_NAME, Record, NameAbbrev); - Record.clear(); +void ModuleBitcodeWriter::writeModuleMetadata() { + if (!VE.hasMDs() && M.named_metadata_empty()) + return; - // Write named metadata operands. - for (const MDNode *N : NMD.operands()) - Record.push_back(VE.getMetadataID(N)); - Stream.EmitRecord(bitc::METADATA_NAMED_NODE, Record, 0); - Record.clear(); - } + Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); + SmallVector<uint64_t, 64> Record; + writeMetadataStrings(VE.getMDStrings(), Record); + writeMetadataRecords(VE.getNonMDStrings(), Record); + writeNamedMetadata(Record); + + auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) { + SmallVector<uint64_t, 4> Record; + Record.push_back(VE.getValueID(&GO)); + pushGlobalMetadataAttachment(Record, GO); + Stream.EmitRecord(bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Record); + }; + for (const Function &F : M) + if (F.isDeclaration() && F.hasMetadata()) + AddDeclAttachedMetadata(F); + // FIXME: Only store metadata for declarations here, and move data for global + // variable definitions to a separate block (PR28134). + for (const GlobalVariable &GV : M.globals()) + if (GV.hasMetadata()) + AddDeclAttachedMetadata(GV); Stream.ExitBlock(); } -static void WriteFunctionLocalMetadata(const Function &F, - const ValueEnumerator &VE, - BitstreamWriter &Stream) { - bool StartedMetadataBlock = false; +void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) { + if (!VE.hasMDs()) + return; + + Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); SmallVector<uint64_t, 64> Record; - const SmallVectorImpl<const LocalAsMetadata *> &MDs = - VE.getFunctionLocalMDs(); - for (unsigned i = 0, e = MDs.size(); i != e; ++i) { - assert(MDs[i] && "Expected valid function-local metadata"); - if (!StartedMetadataBlock) { - Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); - StartedMetadataBlock = true; - } - WriteValueAsMetadata(MDs[i], VE, Stream, Record); - } + writeMetadataStrings(VE.getMDStrings(), Record); + writeMetadataRecords(VE.getNonMDStrings(), Record); + Stream.ExitBlock(); +} - if (StartedMetadataBlock) - Stream.ExitBlock(); +void ModuleBitcodeWriter::pushGlobalMetadataAttachment( + SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) { + // [n x [id, mdnode]] + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + GO.getAllMetadata(MDs); + for (const auto &I : MDs) { + Record.push_back(I.first); + Record.push_back(VE.getMetadataID(I.second)); + } } -static void WriteMetadataAttachment(const Function &F, - const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) { Stream.EnterSubblock(bitc::METADATA_ATTACHMENT_ID, 3); SmallVector<uint64_t, 64> Record; - // Write metadata attachments - // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]] - SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; - F.getAllMetadata(MDs); - if (!MDs.empty()) { - for (const auto &I : MDs) { - Record.push_back(I.first); - Record.push_back(VE.getMetadataID(I.second)); - } + if (F.hasMetadata()) { + pushGlobalMetadataAttachment(Record, F); Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0); Record.clear(); } + // Write metadata attachments + // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]] + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { MDs.clear(); @@ -1420,13 +1954,13 @@ static void WriteMetadataAttachment(const Function &F, Stream.ExitBlock(); } -static void WriteModuleMetadataStore(const Module *M, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeModuleMetadataKinds() { SmallVector<uint64_t, 64> Record; // Write metadata kinds // METADATA_KIND - [n x [id, name]] SmallVector<StringRef, 8> Names; - M->getMDKindNames(Names); + M.getMDKindNames(Names); if (Names.empty()) return; @@ -1444,7 +1978,7 @@ static void WriteModuleMetadataStore(const Module *M, BitstreamWriter &Stream) { Stream.ExitBlock(); } -static void WriteOperandBundleTags(const Module *M, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeOperandBundleTags() { // Write metadata kinds // // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG @@ -1452,7 +1986,7 @@ static void WriteOperandBundleTags(const Module *M, BitstreamWriter &Stream) { // OPERAND_BUNDLE_TAG - [strchr x N] SmallVector<StringRef, 8> Tags; - M->getOperandBundleTags(Tags); + M.getOperandBundleTags(Tags); if (Tags.empty()) return; @@ -1478,9 +2012,8 @@ static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) { Vals.push_back((-V << 1) | 1); } -static void WriteConstants(unsigned FirstVal, unsigned LastVal, - const ValueEnumerator &VE, - BitstreamWriter &Stream, bool isGlobal) { +void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, + bool isGlobal) { if (FirstVal == LastVal) return; Stream.EnterSubblock(bitc::CONSTANTS_BLOCK_ID, 4); @@ -1635,8 +2168,7 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, Record.push_back( CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue()); } - } else if (isa<ConstantArray>(C) || isa<ConstantStruct>(C) || - isa<ConstantVector>(C)) { + } else if (isa<ConstantAggregate>(C)) { Code = bitc::CST_CODE_AGGREGATE; for (const Value *Op : C->operands()) Record.push_back(VE.getValueID(Op)); @@ -1646,17 +2178,17 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, default: if (Instruction::isCast(CE->getOpcode())) { Code = bitc::CST_CODE_CE_CAST; - Record.push_back(GetEncodedCastOpcode(CE->getOpcode())); + Record.push_back(getEncodedCastOpcode(CE->getOpcode())); Record.push_back(VE.getTypeID(C->getOperand(0)->getType())); Record.push_back(VE.getValueID(C->getOperand(0))); AbbrevToUse = CONSTANTS_CE_CAST_Abbrev; } else { assert(CE->getNumOperands() == 2 && "Unknown constant expr!"); Code = bitc::CST_CODE_CE_BINOP; - Record.push_back(GetEncodedBinaryOpcode(CE->getOpcode())); + Record.push_back(getEncodedBinaryOpcode(CE->getOpcode())); Record.push_back(VE.getValueID(C->getOperand(0))); Record.push_back(VE.getValueID(C->getOperand(1))); - uint64_t Flags = GetOptimizationFlags(CE); + uint64_t Flags = getOptimizationFlags(CE); if (Flags != 0) Record.push_back(Flags); } @@ -1735,21 +2267,20 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, Stream.ExitBlock(); } -static void WriteModuleConstants(const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeModuleConstants() { const ValueEnumerator::ValueList &Vals = VE.getValues(); // Find the first constant to emit, which is the first non-globalvalue value. // We know globalvalues have been emitted by WriteModuleInfo. for (unsigned i = 0, e = Vals.size(); i != e; ++i) { if (!isa<GlobalValue>(Vals[i].first)) { - WriteConstants(i, Vals.size(), VE, Stream, true); + writeConstants(i, Vals.size(), true); return; } } } -/// PushValueAndType - The file has to encode both the value and type id for +/// pushValueAndType - The file has to encode both the value and type id for /// many values, because we need to know what type to create for forward /// references. However, most operands are not forward references, so this type /// field is not needed. @@ -1757,9 +2288,8 @@ static void WriteModuleConstants(const ValueEnumerator &VE, /// This function adds V's value ID to Vals. If the value ID is higher than the /// instruction ID, then it is a forward reference, and it also includes the /// type ID. The value ID that is written is encoded relative to the InstID. -static bool PushValueAndType(const Value *V, unsigned InstID, - SmallVectorImpl<unsigned> &Vals, - ValueEnumerator &VE) { +bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals) { unsigned ValID = VE.getValueID(V); // Make encoding relative to the InstID. Vals.push_back(InstID - ValID); @@ -1770,8 +2300,8 @@ static bool PushValueAndType(const Value *V, unsigned InstID, return false; } -static void WriteOperandBundles(BitstreamWriter &Stream, ImmutableCallSite CS, - unsigned InstID, ValueEnumerator &VE) { +void ModuleBitcodeWriter::writeOperandBundles(ImmutableCallSite CS, + unsigned InstID) { SmallVector<unsigned, 64> Record; LLVMContext &C = CS.getInstruction()->getContext(); @@ -1780,34 +2310,32 @@ static void WriteOperandBundles(BitstreamWriter &Stream, ImmutableCallSite CS, Record.push_back(C.getOperandBundleTagID(Bundle.getTagName())); for (auto &Input : Bundle.Inputs) - PushValueAndType(Input, InstID, Record, VE); + pushValueAndType(Input, InstID, Record); Stream.EmitRecord(bitc::FUNC_CODE_OPERAND_BUNDLE, Record); Record.clear(); } } -/// pushValue - Like PushValueAndType, but where the type of the value is +/// pushValue - Like pushValueAndType, but where the type of the value is /// omitted (perhaps it was already encoded in an earlier operand). -static void pushValue(const Value *V, unsigned InstID, - SmallVectorImpl<unsigned> &Vals, - ValueEnumerator &VE) { +void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals) { unsigned ValID = VE.getValueID(V); Vals.push_back(InstID - ValID); } -static void pushValueSigned(const Value *V, unsigned InstID, - SmallVectorImpl<uint64_t> &Vals, - ValueEnumerator &VE) { +void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID, + SmallVectorImpl<uint64_t> &Vals) { unsigned ValID = VE.getValueID(V); int64_t diff = ((int32_t)InstID - (int32_t)ValID); emitSignedInt64(Vals, diff); } /// WriteInstruction - Emit an instruction to the specified stream. -static void WriteInstruction(const Instruction &I, unsigned InstID, - ValueEnumerator &VE, BitstreamWriter &Stream, - SmallVectorImpl<unsigned> &Vals) { +void ModuleBitcodeWriter::writeInstruction(const Instruction &I, + unsigned InstID, + SmallVectorImpl<unsigned> &Vals) { unsigned Code = 0; unsigned AbbrevToUse = 0; VE.setInstructionID(&I); @@ -1815,18 +2343,18 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, default: if (Instruction::isCast(I.getOpcode())) { Code = bitc::FUNC_CODE_INST_CAST; - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) AbbrevToUse = FUNCTION_INST_CAST_ABBREV; Vals.push_back(VE.getTypeID(I.getType())); - Vals.push_back(GetEncodedCastOpcode(I.getOpcode())); + Vals.push_back(getEncodedCastOpcode(I.getOpcode())); } else { assert(isa<BinaryOperator>(I) && "Unknown instruction!"); Code = bitc::FUNC_CODE_INST_BINOP; - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) AbbrevToUse = FUNCTION_INST_BINOP_ABBREV; - pushValue(I.getOperand(1), InstID, Vals, VE); - Vals.push_back(GetEncodedBinaryOpcode(I.getOpcode())); - uint64_t Flags = GetOptimizationFlags(&I); + pushValue(I.getOperand(1), InstID, Vals); + Vals.push_back(getEncodedBinaryOpcode(I.getOpcode())); + uint64_t Flags = getOptimizationFlags(&I); if (Flags != 0) { if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV) AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV; @@ -1842,55 +2370,55 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(GEPInst.isInBounds()); Vals.push_back(VE.getTypeID(GEPInst.getSourceElementType())); for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) - PushValueAndType(I.getOperand(i), InstID, Vals, VE); + pushValueAndType(I.getOperand(i), InstID, Vals); break; } case Instruction::ExtractValue: { Code = bitc::FUNC_CODE_INST_EXTRACTVAL; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); const ExtractValueInst *EVI = cast<ExtractValueInst>(&I); Vals.append(EVI->idx_begin(), EVI->idx_end()); break; } case Instruction::InsertValue: { Code = bitc::FUNC_CODE_INST_INSERTVAL; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - PushValueAndType(I.getOperand(1), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValueAndType(I.getOperand(1), InstID, Vals); const InsertValueInst *IVI = cast<InsertValueInst>(&I); Vals.append(IVI->idx_begin(), IVI->idx_end()); break; } case Instruction::Select: Code = bitc::FUNC_CODE_INST_VSELECT; - PushValueAndType(I.getOperand(1), InstID, Vals, VE); - pushValue(I.getOperand(2), InstID, Vals, VE); - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(1), InstID, Vals); + pushValue(I.getOperand(2), InstID, Vals); + pushValueAndType(I.getOperand(0), InstID, Vals); break; case Instruction::ExtractElement: Code = bitc::FUNC_CODE_INST_EXTRACTELT; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - PushValueAndType(I.getOperand(1), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValueAndType(I.getOperand(1), InstID, Vals); break; case Instruction::InsertElement: Code = bitc::FUNC_CODE_INST_INSERTELT; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - pushValue(I.getOperand(1), InstID, Vals, VE); - PushValueAndType(I.getOperand(2), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValue(I.getOperand(1), InstID, Vals); + pushValueAndType(I.getOperand(2), InstID, Vals); break; case Instruction::ShuffleVector: Code = bitc::FUNC_CODE_INST_SHUFFLEVEC; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - pushValue(I.getOperand(1), InstID, Vals, VE); - pushValue(I.getOperand(2), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValue(I.getOperand(1), InstID, Vals); + pushValue(I.getOperand(2), InstID, Vals); break; case Instruction::ICmp: case Instruction::FCmp: { // compare returning Int1Ty or vector of Int1Ty Code = bitc::FUNC_CODE_INST_CMP2; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - pushValue(I.getOperand(1), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValue(I.getOperand(1), InstID, Vals); Vals.push_back(cast<CmpInst>(I).getPredicate()); - uint64_t Flags = GetOptimizationFlags(&I); + uint64_t Flags = getOptimizationFlags(&I); if (Flags != 0) Vals.push_back(Flags); break; @@ -1903,11 +2431,11 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, if (NumOperands == 0) AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV; else if (NumOperands == 1) { - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV; } else { for (unsigned i = 0, e = NumOperands; i != e; ++i) - PushValueAndType(I.getOperand(i), InstID, Vals, VE); + pushValueAndType(I.getOperand(i), InstID, Vals); } } break; @@ -1918,7 +2446,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(VE.getValueID(II.getSuccessor(0))); if (II.isConditional()) { Vals.push_back(VE.getValueID(II.getSuccessor(1))); - pushValue(II.getCondition(), InstID, Vals, VE); + pushValue(II.getCondition(), InstID, Vals); } } break; @@ -1927,7 +2455,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Code = bitc::FUNC_CODE_INST_SWITCH; const SwitchInst &SI = cast<SwitchInst>(I); Vals.push_back(VE.getTypeID(SI.getCondition()->getType())); - pushValue(SI.getCondition(), InstID, Vals, VE); + pushValue(SI.getCondition(), InstID, Vals); Vals.push_back(VE.getValueID(SI.getDefaultDest())); for (SwitchInst::ConstCaseIt Case : SI.cases()) { Vals.push_back(VE.getValueID(Case.getCaseValue())); @@ -1939,7 +2467,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Code = bitc::FUNC_CODE_INST_INDIRECTBR; Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); // Encode the address operand as relative, but not the basic blocks. - pushValue(I.getOperand(0), InstID, Vals, VE); + pushValue(I.getOperand(0), InstID, Vals); for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) Vals.push_back(VE.getValueID(I.getOperand(i))); break; @@ -1950,7 +2478,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, FunctionType *FTy = II->getFunctionType(); if (II->hasOperandBundles()) - WriteOperandBundles(Stream, II, InstID, VE); + writeOperandBundles(II, InstID); Code = bitc::FUNC_CODE_INST_INVOKE; @@ -1959,28 +2487,28 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(VE.getValueID(II->getNormalDest())); Vals.push_back(VE.getValueID(II->getUnwindDest())); Vals.push_back(VE.getTypeID(FTy)); - PushValueAndType(Callee, InstID, Vals, VE); + pushValueAndType(Callee, InstID, Vals); // Emit value #'s for the fixed parameters. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) - pushValue(I.getOperand(i), InstID, Vals, VE); // fixed param. + pushValue(I.getOperand(i), InstID, Vals); // fixed param. // Emit type/value pairs for varargs params. if (FTy->isVarArg()) { for (unsigned i = FTy->getNumParams(), e = I.getNumOperands()-3; i != e; ++i) - PushValueAndType(I.getOperand(i), InstID, Vals, VE); // vararg + pushValueAndType(I.getOperand(i), InstID, Vals); // vararg } break; } case Instruction::Resume: Code = bitc::FUNC_CODE_INST_RESUME; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); break; case Instruction::CleanupRet: { Code = bitc::FUNC_CODE_INST_CLEANUPRET; const auto &CRI = cast<CleanupReturnInst>(I); - pushValue(CRI.getCleanupPad(), InstID, Vals, VE); + pushValue(CRI.getCleanupPad(), InstID, Vals); if (CRI.hasUnwindDest()) Vals.push_back(VE.getValueID(CRI.getUnwindDest())); break; @@ -1988,7 +2516,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, case Instruction::CatchRet: { Code = bitc::FUNC_CODE_INST_CATCHRET; const auto &CRI = cast<CatchReturnInst>(I); - pushValue(CRI.getCatchPad(), InstID, Vals, VE); + pushValue(CRI.getCatchPad(), InstID, Vals); Vals.push_back(VE.getValueID(CRI.getSuccessor())); break; } @@ -1997,19 +2525,19 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, const auto &FuncletPad = cast<FuncletPadInst>(I); Code = isa<CatchPadInst>(FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD : bitc::FUNC_CODE_INST_CLEANUPPAD; - pushValue(FuncletPad.getParentPad(), InstID, Vals, VE); + pushValue(FuncletPad.getParentPad(), InstID, Vals); unsigned NumArgOperands = FuncletPad.getNumArgOperands(); Vals.push_back(NumArgOperands); for (unsigned Op = 0; Op != NumArgOperands; ++Op) - PushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals, VE); + pushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals); break; } case Instruction::CatchSwitch: { Code = bitc::FUNC_CODE_INST_CATCHSWITCH; const auto &CatchSwitch = cast<CatchSwitchInst>(I); - pushValue(CatchSwitch.getParentPad(), InstID, Vals, VE); + pushValue(CatchSwitch.getParentPad(), InstID, Vals); unsigned NumHandlers = CatchSwitch.getNumHandlers(); Vals.push_back(NumHandlers); @@ -2034,7 +2562,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, SmallVector<uint64_t, 128> Vals64; Vals64.push_back(VE.getTypeID(PN.getType())); for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) { - pushValueSigned(PN.getIncomingValue(i), InstID, Vals64, VE); + pushValueSigned(PN.getIncomingValue(i), InstID, Vals64); Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i))); } // Emit a Vals64 vector and exit. @@ -2054,7 +2582,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(LandingPadInst::Catch); else Vals.push_back(LandingPadInst::Filter); - PushValueAndType(LP.getClause(I), InstID, Vals, VE); + pushValueAndType(LP.getClause(I), InstID, Vals); } break; } @@ -2071,8 +2599,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, assert(AlignRecord < 1 << 5 && "alignment greater than 1 << 64"); AlignRecord |= AI.isUsedWithInAlloca() << 5; AlignRecord |= 1 << 6; - // Reserve bit 7 for SwiftError flag. - // AlignRecord |= AI.isSwiftError() << 7; + AlignRecord |= AI.isSwiftError() << 7; Vals.push_back(AlignRecord); break; } @@ -2080,18 +2607,18 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, case Instruction::Load: if (cast<LoadInst>(I).isAtomic()) { Code = bitc::FUNC_CODE_INST_LOADATOMIC; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); } else { Code = bitc::FUNC_CODE_INST_LOAD; - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) // ptr + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr AbbrevToUse = FUNCTION_INST_LOAD_ABBREV; } Vals.push_back(VE.getTypeID(I.getType())); Vals.push_back(Log2_32(cast<LoadInst>(I).getAlignment())+1); Vals.push_back(cast<LoadInst>(I).isVolatile()); if (cast<LoadInst>(I).isAtomic()) { - Vals.push_back(GetEncodedOrdering(cast<LoadInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope(cast<LoadInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering())); + Vals.push_back(getEncodedSynchScope(cast<LoadInst>(I).getSynchScope())); } break; case Instruction::Store: @@ -2099,57 +2626,57 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Code = bitc::FUNC_CODE_INST_STOREATOMIC; else Code = bitc::FUNC_CODE_INST_STORE; - PushValueAndType(I.getOperand(1), InstID, Vals, VE); // ptrty + ptr - PushValueAndType(I.getOperand(0), InstID, Vals, VE); // valty + val + pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr + pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1); Vals.push_back(cast<StoreInst>(I).isVolatile()); if (cast<StoreInst>(I).isAtomic()) { - Vals.push_back(GetEncodedOrdering(cast<StoreInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope(cast<StoreInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering())); + Vals.push_back(getEncodedSynchScope(cast<StoreInst>(I).getSynchScope())); } break; case Instruction::AtomicCmpXchg: Code = bitc::FUNC_CODE_INST_CMPXCHG; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); // ptrty + ptr - PushValueAndType(I.getOperand(1), InstID, Vals, VE); // cmp. - pushValue(I.getOperand(2), InstID, Vals, VE); // newval. + pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr + pushValueAndType(I.getOperand(1), InstID, Vals); // cmp. + pushValue(I.getOperand(2), InstID, Vals); // newval. Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile()); - Vals.push_back(GetEncodedOrdering( - cast<AtomicCmpXchgInst>(I).getSuccessOrdering())); - Vals.push_back(GetEncodedSynchScope( - cast<AtomicCmpXchgInst>(I).getSynchScope())); - Vals.push_back(GetEncodedOrdering( - cast<AtomicCmpXchgInst>(I).getFailureOrdering())); + Vals.push_back( + getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getSuccessOrdering())); + Vals.push_back( + getEncodedSynchScope(cast<AtomicCmpXchgInst>(I).getSynchScope())); + Vals.push_back( + getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getFailureOrdering())); Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak()); break; case Instruction::AtomicRMW: Code = bitc::FUNC_CODE_INST_ATOMICRMW; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); // ptrty + ptr - pushValue(I.getOperand(1), InstID, Vals, VE); // val. - Vals.push_back(GetEncodedRMWOperation( - cast<AtomicRMWInst>(I).getOperation())); + pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr + pushValue(I.getOperand(1), InstID, Vals); // val. + Vals.push_back( + getEncodedRMWOperation(cast<AtomicRMWInst>(I).getOperation())); Vals.push_back(cast<AtomicRMWInst>(I).isVolatile()); - Vals.push_back(GetEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope( - cast<AtomicRMWInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering())); + Vals.push_back( + getEncodedSynchScope(cast<AtomicRMWInst>(I).getSynchScope())); break; case Instruction::Fence: Code = bitc::FUNC_CODE_INST_FENCE; - Vals.push_back(GetEncodedOrdering(cast<FenceInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope(cast<FenceInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<FenceInst>(I).getOrdering())); + Vals.push_back(getEncodedSynchScope(cast<FenceInst>(I).getSynchScope())); break; case Instruction::Call: { const CallInst &CI = cast<CallInst>(I); FunctionType *FTy = CI.getFunctionType(); if (CI.hasOperandBundles()) - WriteOperandBundles(Stream, &CI, InstID, VE); + writeOperandBundles(&CI, InstID); Code = bitc::FUNC_CODE_INST_CALL; Vals.push_back(VE.getAttributeID(CI.getAttributes())); - unsigned Flags = GetOptimizationFlags(&I); + unsigned Flags = getOptimizationFlags(&I); Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV | unsigned(CI.isTailCall()) << bitc::CALL_TAIL | unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL | @@ -2160,7 +2687,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(Flags); Vals.push_back(VE.getTypeID(FTy)); - PushValueAndType(CI.getCalledValue(), InstID, Vals, VE); // Callee + pushValueAndType(CI.getCalledValue(), InstID, Vals); // Callee // Emit value #'s for the fixed parameters. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) { @@ -2168,21 +2695,21 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, if (FTy->getParamType(i)->isLabelTy()) Vals.push_back(VE.getValueID(CI.getArgOperand(i))); else - pushValue(CI.getArgOperand(i), InstID, Vals, VE); // fixed param. + pushValue(CI.getArgOperand(i), InstID, Vals); // fixed param. } // Emit type/value pairs for varargs params. if (FTy->isVarArg()) { for (unsigned i = FTy->getNumParams(), e = CI.getNumArgOperands(); i != e; ++i) - PushValueAndType(CI.getArgOperand(i), InstID, Vals, VE); // varargs + pushValueAndType(CI.getArgOperand(i), InstID, Vals); // varargs } break; } case Instruction::VAArg: Code = bitc::FUNC_CODE_INST_VAARG; Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); // valistty - pushValue(I.getOperand(0), InstID, Vals, VE); // valist. + pushValue(I.getOperand(0), InstID, Vals); // valist. Vals.push_back(VE.getTypeID(I.getType())); // restype. break; } @@ -2191,49 +2718,27 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.clear(); } -enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 }; - -/// Determine the encoding to use for the given string name and length. -static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { - bool isChar6 = true; - for (const char *C = Str, *E = C + StrLen; C != E; ++C) { - if (isChar6) - isChar6 = BitCodeAbbrevOp::isChar6(*C); - if ((unsigned char)*C & 128) - // don't bother scanning the rest. - return SE_Fixed8; - } - if (isChar6) - return SE_Char6; - else - return SE_Fixed7; -} - -/// Emit names for globals/functions etc. The VSTOffsetPlaceholder, -/// BitcodeStartBit and FunctionIndex are only passed for the module-level -/// VST, where we are including a function bitcode index and need to -/// backpatch the VST forward declaration record. -static void WriteValueSymbolTable( - const ValueSymbolTable &VST, const ValueEnumerator &VE, - BitstreamWriter &Stream, uint64_t VSTOffsetPlaceholder = 0, - uint64_t BitcodeStartBit = 0, - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> *FunctionIndex = - nullptr) { +/// Emit names for globals/functions etc. \p IsModuleLevel is true when +/// we are writing the module-level VST, where we are including a function +/// bitcode index and need to backpatch the VST forward declaration record. +void ModuleBitcodeWriter::writeValueSymbolTable( + const ValueSymbolTable &VST, bool IsModuleLevel, + DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex) { if (VST.empty()) { - // WriteValueSymbolTableForwardDecl should have returned early as + // writeValueSymbolTableForwardDecl should have returned early as // well. Ensure this handling remains in sync by asserting that // the placeholder offset is not set. - assert(VSTOffsetPlaceholder == 0); + assert(!IsModuleLevel || !hasVSTOffsetPlaceholder()); return; } - if (VSTOffsetPlaceholder > 0) { + if (IsModuleLevel && hasVSTOffsetPlaceholder()) { // Get the offset of the VST we are writing, and backpatch it into // the VST forward declaration record. uint64_t VSTOffset = Stream.GetCurrentBitNo(); // The BitcodeStartBit was the stream offset of the actual bitcode // (e.g. excluding any initial darwin header). - VSTOffset -= BitcodeStartBit; + VSTOffset -= bitcodeStartBit(); assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); } @@ -2245,8 +2750,9 @@ static void WriteValueSymbolTable( unsigned FnEntry8BitAbbrev; unsigned FnEntry7BitAbbrev; unsigned FnEntry6BitAbbrev; - if (VSTOffsetPlaceholder > 0) { - // 8-bit fixed-width VST_FNENTRY function strings. + unsigned GUIDEntryAbbrev; + if (IsModuleLevel && hasVSTOffsetPlaceholder()) { + // 8-bit fixed-width VST_CODE_FNENTRY function strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id @@ -2255,7 +2761,7 @@ static void WriteValueSymbolTable( Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); FnEntry8BitAbbrev = Stream.EmitAbbrev(Abbv); - // 7-bit fixed width VST_FNENTRY function strings. + // 7-bit fixed width VST_CODE_FNENTRY function strings. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id @@ -2264,7 +2770,7 @@ static void WriteValueSymbolTable( Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); FnEntry7BitAbbrev = Stream.EmitAbbrev(Abbv); - // 6-bit char6 VST_FNENTRY function strings. + // 6-bit char6 VST_CODE_FNENTRY function strings. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id @@ -2272,11 +2778,19 @@ static void WriteValueSymbolTable( Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); FnEntry6BitAbbrev = Stream.EmitAbbrev(Abbv); + + // FIXME: Change the name of this record as it is now used by + // the per-module index as well. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid + GUIDEntryAbbrev = Stream.EmitAbbrev(Abbv); } // FIXME: Set up the abbrev, we know how many values there are! // FIXME: We know if the type names can use 7-bit ascii. - SmallVector<unsigned, 64> NameVals; + SmallVector<uint64_t, 64> NameVals; for (const ValueName &Name : VST) { // Figure out the encoding to use for the name. @@ -2295,9 +2809,9 @@ static void WriteValueSymbolTable( F = dyn_cast<Function>(GA->getBaseObject()); } - // VST_ENTRY: [valueid, namechar x N] - // VST_FNENTRY: [valueid, funcoffset, namechar x N] - // VST_BBENTRY: [bbid, namechar x N] + // VST_CODE_ENTRY: [valueid, namechar x N] + // VST_CODE_FNENTRY: [valueid, funcoffset, namechar x N] + // VST_CODE_BBENTRY: [bbid, namechar x N] unsigned Code; if (isa<BasicBlock>(Name.getValue())) { Code = bitc::VST_CODE_BBENTRY; @@ -2307,14 +2821,12 @@ static void WriteValueSymbolTable( // Must be the module-level VST, where we pass in the Index and // have a VSTOffsetPlaceholder. The function-level VST should not // contain any Function symbols. - assert(FunctionIndex); - assert(VSTOffsetPlaceholder > 0); + assert(FunctionToBitcodeIndex); + assert(hasVSTOffsetPlaceholder()); // Save the word offset of the function (from the start of the // actual bitcode written to the stream). - assert(FunctionIndex->count(F) == 1); - uint64_t BitcodeIndex = - (*FunctionIndex)[F]->bitcodeIndex() - BitcodeStartBit; + uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - bitcodeStartBit(); assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); NameVals.push_back(BitcodeIndex / 32); @@ -2339,71 +2851,51 @@ static void WriteValueSymbolTable( Stream.EmitRecord(Code, NameVals, AbbrevToUse); NameVals.clear(); } + // Emit any GUID valueIDs created for indirect call edges into the + // module-level VST. + if (IsModuleLevel && hasVSTOffsetPlaceholder()) + for (const auto &GI : valueIds()) { + NameVals.push_back(GI.second); + NameVals.push_back(GI.first); + Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, + GUIDEntryAbbrev); + NameVals.clear(); + } Stream.ExitBlock(); } /// Emit function names and summary offsets for the combined index /// used by ThinLTO. -static void WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index, - BitstreamWriter &Stream) { +void IndexBitcodeWriter::writeCombinedValueSymbolTable() { + assert(hasVSTOffsetPlaceholder() && "Expected non-zero VSTOffsetPlaceholder"); + // Get the offset of the VST we are writing, and backpatch it into + // the VST forward declaration record. + uint64_t VSTOffset = Stream.GetCurrentBitNo(); + assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); - // 8-bit fixed-width VST_COMBINED_FNENTRY function strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - unsigned FnEntry8BitAbbrev = Stream.EmitAbbrev(Abbv); - - // 7-bit fixed width VST_COMBINED_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - unsigned FnEntry7BitAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid + unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv); - // 6-bit char6 VST_COMBINED_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - unsigned FnEntry6BitAbbrev = Stream.EmitAbbrev(Abbv); - - // FIXME: We know if the type names can use 7-bit ascii. - SmallVector<unsigned, 64> NameVals; + SmallVector<uint64_t, 64> NameVals; + for (const auto &GVI : valueIds()) { + // VST_CODE_COMBINED_ENTRY: [valueid, refguid] + NameVals.push_back(GVI.second); + NameVals.push_back(GVI.first); - for (const auto &FII : Index) { - for (const auto &FI : FII.getValue()) { - NameVals.push_back(FI->bitcodeIndex()); - - StringRef FuncName = FII.first(); - - // Figure out the encoding to use for the name. - StringEncoding Bits = getStringEncoding(FuncName.data(), FuncName.size()); - - // VST_COMBINED_FNENTRY: [funcsumoffset, namechar x N] - unsigned AbbrevToUse = FnEntry8BitAbbrev; - if (Bits == SE_Char6) - AbbrevToUse = FnEntry6BitAbbrev; - else if (Bits == SE_Fixed7) - AbbrevToUse = FnEntry7BitAbbrev; - - for (const auto P : FuncName) - NameVals.push_back((unsigned char)P); - - // Emit the finished record. - Stream.EmitRecord(bitc::VST_CODE_COMBINED_FNENTRY, NameVals, AbbrevToUse); - NameVals.clear(); - } + // Emit the finished record. + Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev); + NameVals.clear(); } Stream.ExitBlock(); } -static void WriteUseList(ValueEnumerator &VE, UseListOrder &&Order, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) { assert(Order.Shuffle.size() >= 2 && "Shuffle too small"); unsigned Code; if (isa<BasicBlock>(Order.V)) @@ -2416,8 +2908,7 @@ static void WriteUseList(ValueEnumerator &VE, UseListOrder &&Order, Stream.EmitRecord(Code, Record); } -static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeUseListBlock(const Function *F) { assert(VE.shouldPreserveUseListOrder() && "Expected to be preserving use-list order"); @@ -2430,39 +2921,19 @@ static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, Stream.EnterSubblock(bitc::USELIST_BLOCK_ID, 3); while (hasMore()) { - WriteUseList(VE, std::move(VE.UseListOrders.back()), Stream); + writeUseList(std::move(VE.UseListOrders.back())); VE.UseListOrders.pop_back(); } Stream.ExitBlock(); } -/// \brief Save information for the given function into the function index. -/// -/// At a minimum this saves the bitcode index of the function record that -/// was just written. However, if we are emitting function summary information, -/// for example for ThinLTO, then a \a FunctionSummary object is created -/// to hold the provided summary information. -static void SaveFunctionInfo( - const Function &F, - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex, - unsigned NumInsts, uint64_t BitcodeIndex, bool EmitFunctionSummary) { - std::unique_ptr<FunctionSummary> FuncSummary; - if (EmitFunctionSummary) { - FuncSummary = llvm::make_unique<FunctionSummary>(NumInsts); - FuncSummary->setLocalFunction(F.hasLocalLinkage()); - } - FunctionIndex[&F] = - llvm::make_unique<FunctionInfo>(BitcodeIndex, std::move(FuncSummary)); -} - /// Emit a function body to the module stream. -static void WriteFunction( - const Function &F, ValueEnumerator &VE, BitstreamWriter &Stream, - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex, - bool EmitFunctionSummary) { +void ModuleBitcodeWriter::writeFunction( + const Function &F, + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) { // Save the bitcode index of the start of this function block for recording // in the VST. - uint64_t BitcodeIndex = Stream.GetCurrentBitNo(); + FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo(); Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4); VE.incorporateFunction(F); @@ -2478,10 +2949,10 @@ static void WriteFunction( // If there are function-local constants, emit them now. unsigned CstStart, CstEnd; VE.getFunctionConstantRange(CstStart, CstEnd); - WriteConstants(CstStart, CstEnd, VE, Stream, false); + writeConstants(CstStart, CstEnd, false); // If there is function-local metadata, emit it now. - WriteFunctionLocalMetadata(F, VE, Stream); + writeFunctionMetadata(F); // Keep a running idea of what the instruction ID is. unsigned InstID = CstEnd; @@ -2489,16 +2960,11 @@ static void WriteFunction( bool NeedsMetadataAttachment = F.hasMetadata(); DILocation *LastDL = nullptr; - unsigned NumInsts = 0; - // Finally, emit all the instructions, in order. for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) { - WriteInstruction(*I, InstID, VE, Stream, Vals); - - if (!isa<DbgInfoIntrinsic>(I)) - ++NumInsts; + writeInstruction(*I, InstID, Vals); if (!I->getType()->isVoidTy()) ++InstID; @@ -2528,65 +2994,62 @@ static void WriteFunction( } // Emit names for all the instructions etc. - WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream); + writeValueSymbolTable(F.getValueSymbolTable()); if (NeedsMetadataAttachment) - WriteMetadataAttachment(F, VE, Stream); + writeFunctionMetadataAttachment(F); if (VE.shouldPreserveUseListOrder()) - WriteUseListBlock(&F, VE, Stream); + writeUseListBlock(&F); VE.purgeFunction(); Stream.ExitBlock(); - - SaveFunctionInfo(F, FunctionIndex, NumInsts, BitcodeIndex, - EmitFunctionSummary); } // Emit blockinfo, which defines the standard abbreviations etc. -static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeBlockInfo() { // We only want to emit block info records for blocks that have multiple // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK. // Other blocks can define their abbrevs inline. Stream.EnterBlockInfoBlock(2); - { // 8-bit fixed-width VST_ENTRY/VST_BBENTRY strings. + { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_ENTRY_8_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_ENTRY_8_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - { // 7-bit fixed width VST_ENTRY strings. + { // 7-bit fixed width VST_CODE_ENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_ENTRY_7_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_ENTRY_7_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - { // 6-bit char6 VST_ENTRY strings. + { // 6-bit char6 VST_CODE_ENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_ENTRY_6_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_ENTRY_6_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - { // 6-bit char6 VST_BBENTRY strings. + { // 6-bit char6 VST_CODE_BBENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_BBENTRY_6_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_BBENTRY_6_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2597,8 +3060,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, VE.computeBitsRequiredForTypeIndicies())); - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_SETTYPE_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_SETTYPE_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2606,8 +3069,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_INTEGER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_INTEGER_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_INTEGER_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2619,15 +3082,15 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { VE.computeBitsRequiredForTypeIndicies())); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_CE_CAST_Abbrev) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_CE_CAST_Abbrev) llvm_unreachable("Unexpected abbrev ordering!"); } { // NULL abbrev for CONSTANTS_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_NULL)); - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_NULL_Abbrev) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_NULL_Abbrev) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2641,8 +3104,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { VE.computeBitsRequiredForTypeIndicies())); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_LOAD_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_LOAD_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP abbrev for FUNCTION_BLOCK. @@ -2651,8 +3114,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_BINOP_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_BINOP_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK. @@ -2662,8 +3125,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); // flags - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_BINOP_FLAGS_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_BINOP_FLAGS_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_CAST abbrev for FUNCTION_BLOCK. @@ -2673,31 +3136,31 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty VE.computeBitsRequiredForTypeIndicies())); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_CAST_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_CAST_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_RET abbrev for FUNCTION_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_RET_VOID_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_RET_VOID_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_RET abbrev for FUNCTION_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_RET_VAL_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_RET_VAL_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE)); - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_UNREACHABLE_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_UNREACHABLE_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { @@ -2718,8 +3181,7 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { /// Write the module path strings, currently only used when generating /// a combined index file. -static void WriteModStrings(const FunctionInfoIndex &I, - BitstreamWriter &Stream) { +void IndexBitcodeWriter::writeModStrings() { Stream.EnterSubblock(bitc::MODULE_STRTAB_BLOCK_ID, 3); // TODO: See which abbrev sizes we actually need to emit @@ -2748,8 +3210,20 @@ static void WriteModStrings(const FunctionInfoIndex &I, Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv); - SmallVector<unsigned, 64> NameVals; - for (const StringMapEntry<uint64_t> &MPSE : I.modPathStringEntries()) { + // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + unsigned AbbrevHash = Stream.EmitAbbrev(Abbv); + + SmallVector<unsigned, 64> Vals; + for (const auto &MPSE : Index.modulePaths()) { + if (!doIncludeModule(MPSE.getKey())) + continue; StringEncoding Bits = getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size()); unsigned AbbrevToUse = Abbrev8Bit; @@ -2758,116 +3232,355 @@ static void WriteModStrings(const FunctionInfoIndex &I, else if (Bits == SE_Fixed7) AbbrevToUse = Abbrev7Bit; - NameVals.push_back(MPSE.getValue()); + Vals.push_back(MPSE.getValue().first); for (const auto P : MPSE.getKey()) - NameVals.push_back((unsigned char)P); + Vals.push_back((unsigned char)P); // Emit the finished record. - Stream.EmitRecord(bitc::MST_CODE_ENTRY, NameVals, AbbrevToUse); - NameVals.clear(); + Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse); + + Vals.clear(); + // Emit an optional hash for the module now + auto &Hash = MPSE.getValue().second; + bool AllZero = true; // Detect if the hash is empty, and do not generate it + for (auto Val : Hash) { + if (Val) + AllZero = false; + Vals.push_back(Val); + } + if (!AllZero) { + // Emit the hash record. + Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash); + } + + Vals.clear(); } Stream.ExitBlock(); } // Helper to emit a single function summary record. -static void WritePerModuleFunctionSummaryRecord( - SmallVector<unsigned, 64> &NameVals, FunctionSummary *FS, unsigned ValueID, - unsigned FSAbbrev, BitstreamWriter &Stream) { - assert(FS); +void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( + SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, + unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, + const Function &F) { NameVals.push_back(ValueID); - NameVals.push_back(FS->isLocalFunction()); + + FunctionSummary *FS = cast<FunctionSummary>(Summary); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + unsigned SizeBeforeRefs = NameVals.size(); + for (auto &RI : FS->refs()) + NameVals.push_back(VE.getValueID(RI.getValue())); + // Sort the refs for determinism output, the vector returned by FS->refs() has + // been initialized from a DenseSet. + std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); + + std::vector<FunctionSummary::EdgeTy> Calls = FS->calls(); + std::sort(Calls.begin(), Calls.end(), + [this](const FunctionSummary::EdgeTy &L, + const FunctionSummary::EdgeTy &R) { + return getValueId(L.first) < getValueId(R.first); + }); + bool HasProfileData = F.getEntryCount().hasValue(); + for (auto &ECI : Calls) { + NameVals.push_back(getValueId(ECI.first)); + assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(ECI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(ECI.second.ProfileCount); + } + + unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE); // Emit the finished record. - Stream.EmitRecord(bitc::FS_CODE_PERMODULE_ENTRY, NameVals, FSAbbrev); + Stream.EmitRecord(Code, NameVals, FSAbbrev); NameVals.clear(); } -/// Emit the per-module function summary section alongside the rest of +// Collect the global value references in the given variable's initializer, +// and emit them in a summary record. +void ModuleBitcodeWriter::writeModuleLevelReferences( + const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, + unsigned FSModRefsAbbrev) { + // Only interested in recording variable defs in the summary. + if (V.isDeclaration()) + return; + NameVals.push_back(VE.getValueID(&V)); + NameVals.push_back(getEncodedGVSummaryFlags(V)); + auto *Summary = Index->getGlobalValueSummary(V); + GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary); + + unsigned SizeBeforeRefs = NameVals.size(); + for (auto &RI : VS->refs()) + NameVals.push_back(VE.getValueID(RI.getValue())); + // Sort the refs for determinism output, the vector returned by FS->refs() has + // been initialized from a DenseSet. + std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); + + Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); + NameVals.clear(); +} + +// Current version for the summary. +// This is bumped whenever we introduce changes in the way some record are +// interpreted, like flags for instance. +static const uint64_t INDEX_VERSION = 1; + +/// Emit the per-module summary section alongside the rest of /// the module's bitcode. -static void WritePerModuleFunctionSummary( - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex, - const Module *M, const ValueEnumerator &VE, BitstreamWriter &Stream) { - Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3); +void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { + if (Index->begin() == Index->end()) + return; - // Abbrev for FS_CODE_PERMODULE_ENTRY. + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); + + Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + + // Abbrev for FS_PERMODULE. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_PERMODULE_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // islocal + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - unsigned FSAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_PERMODULE_PROFILE. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount, profilecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); - SmallVector<unsigned, 64> NameVals; - for (auto &I : FunctionIndex) { - // Skip anonymous functions. We will emit a function summary for - // any aliases below. - if (!I.first->hasName()) + // Abbrev for FS_ALIAS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + + SmallVector<uint64_t, 64> NameVals; + // Iterate over the list of functions instead of the Index to + // ensure the ordering is stable. + for (const Function &F : M) { + if (F.isDeclaration()) continue; + // Summary emission does not support anonymous functions, they have to + // renamed using the anonymous function renaming pass. + if (!F.hasName()) + report_fatal_error("Unexpected anonymous function when writing summary"); - WritePerModuleFunctionSummaryRecord( - NameVals, I.second->functionSummary(), - VE.getValueID(M->getValueSymbolTable().lookup(I.first->getName())), - FSAbbrev, Stream); + auto *Summary = Index->getGlobalValueSummary(F); + writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), + FSCallsAbbrev, FSCallsProfileAbbrev, F); } - for (const GlobalAlias &A : M->aliases()) { - if (!A.getBaseObject()) - continue; - const Function *F = dyn_cast<Function>(A.getBaseObject()); - if (!F || F->isDeclaration()) - continue; + // Capture references from GlobalVariable initializers, which are outside + // of a function scope. + for (const GlobalVariable &G : M.globals()) + writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev); - assert(FunctionIndex.count(F) == 1); - WritePerModuleFunctionSummaryRecord( - NameVals, FunctionIndex[F]->functionSummary(), - VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), FSAbbrev, - Stream); + for (const GlobalAlias &A : M.aliases()) { + auto *Aliasee = A.getBaseObject(); + if (!Aliasee->hasName()) + // Nameless function don't have an entry in the summary, skip it. + continue; + auto AliasId = VE.getValueID(&A); + auto AliaseeId = VE.getValueID(Aliasee); + NameVals.push_back(AliasId); + NameVals.push_back(getEncodedGVSummaryFlags(A)); + NameVals.push_back(AliaseeId); + Stream.EmitRecord(bitc::FS_ALIAS, NameVals, FSAliasAbbrev); + NameVals.clear(); } Stream.ExitBlock(); } -/// Emit the combined function summary section into the combined index -/// file. -static void WriteCombinedFunctionSummary(const FunctionInfoIndex &I, - BitstreamWriter &Stream) { - Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3); +/// Emit the combined summary section into the combined index file. +void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); + Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); - // Abbrev for FS_CODE_COMBINED_ENTRY. + // Abbrev for FS_COMBINED. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - unsigned FSAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_PROFILE. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount, profilecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_ALIAS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + + // The aliases are emitted as a post-pass, and will point to the value + // id of the aliasee. Save them in a vector for post-processing. + SmallVector<AliasSummary *, 64> Aliases; - SmallVector<unsigned, 64> NameVals; - for (const auto &FII : I) { - for (auto &FI : FII.getValue()) { - FunctionSummary *FS = FI->functionSummary(); - assert(FS); + // Save the value id for each summary for alias emission. + DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap; - NameVals.push_back(I.getModuleId(FS->modulePath())); - NameVals.push_back(FS->instCount()); + SmallVector<uint64_t, 64> NameVals; - // Record the starting offset of this summary entry for use - // in the VST entry. Add the current code size since the - // reader will invoke readRecord after the abbrev id read. - FI->setBitcodeIndex(Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth()); + // For local linkage, we also emit the original name separately + // immediately after the record. + auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) { + if (!GlobalValue::isLocalLinkage(S.linkage())) + return; + NameVals.push_back(S.getOriginalName()); + Stream.EmitRecord(bitc::FS_COMBINED_ORIGINAL_NAME, NameVals); + NameVals.clear(); + }; + + for (const auto &I : *this) { + GlobalValueSummary *S = I.second; + assert(S); + + assert(hasValueId(I.first)); + unsigned ValueId = getValueId(I.first); + SummaryToValueIdMap[S] = ValueId; + + if (auto *AS = dyn_cast<AliasSummary>(S)) { + // Will process aliases as a post-pass because the reader wants all + // global to be loaded first. + Aliases.push_back(AS); + continue; + } + + if (auto *VS = dyn_cast<GlobalVarSummary>(S)) { + NameVals.push_back(ValueId); + NameVals.push_back(Index.getModuleId(VS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); + for (auto &RI : VS->refs()) { + NameVals.push_back(getValueId(RI.getGUID())); + } // Emit the finished record. - Stream.EmitRecord(bitc::FS_CODE_COMBINED_ENTRY, NameVals, FSAbbrev); + Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); NameVals.clear(); + MaybeEmitOriginalName(*S); + continue; + } + + auto *FS = cast<FunctionSummary>(S); + NameVals.push_back(ValueId); + NameVals.push_back(Index.getModuleId(FS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); + NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + for (auto &RI : FS->refs()) { + NameVals.push_back(getValueId(RI.getGUID())); } + + bool HasProfileData = false; + for (auto &EI : FS->calls()) { + HasProfileData |= EI.second.ProfileCount != 0; + if (HasProfileData) + break; + } + + for (auto &EI : FS->calls()) { + // If this GUID doesn't have a value id, it doesn't have a function + // summary and we don't need to record any calls to it. + if (!hasValueId(EI.first.getGUID())) + continue; + NameVals.push_back(getValueId(EI.first.getGUID())); + assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(EI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(EI.second.ProfileCount); + } + + unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED); + + // Emit the finished record. + Stream.EmitRecord(Code, NameVals, FSAbbrev); + NameVals.clear(); + MaybeEmitOriginalName(*S); + } + + for (auto *AS : Aliases) { + auto AliasValueId = SummaryToValueIdMap[AS]; + assert(AliasValueId); + NameVals.push_back(AliasValueId); + NameVals.push_back(Index.getModuleId(AS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(AS->flags())); + auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()]; + assert(AliaseeValueId); + NameVals.push_back(AliaseeValueId); + + // Emit the finished record. + Stream.EmitRecord(bitc::FS_COMBINED_ALIAS, NameVals, FSAliasAbbrev); + NameVals.clear(); + MaybeEmitOriginalName(*AS); } Stream.ExitBlock(); } -// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the -// current llvm version, and a record for the epoch number. -static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeIdentificationBlock() { Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5); // Write the "user readable" string identifying the bitcode producer @@ -2876,8 +3589,8 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); auto StringAbbrev = Stream.EmitAbbrev(Abbv); - WriteStringRecord(bitc::IDENTIFICATION_CODE_STRING, - "LLVM" LLVM_VERSION_STRING, StringAbbrev, Stream); + writeStringRecord(bitc::IDENTIFICATION_CODE_STRING, + "LLVM" LLVM_VERSION_STRING, StringAbbrev); // Write the epoch version Abbv = new BitCodeAbbrev(); @@ -2889,71 +3602,114 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { Stream.ExitBlock(); } -/// WriteModule - Emit the specified module to the bitstream. -static void WriteModule(const Module *M, BitstreamWriter &Stream, - bool ShouldPreserveUseListOrder, - uint64_t BitcodeStartBit, bool EmitFunctionSummary) { +void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { + // Emit the module's hash. + // MODULE_CODE_HASH: [5*i32] + SHA1 Hasher; + Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], + Buffer.size() - BlockStartPos)); + auto Hash = Hasher.result(); + SmallVector<uint64_t, 20> Vals; + auto LShift = [&](unsigned char Val, unsigned Amount) + -> uint64_t { return ((uint64_t)Val) << Amount; }; + for (int Pos = 0; Pos < 20; Pos += 4) { + uint32_t SubHash = LShift(Hash[Pos + 0], 24); + SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) | + (unsigned)(unsigned char)Hash[Pos + 3]; + Vals.push_back(SubHash); + } + + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); +} + +void BitcodeWriter::write() { + // Emit the file header first. + writeBitcodeHeader(); + + writeBlocks(); +} + +void ModuleBitcodeWriter::writeBlocks() { + writeIdentificationBlock(); + writeModule(); +} + +void IndexBitcodeWriter::writeBlocks() { + // Index contains only a single outer (module) block. + writeIndex(); +} + +void ModuleBitcodeWriter::writeModule() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); + size_t BlockStartPos = Buffer.size(); SmallVector<unsigned, 1> Vals; unsigned CurVersion = 1; Vals.push_back(CurVersion); Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); - // Analyze the module, enumerating globals, functions, etc. - ValueEnumerator VE(*M, ShouldPreserveUseListOrder); - // Emit blockinfo, which defines the standard abbreviations etc. - WriteBlockInfo(VE, Stream); + writeBlockInfo(); // Emit information about attribute groups. - WriteAttributeGroupTable(VE, Stream); + writeAttributeGroupTable(); // Emit information about parameter attributes. - WriteAttributeTable(VE, Stream); + writeAttributeTable(); // Emit information describing all of the types in the module. - WriteTypeTable(VE, Stream); + writeTypeTable(); - writeComdats(VE, Stream); + writeComdats(); // Emit top-level description of module, including target triple, inline asm, // descriptors for global variables, and function prototype info. - uint64_t VSTOffsetPlaceholder = WriteModuleInfo(M, VE, Stream); + writeModuleInfo(); // Emit constants. - WriteModuleConstants(VE, Stream); + writeModuleConstants(); - // Emit metadata. - WriteModuleMetadata(M, VE, Stream); + // Emit metadata kind names. + writeModuleMetadataKinds(); // Emit metadata. - WriteModuleMetadataStore(M, Stream); + writeModuleMetadata(); // Emit module-level use-lists. if (VE.shouldPreserveUseListOrder()) - WriteUseListBlock(nullptr, VE, Stream); + writeUseListBlock(nullptr); - WriteOperandBundleTags(M, Stream); + writeOperandBundleTags(); // Emit function bodies. - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> FunctionIndex; - for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) + DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex; + for (Module::const_iterator F = M.begin(), E = M.end(); F != E; ++F) if (!F->isDeclaration()) - WriteFunction(*F, VE, Stream, FunctionIndex, EmitFunctionSummary); + writeFunction(*F, FunctionToBitcodeIndex); // Need to write after the above call to WriteFunction which populates // the summary information in the index. - if (EmitFunctionSummary) - WritePerModuleFunctionSummary(FunctionIndex, M, VE, Stream); + if (Index) + writePerModuleGlobalValueSummary(); - WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream, - VSTOffsetPlaceholder, BitcodeStartBit, &FunctionIndex); + writeValueSymbolTable(M.getValueSymbolTable(), + /* IsModuleLevel */ true, &FunctionToBitcodeIndex); + + if (GenerateHash) { + writeModuleHash(BlockStartPos); + } Stream.ExitBlock(); } -/// EmitDarwinBCHeader - If generating a bc file on darwin, we have to emit a +static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer, + uint32_t &Position) { + support::endian::write32le(&Buffer[Position], Value); + Position += 4; +} + +/// If generating a bc file on darwin, we have to emit a /// header and trailer to make it compatible with the system archiver. To do /// this we emit the following header, and then emit a trailer that pads the /// file out to be a multiple of 16 bytes. @@ -2966,18 +3722,7 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream, /// uint32_t CPUType; // CPU specifier. /// ... potentially more later ... /// }; -enum { - DarwinBCSizeFieldOffset = 3*4, // Offset to bitcode_size. - DarwinBCHeaderSize = 5*4 -}; - -static void WriteInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer, - uint32_t &Position) { - support::endian::write32le(&Buffer[Position], Value); - Position += 4; -} - -static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, +static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, const Triple &TT) { unsigned CPUType = ~0U; @@ -3005,18 +3750,18 @@ static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, CPUType = DARWIN_CPU_TYPE_ARM; // Traditional Bitcode starts after header. - assert(Buffer.size() >= DarwinBCHeaderSize && + assert(Buffer.size() >= BWH_HeaderSize && "Expected header size to be reserved"); - unsigned BCOffset = DarwinBCHeaderSize; - unsigned BCSize = Buffer.size()-DarwinBCHeaderSize; + unsigned BCOffset = BWH_HeaderSize; + unsigned BCSize = Buffer.size() - BWH_HeaderSize; // Write the magic and version. unsigned Position = 0; - WriteInt32ToBuffer(0x0B17C0DE , Buffer, Position); - WriteInt32ToBuffer(0 , Buffer, Position); // Version. - WriteInt32ToBuffer(BCOffset , Buffer, Position); - WriteInt32ToBuffer(BCSize , Buffer, Position); - WriteInt32ToBuffer(CPUType , Buffer, Position); + writeInt32ToBuffer(0x0B17C0DE, Buffer, Position); + writeInt32ToBuffer(0, Buffer, Position); // Version. + writeInt32ToBuffer(BCOffset, Buffer, Position); + writeInt32ToBuffer(BCSize, Buffer, Position); + writeInt32ToBuffer(CPUType, Buffer, Position); // If the file is not a multiple of 16 bytes, insert dummy padding. while (Buffer.size() & 15) @@ -3024,7 +3769,7 @@ static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, } /// Helper to write the header common to all bitcode files. -static void WriteBitcodeHeader(BitstreamWriter &Stream) { +void BitcodeWriter::writeBitcodeHeader() { // Emit the file header. Stream.Emit((unsigned)'B', 8); Stream.Emit((unsigned)'C', 8); @@ -3038,55 +3783,30 @@ static void WriteBitcodeHeader(BitstreamWriter &Stream) { /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, - bool EmitFunctionSummary) { + const ModuleSummaryIndex *Index, + bool GenerateHash) { SmallVector<char, 0> Buffer; Buffer.reserve(256*1024); // If this is darwin or another generic macho target, reserve space for the // header. Triple TT(M->getTargetTriple()); - if (TT.isOSDarwin()) - Buffer.insert(Buffer.begin(), DarwinBCHeaderSize, 0); + if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) + Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); // Emit the module into the buffer. - { - BitstreamWriter Stream(Buffer); - // Save the start bit of the actual bitcode, in case there is space - // saved at the start for the darwin header above. The reader stream - // will start at the bitcode, and we need the offset of the VST - // to line up. - uint64_t BitcodeStartBit = Stream.GetCurrentBitNo(); - - // Emit the file header. - WriteBitcodeHeader(Stream); - - WriteIdentificationBlock(M, Stream); - - // Emit the module. - WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit, - EmitFunctionSummary); - } + ModuleBitcodeWriter ModuleWriter(M, Buffer, ShouldPreserveUseListOrder, Index, + GenerateHash); + ModuleWriter.write(); - if (TT.isOSDarwin()) - EmitDarwinBCHeaderAndTrailer(Buffer, TT); + if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) + emitDarwinBCHeaderAndTrailer(Buffer, TT); // Write the generated bitstream to "Out". Out.write((char*)&Buffer.front(), Buffer.size()); } -// Write the specified function summary index to the given raw output stream, -// where it will be written in a new bitcode block. This is used when -// writing the combined index file for ThinLTO. -void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, - raw_ostream &Out) { - SmallVector<char, 0> Buffer; - Buffer.reserve(256 * 1024); - - BitstreamWriter Stream(Buffer); - - // Emit the bitcode header. - WriteBitcodeHeader(Stream); - +void IndexBitcodeWriter::writeIndex() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); SmallVector<unsigned, 1> Vals; @@ -3094,17 +3814,34 @@ void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, Vals.push_back(CurVersion); Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); + // If we have a VST, write the VSTOFFSET record placeholder. + writeValueSymbolTableForwardDecl(); + // Write the module paths in the combined index. - WriteModStrings(Index, Stream); + writeModStrings(); - // Write the function summary combined index records. - WriteCombinedFunctionSummary(Index, Stream); + // Write the summary combined index records. + writeCombinedGlobalValueSummary(); // Need a special VST writer for the combined index (we don't have a // real VST and real values when this is invoked). - WriteCombinedValueSymbolTable(Index, Stream); + writeCombinedValueSymbolTable(); Stream.ExitBlock(); +} + +// Write the specified module summary index to the given raw output stream, +// where it will be written in a new bitcode block. This is used when +// writing the combined index file for ThinLTO. When writing a subset of the +// index for a distributed backend, provide a \p ModuleToSummariesForIndex map. +void llvm::WriteIndexToFile( + const ModuleSummaryIndex &Index, raw_ostream &Out, + std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { + SmallVector<char, 0> Buffer; + Buffer.reserve(256 * 1024); + + IndexBitcodeWriter IndexWriter(Buffer, Index, ModuleToSummariesForIndex); + IndexWriter.write(); Out.write((char *)&Buffer.front(), Buffer.size()); } diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 24de99a..3e89ade 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -12,14 +12,19 @@ //===----------------------------------------------------------------------===// #include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" using namespace llvm; -PreservedAnalyses BitcodeWriterPass::run(Module &M) { - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, EmitFunctionSummary); +PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &) { + std::unique_ptr<ModuleSummaryIndex> Index; + if (EmitSummaryIndex) + Index = ModuleSummaryIndexBuilder(&M).takeIndex(); + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index.get(), + EmitModuleHash); return PreservedAnalyses::all(); } @@ -27,31 +32,52 @@ namespace { class WriteBitcodePass : public ModulePass { raw_ostream &OS; // raw_ostream to print on bool ShouldPreserveUseListOrder; - bool EmitFunctionSummary; + bool EmitSummaryIndex; + bool EmitModuleHash; public: static char ID; // Pass identification, replacement for typeid + WriteBitcodePass() : ModulePass(ID), OS(dbgs()) { + initializeWriteBitcodePassPass(*PassRegistry::getPassRegistry()); + } + explicit WriteBitcodePass(raw_ostream &o, bool ShouldPreserveUseListOrder, - bool EmitFunctionSummary) + bool EmitSummaryIndex, bool EmitModuleHash) : ModulePass(ID), OS(o), ShouldPreserveUseListOrder(ShouldPreserveUseListOrder), - EmitFunctionSummary(EmitFunctionSummary) {} + EmitSummaryIndex(EmitSummaryIndex), EmitModuleHash(EmitModuleHash) { + initializeWriteBitcodePassPass(*PassRegistry::getPassRegistry()); + } const char *getPassName() const override { return "Bitcode Writer"; } bool runOnModule(Module &M) override { - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, - EmitFunctionSummary); + const ModuleSummaryIndex *Index = + EmitSummaryIndex + ? &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex()) + : nullptr; + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, + EmitModuleHash); return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + if (EmitSummaryIndex) + AU.addRequired<ModuleSummaryIndexWrapperPass>(); + } }; } char WriteBitcodePass::ID = 0; +INITIALIZE_PASS_BEGIN(WriteBitcodePass, "write-bitcode", "Write Bitcode", false, + true) +INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass) +INITIALIZE_PASS_END(WriteBitcodePass, "write-bitcode", "Write Bitcode", false, + true) ModulePass *llvm::createBitcodeWriterPass(raw_ostream &Str, bool ShouldPreserveUseListOrder, - bool EmitFunctionSummary) { + bool EmitSummaryIndex, bool EmitModuleHash) { return new WriteBitcodePass(Str, ShouldPreserveUseListOrder, - EmitFunctionSummary); + EmitSummaryIndex, EmitModuleHash); } diff --git a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index e07563b..5d5bfab 100644 --- a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -86,6 +86,9 @@ static OrderMap orderModule(const Module &M) { for (const GlobalAlias &A : M.aliases()) if (!isa<GlobalValue>(A.getAliasee())) orderValue(A.getAliasee(), OM); + for (const GlobalIFunc &I : M.ifuncs()) + if (!isa<GlobalValue>(I.getResolver())) + orderValue(I.getResolver(), OM); for (const Function &F : M) { for (const Use &U : F.operands()) if (!isa<GlobalValue>(U.get())) @@ -105,6 +108,8 @@ static OrderMap orderModule(const Module &M) { orderValue(&F, OM); for (const GlobalAlias &A : M.aliases()) orderValue(&A, OM); + for (const GlobalIFunc &I : M.ifuncs()) + orderValue(&I, OM); for (const GlobalVariable &G : M.globals()) orderValue(&G, OM); OM.LastGlobalValueID = OM.size(); @@ -261,11 +266,15 @@ static UseListOrderStack predictUseListOrder(const Module &M) { predictValueUseListOrder(&F, nullptr, OM, Stack); for (const GlobalAlias &A : M.aliases()) predictValueUseListOrder(&A, nullptr, OM, Stack); + for (const GlobalIFunc &I : M.ifuncs()) + predictValueUseListOrder(&I, nullptr, OM, Stack); for (const GlobalVariable &G : M.globals()) if (G.hasInitializer()) predictValueUseListOrder(G.getInitializer(), nullptr, OM, Stack); for (const GlobalAlias &A : M.aliases()) predictValueUseListOrder(A.getAliasee(), nullptr, OM, Stack); + for (const GlobalIFunc &I : M.ifuncs()) + predictValueUseListOrder(I.getResolver(), nullptr, OM, Stack); for (const Function &F : M) { for (const Use &U : F.operands()) predictValueUseListOrder(U.get(), nullptr, OM, Stack); @@ -280,8 +289,7 @@ static bool isIntOrIntVectorValue(const std::pair<const Value*, unsigned> &V) { ValueEnumerator::ValueEnumerator(const Module &M, bool ShouldPreserveUseListOrder) - : HasMDString(false), HasDILocation(false), HasGenericDINode(false), - ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) { + : ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) { if (ShouldPreserveUseListOrder) UseListOrders = predictUseListOrder(M); @@ -299,6 +307,10 @@ ValueEnumerator::ValueEnumerator(const Module &M, for (const GlobalAlias &GA : M.aliases()) EnumerateValue(&GA); + // Enumerate the ifuncs. + for (const GlobalIFunc &GIF : M.ifuncs()) + EnumerateValue(&GIF); + // Remember what is the cutoff between globalvalue's and other constants. unsigned FirstConstant = Values.size(); @@ -311,6 +323,10 @@ ValueEnumerator::ValueEnumerator(const Module &M, for (const GlobalAlias &GA : M.aliases()) EnumerateValue(GA.getAliasee()); + // Enumerate the ifunc resolvers. + for (const GlobalIFunc &GIF : M.ifuncs()) + EnumerateValue(GIF.getResolver()); + // Enumerate any optional Function data. for (const Function &F : M) for (const Use &U : F.operands()) @@ -328,6 +344,15 @@ ValueEnumerator::ValueEnumerator(const Module &M, EnumerateNamedMetadata(M); SmallVector<std::pair<unsigned, MDNode *>, 8> MDs; + for (const GlobalVariable &GV : M.globals()) { + MDs.clear(); + GV.getAllMetadata(MDs); + for (const auto &I : MDs) + // FIXME: Pass GV to EnumerateMetadata and arrange for the bitcode writer + // to write metadata to the global variable's own metadata block + // (PR28134). + EnumerateMetadata(nullptr, I.second); + } // Enumerate types used by function bodies and argument lists. for (const Function &F : M) { @@ -335,9 +360,10 @@ ValueEnumerator::ValueEnumerator(const Module &M, EnumerateType(A.getType()); // Enumerate metadata attached to this function. + MDs.clear(); F.getAllMetadata(MDs); for (const auto &I : MDs) - EnumerateMetadata(I.second); + EnumerateMetadata(F.isDeclaration() ? nullptr : &F, I.second); for (const BasicBlock &BB : F) for (const Instruction &I : BB) { @@ -352,7 +378,7 @@ ValueEnumerator::ValueEnumerator(const Module &M, if (isa<LocalAsMetadata>(MD->getMetadata())) continue; - EnumerateMetadata(MD->getMetadata()); + EnumerateMetadata(&F, MD->getMetadata()); } EnumerateType(I.getType()); if (const CallInst *CI = dyn_cast<CallInst>(&I)) @@ -364,17 +390,21 @@ ValueEnumerator::ValueEnumerator(const Module &M, MDs.clear(); I.getAllMetadataOtherThanDebugLoc(MDs); for (unsigned i = 0, e = MDs.size(); i != e; ++i) - EnumerateMetadata(MDs[i].second); + EnumerateMetadata(&F, MDs[i].second); // Don't enumerate the location directly -- it has a special record // type -- but enumerate its operands. if (DILocation *L = I.getDebugLoc()) - EnumerateMDNodeOperands(L); + for (const Metadata *Op : L->operands()) + EnumerateMetadata(&F, Op); } } // Optimize constant ordering. OptimizeConstants(FirstConstant, Values.size()); + + // Organize metadata ordering. + organizeMetadata(); } unsigned ValueEnumerator::getInstructionID(const Instruction *Inst) const { @@ -402,7 +432,7 @@ unsigned ValueEnumerator::getValueID(const Value *V) const { return I->second-1; } -void ValueEnumerator::dump() const { +LLVM_DUMP_METHOD void ValueEnumerator::dump() const { print(dbgs(), ValueMap, "Default"); dbgs() << '\n'; print(dbgs(), MetadataMap, "MetaData"); @@ -445,8 +475,10 @@ void ValueEnumerator::print(raw_ostream &OS, const MetadataMapType &Map, OS << "Size: " << Map.size() << "\n"; for (auto I = Map.begin(), E = Map.end(); I != E; ++I) { const Metadata *MD = I->first; - OS << "Metadata: slot = " << I->second << "\n"; + OS << "Metadata: slot = " << I->second.ID << "\n"; + OS << "Metadata: function = " << I->second.F << "\n"; MD->print(OS); + OS << "\n"; } } @@ -472,8 +504,8 @@ void ValueEnumerator::OptimizeConstants(unsigned CstStart, unsigned CstEnd) { // Ensure that integer and vector of integer constants are at the start of the // constant pool. This is important so that GEP structure indices come before // gep constant exprs. - std::partition(Values.begin()+CstStart, Values.begin()+CstEnd, - isIntOrIntVectorValue); + std::stable_partition(Values.begin() + CstStart, Values.begin() + CstEnd, + isIntOrIntVectorValue); // Rebuild the modified portion of ValueMap. for (; CstStart != CstEnd; ++CstStart) @@ -498,65 +530,244 @@ void ValueEnumerator::EnumerateNamedMetadata(const Module &M) { void ValueEnumerator::EnumerateNamedMDNode(const NamedMDNode *MD) { for (unsigned i = 0, e = MD->getNumOperands(); i != e; ++i) - EnumerateMetadata(MD->getOperand(i)); + EnumerateMetadata(nullptr, MD->getOperand(i)); +} + +unsigned ValueEnumerator::getMetadataFunctionID(const Function *F) const { + return F ? getValueID(F) + 1 : 0; +} + +void ValueEnumerator::EnumerateMetadata(const Function *F, const Metadata *MD) { + EnumerateMetadata(getMetadataFunctionID(F), MD); +} + +void ValueEnumerator::EnumerateFunctionLocalMetadata( + const Function &F, const LocalAsMetadata *Local) { + EnumerateFunctionLocalMetadata(getMetadataFunctionID(&F), Local); +} + +void ValueEnumerator::dropFunctionFromMetadata( + MetadataMapType::value_type &FirstMD) { + SmallVector<const MDNode *, 64> Worklist; + auto push = [this, &Worklist](MetadataMapType::value_type &MD) { + auto &Entry = MD.second; + + // Nothing to do if this metadata isn't tagged. + if (!Entry.F) + return; + + // Drop the function tag. + Entry.F = 0; + + // If this is has an ID and is an MDNode, then its operands have entries as + // well. We need to drop the function from them too. + if (Entry.ID) + if (auto *N = dyn_cast<MDNode>(MD.first)) + Worklist.push_back(N); + }; + push(FirstMD); + while (!Worklist.empty()) + for (const Metadata *Op : Worklist.pop_back_val()->operands()) { + if (!Op) + continue; + auto MD = MetadataMap.find(Op); + if (MD != MetadataMap.end()) + push(*MD); + } } -/// EnumerateMDNodeOperands - Enumerate all non-function-local values -/// and types referenced by the given MDNode. -void ValueEnumerator::EnumerateMDNodeOperands(const MDNode *N) { - for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { - Metadata *MD = N->getOperand(i); - if (!MD) +void ValueEnumerator::EnumerateMetadata(unsigned F, const Metadata *MD) { + // It's vital for reader efficiency that uniqued subgraphs are done in + // post-order; it's expensive when their operands have forward references. + // If a distinct node is referenced from a uniqued node, it'll be delayed + // until the uniqued subgraph has been completely traversed. + SmallVector<const MDNode *, 32> DelayedDistinctNodes; + + // Start by enumerating MD, and then work through its transitive operands in + // post-order. This requires a depth-first search. + SmallVector<std::pair<const MDNode *, MDNode::op_iterator>, 32> Worklist; + if (const MDNode *N = enumerateMetadataImpl(F, MD)) + Worklist.push_back(std::make_pair(N, N->op_begin())); + + while (!Worklist.empty()) { + const MDNode *N = Worklist.back().first; + + // Enumerate operands until we hit a new node. We need to traverse these + // nodes' operands before visiting the rest of N's operands. + MDNode::op_iterator I = std::find_if( + Worklist.back().second, N->op_end(), + [&](const Metadata *MD) { return enumerateMetadataImpl(F, MD); }); + if (I != N->op_end()) { + auto *Op = cast<MDNode>(*I); + Worklist.back().second = ++I; + + // Delay traversing Op if it's a distinct node and N is uniqued. + if (Op->isDistinct() && !N->isDistinct()) + DelayedDistinctNodes.push_back(Op); + else + Worklist.push_back(std::make_pair(Op, Op->op_begin())); continue; - assert(!isa<LocalAsMetadata>(MD) && "MDNodes cannot be function-local"); - EnumerateMetadata(MD); + } + + // All the operands have been visited. Now assign an ID. + Worklist.pop_back(); + MDs.push_back(N); + MetadataMap[N].ID = MDs.size(); + + // Flush out any delayed distinct nodes; these are all the distinct nodes + // that are leaves in last uniqued subgraph. + if (Worklist.empty() || Worklist.back().first->isDistinct()) { + for (const MDNode *N : DelayedDistinctNodes) + Worklist.push_back(std::make_pair(N, N->op_begin())); + DelayedDistinctNodes.clear(); + } } } -void ValueEnumerator::EnumerateMetadata(const Metadata *MD) { +const MDNode *ValueEnumerator::enumerateMetadataImpl(unsigned F, const Metadata *MD) { + if (!MD) + return nullptr; + assert( (isa<MDNode>(MD) || isa<MDString>(MD) || isa<ConstantAsMetadata>(MD)) && "Invalid metadata kind"); - // Insert a dummy ID to block the co-recursive call to - // EnumerateMDNodeOperands() from re-visiting MD in a cyclic graph. - // - // Return early if there's already an ID. - if (!MetadataMap.insert(std::make_pair(MD, 0)).second) - return; + auto Insertion = MetadataMap.insert(std::make_pair(MD, MDIndex(F))); + MDIndex &Entry = Insertion.first->second; + if (!Insertion.second) { + // Already mapped. If F doesn't match the function tag, drop it. + if (Entry.hasDifferentFunction(F)) + dropFunctionFromMetadata(*Insertion.first); + return nullptr; + } - // Visit operands first to minimize RAUW. + // Don't assign IDs to metadata nodes. if (auto *N = dyn_cast<MDNode>(MD)) - EnumerateMDNodeOperands(N); - else if (auto *C = dyn_cast<ConstantAsMetadata>(MD)) - EnumerateValue(C->getValue()); + return N; - HasMDString |= isa<MDString>(MD); - HasDILocation |= isa<DILocation>(MD); - HasGenericDINode |= isa<GenericDINode>(MD); - - // Replace the dummy ID inserted above with the correct one. MetadataMap may - // have changed by inserting operands, so we need a fresh lookup here. + // Save the metadata. MDs.push_back(MD); - MetadataMap[MD] = MDs.size(); + Entry.ID = MDs.size(); + + // Enumerate the constant, if any. + if (auto *C = dyn_cast<ConstantAsMetadata>(MD)) + EnumerateValue(C->getValue()); + + return nullptr; } /// EnumerateFunctionLocalMetadataa - Incorporate function-local metadata /// information reachable from the metadata. void ValueEnumerator::EnumerateFunctionLocalMetadata( - const LocalAsMetadata *Local) { + unsigned F, const LocalAsMetadata *Local) { + assert(F && "Expected a function"); + // Check to see if it's already in! - unsigned &MetadataID = MetadataMap[Local]; - if (MetadataID) + MDIndex &Index = MetadataMap[Local]; + if (Index.ID) { + assert(Index.F == F && "Expected the same function"); return; + } MDs.push_back(Local); - MetadataID = MDs.size(); + Index.F = F; + Index.ID = MDs.size(); EnumerateValue(Local->getValue()); +} + +static unsigned getMetadataTypeOrder(const Metadata *MD) { + // Strings are emitted in bulk and must come first. + if (isa<MDString>(MD)) + return 0; + + // ConstantAsMetadata doesn't reference anything. We may as well shuffle it + // to the front since we can detect it. + auto *N = dyn_cast<MDNode>(MD); + if (!N) + return 1; + + // The reader is fast forward references for distinct node operands, but slow + // when uniqued operands are unresolved. + return N->isDistinct() ? 2 : 3; +} + +void ValueEnumerator::organizeMetadata() { + assert(MetadataMap.size() == MDs.size() && + "Metadata map and vector out of sync"); + + if (MDs.empty()) + return; + + // Copy out the index information from MetadataMap in order to choose a new + // order. + SmallVector<MDIndex, 64> Order; + Order.reserve(MetadataMap.size()); + for (const Metadata *MD : MDs) + Order.push_back(MetadataMap.lookup(MD)); + + // Partition: + // - by function, then + // - by isa<MDString> + // and then sort by the original/current ID. Since the IDs are guaranteed to + // be unique, the result of std::sort will be deterministic. There's no need + // for std::stable_sort. + std::sort(Order.begin(), Order.end(), [this](MDIndex LHS, MDIndex RHS) { + return std::make_tuple(LHS.F, getMetadataTypeOrder(LHS.get(MDs)), LHS.ID) < + std::make_tuple(RHS.F, getMetadataTypeOrder(RHS.get(MDs)), RHS.ID); + }); + + // Rebuild MDs, index the metadata ranges for each function in FunctionMDs, + // and fix up MetadataMap. + std::vector<const Metadata *> OldMDs = std::move(MDs); + MDs.reserve(OldMDs.size()); + for (unsigned I = 0, E = Order.size(); I != E && !Order[I].F; ++I) { + auto *MD = Order[I].get(OldMDs); + MDs.push_back(MD); + MetadataMap[MD].ID = I + 1; + if (isa<MDString>(MD)) + ++NumMDStrings; + } - // Also, collect all function-local metadata for easy access. - FunctionLocalMDs.push_back(Local); + // Return early if there's nothing for the functions. + if (MDs.size() == Order.size()) + return; + + // Build the function metadata ranges. + MDRange R; + FunctionMDs.reserve(OldMDs.size()); + unsigned PrevF = 0; + for (unsigned I = MDs.size(), E = Order.size(), ID = MDs.size(); I != E; + ++I) { + unsigned F = Order[I].F; + if (!PrevF) { + PrevF = F; + } else if (PrevF != F) { + R.Last = FunctionMDs.size(); + std::swap(R, FunctionMDInfo[PrevF]); + R.First = FunctionMDs.size(); + + ID = MDs.size(); + PrevF = F; + } + + auto *MD = Order[I].get(OldMDs); + FunctionMDs.push_back(MD); + MetadataMap[MD].ID = ++ID; + if (isa<MDString>(MD)) + ++R.NumStrings; + } + R.Last = FunctionMDs.size(); + FunctionMDInfo[PrevF] = R; +} + +void ValueEnumerator::incorporateFunctionMetadata(const Function &F) { + NumModuleMDs = MDs.size(); + + auto R = FunctionMDInfo.lookup(getValueID(&F) + 1); + NumMDStrings = R.NumStrings; + MDs.insert(MDs.end(), FunctionMDs.begin() + R.First, + FunctionMDs.begin() + R.Last); } void ValueEnumerator::EnumerateValue(const Value *V) { @@ -650,13 +861,7 @@ void ValueEnumerator::EnumerateType(Type *Ty) { void ValueEnumerator::EnumerateOperandType(const Value *V) { EnumerateType(V->getType()); - if (auto *MD = dyn_cast<MetadataAsValue>(V)) { - assert(!isa<LocalAsMetadata>(MD->getMetadata()) && - "Function-local metadata should be left for later"); - - EnumerateMetadata(MD->getMetadata()); - return; - } + assert(!isa<MetadataAsValue>(V) && "Unexpected metadata operand"); const Constant *C = dyn_cast<Constant>(V); if (!C) @@ -704,7 +909,10 @@ void ValueEnumerator::EnumerateAttributes(AttributeSet PAL) { void ValueEnumerator::incorporateFunction(const Function &F) { InstructionCount = 0; NumModuleValues = Values.size(); - NumModuleMDs = MDs.size(); + + // Add global metadata to the function block. This doesn't include + // LocalAsMetadata. + incorporateFunctionMetadata(F); // Adding function arguments to the value table. for (const auto &I : F.args()) @@ -749,8 +957,13 @@ void ValueEnumerator::incorporateFunction(const Function &F) { } // Add all of the function-local metadata. - for (unsigned i = 0, e = FnLocalMDVector.size(); i != e; ++i) - EnumerateFunctionLocalMetadata(FnLocalMDVector[i]); + for (unsigned i = 0, e = FnLocalMDVector.size(); i != e; ++i) { + // At this point, every local values have been incorporated, we shouldn't + // have a metadata operand that references a value that hasn't been seen. + assert(ValueMap.count(FnLocalMDVector[i]->getValue()) && + "Missing value for metadata operand"); + EnumerateFunctionLocalMetadata(F, FnLocalMDVector[i]); + } } void ValueEnumerator::purgeFunction() { @@ -765,7 +978,7 @@ void ValueEnumerator::purgeFunction() { Values.resize(NumModuleValues); MDs.resize(NumModuleMDs); BasicBlocks.clear(); - FunctionLocalMDs.clear(); + NumMDStrings = 0; } static void IncorporateFunctionInfoGlobalBBIDs(const Function *F, diff --git a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h index 9fb8325..a8d6cf9 100644 --- a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h +++ b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h @@ -15,9 +15,10 @@ #define LLVM_LIB_BITCODE_WRITER_VALUEENUMERATOR_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/UniqueVector.h" #include "llvm/IR/Attributes.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Type.h" #include "llvm/IR/UseListOrder.h" #include <vector> @@ -33,6 +34,7 @@ class Module; class Metadata; class LocalAsMetadata; class MDNode; +class MDOperand; class NamedMDNode; class AttributeSet; class ValueSymbolTable; @@ -61,12 +63,43 @@ private: ComdatSetType Comdats; std::vector<const Metadata *> MDs; - SmallVector<const LocalAsMetadata *, 8> FunctionLocalMDs; - typedef DenseMap<const Metadata *, unsigned> MetadataMapType; + std::vector<const Metadata *> FunctionMDs; + + /// Index of information about a piece of metadata. + struct MDIndex { + unsigned F = 0; ///< The ID of the function for this metadata, if any. + unsigned ID = 0; ///< The implicit ID of this metadata in bitcode. + + MDIndex() = default; + explicit MDIndex(unsigned F) : F(F) {} + + /// Check if this has a function tag, and it's different from NewF. + bool hasDifferentFunction(unsigned NewF) const { return F && F != NewF; } + + /// Fetch the MD this references out of the given metadata array. + const Metadata *get(ArrayRef<const Metadata *> MDs) const { + assert(ID && "Expected non-zero ID"); + assert(ID <= MDs.size() && "Expected valid ID"); + return MDs[ID - 1]; + } + }; + + typedef DenseMap<const Metadata *, MDIndex> MetadataMapType; MetadataMapType MetadataMap; - bool HasMDString; - bool HasDILocation; - bool HasGenericDINode; + + /// Range of metadata IDs, as a half-open range. + struct MDRange { + unsigned First = 0; + unsigned Last = 0; + + /// Number of strings in the prefix of the metadata range. + unsigned NumStrings = 0; + + MDRange() {} + explicit MDRange(unsigned First) : First(First) {} + }; + SmallDenseMap<unsigned, MDRange, 1> FunctionMDInfo; + bool ShouldPreserveUseListOrder; typedef DenseMap<AttributeSet, unsigned> AttributeGroupMapType; @@ -95,7 +128,8 @@ private: /// When a function is incorporated, this is the size of the Metadatas list /// before incorporation. - unsigned NumModuleMDs; + unsigned NumModuleMDs = 0; + unsigned NumMDStrings = 0; unsigned FirstFuncConstantID; unsigned FirstInstID; @@ -117,14 +151,10 @@ public: return ID - 1; } unsigned getMetadataOrNullID(const Metadata *MD) const { - return MetadataMap.lookup(MD); + return MetadataMap.lookup(MD).ID; } unsigned numMDs() const { return MDs.size(); } - bool hasMDString() const { return HasMDString; } - bool hasDILocation() const { return HasDILocation; } - bool hasGenericDINode() const { return HasGenericDINode; } - bool shouldPreserveUseListOrder() const { return ShouldPreserveUseListOrder; } unsigned getTypeID(Type *T) const { @@ -158,10 +188,20 @@ public: } const ValueList &getValues() const { return Values; } - const std::vector<const Metadata *> &getMDs() const { return MDs; } - const SmallVectorImpl<const LocalAsMetadata *> &getFunctionLocalMDs() const { - return FunctionLocalMDs; + + /// Check whether the current block has any metadata to emit. + bool hasMDs() const { return NumModuleMDs < MDs.size(); } + + /// Get the MDString metadata for this block. + ArrayRef<const Metadata *> getMDStrings() const { + return makeArrayRef(MDs).slice(NumModuleMDs, NumMDStrings); + } + + /// Get the non-MDString metadata for this block. + ArrayRef<const Metadata *> getNonMDStrings() const { + return makeArrayRef(MDs).slice(NumModuleMDs).slice(NumMDStrings); } + const TypeList &getTypes() const { return Types; } const std::vector<const BasicBlock*> &getBasicBlocks() const { return BasicBlocks; @@ -191,9 +231,54 @@ public: private: void OptimizeConstants(unsigned CstStart, unsigned CstEnd); - void EnumerateMDNodeOperands(const MDNode *N); - void EnumerateMetadata(const Metadata *MD); - void EnumerateFunctionLocalMetadata(const LocalAsMetadata *Local); + /// Reorder the reachable metadata. + /// + /// This is not just an optimization, but is mandatory for emitting MDString + /// correctly. + void organizeMetadata(); + + /// Drop the function tag from the transitive operands of the given node. + void dropFunctionFromMetadata(MetadataMapType::value_type &FirstMD); + + /// Incorporate the function metadata. + /// + /// This should be called before enumerating LocalAsMetadata for the + /// function. + void incorporateFunctionMetadata(const Function &F); + + /// Enumerate a single instance of metadata with the given function tag. + /// + /// If \c MD has already been enumerated, check that \c F matches its + /// function tag. If not, call \a dropFunctionFromMetadata(). + /// + /// Otherwise, mark \c MD as visited. Assign it an ID, or just return it if + /// it's an \a MDNode. + const MDNode *enumerateMetadataImpl(unsigned F, const Metadata *MD); + + unsigned getMetadataFunctionID(const Function *F) const; + + /// Enumerate reachable metadata in (almost) post-order. + /// + /// Enumerate all the metadata reachable from MD. We want to minimize the + /// cost of reading bitcode records, and so the primary consideration is that + /// operands of uniqued nodes are resolved before the nodes are read. This + /// avoids re-uniquing them on the context and factors away RAUW support. + /// + /// This algorithm guarantees that subgraphs of uniqued nodes are in + /// post-order. Distinct subgraphs reachable only from a single uniqued node + /// will be in post-order. + /// + /// \note The relative order of a distinct and uniqued node is irrelevant. + /// \a organizeMetadata() will later partition distinct nodes ahead of + /// uniqued ones. + ///{ + void EnumerateMetadata(const Function *F, const Metadata *MD); + void EnumerateMetadata(unsigned F, const Metadata *MD); + ///} + + void EnumerateFunctionLocalMetadata(const Function &F, + const LocalAsMetadata *Local); + void EnumerateFunctionLocalMetadata(unsigned F, const LocalAsMetadata *Local); void EnumerateNamedMDNode(const NamedMDNode *NMD); void EnumerateValue(const Value *V); void EnumerateType(Type *T); diff --git a/contrib/llvm/lib/Bitcode/module.modulemap b/contrib/llvm/lib/Bitcode/module.modulemap deleted file mode 100644 index 7df1a0a..0000000 --- a/contrib/llvm/lib/Bitcode/module.modulemap +++ /dev/null @@ -1 +0,0 @@ -module Bitcode { requires cplusplus umbrella "." module * { export * } } |
