diff options
Diffstat (limited to 'contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp')
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 636 |
1 files changed, 375 insertions, 261 deletions
diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index dcb8b58..ebb2022 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Bitcode/BitcodeWriter.h" #include "ValueEnumerator.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" -#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -38,6 +38,11 @@ using namespace llvm; namespace { + +cl::opt<unsigned> + IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(25), + cl::desc("Number of metadatas above which we emit an index " + "to enable lazy-loading")); /// These are manifest constants used by the bitcode writer. They do not need to /// be kept in sync with the reader, but need to be consistent within this file. enum { @@ -65,36 +70,20 @@ enum { }; /// Abstract class to manage the bitcode writing, subclassed for each bitcode -/// file type. Owns the BitstreamWriter, and includes the main entry point for -/// writing. -class BitcodeWriter { +/// file type. +class BitcodeWriterBase { protected: - /// Pointer to the buffer allocated by caller for bitcode writing. - const SmallVectorImpl<char> &Buffer; - - /// The stream created and owned by the BitodeWriter. - BitstreamWriter Stream; + /// The stream created and owned by the client. + BitstreamWriter &Stream; /// Saves the offset of the VSTOffset record that must eventually be /// backpatched with the offset of the actual VST. uint64_t VSTOffsetPlaceholder = 0; public: - /// Constructs a BitcodeWriter object, and initializes a BitstreamRecord, - /// writing to the provided \p Buffer. - BitcodeWriter(SmallVectorImpl<char> &Buffer) - : Buffer(Buffer), Stream(Buffer) {} - - virtual ~BitcodeWriter() = default; - - /// Main entry point to write the bitcode file, which writes the bitcode - /// header and will then invoke the virtual writeBlocks() method. - void write(); - -private: - /// Derived classes must implement this to write the corresponding blocks for - /// that bitcode file type. - virtual void writeBlocks() = 0; + /// Constructs a BitcodeWriterBase object that writes to the provided + /// \p Stream. + BitcodeWriterBase(BitstreamWriter &Stream) : Stream(Stream) {} protected: bool hasVSTOffsetPlaceholder() { return VSTOffsetPlaceholder != 0; } @@ -103,7 +92,10 @@ protected: }; /// Class to manage the bitcode writing for a module. -class ModuleBitcodeWriter : public BitcodeWriter { +class ModuleBitcodeWriter : public BitcodeWriterBase { + /// Pointer to the buffer allocated by caller for bitcode writing. + const SmallVectorImpl<char> &Buffer; + /// The Module to write to bitcode. const Module &M; @@ -116,8 +108,8 @@ class ModuleBitcodeWriter : public BitcodeWriter { /// True if a module hash record should be written. bool GenerateHash; - /// The start bit of the module block, for use in generating a module hash - uint64_t BitcodeStartBit = 0; + /// The start bit of the identification block. + uint64_t BitcodeStartBit; /// Map that holds the correspondence between GUIDs in the summary index, /// that came from indirect call profiles, and a value id generated by this @@ -131,51 +123,38 @@ public: /// Constructs a ModuleBitcodeWriter object for the given Module, /// writing to the provided \p Buffer. ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, - bool ShouldPreserveUseListOrder, + BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, bool GenerateHash) - : BitcodeWriter(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), - Index(Index), GenerateHash(GenerateHash) { - // Save the start bit of the actual bitcode, in case there is space - // saved at the start for the darwin header above. The reader stream - // will start at the bitcode, and we need the offset of the VST - // to line up. - BitcodeStartBit = Stream.GetCurrentBitNo(); - + : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M), + VE(*M, ShouldPreserveUseListOrder), Index(Index), + GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). // The starting ValueId is just after the number of values in the // ValueEnumerator, so that they can be emitted in the VST. GlobalValueId = VE.getValues().size(); - if (Index) - for (const auto &GUIDSummaryLists : *Index) - // Examine all summaries for this GUID. - for (auto &Summary : GUIDSummaryLists.second) - if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) - // For each call in the function summary, see if the call - // is to a GUID (which means it is for an indirect call, - // otherwise we would have a Value for it). If so, synthesize - // a value id. - for (auto &CallEdge : FS->calls()) - if (CallEdge.first.isGUID()) - assignValueId(CallEdge.first.getGUID()); + if (!Index) + return; + for (const auto &GUIDSummaryLists : *Index) + // Examine all summaries for this GUID. + for (auto &Summary : GUIDSummaryLists.second) + if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) + // For each call in the function summary, see if the call + // is to a GUID (which means it is for an indirect call, + // otherwise we would have a Value for it). If so, synthesize + // a value id. + for (auto &CallEdge : FS->calls()) + if (CallEdge.first.isGUID()) + assignValueId(CallEdge.first.getGUID()); } -private: - /// Main entry point for writing a module to bitcode, invoked by - /// BitcodeWriter::write() after it writes the header. - void writeBlocks() override; - - /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the - /// current llvm version, and a record for the epoch number. - void writeIdentificationBlock(); - /// Emit the current module to the bitstream. - void writeModule(); + void write(); +private: uint64_t bitcodeStartBit() { return BitcodeStartBit; } - void writeStringRecord(unsigned Code, StringRef Str, unsigned AbbrevToUse); void writeAttributeGroupTable(); void writeAttributeTable(); void writeTypeTable(); @@ -236,6 +215,9 @@ private: SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); void writeDIExpression(const DIExpression *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); void writeDIObjCProperty(const DIObjCProperty *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); void writeDIImportedEntity(const DIImportedEntity *N, @@ -247,7 +229,9 @@ private: void writeMetadataStrings(ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record); void writeMetadataRecords(ArrayRef<const Metadata *> MDs, - SmallVectorImpl<uint64_t> &Record); + SmallVectorImpl<uint64_t> &Record, + std::vector<unsigned> *MDAbbrevs = nullptr, + std::vector<uint64_t> *IndexPos = nullptr); void writeModuleMetadata(); void writeFunctionMetadata(const Function &F); void writeFunctionMetadataAttachment(const Function &F); @@ -293,7 +277,10 @@ private: } unsigned getValueId(GlobalValue::GUID ValGUID) { const auto &VMI = GUIDToValueIdMap.find(ValGUID); - assert(VMI != GUIDToValueIdMap.end()); + // Expect that any GUID value had a value Id assigned by an + // earlier call to assignValueId. + assert(VMI != GUIDToValueIdMap.end() && + "GUID does not have assigned value Id"); return VMI->second; } // Helper to get the valueId for the type of value recorded in VI. @@ -306,13 +293,13 @@ private: }; /// Class to manage the bitcode writing for a combined index. -class IndexBitcodeWriter : public BitcodeWriter { +class IndexBitcodeWriter : public BitcodeWriterBase { /// The combined index to write to bitcode. const ModuleSummaryIndex &Index; /// When writing a subset of the index for distributed backends, client /// provides a map of modules to the corresponding GUIDs/summaries to write. - std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex; + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex; /// Map that holds the correspondence between the GUID used in the combined /// index and a value id generated by this class to use in references. @@ -325,11 +312,10 @@ public: /// Constructs a IndexBitcodeWriter object for the given combined index, /// writing to the provided \p Buffer. When writing a subset of the index /// for a distributed backend, provide a \p ModuleToSummariesForIndex map. - IndexBitcodeWriter(SmallVectorImpl<char> &Buffer, - const ModuleSummaryIndex &Index, - std::map<std::string, GVSummaryMapTy> + IndexBitcodeWriter(BitstreamWriter &Stream, const ModuleSummaryIndex &Index, + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex = nullptr) - : BitcodeWriter(Buffer), Index(Index), + : BitcodeWriterBase(Stream), Index(Index), ModuleToSummariesForIndex(ModuleToSummariesForIndex) { // Assign unique value ids to all summaries to be written, for use // in writing out the call graph edges. Save the mapping from GUID @@ -355,11 +341,11 @@ public: // ModuleToSummariesForIndex map: /// Points to the last element in outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesBack; + std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesBack; /// Iterator on outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesIter; + std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesIter; /// Iterator on an inner global variable summary map. - GVSummaryMapTy::iterator ModuleGVSummariesIter; + GVSummaryMapTy::const_iterator ModuleGVSummariesIter; // Iterators used when writing all summaries in the index: @@ -476,11 +462,10 @@ public: /// Obtain the end iterator over the summaries to be written. iterator end() { return iterator(*this, /*IsAtEnd=*/true); } -private: - /// Main entry point for writing a combined index to bitcode, invoked by - /// BitcodeWriter::write() after it writes the header. - void writeBlocks() override; + /// Main entry point for writing a combined index to bitcode. + void write(); +private: void writeIndex(); void writeModStrings(); void writeCombinedValueSymbolTable(); @@ -593,8 +578,8 @@ static unsigned getEncodedSynchScope(SynchronizationScope SynchScope) { llvm_unreachable("Invalid synch scope"); } -void ModuleBitcodeWriter::writeStringRecord(unsigned Code, StringRef Str, - unsigned AbbrevToUse) { +static void writeStringRecord(BitstreamWriter &Stream, unsigned Code, + StringRef Str, unsigned AbbrevToUse) { SmallVector<unsigned, 64> Vals; // Code: [strchar x N] @@ -799,53 +784,53 @@ void ModuleBitcodeWriter::writeTypeTable() { uint64_t NumBits = VE.computeBitsRequiredForTypeIndicies(); // Abbrev for TYPE_CODE_POINTER. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_POINTER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); Abbv->Add(BitCodeAbbrevOp(0)); // Addrspace = 0 - unsigned PtrAbbrev = Stream.EmitAbbrev(Abbv); + unsigned PtrAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_FUNCTION. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FunctionAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_ANON. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv); + unsigned StructAnonAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_NAME. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv); + unsigned StructNameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_STRUCT_NAMED. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv); + unsigned StructNamedAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_ARRAY. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv); + unsigned ArrayAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Emit an entry count so the reader can reserve space. TypeVals.push_back(TypeList.size()); @@ -918,7 +903,7 @@ void ModuleBitcodeWriter::writeTypeTable() { // Emit the name if it is present. if (!ST->getName().empty()) - writeStringRecord(bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), + writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), StructNameAbbrev); } break; @@ -986,8 +971,8 @@ static unsigned getEncodedLinkage(const GlobalValue &GV) { static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { uint64_t RawFlags = 0; - RawFlags |= Flags.HasSection; // bool - + RawFlags |= Flags.NotEligibleToImport; // bool + RawFlags |= (Flags.LiveRoot << 1); // Linkage don't need to be remapped at that time for the summary. Any future // change to the getEncodedLinkage() function will need to be taken into // account here as well. @@ -1068,18 +1053,18 @@ void ModuleBitcodeWriter::writeComdats() { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched /// after the real VST is written. Saves the bit offset to backpatch. -void BitcodeWriter::writeValueSymbolTableForwardDecl() { +void BitcodeWriterBase::writeValueSymbolTableForwardDecl() { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be // updated when the real VST is written. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET)); // Blocks are 32-bit aligned, so we can use a 32-bit word offset to // hold the real VST offset. Must use fixed instead of VBR as we don't // know how many VBR chunks to reserve ahead of time. Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); - unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv); + unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Emit the placeholder uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0}; @@ -1115,13 +1100,13 @@ static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { void ModuleBitcodeWriter::writeModuleInfo() { // Emit various pieces of data attached to a module. if (!M.getTargetTriple().empty()) - writeStringRecord(bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(), + writeStringRecord(Stream, bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(), 0 /*TODO*/); const std::string &DL = M.getDataLayoutStr(); if (!DL.empty()) - writeStringRecord(bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/); + writeStringRecord(Stream, bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/); if (!M.getModuleInlineAsm().empty()) - writeStringRecord(bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(), + writeStringRecord(Stream, bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(), 0 /*TODO*/); // Emit information about sections and GC, computing how many there are. Also @@ -1137,7 +1122,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Give section names unique ID's. unsigned &Entry = SectionMap[GV.getSection()]; if (!Entry) { - writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), + writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), 0 /*TODO*/); Entry = SectionMap.size(); } @@ -1149,7 +1134,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Give section names unique ID's. unsigned &Entry = SectionMap[F.getSection()]; if (!Entry) { - writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, F.getSection(), + writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, F.getSection(), 0 /*TODO*/); Entry = SectionMap.size(); } @@ -1158,7 +1143,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Same for GC names. unsigned &Entry = GCMap[F.getGC()]; if (!Entry) { - writeStringRecord(bitc::MODULE_CODE_GCNAME, F.getGC(), 0 /*TODO*/); + writeStringRecord(Stream, bitc::MODULE_CODE_GCNAME, F.getGC(), + 0 /*TODO*/); Entry = GCMap.size(); } } @@ -1168,7 +1154,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { unsigned SimpleGVarAbbrev = 0; if (!M.global_empty()) { // Add an abbrev for common globals with no visibility or thread localness. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(MaxGlobalType+1))); @@ -1190,7 +1176,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(SectionMap.size()+1))); // Don't bother emitting vis + thread local. - SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv); + SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } // Emit the global variable information. @@ -1298,11 +1284,11 @@ void ModuleBitcodeWriter::writeModuleInfo() { AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); // MODULE_CODE_SOURCE_FILENAME: [namechar x N] - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(AbbrevOpToUse); - unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); for (const auto P : M.getSourceFileName()) Vals.push_back((unsigned char)P); @@ -1373,14 +1359,14 @@ void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N, unsigned ModuleBitcodeWriter::createDILocationAbbrev() { // Assume the column is usually under 128, and always output the inlined-at // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } void ModuleBitcodeWriter::writeDILocation(const DILocation *N, @@ -1402,7 +1388,7 @@ void ModuleBitcodeWriter::writeDILocation(const DILocation *N, unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() { // Assume the column is usually under 128, and always output the inlined-at // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); @@ -1410,7 +1396,7 @@ unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N, @@ -1535,6 +1521,8 @@ void ModuleBitcodeWriter::writeDIFile(const DIFile *N, Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawFilename())); Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory())); + Record.push_back(N->getChecksumKind()); + Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum())); Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev); Record.clear(); @@ -1560,6 +1548,7 @@ void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get())); Record.push_back(N->getDWOId()); Record.push_back(VE.getMetadataOrNullID(N->getMacros().get())); + Record.push_back(N->getSplitDebugInlining()); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -1622,7 +1611,7 @@ void ModuleBitcodeWriter::writeDILexicalBlockFile( void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + Record.push_back(N->isDistinct() | N->getExportSymbols() << 1); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1696,7 +1685,8 @@ void ModuleBitcodeWriter::writeDITemplateValueParameter( void ModuleBitcodeWriter::writeDIGlobalVariable( const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + const uint64_t Version = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | Version); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName())); @@ -1705,8 +1695,9 @@ void ModuleBitcodeWriter::writeDIGlobalVariable( Record.push_back(VE.getMetadataOrNullID(N->getType())); Record.push_back(N->isLocalToUnit()); Record.push_back(N->isDefinition()); - Record.push_back(VE.getMetadataOrNullID(N->getRawVariable())); + Record.push_back(/* expr */ 0); Record.push_back(VE.getMetadataOrNullID(N->getStaticDataMemberDeclaration())); + Record.push_back(N->getAlignInBits()); Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR, Record, Abbrev); Record.clear(); @@ -1715,7 +1706,21 @@ void ModuleBitcodeWriter::writeDIGlobalVariable( void ModuleBitcodeWriter::writeDILocalVariable( const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back(N->isDistinct()); + // In order to support all possible bitcode formats in BitcodeReader we need + // to distinguish the following cases: + // 1) Record has no artificial tag (Record[1]), + // has no obsolete inlinedAt field (Record[9]). + // In this case Record size will be 8, HasAlignment flag is false. + // 2) Record has artificial tag (Record[1]), + // has no obsolete inlignedAt field (Record[9]). + // In this case Record size will be 9, HasAlignment flag is false. + // 3) Record has both artificial tag (Record[1]) and + // obsolete inlignedAt field (Record[9]). + // In this case Record size will be 10, HasAlignment flag is false. + // 4) Record has neither artificial tag, nor inlignedAt field, but + // HasAlignment flag is true and Record[8] contains alignment value. + const uint64_t HasAlignmentFlag = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | HasAlignmentFlag); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1723,6 +1728,7 @@ void ModuleBitcodeWriter::writeDILocalVariable( Record.push_back(VE.getMetadataOrNullID(N->getType())); Record.push_back(N->getArg()); Record.push_back(N->getFlags()); + Record.push_back(N->getAlignInBits()); Stream.EmitRecord(bitc::METADATA_LOCAL_VAR, Record, Abbrev); Record.clear(); @@ -1733,13 +1739,25 @@ void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, unsigned Abbrev) { Record.reserve(N->getElements().size() + 1); - Record.push_back(N->isDistinct()); + const uint64_t HasOpFragmentFlag = 1 << 1; + Record.push_back((uint64_t)N->isDistinct() | HasOpFragmentFlag); Record.append(N->elements_begin(), N->elements_end()); Stream.EmitRecord(bitc::METADATA_EXPRESSION, Record, Abbrev); Record.clear(); } +void ModuleBitcodeWriter::writeDIGlobalVariableExpression( + const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + Record.push_back(N->isDistinct()); + Record.push_back(VE.getMetadataOrNullID(N->getVariable())); + Record.push_back(VE.getMetadataOrNullID(N->getExpression())); + + Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR_EXPR, Record, Abbrev); + Record.clear(); +} + void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { @@ -1771,11 +1789,11 @@ void ModuleBitcodeWriter::writeDIImportedEntity( } unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } void ModuleBitcodeWriter::writeNamedMetadata( @@ -1800,12 +1818,12 @@ void ModuleBitcodeWriter::writeNamedMetadata( } unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRINGS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - return Stream.EmitAbbrev(Abbv); + return Stream.EmitAbbrev(std::move(Abbv)); } /// Write out a record for MDString. @@ -1842,8 +1860,16 @@ void ModuleBitcodeWriter::writeMetadataStrings( Record.clear(); } +// Generates an enum to use as an index in the Abbrev array of Metadata record. +enum MetadataAbbrev : unsigned { +#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID, +#include "llvm/IR/Metadata.def" + LastPlusOne +}; + void ModuleBitcodeWriter::writeMetadataRecords( - ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record) { + ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record, + std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) { if (MDs.empty()) return; @@ -1852,6 +1878,8 @@ void ModuleBitcodeWriter::writeMetadataRecords( #include "llvm/IR/Metadata.def" for (const Metadata *MD : MDs) { + if (IndexPos) + IndexPos->push_back(Stream.GetCurrentBitNo()); if (const MDNode *N = dyn_cast<MDNode>(MD)) { assert(N->isResolved() && "Expected forward references to be resolved"); @@ -1860,7 +1888,11 @@ void ModuleBitcodeWriter::writeMetadataRecords( llvm_unreachable("Invalid MDNode subclass"); #define HANDLE_MDNODE_LEAF(CLASS) \ case Metadata::CLASS##Kind: \ - write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \ + if (MDAbbrevs) \ + write##CLASS(cast<CLASS>(N), Record, \ + (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \ + else \ + write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \ continue; #include "llvm/IR/Metadata.def" } @@ -1873,10 +1905,77 @@ void ModuleBitcodeWriter::writeModuleMetadata() { if (!VE.hasMDs() && M.named_metadata_empty()) return; - Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); + Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 4); SmallVector<uint64_t, 64> Record; + + // Emit all abbrevs upfront, so that the reader can jump in the middle of the + // block and load any metadata. + std::vector<unsigned> MDAbbrevs; + + MDAbbrevs.resize(MetadataAbbrev::LastPlusOne); + MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev(); + MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] = + createGenericDINodeAbbrev(); + + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + unsigned OffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + unsigned IndexAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + // Emit MDStrings together upfront. writeMetadataStrings(VE.getMDStrings(), Record); - writeMetadataRecords(VE.getNonMDStrings(), Record); + + // We only emit an index for the metadata record if we have more than a given + // (naive) threshold of metadatas, otherwise it is not worth it. + if (VE.getNonMDStrings().size() > IndexThreshold) { + // Write a placeholder value in for the offset of the metadata index, + // which is written after the records, so that it can include + // the offset of each entry. The placeholder offset will be + // updated after all records are emitted. + uint64_t Vals[] = {0, 0}; + Stream.EmitRecord(bitc::METADATA_INDEX_OFFSET, Vals, OffsetAbbrev); + } + + // Compute and save the bit offset to the current position, which will be + // patched when we emit the index later. We can simply subtract the 64-bit + // fixed size from the current bit number to get the location to backpatch. + uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo(); + + // This index will contain the bitpos for each individual record. + std::vector<uint64_t> IndexPos; + IndexPos.reserve(VE.getNonMDStrings().size()); + + // Write all the records + writeMetadataRecords(VE.getNonMDStrings(), Record, &MDAbbrevs, &IndexPos); + + if (VE.getNonMDStrings().size() > IndexThreshold) { + // Now that we have emitted all the records we will emit the index. But + // first + // backpatch the forward reference so that the reader can skip the records + // efficiently. + Stream.BackpatchWord64(IndexOffsetRecordBitPos - 64, + Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos); + + // Delta encode the index. + uint64_t PreviousValue = IndexOffsetRecordBitPos; + for (auto &Elt : IndexPos) { + auto EltDelta = Elt - PreviousValue; + PreviousValue = Elt; + Elt = EltDelta; + } + // Emit the index record. + Stream.EmitRecord(bitc::METADATA_INDEX, IndexPos, IndexAbbrev); + IndexPos.clear(); + } + + // Write the named metadata now. writeNamedMetadata(Record); auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) { @@ -2025,30 +2124,30 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, // If this is a constant pool for the module, emit module-specific abbrevs. if (isGlobal) { // Abbrev for CST_CODE_AGGREGATE. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(LastVal+1))); - AggregateAbbrev = Stream.EmitAbbrev(Abbv); + AggregateAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_STRING. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_STRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - String8Abbrev = Stream.EmitAbbrev(Abbv); + String8Abbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_CSTRING. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - CString7Abbrev = Stream.EmitAbbrev(Abbv); + CString7Abbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_CSTRING. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - CString6Abbrev = Stream.EmitAbbrev(Abbv); + CString6Abbrev = Stream.EmitAbbrev(std::move(Abbv)); } SmallVector<uint64_t, 64> Record; @@ -2196,9 +2295,12 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, case Instruction::GetElementPtr: { Code = bitc::CST_CODE_CE_GEP; const auto *GO = cast<GEPOperator>(C); - if (GO->isInBounds()) - Code = bitc::CST_CODE_CE_INBOUNDS_GEP; Record.push_back(VE.getTypeID(GO->getSourceElementType())); + if (Optional<unsigned> Idx = GO->getInRangeIndex()) { + Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX; + Record.push_back((*Idx << 1) | GO->isInBounds()); + } else if (GO->isInBounds()) + Code = bitc::CST_CODE_CE_INBOUNDS_GEP; for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) { Record.push_back(VE.getTypeID(C->getOperand(i)->getType())); Record.push_back(VE.getValueID(C->getOperand(i))); @@ -2495,7 +2597,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, // Emit type/value pairs for varargs params. if (FTy->isVarArg()) { - for (unsigned i = FTy->getNumParams(), e = I.getNumOperands()-3; + for (unsigned i = FTy->getNumParams(), e = II->getNumArgOperands(); i != e; ++i) pushValueAndType(I.getOperand(i), InstID, Vals); // vararg } @@ -2736,11 +2838,13 @@ void ModuleBitcodeWriter::writeValueSymbolTable( // Get the offset of the VST we are writing, and backpatch it into // the VST forward declaration record. uint64_t VSTOffset = Stream.GetCurrentBitNo(); - // The BitcodeStartBit was the stream offset of the actual bitcode - // (e.g. excluding any initial darwin header). + // The BitcodeStartBit was the stream offset of the identification block. VSTOffset -= bitcodeStartBit(); assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); - Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); + // Note that we add 1 here because the offset is relative to one word + // before the start of the identification block, which was historically + // always the start of the regular bitcode header. + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1); } Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); @@ -2753,39 +2857,39 @@ void ModuleBitcodeWriter::writeValueSymbolTable( unsigned GUIDEntryAbbrev; if (IsModuleLevel && hasVSTOffsetPlaceholder()) { // 8-bit fixed-width VST_CODE_FNENTRY function strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - FnEntry8BitAbbrev = Stream.EmitAbbrev(Abbv); + FnEntry8BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // 7-bit fixed width VST_CODE_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - FnEntry7BitAbbrev = Stream.EmitAbbrev(Abbv); + FnEntry7BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // 6-bit char6 VST_CODE_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - FnEntry6BitAbbrev = Stream.EmitAbbrev(Abbv); + FnEntry6BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // FIXME: Change the name of this record as it is now used by // the per-module index as well. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - GUIDEntryAbbrev = Stream.EmitAbbrev(Abbv); + GUIDEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } // FIXME: Set up the abbrev, we know how many values there are! @@ -2828,7 +2932,10 @@ void ModuleBitcodeWriter::writeValueSymbolTable( // actual bitcode written to the stream). uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - bitcodeStartBit(); assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); - NameVals.push_back(BitcodeIndex / 32); + // Note that we add 1 here because the offset is relative to one word + // before the start of the identification block, which was historically + // always the start of the regular bitcode header. + NameVals.push_back(BitcodeIndex / 32 + 1); Code = bitc::VST_CODE_FNENTRY; AbbrevToUse = FnEntry8BitAbbrev; @@ -2876,11 +2983,11 @@ void IndexBitcodeWriter::writeCombinedValueSymbolTable() { Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv); + unsigned EntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<uint64_t, 64> NameVals; for (const auto &GVI : valueIds()) { @@ -2994,7 +3101,8 @@ void ModuleBitcodeWriter::writeFunction( } // Emit names for all the instructions etc. - writeValueSymbolTable(F.getValueSymbolTable()); + if (auto *Symtab = F.getValueSymbolTable()) + writeValueSymbolTable(*Symtab); if (NeedsMetadataAttachment) writeFunctionMetadataAttachment(F); @@ -3009,10 +3117,10 @@ void ModuleBitcodeWriter::writeBlockInfo() { // We only want to emit block info records for blocks that have multiple // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK. // Other blocks can define their abbrevs inline. - Stream.EnterBlockInfoBlock(2); + Stream.EnterBlockInfoBlock(); { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3023,7 +3131,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // 7-bit fixed width VST_CODE_ENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3033,7 +3141,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // 6-bit char6 VST_CODE_ENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3043,7 +3151,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // 6-bit char6 VST_CODE_BBENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3056,7 +3164,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { { // SETTYPE abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, VE.computeBitsRequiredForTypeIndicies())); @@ -3066,7 +3174,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // INTEGER abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_INTEGER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != @@ -3075,7 +3183,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // CE_CAST abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid @@ -3087,7 +3195,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // NULL abbrev for CONSTANTS_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_NULL)); if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != CONSTANTS_NULL_Abbrev) @@ -3097,7 +3205,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { // FIXME: This should only use space for first class types! { // INST_LOAD abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty @@ -3109,7 +3217,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS @@ -3119,7 +3227,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS @@ -3130,7 +3238,7 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_CAST abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty @@ -3142,14 +3250,14 @@ void ModuleBitcodeWriter::writeBlockInfo() { } { // INST_RET abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_RET_VOID_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_RET abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != @@ -3157,14 +3265,14 @@ void ModuleBitcodeWriter::writeBlockInfo() { llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE)); if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_UNREACHABLE_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty @@ -3187,38 +3295,38 @@ void IndexBitcodeWriter::writeModStrings() { // TODO: See which abbrev sizes we actually need to emit // 8-bit fixed-width MST_ENTRY strings. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv); + unsigned Abbrev8Bit = Stream.EmitAbbrev(std::move(Abbv)); // 7-bit fixed width MST_ENTRY strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv); + unsigned Abbrev7Bit = Stream.EmitAbbrev(std::move(Abbv)); // 6-bit char6 MST_ENTRY strings. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv); + unsigned Abbrev6Bit = Stream.EmitAbbrev(std::move(Abbv)); // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); - unsigned AbbrevHash = Stream.EmitAbbrev(Abbv); + unsigned AbbrevHash = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<unsigned, 64> Vals; for (const auto &MPSE : Index.modulePaths()) { @@ -3267,30 +3375,21 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( NameVals.push_back(ValueID); FunctionSummary *FS = cast<FunctionSummary>(Summary); + if (!FS->type_tests().empty()) + Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); NameVals.push_back(FS->refs().size()); - unsigned SizeBeforeRefs = NameVals.size(); for (auto &RI : FS->refs()) NameVals.push_back(VE.getValueID(RI.getValue())); - // Sort the refs for determinism output, the vector returned by FS->refs() has - // been initialized from a DenseSet. - std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); - std::vector<FunctionSummary::EdgeTy> Calls = FS->calls(); - std::sort(Calls.begin(), Calls.end(), - [this](const FunctionSummary::EdgeTy &L, - const FunctionSummary::EdgeTy &R) { - return getValueId(L.first) < getValueId(R.first); - }); bool HasProfileData = F.getEntryCount().hasValue(); - for (auto &ECI : Calls) { + for (auto &ECI : FS->calls()) { NameVals.push_back(getValueId(ECI.first)); - assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite"); - NameVals.push_back(ECI.second.CallsiteCount); if (HasProfileData) - NameVals.push_back(ECI.second.ProfileCount); + NameVals.push_back(static_cast<uint8_t>(ECI.second.Hotness)); } unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); @@ -3307,13 +3406,18 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( void ModuleBitcodeWriter::writeModuleLevelReferences( const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev) { - // Only interested in recording variable defs in the summary. - if (V.isDeclaration()) + auto Summaries = + Index->findGlobalValueSummaryList(GlobalValue::getGUID(V.getName())); + if (Summaries == Index->end()) { + // Only declarations should not have a summary (a declaration might however + // have a summary if the def was in module level asm). + assert(V.isDeclaration()); return; + } + auto *Summary = Summaries->second.front().get(); NameVals.push_back(VE.getValueID(&V)); - NameVals.push_back(getEncodedGVSummaryFlags(V)); - auto *Summary = Index->getGlobalValueSummary(V); GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary); + NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); unsigned SizeBeforeRefs = NameVals.size(); for (auto &RI : VS->refs()) @@ -3330,71 +3434,79 @@ void ModuleBitcodeWriter::writeModuleLevelReferences( // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 1; +static const uint64_t INDEX_VERSION = 3; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { - if (Index->begin() == Index->end()) - return; - Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + if (Index->begin() == Index->end()) { + Stream.ExitBlock(); + return; + } + // Abbrev for FS_PERMODULE. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount) + // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_PERMODULE_PROFILE. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount, profilecount) + // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_ALIAS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<uint64_t, 64> NameVals; // Iterate over the list of functions instead of the Index to // ensure the ordering is stable. for (const Function &F : M) { - if (F.isDeclaration()) - continue; // Summary emission does not support anonymous functions, they have to // renamed using the anonymous function renaming pass. if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - auto *Summary = Index->getGlobalValueSummary(F); + auto Summaries = + Index->findGlobalValueSummaryList(GlobalValue::getGUID(F.getName())); + if (Summaries == Index->end()) { + // Only declarations should not have a summary (a declaration might + // however have a summary if the def was in module level asm). + assert(F.isDeclaration()); + continue; + } + auto *Summary = Summaries->second.front().get(); writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), FSCallsAbbrev, FSCallsProfileAbbrev, F); } @@ -3412,7 +3524,9 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { auto AliasId = VE.getValueID(&A); auto AliaseeId = VE.getValueID(Aliasee); NameVals.push_back(AliasId); - NameVals.push_back(getEncodedGVSummaryFlags(A)); + auto *Summary = Index->getGlobalValueSummary(A); + AliasSummary *AS = cast<AliasSummary>(Summary); + NameVals.push_back(getEncodedGVSummaryFlags(AS->flags())); NameVals.push_back(AliaseeId); Stream.EmitRecord(bitc::FS_ALIAS, NameVals, FSAliasAbbrev); NameVals.clear(); @@ -3427,49 +3541,49 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); // Abbrev for FS_COMBINED. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount) + // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_PROFILE. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - // numrefs x valueid, n x (valueid, callsitecount, profilecount) + // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_ALIAS. - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // The aliases are emitted as a post-pass, and will point to the value // id of the aliasee. Save them in a vector for post-processing. @@ -3522,6 +3636,9 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { } auto *FS = cast<FunctionSummary>(S); + if (!FS->type_tests().empty()) + Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + NameVals.push_back(ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); @@ -3534,7 +3651,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { bool HasProfileData = false; for (auto &EI : FS->calls()) { - HasProfileData |= EI.second.ProfileCount != 0; + HasProfileData |= EI.second.Hotness != CalleeInfo::HotnessType::Unknown; if (HasProfileData) break; } @@ -3545,10 +3662,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { if (!hasValueId(EI.first.getGUID())) continue; NameVals.push_back(getValueId(EI.first.getGUID())); - assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); - NameVals.push_back(EI.second.CallsiteCount); if (HasProfileData) - NameVals.push_back(EI.second.ProfileCount); + NameVals.push_back(static_cast<uint8_t>(EI.second.Hotness)); } unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); @@ -3580,23 +3695,25 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.ExitBlock(); } -void ModuleBitcodeWriter::writeIdentificationBlock() { +/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the +/// current llvm version, and a record for the epoch number. +void writeIdentificationBlock(BitstreamWriter &Stream) { Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5); // Write the "user readable" string identifying the bitcode producer - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - auto StringAbbrev = Stream.EmitAbbrev(Abbv); - writeStringRecord(bitc::IDENTIFICATION_CODE_STRING, + auto StringAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + writeStringRecord(Stream, bitc::IDENTIFICATION_CODE_STRING, "LLVM" LLVM_VERSION_STRING, StringAbbrev); // Write the epoch version - Abbv = new BitCodeAbbrev(); + Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - auto EpochAbbrev = Stream.EmitAbbrev(Abbv); + auto EpochAbbrev = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<unsigned, 1> Vals = {bitc::BITCODE_CURRENT_EPOCH}; Stream.EmitRecord(bitc::IDENTIFICATION_CODE_EPOCH, Vals, EpochAbbrev); Stream.ExitBlock(); @@ -3608,39 +3725,19 @@ void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { SHA1 Hasher; Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], Buffer.size() - BlockStartPos)); - auto Hash = Hasher.result(); - SmallVector<uint64_t, 20> Vals; - auto LShift = [&](unsigned char Val, unsigned Amount) - -> uint64_t { return ((uint64_t)Val) << Amount; }; + StringRef Hash = Hasher.result(); + uint32_t Vals[5]; for (int Pos = 0; Pos < 20; Pos += 4) { - uint32_t SubHash = LShift(Hash[Pos + 0], 24); - SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) | - (unsigned)(unsigned char)Hash[Pos + 3]; - Vals.push_back(SubHash); + Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); } // Emit the finished record. Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); } -void BitcodeWriter::write() { - // Emit the file header first. - writeBitcodeHeader(); - - writeBlocks(); -} - -void ModuleBitcodeWriter::writeBlocks() { - writeIdentificationBlock(); - writeModule(); -} - -void IndexBitcodeWriter::writeBlocks() { - // Index contains only a single outer (module) block. - writeIndex(); -} +void ModuleBitcodeWriter::write() { + writeIdentificationBlock(Stream); -void ModuleBitcodeWriter::writeModule() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); size_t BlockStartPos = Buffer.size(); @@ -3769,7 +3866,7 @@ static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, } /// Helper to write the header common to all bitcode files. -void BitcodeWriter::writeBitcodeHeader() { +static void writeBitcodeHeader(BitstreamWriter &Stream) { // Emit the file header. Stream.Emit((unsigned)'B', 8); Stream.Emit((unsigned)'C', 8); @@ -3779,6 +3876,22 @@ void BitcodeWriter::writeBitcodeHeader() { Stream.Emit(0xD, 4); } +BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer) + : Buffer(Buffer), Stream(new BitstreamWriter(Buffer)) { + writeBitcodeHeader(*Stream); +} + +BitcodeWriter::~BitcodeWriter() = default; + +void BitcodeWriter::writeModule(const Module *M, + bool ShouldPreserveUseListOrder, + const ModuleSummaryIndex *Index, + bool GenerateHash) { + ModuleBitcodeWriter ModuleWriter( + M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash); + ModuleWriter.write(); +} + /// WriteBitcodeToFile - Write the specified module to the specified output /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, @@ -3794,10 +3907,8 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); - // Emit the module into the buffer. - ModuleBitcodeWriter ModuleWriter(M, Buffer, ShouldPreserveUseListOrder, Index, - GenerateHash); - ModuleWriter.write(); + BitcodeWriter Writer(Buffer); + Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) emitDarwinBCHeaderAndTrailer(Buffer, TT); @@ -3806,7 +3917,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, Out.write((char*)&Buffer.front(), Buffer.size()); } -void IndexBitcodeWriter::writeIndex() { +void IndexBitcodeWriter::write() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); SmallVector<unsigned, 1> Vals; @@ -3836,11 +3947,14 @@ void IndexBitcodeWriter::writeIndex() { // index for a distributed backend, provide a \p ModuleToSummariesForIndex map. void llvm::WriteIndexToFile( const ModuleSummaryIndex &Index, raw_ostream &Out, - std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { SmallVector<char, 0> Buffer; Buffer.reserve(256 * 1024); - IndexBitcodeWriter IndexWriter(Buffer, Index, ModuleToSummariesForIndex); + BitstreamWriter Stream(Buffer); + writeBitcodeHeader(Stream); + + IndexBitcodeWriter IndexWriter(Stream, Index, ModuleToSummariesForIndex); IndexWriter.write(); Out.write((char *)&Buffer.front(), Buffer.size()); |
