diff options
Diffstat (limited to 'contrib/llvm/lib/Bitcode')
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 1301 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp | 240 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h | 3 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/ValueList.cpp | 6 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Reader/ValueList.h | 2 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 1058 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 33 | ||||
| -rw-r--r-- | contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h | 37 |
8 files changed, 1611 insertions, 1069 deletions
diff --git a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index a46e49c..048e367 100644 --- a/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -28,8 +28,8 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallingConv.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/CallingConv.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" @@ -40,13 +40,13 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalIFunc.h" #include "llvm/IR/GlobalIndirectSymbol.h" #include "llvm/IR/GlobalObject.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/InstrTypes.h" @@ -264,7 +264,7 @@ Expected<bool> hasObjCCategoryInModule(BitstreamCursor &Stream) { if (convertToString(Record, 0, S)) return error("Invalid record"); // Check for the i386 and other (x86_64, ARM) conventions - if (S.find("__DATA, __objc_catlist") != std::string::npos || + if (S.find("__DATA,__objc_catlist") != std::string::npos || S.find("__OBJC,__category") != std::string::npos) return true; break; @@ -372,12 +372,26 @@ Expected<std::string> readTriple(BitstreamCursor &Stream) { class BitcodeReaderBase { protected: - BitcodeReaderBase(BitstreamCursor Stream) : Stream(std::move(Stream)) { + BitcodeReaderBase(BitstreamCursor Stream, StringRef Strtab) + : Stream(std::move(Stream)), Strtab(Strtab) { this->Stream.setBlockInfo(&BlockInfo); } BitstreamBlockInfo BlockInfo; BitstreamCursor Stream; + StringRef Strtab; + + /// In version 2 of the bitcode we store names of global values and comdats in + /// a string table rather than in the VST. + bool UseStrtab = false; + + Expected<unsigned> parseVersionRecord(ArrayRef<uint64_t> Record); + + /// If this module uses a string table, pop the reference to the string table + /// and return the referenced string and the rest of the record. Otherwise + /// just return the record itself. + std::pair<StringRef, ArrayRef<uint64_t>> + readNameFromStrtab(ArrayRef<uint64_t> Record); bool readBlockInfo(); @@ -395,6 +409,27 @@ Error BitcodeReaderBase::error(const Twine &Message) { return ::error(FullMsg); } +Expected<unsigned> +BitcodeReaderBase::parseVersionRecord(ArrayRef<uint64_t> Record) { + if (Record.size() < 1) + return error("Invalid record"); + unsigned ModuleVersion = Record[0]; + if (ModuleVersion > 2) + return error("Invalid value"); + UseStrtab = ModuleVersion >= 2; + return ModuleVersion; +} + +std::pair<StringRef, ArrayRef<uint64_t>> +BitcodeReaderBase::readNameFromStrtab(ArrayRef<uint64_t> Record) { + if (!UseStrtab) + return {"", Record}; + // Invalid reference. Let the caller complain about the record being empty. + if (Record[0] + Record[1] > Strtab.size()) + return {"", {}}; + return {StringRef(Strtab.data() + Record[0], Record[1]), Record.slice(2)}; +} + class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { LLVMContext &Context; Module *TheModule = nullptr; @@ -405,6 +440,9 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { bool SeenValueSymbolTable = false; uint64_t VSTOffset = 0; + std::vector<std::string> SectionTable; + std::vector<std::string> GCTable; + std::vector<Type*> TypeList; BitcodeReaderValueList ValueList; Optional<MetadataLoader> MDLoader; @@ -419,10 +457,10 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { /// The set of attributes by index. Index zero in the file is for null, and /// is thus not represented here. As such all indices are off by one. - std::vector<AttributeSet> MAttributes; + std::vector<AttributeList> MAttributes; /// The set of attribute groups. - std::map<unsigned, AttributeSet> MAttributeGroups; + std::map<unsigned, AttributeList> MAttributeGroups; /// While parsing a function body, this is a list of the basic blocks for the /// function. @@ -475,10 +513,11 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { TBAAVerifier TBAAVerifyHelper; std::vector<std::string> BundleTags; + SmallVector<SyncScope::ID, 8> SSIDs; public: - BitcodeReader(BitstreamCursor Stream, StringRef ProducerIdentification, - LLVMContext &Context); + BitcodeReader(BitstreamCursor Stream, StringRef Strtab, + StringRef ProducerIdentification, LLVMContext &Context); Error materializeForwardReferencedFunctions(); @@ -520,10 +559,10 @@ private: return FunctionBBs[ID]; } - AttributeSet getAttributes(unsigned i) const { + AttributeList getAttributes(unsigned i) const { if (i-1 < MAttributes.size()) return MAttributes[i-1]; - return AttributeSet(); + return AttributeList(); } /// Read a value/type pair out of the specified record from slot 'Slot'. @@ -598,15 +637,26 @@ private: Error parseAlignmentValue(uint64_t Exponent, unsigned &Alignment); Error parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind); Error parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata = false); + + Error parseComdatRecord(ArrayRef<uint64_t> Record); + Error parseGlobalVarRecord(ArrayRef<uint64_t> Record); + Error parseFunctionRecord(ArrayRef<uint64_t> Record); + Error parseGlobalIndirectSymbolRecord(unsigned BitCode, + ArrayRef<uint64_t> Record); + Error parseAttributeBlock(); Error parseAttributeGroupBlock(); Error parseTypeTable(); Error parseTypeTableBody(); Error parseOperandBundleTags(); + Error parseSyncScopeNames(); Expected<Value *> recordValue(SmallVectorImpl<uint64_t> &Record, unsigned NameIndex, Triple &TT); + void setDeferredFunctionInfo(unsigned FuncBitcodeOffsetDelta, Function *F, + ArrayRef<uint64_t> Record); Error parseValueSymbolTable(uint64_t Offset = 0); + Error parseGlobalValueSymbolTable(); Error parseConstants(); Error rememberAndSkipFunctionBodies(); Error rememberAndSkipFunctionBody(); @@ -620,6 +670,8 @@ private: Error findFunctionInStream( Function *F, DenseMap<Function *, uint64_t>::iterator DeferredFunctionInfoIterator); + + SyncScope::ID getDecodedSyncScopeID(unsigned Val); }; /// Class to manage reading and parsing function summary index bitcode @@ -639,15 +691,16 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase { /// Used to enable on-demand parsing of the VST. uint64_t VSTOffset = 0; - // Map to save ValueId to GUID association that was recorded in the + // Map to save ValueId to ValueInfo association that was recorded in the // ValueSymbolTable. It is used after the VST is parsed to convert // call graph edges read from the function summary from referencing - // callees by their ValueId to using the GUID instead, which is how + // callees by their ValueId to using the ValueInfo instead, which is how // they are recorded in the summary index being built. - // We save a second GUID which is the same as the first one, but ignoring the - // linkage, i.e. for value other than local linkage they are identical. - DenseMap<unsigned, std::pair<GlobalValue::GUID, GlobalValue::GUID>> - ValueIdToCallGraphGUIDMap; + // We save a GUID which refers to the same global as the ValueInfo, but + // ignoring the linkage, i.e. for values other than local linkage they are + // identical. + DenseMap<unsigned, std::pair<ValueInfo, GlobalValue::GUID>> + ValueIdToValueInfoMap; /// Map populated during module path string table parsing, from the /// module ID to a string reference owned by the index's module @@ -658,13 +711,25 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase { /// Original source file name recorded in a bitcode record. std::string SourceFileName; + /// The string identifier given to this module by the client, normally the + /// path to the bitcode file. + StringRef ModulePath; + + /// For per-module summary indexes, the unique numerical identifier given to + /// this module by the client. + unsigned ModuleId; + public: - ModuleSummaryIndexBitcodeReader( - BitstreamCursor Stream, ModuleSummaryIndex &TheIndex); + ModuleSummaryIndexBitcodeReader(BitstreamCursor Stream, StringRef Strtab, + ModuleSummaryIndex &TheIndex, + StringRef ModulePath, unsigned ModuleId); - Error parseModule(StringRef ModulePath); + Error parseModule(); private: + void setValueGUID(uint64_t ValueID, StringRef ValueName, + GlobalValue::LinkageTypes Linkage, + StringRef SourceFileName); Error parseValueSymbolTable( uint64_t Offset, DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap); @@ -672,11 +737,13 @@ private: std::vector<FunctionSummary::EdgeTy> makeCallList(ArrayRef<uint64_t> Record, bool IsOldProfileFormat, bool HasProfile); - Error parseEntireSummary(StringRef ModulePath); + Error parseEntireSummary(unsigned ID); Error parseModuleStringTable(); - std::pair<GlobalValue::GUID, GlobalValue::GUID> - getGUIDFromValueId(unsigned ValueId); + std::pair<ValueInfo, GlobalValue::GUID> + getValueInfoFromValueId(unsigned ValueId); + + ModuleSummaryIndex::ModuleInfo *addThisModule(); }; } // end anonymous namespace @@ -694,10 +761,10 @@ std::error_code llvm::errorToErrorCodeAndEmitErrors(LLVMContext &Ctx, return std::error_code(); } -BitcodeReader::BitcodeReader(BitstreamCursor Stream, +BitcodeReader::BitcodeReader(BitstreamCursor Stream, StringRef Strtab, StringRef ProducerIdentification, LLVMContext &Context) - : BitcodeReaderBase(std::move(Stream)), Context(Context), + : BitcodeReaderBase(std::move(Stream), Strtab), Context(Context), ValueList(Context) { this->ProducerIdentification = ProducerIdentification; } @@ -802,11 +869,11 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, auto Linkage = GlobalValue::LinkageTypes(RawFlags & 0xF); // 4 bits RawFlags = RawFlags >> 4; bool NotEligibleToImport = (RawFlags & 0x1) || Version < 3; - // The LiveRoot flag wasn't introduced until version 3. For dead stripping + // The Live flag wasn't introduced until version 3. For dead stripping // to work correctly on earlier versions, we must conservatively treat all // values as live. - bool LiveRoot = (RawFlags & 0x2) || Version < 3; - return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, LiveRoot); + bool Live = (RawFlags & 0x2) || Version < 3; + return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live); } static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { @@ -935,14 +1002,6 @@ static AtomicOrdering getDecodedOrdering(unsigned Val) { } } -static SynchronizationScope getDecodedSynchScope(unsigned Val) { - switch (Val) { - case bitc::SYNCHSCOPE_SINGLETHREAD: return SingleThread; - default: // Map unknown scopes to cross-thread. - case bitc::SYNCHSCOPE_CROSSTHREAD: return CrossThread; - } -} - static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) { switch (Val) { default: // Map unknown selection kinds to any. @@ -971,6 +1030,8 @@ static FastMathFlags getDecodedFastMathFlags(unsigned Val) { FMF.setNoSignedZeros(); if (0 != (Val & FastMathFlags::AllowReciprocal)) FMF.setAllowReciprocal(); + if (0 != (Val & FastMathFlags::AllowContract)) + FMF.setAllowContract(true); return FMF; } @@ -1066,6 +1127,7 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { case Attribute::SwiftSelf: return 1ULL << 51; case Attribute::SwiftError: return 1ULL << 52; case Attribute::WriteOnly: return 1ULL << 53; + case Attribute::Speculatable: return 1ULL << 54; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1132,7 +1194,7 @@ Error BitcodeReader::parseAttributeBlock() { SmallVector<uint64_t, 64> Record; - SmallVector<AttributeSet, 8> Attrs; + SmallVector<AttributeList, 8> Attrs; // Read all the records. while (true) { @@ -1162,10 +1224,10 @@ Error BitcodeReader::parseAttributeBlock() { for (unsigned i = 0, e = Record.size(); i != e; i += 2) { AttrBuilder B; decodeLLVMAttributesForBitcode(B, Record[i+1]); - Attrs.push_back(AttributeSet::get(Context, Record[i], B)); + Attrs.push_back(AttributeList::get(Context, Record[i], B)); } - MAttributes.push_back(AttributeSet::get(Context, Attrs)); + MAttributes.push_back(AttributeList::get(Context, Attrs)); Attrs.clear(); break; } @@ -1173,7 +1235,7 @@ Error BitcodeReader::parseAttributeBlock() { for (unsigned i = 0, e = Record.size(); i != e; ++i) Attrs.push_back(MAttributeGroups[Record[i]]); - MAttributes.push_back(AttributeSet::get(Context, Attrs)); + MAttributes.push_back(AttributeList::get(Context, Attrs)); Attrs.clear(); break; } @@ -1262,6 +1324,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::ReturnsTwice; case bitc::ATTR_KIND_S_EXT: return Attribute::SExt; + case bitc::ATTR_KIND_SPECULATABLE: + return Attribute::Speculatable; case bitc::ATTR_KIND_STACK_ALIGNMENT: return Attribute::StackAlignment; case bitc::ATTR_KIND_STACK_PROTECT: @@ -1391,7 +1455,7 @@ Error BitcodeReader::parseAttributeGroupBlock() { } } - MAttributeGroups[GrpID] = AttributeSet::get(Context, Idx, B); + MAttributeGroups[GrpID] = AttributeList::get(Context, Idx, B); break; } } @@ -1677,6 +1741,44 @@ Error BitcodeReader::parseOperandBundleTags() { } } +Error BitcodeReader::parseSyncScopeNames() { + if (Stream.EnterSubBlock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID)) + return error("Invalid record"); + + if (!SSIDs.empty()) + return error("Invalid multiple synchronization scope names blocks"); + + SmallVector<uint64_t, 64> Record; + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + if (SSIDs.empty()) + return error("Invalid empty synchronization scope names block"); + return Error::success(); + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Synchronization scope names are implicitly mapped to synchronization + // scope IDs by their order. + + if (Stream.readRecord(Entry.ID, Record) != bitc::SYNC_SCOPE_NAME) + return error("Invalid record"); + + SmallString<16> SSN; + if (convertToString(Record, 0, SSN)) + return error("Invalid record"); + + SSIDs.push_back(Context.getOrInsertSyncScopeID(SSN)); + Record.clear(); + } +} + /// Associate a value with its name from the given index in the provided record. Expected<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record, unsigned NameIndex, Triple &TT) { @@ -1725,6 +1827,54 @@ static uint64_t jumpToValueSymbolTable(uint64_t Offset, return CurrentBit; } +void BitcodeReader::setDeferredFunctionInfo(unsigned FuncBitcodeOffsetDelta, + Function *F, + ArrayRef<uint64_t> Record) { + // Note that we subtract 1 here because the offset is relative to one word + // before the start of the identification or module block, which was + // historically always the start of the regular bitcode header. + uint64_t FuncWordOffset = Record[1] - 1; + uint64_t FuncBitOffset = FuncWordOffset * 32; + DeferredFunctionInfo[F] = FuncBitOffset + FuncBitcodeOffsetDelta; + // Set the LastFunctionBlockBit to point to the last function block. + // Later when parsing is resumed after function materialization, + // we can simply skip that last function block. + if (FuncBitOffset > LastFunctionBlockBit) + LastFunctionBlockBit = FuncBitOffset; +} + +/// Read a new-style GlobalValue symbol table. +Error BitcodeReader::parseGlobalValueSymbolTable() { + unsigned FuncBitcodeOffsetDelta = + Stream.getAbbrevIDWidth() + bitc::BlockIDWidth; + + if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) + return error("Invalid record"); + + SmallVector<uint64_t, 64> Record; + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + return Error::success(); + case BitstreamEntry::Record: + break; + } + + Record.clear(); + switch (Stream.readRecord(Entry.ID, Record)) { + case bitc::VST_CODE_FNENTRY: // [valueid, offset] + setDeferredFunctionInfo(FuncBitcodeOffsetDelta, + cast<Function>(ValueList[Record[0]]), Record); + break; + } + } +} + /// Parse the value symbol table at either the current parsing location or /// at the given bit offset if provided. Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { @@ -1732,8 +1882,18 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { // Pass in the Offset to distinguish between calling for the module-level // VST (where we want to jump to the VST offset) and the function-level // VST (where we don't). - if (Offset > 0) + if (Offset > 0) { CurrentBit = jumpToValueSymbolTable(Offset, Stream); + // If this module uses a string table, read this as a module-level VST. + if (UseStrtab) { + if (Error Err = parseGlobalValueSymbolTable()) + return Err; + Stream.JumpToBit(CurrentBit); + return Error::success(); + } + // Otherwise, the VST will be in a similar format to a function-level VST, + // and will contain symbol names. + } // Compute the delta between the bitcode indices in the VST (the word offset // to the word-aligned ENTER_SUBBLOCK for the function block, and that @@ -1794,29 +1954,10 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { return Err; Value *V = ValOrErr.get(); - auto *GO = dyn_cast<GlobalObject>(V); - if (!GO) { - // If this is an alias, need to get the actual Function object - // it aliases, in order to set up the DeferredFunctionInfo entry below. - auto *GA = dyn_cast<GlobalAlias>(V); - if (GA) - GO = GA->getBaseObject(); - assert(GO); - } - - // Note that we subtract 1 here because the offset is relative to one word - // before the start of the identification or module block, which was - // historically always the start of the regular bitcode header. - uint64_t FuncWordOffset = Record[1] - 1; - Function *F = dyn_cast<Function>(GO); - assert(F); - uint64_t FuncBitOffset = FuncWordOffset * 32; - DeferredFunctionInfo[F] = FuncBitOffset + FuncBitcodeOffsetDelta; - // Set the LastFunctionBlockBit to point to the last function block. - // Later when parsing is resumed after function materialization, - // we can simply skip that last function block. - if (FuncBitOffset > LastFunctionBlockBit) - LastFunctionBlockBit = FuncBitOffset; + // Ignore function offsets emitted for aliases of functions in older + // versions of LLVM. + if (auto *F = dyn_cast<Function>(V)) + setDeferredFunctionInfo(FuncBitcodeOffsetDelta, F, Record); break; } case bitc::VST_CODE_BBENTRY: { @@ -2501,6 +2642,16 @@ Error BitcodeReader::materializeMetadata() { if (Error Err = MDLoader->parseModuleMetadata()) return Err; } + + // Upgrade "Linker Options" module flag to "llvm.linker.options" module-level + // metadata. + if (Metadata *Val = TheModule->getModuleFlag("Linker Options")) { + NamedMDNode *LinkerOpts = + TheModule->getOrInsertNamedMetadata("llvm.linker.options"); + for (const MDOperand &MDOptions : cast<MDNode>(Val)->operands()) + LinkerOpts->addOperand(cast<MDNode>(MDOptions)); + } + DeferredMetadataInfo.clear(); return Error::success(); } @@ -2539,6 +2690,7 @@ Error BitcodeReader::globalCleanup() { // Look for intrinsic functions which need to be upgraded at some point for (Function &F : *TheModule) { + MDLoader->upgradeDebugIntrinsics(F); Function *NewFn; if (UpgradeIntrinsicFunction(&F, NewFn)) UpgradedIntrinsics[&F] = NewFn; @@ -2607,6 +2759,272 @@ bool BitcodeReaderBase::readBlockInfo() { return false; } +Error BitcodeReader::parseComdatRecord(ArrayRef<uint64_t> Record) { + // v1: [selection_kind, name] + // v2: [strtab_offset, strtab_size, selection_kind] + StringRef Name; + std::tie(Name, Record) = readNameFromStrtab(Record); + + if (Record.size() < 1) + return error("Invalid record"); + Comdat::SelectionKind SK = getDecodedComdatSelectionKind(Record[0]); + std::string OldFormatName; + if (!UseStrtab) { + if (Record.size() < 2) + return error("Invalid record"); + unsigned ComdatNameSize = Record[1]; + OldFormatName.reserve(ComdatNameSize); + for (unsigned i = 0; i != ComdatNameSize; ++i) + OldFormatName += (char)Record[2 + i]; + Name = OldFormatName; + } + Comdat *C = TheModule->getOrInsertComdat(Name); + C->setSelectionKind(SK); + ComdatList.push_back(C); + return Error::success(); +} + +Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) { + // v1: [pointer type, isconst, initid, linkage, alignment, section, + // visibility, threadlocal, unnamed_addr, externally_initialized, + // dllstorageclass, comdat, attributes] (name in VST) + // v2: [strtab_offset, strtab_size, v1] + StringRef Name; + std::tie(Name, Record) = readNameFromStrtab(Record); + + if (Record.size() < 6) + return error("Invalid record"); + Type *Ty = getTypeByID(Record[0]); + if (!Ty) + return error("Invalid record"); + bool isConstant = Record[1] & 1; + bool explicitType = Record[1] & 2; + unsigned AddressSpace; + if (explicitType) { + AddressSpace = Record[1] >> 2; + } else { + if (!Ty->isPointerTy()) + return error("Invalid type for value"); + AddressSpace = cast<PointerType>(Ty)->getAddressSpace(); + Ty = cast<PointerType>(Ty)->getElementType(); + } + + uint64_t RawLinkage = Record[3]; + GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); + unsigned Alignment; + if (Error Err = parseAlignmentValue(Record[4], Alignment)) + return Err; + std::string Section; + if (Record[5]) { + if (Record[5] - 1 >= SectionTable.size()) + return error("Invalid ID"); + Section = SectionTable[Record[5] - 1]; + } + GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility; + // Local linkage must have default visibility. + if (Record.size() > 6 && !GlobalValue::isLocalLinkage(Linkage)) + // FIXME: Change to an error if non-default in 4.0. + Visibility = getDecodedVisibility(Record[6]); + + GlobalVariable::ThreadLocalMode TLM = GlobalVariable::NotThreadLocal; + if (Record.size() > 7) + TLM = getDecodedThreadLocalMode(Record[7]); + + GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; + if (Record.size() > 8) + UnnamedAddr = getDecodedUnnamedAddrType(Record[8]); + + bool ExternallyInitialized = false; + if (Record.size() > 9) + ExternallyInitialized = Record[9]; + + GlobalVariable *NewGV = + new GlobalVariable(*TheModule, Ty, isConstant, Linkage, nullptr, Name, + nullptr, TLM, AddressSpace, ExternallyInitialized); + NewGV->setAlignment(Alignment); + if (!Section.empty()) + NewGV->setSection(Section); + NewGV->setVisibility(Visibility); + NewGV->setUnnamedAddr(UnnamedAddr); + + if (Record.size() > 10) + NewGV->setDLLStorageClass(getDecodedDLLStorageClass(Record[10])); + else + upgradeDLLImportExportLinkage(NewGV, RawLinkage); + + ValueList.push_back(NewGV); + + // Remember which value to use for the global initializer. + if (unsigned InitID = Record[2]) + GlobalInits.push_back(std::make_pair(NewGV, InitID - 1)); + + if (Record.size() > 11) { + if (unsigned ComdatID = Record[11]) { + if (ComdatID > ComdatList.size()) + return error("Invalid global variable comdat ID"); + NewGV->setComdat(ComdatList[ComdatID - 1]); + } + } else if (hasImplicitComdat(RawLinkage)) { + NewGV->setComdat(reinterpret_cast<Comdat *>(1)); + } + + if (Record.size() > 12) { + auto AS = getAttributes(Record[12]).getFnAttributes(); + NewGV->setAttributes(AS); + } + return Error::success(); +} + +Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) { + // v1: [type, callingconv, isproto, linkage, paramattr, alignment, section, + // visibility, gc, unnamed_addr, prologuedata, dllstorageclass, comdat, + // prefixdata] (name in VST) + // v2: [strtab_offset, strtab_size, v1] + StringRef Name; + std::tie(Name, Record) = readNameFromStrtab(Record); + + if (Record.size() < 8) + return error("Invalid record"); + Type *Ty = getTypeByID(Record[0]); + if (!Ty) + return error("Invalid record"); + if (auto *PTy = dyn_cast<PointerType>(Ty)) + Ty = PTy->getElementType(); + auto *FTy = dyn_cast<FunctionType>(Ty); + if (!FTy) + return error("Invalid type for value"); + auto CC = static_cast<CallingConv::ID>(Record[1]); + if (CC & ~CallingConv::MaxID) + return error("Invalid calling convention ID"); + + Function *Func = + Function::Create(FTy, GlobalValue::ExternalLinkage, Name, TheModule); + + Func->setCallingConv(CC); + bool isProto = Record[2]; + uint64_t RawLinkage = Record[3]; + Func->setLinkage(getDecodedLinkage(RawLinkage)); + Func->setAttributes(getAttributes(Record[4])); + + unsigned Alignment; + if (Error Err = parseAlignmentValue(Record[5], Alignment)) + return Err; + Func->setAlignment(Alignment); + if (Record[6]) { + if (Record[6] - 1 >= SectionTable.size()) + return error("Invalid ID"); + Func->setSection(SectionTable[Record[6] - 1]); + } + // Local linkage must have default visibility. + if (!Func->hasLocalLinkage()) + // FIXME: Change to an error if non-default in 4.0. + Func->setVisibility(getDecodedVisibility(Record[7])); + if (Record.size() > 8 && Record[8]) { + if (Record[8] - 1 >= GCTable.size()) + return error("Invalid ID"); + Func->setGC(GCTable[Record[8] - 1]); + } + GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; + if (Record.size() > 9) + UnnamedAddr = getDecodedUnnamedAddrType(Record[9]); + Func->setUnnamedAddr(UnnamedAddr); + if (Record.size() > 10 && Record[10] != 0) + FunctionPrologues.push_back(std::make_pair(Func, Record[10] - 1)); + + if (Record.size() > 11) + Func->setDLLStorageClass(getDecodedDLLStorageClass(Record[11])); + else + upgradeDLLImportExportLinkage(Func, RawLinkage); + + if (Record.size() > 12) { + if (unsigned ComdatID = Record[12]) { + if (ComdatID > ComdatList.size()) + return error("Invalid function comdat ID"); + Func->setComdat(ComdatList[ComdatID - 1]); + } + } else if (hasImplicitComdat(RawLinkage)) { + Func->setComdat(reinterpret_cast<Comdat *>(1)); + } + + if (Record.size() > 13 && Record[13] != 0) + FunctionPrefixes.push_back(std::make_pair(Func, Record[13] - 1)); + + if (Record.size() > 14 && Record[14] != 0) + FunctionPersonalityFns.push_back(std::make_pair(Func, Record[14] - 1)); + + ValueList.push_back(Func); + + // If this is a function with a body, remember the prototype we are + // creating now, so that we can match up the body with them later. + if (!isProto) { + Func->setIsMaterializable(true); + FunctionsWithBodies.push_back(Func); + DeferredFunctionInfo[Func] = 0; + } + return Error::success(); +} + +Error BitcodeReader::parseGlobalIndirectSymbolRecord( + unsigned BitCode, ArrayRef<uint64_t> Record) { + // v1 ALIAS_OLD: [alias type, aliasee val#, linkage] (name in VST) + // v1 ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility, + // dllstorageclass] (name in VST) + // v1 IFUNC: [alias type, addrspace, aliasee val#, linkage, + // visibility, dllstorageclass] (name in VST) + // v2: [strtab_offset, strtab_size, v1] + StringRef Name; + std::tie(Name, Record) = readNameFromStrtab(Record); + + bool NewRecord = BitCode != bitc::MODULE_CODE_ALIAS_OLD; + if (Record.size() < (3 + (unsigned)NewRecord)) + return error("Invalid record"); + unsigned OpNum = 0; + Type *Ty = getTypeByID(Record[OpNum++]); + if (!Ty) + return error("Invalid record"); + + unsigned AddrSpace; + if (!NewRecord) { + auto *PTy = dyn_cast<PointerType>(Ty); + if (!PTy) + return error("Invalid type for value"); + Ty = PTy->getElementType(); + AddrSpace = PTy->getAddressSpace(); + } else { + AddrSpace = Record[OpNum++]; + } + + auto Val = Record[OpNum++]; + auto Linkage = Record[OpNum++]; + GlobalIndirectSymbol *NewGA; + if (BitCode == bitc::MODULE_CODE_ALIAS || + BitCode == bitc::MODULE_CODE_ALIAS_OLD) + NewGA = GlobalAlias::create(Ty, AddrSpace, getDecodedLinkage(Linkage), Name, + TheModule); + else + NewGA = GlobalIFunc::create(Ty, AddrSpace, getDecodedLinkage(Linkage), Name, + nullptr, TheModule); + // Old bitcode files didn't have visibility field. + // Local linkage must have default visibility. + if (OpNum != Record.size()) { + auto VisInd = OpNum++; + if (!NewGA->hasLocalLinkage()) + // FIXME: Change to an error if non-default in 4.0. + NewGA->setVisibility(getDecodedVisibility(Record[VisInd])); + } + if (OpNum != Record.size()) + NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++])); + else + upgradeDLLImportExportLinkage(NewGA, Linkage); + if (OpNum != Record.size()) + NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++])); + if (OpNum != Record.size()) + NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++])); + ValueList.push_back(NewGA); + IndirectSymbolInits.push_back(std::make_pair(NewGA, Val)); + return Error::success(); +} + Error BitcodeReader::parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata) { if (ResumeBit) @@ -2615,8 +3033,6 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, return error("Invalid record"); SmallVector<uint64_t, 64> Record; - std::vector<std::string> SectionTable; - std::vector<std::string> GCTable; // Read all the records for this module. while (true) { @@ -2750,6 +3166,10 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, if (Error Err = parseOperandBundleTags()) return Err; break; + case bitc::SYNC_SCOPE_NAMES_BLOCK_ID: + if (Error Err = parseSyncScopeNames()) + return Err; + break; } continue; @@ -2762,21 +3182,11 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, auto BitCode = Stream.readRecord(Entry.ID, Record); switch (BitCode) { default: break; // Default behavior, ignore unknown content. - case bitc::MODULE_CODE_VERSION: { // VERSION: [version#] - if (Record.size() < 1) - return error("Invalid record"); - // Only version #0 and #1 are supported so far. - unsigned module_version = Record[0]; - switch (module_version) { - default: - return error("Invalid value"); - case 0: - UseRelativeIDs = false; - break; - case 1: - UseRelativeIDs = true; - break; - } + case bitc::MODULE_CODE_VERSION: { + Expected<unsigned> VersionOrErr = parseVersionRecord(Record); + if (!VersionOrErr) + return VersionOrErr.takeError(); + UseRelativeIDs = *VersionOrErr >= 1; break; } case bitc::MODULE_CODE_TRIPLE: { // TRIPLE: [strchr x N] @@ -2822,249 +3232,28 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, GCTable.push_back(S); break; } - case bitc::MODULE_CODE_COMDAT: { // COMDAT: [selection_kind, name] - if (Record.size() < 2) - return error("Invalid record"); - Comdat::SelectionKind SK = getDecodedComdatSelectionKind(Record[0]); - unsigned ComdatNameSize = Record[1]; - std::string ComdatName; - ComdatName.reserve(ComdatNameSize); - for (unsigned i = 0; i != ComdatNameSize; ++i) - ComdatName += (char)Record[2 + i]; - Comdat *C = TheModule->getOrInsertComdat(ComdatName); - C->setSelectionKind(SK); - ComdatList.push_back(C); - break; - } - // GLOBALVAR: [pointer type, isconst, initid, - // linkage, alignment, section, visibility, threadlocal, - // unnamed_addr, externally_initialized, dllstorageclass, - // comdat] + case bitc::MODULE_CODE_COMDAT: { + if (Error Err = parseComdatRecord(Record)) + return Err; + break; + } case bitc::MODULE_CODE_GLOBALVAR: { - if (Record.size() < 6) - return error("Invalid record"); - Type *Ty = getTypeByID(Record[0]); - if (!Ty) - return error("Invalid record"); - bool isConstant = Record[1] & 1; - bool explicitType = Record[1] & 2; - unsigned AddressSpace; - if (explicitType) { - AddressSpace = Record[1] >> 2; - } else { - if (!Ty->isPointerTy()) - return error("Invalid type for value"); - AddressSpace = cast<PointerType>(Ty)->getAddressSpace(); - Ty = cast<PointerType>(Ty)->getElementType(); - } - - uint64_t RawLinkage = Record[3]; - GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); - unsigned Alignment; - if (Error Err = parseAlignmentValue(Record[4], Alignment)) + if (Error Err = parseGlobalVarRecord(Record)) return Err; - std::string Section; - if (Record[5]) { - if (Record[5]-1 >= SectionTable.size()) - return error("Invalid ID"); - Section = SectionTable[Record[5]-1]; - } - GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility; - // Local linkage must have default visibility. - if (Record.size() > 6 && !GlobalValue::isLocalLinkage(Linkage)) - // FIXME: Change to an error if non-default in 4.0. - Visibility = getDecodedVisibility(Record[6]); - - GlobalVariable::ThreadLocalMode TLM = GlobalVariable::NotThreadLocal; - if (Record.size() > 7) - TLM = getDecodedThreadLocalMode(Record[7]); - - GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; - if (Record.size() > 8) - UnnamedAddr = getDecodedUnnamedAddrType(Record[8]); - - bool ExternallyInitialized = false; - if (Record.size() > 9) - ExternallyInitialized = Record[9]; - - GlobalVariable *NewGV = - new GlobalVariable(*TheModule, Ty, isConstant, Linkage, nullptr, "", nullptr, - TLM, AddressSpace, ExternallyInitialized); - NewGV->setAlignment(Alignment); - if (!Section.empty()) - NewGV->setSection(Section); - NewGV->setVisibility(Visibility); - NewGV->setUnnamedAddr(UnnamedAddr); - - if (Record.size() > 10) - NewGV->setDLLStorageClass(getDecodedDLLStorageClass(Record[10])); - else - upgradeDLLImportExportLinkage(NewGV, RawLinkage); - - ValueList.push_back(NewGV); - - // Remember which value to use for the global initializer. - if (unsigned InitID = Record[2]) - GlobalInits.push_back(std::make_pair(NewGV, InitID-1)); - - if (Record.size() > 11) { - if (unsigned ComdatID = Record[11]) { - if (ComdatID > ComdatList.size()) - return error("Invalid global variable comdat ID"); - NewGV->setComdat(ComdatList[ComdatID - 1]); - } - } else if (hasImplicitComdat(RawLinkage)) { - NewGV->setComdat(reinterpret_cast<Comdat *>(1)); - } - break; } - // FUNCTION: [type, callingconv, isproto, linkage, paramattr, - // alignment, section, visibility, gc, unnamed_addr, - // prologuedata, dllstorageclass, comdat, prefixdata] case bitc::MODULE_CODE_FUNCTION: { - if (Record.size() < 8) - return error("Invalid record"); - Type *Ty = getTypeByID(Record[0]); - if (!Ty) - return error("Invalid record"); - if (auto *PTy = dyn_cast<PointerType>(Ty)) - Ty = PTy->getElementType(); - auto *FTy = dyn_cast<FunctionType>(Ty); - if (!FTy) - return error("Invalid type for value"); - auto CC = static_cast<CallingConv::ID>(Record[1]); - if (CC & ~CallingConv::MaxID) - return error("Invalid calling convention ID"); - - Function *Func = Function::Create(FTy, GlobalValue::ExternalLinkage, - "", TheModule); - - Func->setCallingConv(CC); - bool isProto = Record[2]; - uint64_t RawLinkage = Record[3]; - Func->setLinkage(getDecodedLinkage(RawLinkage)); - Func->setAttributes(getAttributes(Record[4])); - - unsigned Alignment; - if (Error Err = parseAlignmentValue(Record[5], Alignment)) + if (Error Err = parseFunctionRecord(Record)) return Err; - Func->setAlignment(Alignment); - if (Record[6]) { - if (Record[6]-1 >= SectionTable.size()) - return error("Invalid ID"); - Func->setSection(SectionTable[Record[6]-1]); - } - // Local linkage must have default visibility. - if (!Func->hasLocalLinkage()) - // FIXME: Change to an error if non-default in 4.0. - Func->setVisibility(getDecodedVisibility(Record[7])); - if (Record.size() > 8 && Record[8]) { - if (Record[8]-1 >= GCTable.size()) - return error("Invalid ID"); - Func->setGC(GCTable[Record[8] - 1]); - } - GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; - if (Record.size() > 9) - UnnamedAddr = getDecodedUnnamedAddrType(Record[9]); - Func->setUnnamedAddr(UnnamedAddr); - if (Record.size() > 10 && Record[10] != 0) - FunctionPrologues.push_back(std::make_pair(Func, Record[10]-1)); - - if (Record.size() > 11) - Func->setDLLStorageClass(getDecodedDLLStorageClass(Record[11])); - else - upgradeDLLImportExportLinkage(Func, RawLinkage); - - if (Record.size() > 12) { - if (unsigned ComdatID = Record[12]) { - if (ComdatID > ComdatList.size()) - return error("Invalid function comdat ID"); - Func->setComdat(ComdatList[ComdatID - 1]); - } - } else if (hasImplicitComdat(RawLinkage)) { - Func->setComdat(reinterpret_cast<Comdat *>(1)); - } - - if (Record.size() > 13 && Record[13] != 0) - FunctionPrefixes.push_back(std::make_pair(Func, Record[13]-1)); - - if (Record.size() > 14 && Record[14] != 0) - FunctionPersonalityFns.push_back(std::make_pair(Func, Record[14] - 1)); - - ValueList.push_back(Func); - - // If this is a function with a body, remember the prototype we are - // creating now, so that we can match up the body with them later. - if (!isProto) { - Func->setIsMaterializable(true); - FunctionsWithBodies.push_back(Func); - DeferredFunctionInfo[Func] = 0; - } break; } - // ALIAS: [alias type, addrspace, aliasee val#, linkage] - // ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility, dllstorageclass] - // IFUNC: [alias type, addrspace, aliasee val#, linkage, visibility, dllstorageclass] case bitc::MODULE_CODE_IFUNC: case bitc::MODULE_CODE_ALIAS: case bitc::MODULE_CODE_ALIAS_OLD: { - bool NewRecord = BitCode != bitc::MODULE_CODE_ALIAS_OLD; - if (Record.size() < (3 + (unsigned)NewRecord)) - return error("Invalid record"); - unsigned OpNum = 0; - Type *Ty = getTypeByID(Record[OpNum++]); - if (!Ty) - return error("Invalid record"); - - unsigned AddrSpace; - if (!NewRecord) { - auto *PTy = dyn_cast<PointerType>(Ty); - if (!PTy) - return error("Invalid type for value"); - Ty = PTy->getElementType(); - AddrSpace = PTy->getAddressSpace(); - } else { - AddrSpace = Record[OpNum++]; - } - - auto Val = Record[OpNum++]; - auto Linkage = Record[OpNum++]; - GlobalIndirectSymbol *NewGA; - if (BitCode == bitc::MODULE_CODE_ALIAS || - BitCode == bitc::MODULE_CODE_ALIAS_OLD) - NewGA = GlobalAlias::create(Ty, AddrSpace, getDecodedLinkage(Linkage), - "", TheModule); - else - NewGA = GlobalIFunc::create(Ty, AddrSpace, getDecodedLinkage(Linkage), - "", nullptr, TheModule); - // Old bitcode files didn't have visibility field. - // Local linkage must have default visibility. - if (OpNum != Record.size()) { - auto VisInd = OpNum++; - if (!NewGA->hasLocalLinkage()) - // FIXME: Change to an error if non-default in 4.0. - NewGA->setVisibility(getDecodedVisibility(Record[VisInd])); - } - if (OpNum != Record.size()) - NewGA->setDLLStorageClass(getDecodedDLLStorageClass(Record[OpNum++])); - else - upgradeDLLImportExportLinkage(NewGA, Linkage); - if (OpNum != Record.size()) - NewGA->setThreadLocalMode(getDecodedThreadLocalMode(Record[OpNum++])); - if (OpNum != Record.size()) - NewGA->setUnnamedAddr(getDecodedUnnamedAddrType(Record[OpNum++])); - ValueList.push_back(NewGA); - IndirectSymbolInits.push_back(std::make_pair(NewGA, Val)); - break; - } - /// MODULE_CODE_PURGEVALS: [numvals] - case bitc::MODULE_CODE_PURGEVALS: - // Trim down the value list to the specified size. - if (Record.size() < 1 || Record[0] > ValueList.size()) - return error("Invalid record"); - ValueList.shrinkTo(Record[0]); + if (Error Err = parseGlobalIndirectSymbolRecord(BitCode, Record)) + return Err; break; + } /// MODULE_CODE_VSTOFFSET: [offset] case bitc::MODULE_CODE_VSTOFFSET: if (Record.size() < 1) @@ -3840,7 +4029,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (Record.size() < 4) return error("Invalid record"); unsigned OpNum = 0; - AttributeSet PAL = getAttributes(Record[OpNum++]); + AttributeList PAL = getAttributes(Record[OpNum++]); unsigned CCInfo = Record[OpNum++]; BasicBlock *NormalBB = getBasicBlock(Record[OpNum++]); BasicBlock *UnwindBB = getBasicBlock(Record[OpNum++]); @@ -4017,7 +4206,12 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } if (!Ty || !Size) return error("Invalid record"); - AllocaInst *AI = new AllocaInst(Ty, Size, Align); + + // FIXME: Make this an optional field. + const DataLayout &DL = TheModule->getDataLayout(); + unsigned AS = DL.getAllocaAddrSpace(); + + AllocaInst *AI = new AllocaInst(Ty, AS, Size, Align); AI->setUsedWithInAlloca(InAlloca); AI->setSwiftError(SwiftError); I = AI; @@ -4048,7 +4242,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_INST_LOADATOMIC: { - // LOADATOMIC: [opty, op, align, vol, ordering, synchscope] + // LOADATOMIC: [opty, op, align, vol, ordering, ssid] unsigned OpNum = 0; Value *Op; if (getValueTypePair(Record, OpNum, NextValueNo, Op) || @@ -4070,12 +4264,12 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); unsigned Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SynchScope); + I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SSID); InstructionList.push_back(I); break; @@ -4104,7 +4298,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } case bitc::FUNC_CODE_INST_STOREATOMIC: case bitc::FUNC_CODE_INST_STOREATOMIC_OLD: { - // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, synchscope] + // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, ssid] unsigned OpNum = 0; Value *Val, *Ptr; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || @@ -4124,20 +4318,20 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::AcquireRelease) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0) return error("Invalid record"); unsigned Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new StoreInst(Val, Ptr, Record[OpNum+1], Align, Ordering, SynchScope); + I = new StoreInst(Val, Ptr, Record[OpNum+1], Align, Ordering, SSID); InstructionList.push_back(I); break; } case bitc::FUNC_CODE_INST_CMPXCHG_OLD: case bitc::FUNC_CODE_INST_CMPXCHG: { - // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, synchscope, + // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, ssid, // failureordering?, isweak?] unsigned OpNum = 0; Value *Ptr, *Cmp, *New; @@ -4154,7 +4348,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (SuccessOrdering == AtomicOrdering::NotAtomic || SuccessOrdering == AtomicOrdering::Unordered) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 2]); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 2]); if (Error Err = typeCheckLoadStoreInst(Cmp->getType(), Ptr->getType())) return Err; @@ -4166,7 +4360,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { FailureOrdering = getDecodedOrdering(Record[OpNum + 3]); I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering, - SynchScope); + SSID); cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]); if (Record.size() < 8) { @@ -4183,7 +4377,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_INST_ATOMICRMW: { - // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, synchscope] + // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, ssid] unsigned OpNum = 0; Value *Ptr, *Val; if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || @@ -4200,13 +4394,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (Ordering == AtomicOrdering::NotAtomic || Ordering == AtomicOrdering::Unordered) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[OpNum + 3]); - I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SynchScope); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); + I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID); cast<AtomicRMWInst>(I)->setVolatile(Record[OpNum+1]); InstructionList.push_back(I); break; } - case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, synchscope] + case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, ssid] if (2 != Record.size()) return error("Invalid record"); AtomicOrdering Ordering = getDecodedOrdering(Record[0]); @@ -4214,8 +4408,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Ordering == AtomicOrdering::Unordered || Ordering == AtomicOrdering::Monotonic) return error("Invalid record"); - SynchronizationScope SynchScope = getDecodedSynchScope(Record[1]); - I = new FenceInst(Context, Ordering, SynchScope); + SyncScope::ID SSID = getDecodedSyncScopeID(Record[1]); + I = new FenceInst(Context, Ordering, SSID); InstructionList.push_back(I); break; } @@ -4225,7 +4419,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); unsigned OpNum = 0; - AttributeSet PAL = getAttributes(Record[OpNum++]); + AttributeList PAL = getAttributes(Record[OpNum++]); unsigned CCInfo = Record[OpNum++]; FastMathFlags FMF; @@ -4343,11 +4537,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // Add instruction to end of current BB. If there is no current BB, reject // this file. if (!CurBB) { - delete I; + I->deleteValue(); return error("Invalid instruction with no BB"); } if (!OperandBundles.empty()) { - delete I; + I->deleteValue(); return error("Operand bundles found with no consumer"); } CurBB->getInstList().push_back(I); @@ -4411,6 +4605,14 @@ Error BitcodeReader::findFunctionInStream( return Error::success(); } +SyncScope::ID BitcodeReader::getDecodedSyncScopeID(unsigned Val) { + if (Val == SyncScope::SingleThread || Val == SyncScope::System) + return SyncScope::ID(Val); + if (Val >= SSIDs.size()) + return SyncScope::System; // Map unknown synchronization scopes to system. + return SSIDs[Val]; +} + //===----------------------------------------------------------------------===// // GVMaterializer implementation //===----------------------------------------------------------------------===// @@ -4540,14 +4742,37 @@ std::vector<StructType *> BitcodeReader::getIdentifiedStructTypes() const { } ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader( - BitstreamCursor Cursor, ModuleSummaryIndex &TheIndex) - : BitcodeReaderBase(std::move(Cursor)), TheIndex(TheIndex) {} - -std::pair<GlobalValue::GUID, GlobalValue::GUID> -ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) { - auto VGI = ValueIdToCallGraphGUIDMap.find(ValueId); - assert(VGI != ValueIdToCallGraphGUIDMap.end()); - return VGI->second; + BitstreamCursor Cursor, StringRef Strtab, ModuleSummaryIndex &TheIndex, + StringRef ModulePath, unsigned ModuleId) + : BitcodeReaderBase(std::move(Cursor), Strtab), TheIndex(TheIndex), + ModulePath(ModulePath), ModuleId(ModuleId) {} + +ModuleSummaryIndex::ModuleInfo * +ModuleSummaryIndexBitcodeReader::addThisModule() { + return TheIndex.addModule(ModulePath, ModuleId); +} + +std::pair<ValueInfo, GlobalValue::GUID> +ModuleSummaryIndexBitcodeReader::getValueInfoFromValueId(unsigned ValueId) { + auto VGI = ValueIdToValueInfoMap[ValueId]; + assert(VGI.first); + return VGI; +} + +void ModuleSummaryIndexBitcodeReader::setValueGUID( + uint64_t ValueID, StringRef ValueName, GlobalValue::LinkageTypes Linkage, + StringRef SourceFileName) { + std::string GlobalId = + GlobalValue::getGlobalIdentifier(ValueName, Linkage, SourceFileName); + auto ValueGUID = GlobalValue::getGUID(GlobalId); + auto OriginalNameID = ValueGUID; + if (GlobalValue::isLocalLinkage(Linkage)) + OriginalNameID = GlobalValue::getGUID(ValueName); + if (PrintSummaryGUIDs) + dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is " + << ValueName << "\n"; + ValueIdToValueInfoMap[ValueID] = + std::make_pair(TheIndex.getOrInsertValueInfo(ValueGUID), OriginalNameID); } // Specialized value symbol table parser used when reading module index @@ -4556,6 +4781,10 @@ ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) { Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( uint64_t Offset, DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap) { + // With a strtab the VST is not required to parse the summary. + if (UseStrtab) + return Error::success(); + assert(Offset > 0 && "Expected non-zero VST offset"); uint64_t CurrentBit = jumpToValueSymbolTable(Offset, Stream); @@ -4597,17 +4826,7 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( assert(VLI != ValueIdToLinkageMap.end() && "No linkage found for VST entry?"); auto Linkage = VLI->second; - std::string GlobalId = - GlobalValue::getGlobalIdentifier(ValueName, Linkage, SourceFileName); - auto ValueGUID = GlobalValue::getGUID(GlobalId); - auto OriginalNameID = ValueGUID; - if (GlobalValue::isLocalLinkage(Linkage)) - OriginalNameID = GlobalValue::getGUID(ValueName); - if (PrintSummaryGUIDs) - dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is " - << ValueName << "\n"; - ValueIdToCallGraphGUIDMap[ValueID] = - std::make_pair(ValueGUID, OriginalNameID); + setValueGUID(ValueID, ValueName, Linkage, SourceFileName); ValueName.clear(); break; } @@ -4621,18 +4840,7 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( assert(VLI != ValueIdToLinkageMap.end() && "No linkage found for VST entry?"); auto Linkage = VLI->second; - std::string FunctionGlobalId = GlobalValue::getGlobalIdentifier( - ValueName, VLI->second, SourceFileName); - auto FunctionGUID = GlobalValue::getGUID(FunctionGlobalId); - auto OriginalNameID = FunctionGUID; - if (GlobalValue::isLocalLinkage(Linkage)) - OriginalNameID = GlobalValue::getGUID(ValueName); - if (PrintSummaryGUIDs) - dbgs() << "GUID " << FunctionGUID << "(" << OriginalNameID << ") is " - << ValueName << "\n"; - ValueIdToCallGraphGUIDMap[ValueID] = - std::make_pair(FunctionGUID, OriginalNameID); - + setValueGUID(ValueID, ValueName, Linkage, SourceFileName); ValueName.clear(); break; } @@ -4642,7 +4850,8 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( GlobalValue::GUID RefGUID = Record[1]; // The "original name", which is the second value of the pair will be // overriden later by a FS_COMBINED_ORIGINAL_NAME in the combined index. - ValueIdToCallGraphGUIDMap[ValueID] = std::make_pair(RefGUID, RefGUID); + ValueIdToValueInfoMap[ValueID] = + std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID); break; } } @@ -4652,7 +4861,7 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( // Parse just the blocks needed for building the index out of the module. // At the end of this routine the module Index is populated with a map // from global value id to GlobalValueSummary objects. -Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { +Error ModuleSummaryIndexBitcodeReader::parseModule() { if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) return error("Invalid record"); @@ -4691,6 +4900,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { return error("Invalid record"); break; case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: + case bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID: assert(!SeenValueSymbolTable && "Already read VST when parsing summary block?"); // We might not have a VST if there were no values in the @@ -4703,7 +4913,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { SeenValueSymbolTable = true; } SeenGlobalValSummary = true; - if (Error Err = parseEntireSummary(ModulePath)) + if (Error Err = parseEntireSummary(Entry.ID)) return Err; break; case bitc::MODULE_STRTAB_BLOCK_ID: @@ -4719,6 +4929,11 @@ Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { switch (BitCode) { default: break; // Default behavior, ignore unknown content. + case bitc::MODULE_CODE_VERSION: { + if (Error Err = parseVersionRecord(Record).takeError()) + return Err; + break; + } /// MODULE_CODE_SOURCE_FILENAME: [namechar x N] case bitc::MODULE_CODE_SOURCE_FILENAME: { SmallString<128> ValueName; @@ -4731,12 +4946,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { case bitc::MODULE_CODE_HASH: { if (Record.size() != 5) return error("Invalid hash length " + Twine(Record.size()).str()); - if (TheIndex.modulePaths().empty()) - // We always seed the index with the module. - TheIndex.addModulePath(ModulePath, 0); - if (TheIndex.modulePaths().size() != 1) - return error("Don't expect multiple modules defined?"); - auto &Hash = TheIndex.modulePaths().begin()->second.second; + auto &Hash = addThisModule()->second.second; int Pos = 0; for (auto &Val : Record) { assert(!(Val >> 32) && "Unexpected high bits set"); @@ -4753,37 +4963,26 @@ Error ModuleSummaryIndexBitcodeReader::parseModule(StringRef ModulePath) { // was historically always the start of the regular bitcode header. VSTOffset = Record[0] - 1; break; - // GLOBALVAR: [pointer type, isconst, initid, - // linkage, alignment, section, visibility, threadlocal, - // unnamed_addr, externally_initialized, dllstorageclass, - // comdat] - case bitc::MODULE_CODE_GLOBALVAR: { - if (Record.size() < 6) - return error("Invalid record"); - uint64_t RawLinkage = Record[3]; - GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); - ValueIdToLinkageMap[ValueId++] = Linkage; - break; - } - // FUNCTION: [type, callingconv, isproto, linkage, paramattr, - // alignment, section, visibility, gc, unnamed_addr, - // prologuedata, dllstorageclass, comdat, prefixdata] - case bitc::MODULE_CODE_FUNCTION: { - if (Record.size() < 8) - return error("Invalid record"); - uint64_t RawLinkage = Record[3]; - GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); - ValueIdToLinkageMap[ValueId++] = Linkage; - break; - } - // ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility, - // dllstorageclass] + // v1 GLOBALVAR: [pointer type, isconst, initid, linkage, ...] + // v1 FUNCTION: [type, callingconv, isproto, linkage, ...] + // v1 ALIAS: [alias type, addrspace, aliasee val#, linkage, ...] + // v2: [strtab offset, strtab size, v1] + case bitc::MODULE_CODE_GLOBALVAR: + case bitc::MODULE_CODE_FUNCTION: case bitc::MODULE_CODE_ALIAS: { - if (Record.size() < 6) + StringRef Name; + ArrayRef<uint64_t> GVRecord; + std::tie(Name, GVRecord) = readNameFromStrtab(Record); + if (GVRecord.size() <= 3) return error("Invalid record"); - uint64_t RawLinkage = Record[3]; + uint64_t RawLinkage = GVRecord[3]; GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage); - ValueIdToLinkageMap[ValueId++] = Linkage; + if (!UseStrtab) { + ValueIdToLinkageMap[ValueId++] = Linkage; + break; + } + + setValueGUID(ValueId++, Name, Linkage, SourceFileName); break; } } @@ -4798,7 +4997,7 @@ ModuleSummaryIndexBitcodeReader::makeRefList(ArrayRef<uint64_t> Record) { std::vector<ValueInfo> Ret; Ret.reserve(Record.size()); for (uint64_t RefValueId : Record) - Ret.push_back(getGUIDFromValueId(RefValueId).first); + Ret.push_back(getValueInfoFromValueId(RefValueId).first); return Ret; } @@ -4808,23 +5007,22 @@ std::vector<FunctionSummary::EdgeTy> ModuleSummaryIndexBitcodeReader::makeCallLi Ret.reserve(Record.size()); for (unsigned I = 0, E = Record.size(); I != E; ++I) { CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown; - GlobalValue::GUID CalleeGUID = getGUIDFromValueId(Record[I]).first; + ValueInfo Callee = getValueInfoFromValueId(Record[I]).first; if (IsOldProfileFormat) { I += 1; // Skip old callsitecount field if (HasProfile) I += 1; // Skip old profilecount field } else if (HasProfile) Hotness = static_cast<CalleeInfo::HotnessType>(Record[++I]); - Ret.push_back(FunctionSummary::EdgeTy{CalleeGUID, CalleeInfo{Hotness}}); + Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}}); } return Ret; } // Eagerly parse the entire summary block. This populates the GlobalValueSummary // objects in the index. -Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( - StringRef ModulePath) { - if (Stream.EnterSubBlock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID)) +Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { + if (Stream.EnterSubBlock(ID)) return error("Invalid record"); SmallVector<uint64_t, 64> Record; @@ -4846,8 +5044,16 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( // Keep around the last seen summary to be used when we see an optional // "OriginalName" attachement. GlobalValueSummary *LastSeenSummary = nullptr; - bool Combined = false; + GlobalValue::GUID LastSeenGUID = 0; + + // We can expect to see any number of type ID information records before + // each function summary records; these variables store the information + // collected so far so that it can be used to create the summary object. std::vector<GlobalValue::GUID> PendingTypeTests; + std::vector<FunctionSummary::VFuncId> PendingTypeTestAssumeVCalls, + PendingTypeCheckedLoadVCalls; + std::vector<FunctionSummary::ConstVCall> PendingTypeTestAssumeConstVCalls, + PendingTypeCheckedLoadConstVCalls; while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); @@ -4857,16 +5063,6 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - // For a per-module index, remove any entries that still have empty - // summaries. The VST parsing creates entries eagerly for all symbols, - // but not all have associated summaries (e.g. it doesn't know how to - // distinguish between VST_CODE_ENTRY for function declarations vs global - // variables with initializers that end up with a summary). Remove those - // entries now so that we don't need to rely on the combined index merger - // to clean them up (especially since that may not run for the first - // module's index if we merge into that). - if (!Combined) - TheIndex.removeEmptySummaryEntries(); return Error::success(); case BitstreamEntry::Record: // The interesting case. @@ -4885,6 +5081,13 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( switch (BitCode) { default: // Default behavior: ignore. break; + case bitc::FS_VALUE_GUID: { // [valueid, refguid] + uint64_t ValueID = Record[0]; + GlobalValue::GUID RefGUID = Record[1]; + ValueIdToValueInfoMap[ValueID] = + std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID); + break; + } // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid, // n x (valueid)] // FS_PERMODULE_PROFILE: [valueid, flags, instcount, numrefs, @@ -4914,12 +5117,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( IsOldProfileFormat, HasProfile); auto FS = llvm::make_unique<FunctionSummary>( Flags, InstCount, std::move(Refs), std::move(Calls), - std::move(PendingTypeTests)); + std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), + std::move(PendingTypeCheckedLoadVCalls), + std::move(PendingTypeTestAssumeConstVCalls), + std::move(PendingTypeCheckedLoadConstVCalls)); PendingTypeTests.clear(); - auto GUID = getGUIDFromValueId(ValueID); - FS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first()); - FS->setOriginalName(GUID.second); - TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); + PendingTypeTestAssumeVCalls.clear(); + PendingTypeCheckedLoadVCalls.clear(); + PendingTypeTestAssumeConstVCalls.clear(); + PendingTypeCheckedLoadConstVCalls.clear(); + auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID); + FS->setModulePath(addThisModule()->first()); + FS->setOriginalName(VIAndOriginalGUID.second); + TheIndex.addGlobalValueSummary(VIAndOriginalGUID.first, std::move(FS)); break; } // FS_ALIAS: [valueid, flags, valueid] @@ -4937,15 +5147,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. - AS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first()); + AS->setModulePath(addThisModule()->first()); - GlobalValue::GUID AliaseeGUID = getGUIDFromValueId(AliaseeID).first; - auto *AliaseeSummary = TheIndex.getGlobalValueSummary(AliaseeGUID); - if (!AliaseeSummary) + GlobalValue::GUID AliaseeGUID = + getValueInfoFromValueId(AliaseeID).first.getGUID(); + auto AliaseeInModule = + TheIndex.findSummaryInModule(AliaseeGUID, ModulePath); + if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); - AS->setAliasee(AliaseeSummary); + AS->setAliasee(AliaseeInModule); - auto GUID = getGUIDFromValueId(ValueID); + auto GUID = getValueInfoFromValueId(ValueID); AS->setOriginalName(GUID.second); TheIndex.addGlobalValueSummary(GUID.first, std::move(AS)); break; @@ -4958,8 +5170,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( std::vector<ValueInfo> Refs = makeRefList(ArrayRef<uint64_t>(Record).slice(2)); auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs)); - FS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first()); - auto GUID = getGUIDFromValueId(ValueID); + FS->setModulePath(addThisModule()->first()); + auto GUID = getValueInfoFromValueId(ValueID); FS->setOriginalName(GUID.second); TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); break; @@ -4986,15 +5198,22 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( std::vector<FunctionSummary::EdgeTy> Edges = makeCallList( ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile); - GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; + ValueInfo VI = getValueInfoFromValueId(ValueID).first; auto FS = llvm::make_unique<FunctionSummary>( Flags, InstCount, std::move(Refs), std::move(Edges), - std::move(PendingTypeTests)); + std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), + std::move(PendingTypeCheckedLoadVCalls), + std::move(PendingTypeTestAssumeConstVCalls), + std::move(PendingTypeCheckedLoadConstVCalls)); PendingTypeTests.clear(); + PendingTypeTestAssumeVCalls.clear(); + PendingTypeCheckedLoadVCalls.clear(); + PendingTypeTestAssumeConstVCalls.clear(); + PendingTypeCheckedLoadConstVCalls.clear(); LastSeenSummary = FS.get(); + LastSeenGUID = VI.getGUID(); FS->setModulePath(ModuleIdMap[ModuleId]); - TheIndex.addGlobalValueSummary(GUID, std::move(FS)); - Combined = true; + TheIndex.addGlobalValueSummary(VI, std::move(FS)); break; } // FS_COMBINED_ALIAS: [valueid, modid, flags, valueid] @@ -5010,16 +5229,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( LastSeenSummary = AS.get(); AS->setModulePath(ModuleIdMap[ModuleId]); - auto AliaseeGUID = getGUIDFromValueId(AliaseeValueId).first; + auto AliaseeGUID = + getValueInfoFromValueId(AliaseeValueId).first.getGUID(); auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeGUID, AS->modulePath()); if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); AS->setAliasee(AliaseeInModule); - GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; - TheIndex.addGlobalValueSummary(GUID, std::move(AS)); - Combined = true; + ValueInfo VI = getValueInfoFromValueId(ValueID).first; + LastSeenGUID = VI.getGUID(); + TheIndex.addGlobalValueSummary(VI, std::move(AS)); break; } // FS_COMBINED_GLOBALVAR_INIT_REFS: [valueid, modid, flags, n x valueid] @@ -5033,9 +5253,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs)); LastSeenSummary = FS.get(); FS->setModulePath(ModuleIdMap[ModuleId]); - GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; - TheIndex.addGlobalValueSummary(GUID, std::move(FS)); - Combined = true; + ValueInfo VI = getValueInfoFromValueId(ValueID).first; + LastSeenGUID = VI.getGUID(); + TheIndex.addGlobalValueSummary(VI, std::move(FS)); break; } // FS_COMBINED_ORIGINAL_NAME: [original_name] @@ -5044,8 +5264,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( if (!LastSeenSummary) return error("Name attachment that does not follow a combined record"); LastSeenSummary->setOriginalName(OriginalName); + TheIndex.addOriginalName(LastSeenGUID, OriginalName); // Reset the LastSeenSummary LastSeenSummary = nullptr; + LastSeenGUID = 0; break; } case bitc::FS_TYPE_TESTS: { @@ -5054,6 +5276,42 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary( Record.end()); break; } + case bitc::FS_TYPE_TEST_ASSUME_VCALLS: { + assert(PendingTypeTestAssumeVCalls.empty()); + for (unsigned I = 0; I != Record.size(); I += 2) + PendingTypeTestAssumeVCalls.push_back({Record[I], Record[I+1]}); + break; + } + case bitc::FS_TYPE_CHECKED_LOAD_VCALLS: { + assert(PendingTypeCheckedLoadVCalls.empty()); + for (unsigned I = 0; I != Record.size(); I += 2) + PendingTypeCheckedLoadVCalls.push_back({Record[I], Record[I+1]}); + break; + } + case bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL: { + PendingTypeTestAssumeConstVCalls.push_back( + {{Record[0], Record[1]}, {Record.begin() + 2, Record.end()}}); + break; + } + case bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL: { + PendingTypeCheckedLoadConstVCalls.push_back( + {{Record[0], Record[1]}, {Record.begin() + 2, Record.end()}}); + break; + } + case bitc::FS_CFI_FUNCTION_DEFS: { + std::set<std::string> &CfiFunctionDefs = TheIndex.cfiFunctionDefs(); + for (unsigned I = 0; I != Record.size(); I += 2) + CfiFunctionDefs.insert( + {Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])}); + break; + } + case bitc::FS_CFI_FUNCTION_DECLS: { + std::set<std::string> &CfiFunctionDecls = TheIndex.cfiFunctionDecls(); + for (unsigned I = 0; I != Record.size(); I += 2) + CfiFunctionDecls.insert( + {Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])}); + break; + } } } llvm_unreachable("Exit infinite loop"); @@ -5068,7 +5326,7 @@ Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { SmallVector<uint64_t, 64> Record; SmallString<128> ModulePath; - ModulePathStringTableTy::iterator LastSeenModulePath; + ModuleSummaryIndex::ModuleInfo *LastSeenModule = nullptr; while (true) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); @@ -5095,8 +5353,8 @@ Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { if (convertToString(Record, 1, ModulePath)) return error("Invalid record"); - LastSeenModulePath = TheIndex.addModulePath(ModulePath, ModuleId); - ModuleIdMap[ModuleId] = LastSeenModulePath->first(); + LastSeenModule = TheIndex.addModule(ModulePath, ModuleId); + ModuleIdMap[ModuleId] = LastSeenModule->first(); ModulePath.clear(); break; @@ -5105,15 +5363,15 @@ Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { case bitc::MST_CODE_HASH: { if (Record.size() != 5) return error("Invalid hash length " + Twine(Record.size()).str()); - if (LastSeenModulePath == TheIndex.modulePaths().end()) + if (!LastSeenModule) return error("Invalid hash that does not follow a module path"); int Pos = 0; for (auto &Val : Record) { assert(!(Val >> 32) && "Unexpected high bits set"); - LastSeenModulePath->second.second[Pos++] = Val; + LastSeenModule->second.second[Pos++] = Val; } - // Reset LastSeenModulePath to avoid overriding the hash unexpectedly. - LastSeenModulePath = TheIndex.modulePaths().end(); + // Reset LastSeenModule to avoid overriding the hash unexpectedly. + LastSeenModule = nullptr; break; } } @@ -5148,18 +5406,56 @@ const std::error_category &llvm::BitcodeErrorCategory() { return *ErrorCategory; } +static Expected<StringRef> readBlobInRecord(BitstreamCursor &Stream, + unsigned Block, unsigned RecordID) { + if (Stream.EnterSubBlock(Block)) + return error("Invalid record"); + + StringRef Strtab; + while (1) { + BitstreamEntry Entry = Stream.advance(); + switch (Entry.Kind) { + case BitstreamEntry::EndBlock: + return Strtab; + + case BitstreamEntry::Error: + return error("Malformed block"); + + case BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return error("Malformed block"); + break; + + case BitstreamEntry::Record: + StringRef Blob; + SmallVector<uint64_t, 1> Record; + if (Stream.readRecord(Entry.ID, Record, &Blob) == RecordID) + Strtab = Blob; + break; + } + } +} + //===----------------------------------------------------------------------===// // External interface //===----------------------------------------------------------------------===// Expected<std::vector<BitcodeModule>> llvm::getBitcodeModuleList(MemoryBufferRef Buffer) { + auto FOrErr = getBitcodeFileContents(Buffer); + if (!FOrErr) + return FOrErr.takeError(); + return std::move(FOrErr->Mods); +} + +Expected<BitcodeFileContents> +llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { Expected<BitstreamCursor> StreamOrErr = initStream(Buffer); if (!StreamOrErr) return StreamOrErr.takeError(); BitstreamCursor &Stream = *StreamOrErr; - std::vector<BitcodeModule> Modules; + BitcodeFileContents F; while (true) { uint64_t BCBegin = Stream.getCurrentByteNo(); @@ -5167,7 +5463,7 @@ llvm::getBitcodeModuleList(MemoryBufferRef Buffer) { // of the bitcode stream (e.g. Apple's ar tool). If we are close enough to // the end that there cannot possibly be another module, stop looking. if (BCBegin + 8 >= Stream.getBitcodeBytes().size()) - return Modules; + return F; BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { @@ -5193,10 +5489,49 @@ llvm::getBitcodeModuleList(MemoryBufferRef Buffer) { if (Stream.SkipBlock()) return error("Malformed block"); - Modules.push_back({Stream.getBitcodeBytes().slice( - BCBegin, Stream.getCurrentByteNo() - BCBegin), - Buffer.getBufferIdentifier(), IdentificationBit, - ModuleBit}); + F.Mods.push_back({Stream.getBitcodeBytes().slice( + BCBegin, Stream.getCurrentByteNo() - BCBegin), + Buffer.getBufferIdentifier(), IdentificationBit, + ModuleBit}); + continue; + } + + if (Entry.ID == bitc::STRTAB_BLOCK_ID) { + Expected<StringRef> Strtab = + readBlobInRecord(Stream, bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB); + if (!Strtab) + return Strtab.takeError(); + // This string table is used by every preceding bitcode module that does + // not have its own string table. A bitcode file may have multiple + // string tables if it was created by binary concatenation, for example + // with "llvm-cat -b". + for (auto I = F.Mods.rbegin(), E = F.Mods.rend(); I != E; ++I) { + if (!I->Strtab.empty()) + break; + I->Strtab = *Strtab; + } + // Similarly, the string table is used by every preceding symbol table; + // normally there will be just one unless the bitcode file was created + // by binary concatenation. + if (!F.Symtab.empty() && F.StrtabForSymtab.empty()) + F.StrtabForSymtab = *Strtab; + continue; + } + + if (Entry.ID == bitc::SYMTAB_BLOCK_ID) { + Expected<StringRef> SymtabOrErr = + readBlobInRecord(Stream, bitc::SYMTAB_BLOCK_ID, bitc::SYMTAB_BLOB); + if (!SymtabOrErr) + return SymtabOrErr.takeError(); + + // We can expect the bitcode file to have multiple symbol tables if it + // was created by binary concatenation. In that case we silently + // ignore any subsequent symbol tables, which is fine because this is a + // low level function. The client is expected to notice that the number + // of modules in the symbol table does not match the number of modules + // in the input file and regenerate the symbol table. + if (F.Symtab.empty()) + F.Symtab = *SymtabOrErr; continue; } @@ -5236,8 +5571,8 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, } Stream.JumpToBit(ModuleBit); - auto *R = - new BitcodeReader(std::move(Stream), ProducerIdentification, Context); + auto *R = new BitcodeReader(std::move(Stream), Strtab, ProducerIdentification, + Context); std::unique_ptr<Module> M = llvm::make_unique<Module>(ModuleIdentifier, Context); @@ -5266,22 +5601,37 @@ BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, return getModuleImpl(Context, false, ShouldLazyLoadMetadata, IsImporting); } +// Parse the specified bitcode buffer and merge the index into CombinedIndex. +// We don't use ModuleIdentifier here because the client may need to control the +// module path used in the combined summary (e.g. when reading summaries for +// regular LTO modules). +Error BitcodeModule::readSummary(ModuleSummaryIndex &CombinedIndex, + StringRef ModulePath, uint64_t ModuleId) { + BitstreamCursor Stream(Buffer); + Stream.JumpToBit(ModuleBit); + + ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, CombinedIndex, + ModulePath, ModuleId); + return R.parseModule(); +} + // Parse the specified bitcode buffer, returning the function info index. Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() { BitstreamCursor Stream(Buffer); Stream.JumpToBit(ModuleBit); auto Index = llvm::make_unique<ModuleSummaryIndex>(); - ModuleSummaryIndexBitcodeReader R(std::move(Stream), *Index); + ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, *Index, + ModuleIdentifier, 0); - if (Error Err = R.parseModule(ModuleIdentifier)) + if (Error Err = R.parseModule()) return std::move(Err); return std::move(Index); } // Check if the given bitcode buffer contains a global value summary block. -Expected<bool> BitcodeModule::hasSummary() { +Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() { BitstreamCursor Stream(Buffer); Stream.JumpToBit(ModuleBit); @@ -5295,11 +5645,14 @@ Expected<bool> BitcodeModule::hasSummary() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return false; + return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false}; case BitstreamEntry::SubBlock: if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) - return true; + return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true}; + + if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID) + return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true}; // Ignore other sub-blocks. if (Stream.SkipBlock()) @@ -5384,6 +5737,16 @@ Expected<std::string> llvm::getBitcodeProducerString(MemoryBufferRef Buffer) { return readIdentificationCode(*StreamOrErr); } +Error llvm::readModuleSummaryIndex(MemoryBufferRef Buffer, + ModuleSummaryIndex &CombinedIndex, + uint64_t ModuleId) { + Expected<BitcodeModule> BM = getSingleModule(Buffer); + if (!BM) + return BM.takeError(); + + return BM->readSummary(CombinedIndex, BM->getModuleIdentifier(), ModuleId); +} + Expected<std::unique_ptr<ModuleSummaryIndex>> llvm::getModuleSummaryIndex(MemoryBufferRef Buffer) { Expected<BitcodeModule> BM = getSingleModule(Buffer); @@ -5393,10 +5756,22 @@ llvm::getModuleSummaryIndex(MemoryBufferRef Buffer) { return BM->getSummary(); } -Expected<bool> llvm::hasGlobalValueSummary(MemoryBufferRef Buffer) { +Expected<BitcodeLTOInfo> llvm::getBitcodeLTOInfo(MemoryBufferRef Buffer) { Expected<BitcodeModule> BM = getSingleModule(Buffer); if (!BM) return BM.takeError(); - return BM->hasSummary(); + return BM->getLTOInfo(); +} + +Expected<std::unique_ptr<ModuleSummaryIndex>> +llvm::getModuleSummaryIndexForFile(StringRef Path, + bool IgnoreEmptyThinLTOIndexFile) { + ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr = + MemoryBuffer::getFileOrSTDIN(Path); + if (!FileOrErr) + return errorCodeToError(FileOrErr.getError()); + if (IgnoreEmptyThinLTOIndexFile && !(*FileOrErr)->getBufferSize()) + return nullptr; + return getModuleSummaryIndex(**FileOrErr); } diff --git a/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp index b89f5be..10fbcde 100644 --- a/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.cpp @@ -53,6 +53,7 @@ #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -358,6 +359,9 @@ class PlaceholderQueue { std::deque<DistinctMDOperandPlaceholder> PHs; public: + ~PlaceholderQueue() { + assert(empty() && "PlaceholderQueue hasn't been flushed before being destroyed"); + } bool empty() { return PHs.empty(); } DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID); void flush(BitcodeReaderMetadataList &MetadataList); @@ -403,6 +407,11 @@ void PlaceholderQueue::flush(BitcodeReaderMetadataList &MetadataList) { } // anonynous namespace +static Error error(const Twine &Message) { + return make_error<StringError>( + Message, make_error_code(BitcodeError::CorruptedBitcode)); +} + class MetadataLoader::MetadataLoaderImpl { BitcodeReaderMetadataList MetadataList; BitcodeReaderValueList &ValueList; @@ -449,6 +458,7 @@ class MetadataLoader::MetadataLoaderImpl { bool StripTBAA = false; bool HasSeenOldLoopTags = false; bool NeedUpgradeToDIGlobalVariableExpression = false; + bool NeedDeclareExpressionUpgrade = false; /// True if metadata is being parsed for a module being ThinLTO imported. bool IsImporting = false; @@ -457,7 +467,7 @@ class MetadataLoader::MetadataLoaderImpl { PlaceholderQueue &Placeholders, StringRef Blob, unsigned &NextMetadataNo); Error parseMetadataStrings(ArrayRef<uint64_t> Record, StringRef Blob, - std::function<void(StringRef)> CallBack); + function_ref<void(StringRef)> CallBack); Error parseGlobalObjectAttachment(GlobalObject &GO, ArrayRef<uint64_t> Record); Error parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record); @@ -469,8 +479,8 @@ class MetadataLoader::MetadataLoaderImpl { for (auto CU_SP : CUSubprograms) if (auto *SPs = dyn_cast_or_null<MDTuple>(CU_SP.second)) for (auto &Op : SPs->operands()) - if (auto *SP = dyn_cast_or_null<MDNode>(Op)) - SP->replaceOperandWith(7, CU_SP.first); + if (auto *SP = dyn_cast_or_null<DISubprogram>(Op)) + SP->replaceUnit(CU_SP.first); CUSubprograms.clear(); } @@ -495,7 +505,7 @@ class MetadataLoader::MetadataLoaderImpl { // Upgrade variables attached to globals. for (auto &GV : TheModule.globals()) { - SmallVector<MDNode *, 1> MDs, NewMDs; + SmallVector<MDNode *, 1> MDs; GV.getMetadata(LLVMContext::MD_dbg, MDs); GV.eraseMetadata(LLVMContext::MD_dbg); for (auto *MD : MDs) @@ -508,6 +518,108 @@ class MetadataLoader::MetadataLoaderImpl { } } + /// Remove a leading DW_OP_deref from DIExpressions in a dbg.declare that + /// describes a function argument. + void upgradeDeclareExpressions(Function &F) { + if (!NeedDeclareExpressionUpgrade) + return; + + for (auto &BB : F) + for (auto &I : BB) + if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) + if (auto *DIExpr = DDI->getExpression()) + if (DIExpr->startsWithDeref() && + dyn_cast_or_null<Argument>(DDI->getAddress())) { + SmallVector<uint64_t, 8> Ops; + Ops.append(std::next(DIExpr->elements_begin()), + DIExpr->elements_end()); + auto *E = DIExpression::get(Context, Ops); + DDI->setOperand(2, MetadataAsValue::get(Context, E)); + } + } + + /// Upgrade the expression from previous versions. + Error upgradeDIExpression(uint64_t FromVersion, + MutableArrayRef<uint64_t> &Expr, + SmallVectorImpl<uint64_t> &Buffer) { + auto N = Expr.size(); + switch (FromVersion) { + default: + return error("Invalid record"); + case 0: + if (N >= 3 && Expr[N - 3] == dwarf::DW_OP_bit_piece) + Expr[N - 3] = dwarf::DW_OP_LLVM_fragment; + LLVM_FALLTHROUGH; + case 1: + // Move DW_OP_deref to the end. + if (N && Expr[0] == dwarf::DW_OP_deref) { + auto End = Expr.end(); + if (Expr.size() >= 3 && + *std::prev(End, 3) == dwarf::DW_OP_LLVM_fragment) + End = std::prev(End, 3); + std::move(std::next(Expr.begin()), End, Expr.begin()); + *std::prev(End) = dwarf::DW_OP_deref; + } + NeedDeclareExpressionUpgrade = true; + LLVM_FALLTHROUGH; + case 2: { + // Change DW_OP_plus to DW_OP_plus_uconst. + // Change DW_OP_minus to DW_OP_uconst, DW_OP_minus + auto SubExpr = ArrayRef<uint64_t>(Expr); + while (!SubExpr.empty()) { + // Skip past other operators with their operands + // for this version of the IR, obtained from + // from historic DIExpression::ExprOperand::getSize(). + size_t HistoricSize; + switch (SubExpr.front()) { + default: + HistoricSize = 1; + break; + case dwarf::DW_OP_constu: + case dwarf::DW_OP_minus: + case dwarf::DW_OP_plus: + HistoricSize = 2; + break; + case dwarf::DW_OP_LLVM_fragment: + HistoricSize = 3; + break; + } + + // If the expression is malformed, make sure we don't + // copy more elements than we should. + HistoricSize = std::min(SubExpr.size(), HistoricSize); + ArrayRef<uint64_t> Args = SubExpr.slice(1, HistoricSize-1); + + switch (SubExpr.front()) { + case dwarf::DW_OP_plus: + Buffer.push_back(dwarf::DW_OP_plus_uconst); + Buffer.append(Args.begin(), Args.end()); + break; + case dwarf::DW_OP_minus: + Buffer.push_back(dwarf::DW_OP_constu); + Buffer.append(Args.begin(), Args.end()); + Buffer.push_back(dwarf::DW_OP_minus); + break; + default: + Buffer.push_back(*SubExpr.begin()); + Buffer.append(Args.begin(), Args.end()); + break; + } + + // Continue with remaining elements. + SubExpr = SubExpr.slice(HistoricSize); + } + Expr = MutableArrayRef<uint64_t>(Buffer); + LLVM_FALLTHROUGH; + } + case 3: + // Up-to-date! + break; + } + + return Error::success(); + } + void upgradeDebugInfo() { upgradeCUSubprograms(); upgradeCUVariables(); @@ -520,7 +632,7 @@ public: bool IsImporting) : MetadataList(TheModule.getContext()), ValueList(ValueList), Stream(Stream), Context(TheModule.getContext()), TheModule(TheModule), - getTypeByID(getTypeByID), IsImporting(IsImporting) {} + getTypeByID(std::move(getTypeByID)), IsImporting(IsImporting) {} Error parseMetadata(bool ModuleLevel); @@ -562,13 +674,9 @@ public: unsigned size() const { return MetadataList.size(); } void shrinkTo(unsigned N) { MetadataList.shrinkTo(N); } + void upgradeDebugIntrinsics(Function &F) { upgradeDeclareExpressions(F); } }; -Error error(const Twine &Message) { - return make_error<StringError>( - Message, make_error_code(BitcodeError::CorruptedBitcode)); -} - Expected<bool> MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { IndexCursor = Stream; @@ -1107,9 +1215,15 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_DERIVED_TYPE: { - if (Record.size() != 12) + if (Record.size() < 12 || Record.size() > 13) return error("Invalid record"); + // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means + // that there is no DWARF address space associated with DIDerivedType. + Optional<unsigned> DWARFAddressSpace; + if (Record.size() > 12 && Record[12]) + DWARFAddressSpace = Record[12] - 1; + IsDistinct = Record[0]; DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]); MetadataList.assignValue( @@ -1118,7 +1232,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( getMDOrNull(Record[3]), Record[4], getDITypeRefOrNull(Record[5]), getDITypeRefOrNull(Record[6]), Record[7], Record[8], - Record[9], Flags, getDITypeRefOrNull(Record[11]))), + Record[9], DWARFAddressSpace, Flags, + getDITypeRefOrNull(Record[11]))), NextMetadataNo); NextMetadataNo++; break; @@ -1240,7 +1355,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_COMPILE_UNIT: { - if (Record.size() < 14 || Record.size() > 17) + if (Record.size() < 14 || Record.size() > 18) return error("Invalid record"); // Ignore Record[0], which indicates whether this compile unit is @@ -1253,7 +1368,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( getMDOrNull(Record[12]), getMDOrNull(Record[13]), Record.size() <= 15 ? nullptr : getMDOrNull(Record[15]), Record.size() <= 14 ? 0 : Record[14], - Record.size() <= 16 ? true : Record[16]); + Record.size() <= 16 ? true : Record[16], + Record.size() <= 17 ? false : Record[17]); MetadataList.assignValue(CU, NextMetadataNo); NextMetadataNo++; @@ -1264,7 +1380,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_SUBPROGRAM: { - if (Record.size() < 18 || Record.size() > 20) + if (Record.size() < 18 || Record.size() > 21) return error("Invalid record"); IsDistinct = @@ -1280,29 +1396,31 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( unsigned Offset = Record.size() >= 19 ? 1 : 0; bool HasFn = Offset && !HasUnit; bool HasThisAdj = Record.size() >= 20; + bool HasThrownTypes = Record.size() >= 21; DISubprogram *SP = GET_OR_DISTINCT( - DISubprogram, (Context, - getDITypeRefOrNull(Record[1]), // scope - getMDString(Record[2]), // name - getMDString(Record[3]), // linkageName - getMDOrNull(Record[4]), // file - Record[5], // line - getMDOrNull(Record[6]), // type - Record[7], // isLocal - Record[8], // isDefinition - Record[9], // scopeLine - getDITypeRefOrNull(Record[10]), // containingType - Record[11], // virtuality - Record[12], // virtualIndex - HasThisAdj ? Record[19] : 0, // thisAdjustment - static_cast<DINode::DIFlags>(Record[13] // flags - ), - Record[14], // isOptimized - HasUnit ? CUorFn : nullptr, // unit - getMDOrNull(Record[15 + Offset]), // templateParams - getMDOrNull(Record[16 + Offset]), // declaration - getMDOrNull(Record[17 + Offset]) // variables - )); + DISubprogram, + (Context, + getDITypeRefOrNull(Record[1]), // scope + getMDString(Record[2]), // name + getMDString(Record[3]), // linkageName + getMDOrNull(Record[4]), // file + Record[5], // line + getMDOrNull(Record[6]), // type + Record[7], // isLocal + Record[8], // isDefinition + Record[9], // scopeLine + getDITypeRefOrNull(Record[10]), // containingType + Record[11], // virtuality + Record[12], // virtualIndex + HasThisAdj ? Record[19] : 0, // thisAdjustment + static_cast<DINode::DIFlags>(Record[13]), // flags + Record[14], // isOptimized + HasUnit ? CUorFn : nullptr, // unit + getMDOrNull(Record[15 + Offset]), // templateParams + getMDOrNull(Record[16 + Offset]), // declaration + getMDOrNull(Record[17 + Offset]), // variables + HasThrownTypes ? getMDOrNull(Record[20]) : nullptr // thrownTypes + )); MetadataList.assignValue(SP, NextMetadataNo); NextMetadataNo++; @@ -1347,16 +1465,20 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_NAMESPACE: { - if (Record.size() != 5) + // Newer versions of DINamespace dropped file and line. + MDString *Name; + if (Record.size() == 3) + Name = getMDString(Record[2]); + else if (Record.size() == 5) + Name = getMDString(Record[3]); + else return error("Invalid record"); IsDistinct = Record[0] & 1; bool ExportSymbols = Record[0] & 2; MetadataList.assignValue( GET_OR_DISTINCT(DINamespace, - (Context, getMDOrNull(Record[1]), - getMDOrNull(Record[2]), getMDString(Record[3]), - Record[4], ExportSymbols)), + (Context, getMDOrNull(Record[1]), Name, ExportSymbols)), NextMetadataNo); NextMetadataNo++; break; @@ -1433,6 +1555,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( } else if (Version == 0) { // Upgrade old metadata, which stored a global variable reference or a // ConstantInt here. + NeedUpgradeToDIGlobalVariableExpression = true; Metadata *Expr = getMDOrNull(Record[9]); uint32_t AlignInBits = 0; if (Record.size() > 11) { @@ -1463,8 +1586,6 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( DIGlobalVariableExpression *DGVE = nullptr; if (Attach || Expr) DGVE = DIGlobalVariableExpression::getDistinct(Context, DGV, Expr); - else - NeedUpgradeToDIGlobalVariableExpression = true; if (Attach) Attach->addDebugInfo(DGVE); @@ -1485,7 +1606,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( bool HasAlignment = Record[0] & 2; // 2nd field used to be an artificial tag, either DW_TAG_auto_variable or // DW_TAG_arg_variable, if we have alignment flag encoded it means, that - // this is newer version of record which doesn't have artifical tag. + // this is newer version of record which doesn't have artificial tag. bool HasTag = !HasAlignment && Record.size() > 8; DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[7 + HasTag]); uint32_t AlignInBits = 0; @@ -1510,16 +1631,15 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( return error("Invalid record"); IsDistinct = Record[0] & 1; - bool HasOpFragment = Record[0] & 2; + uint64_t Version = Record[0] >> 1; auto Elts = MutableArrayRef<uint64_t>(Record).slice(1); - if (!HasOpFragment) - if (unsigned N = Elts.size()) - if (N >= 3 && Elts[N - 3] == dwarf::DW_OP_bit_piece) - Elts[N - 3] = dwarf::DW_OP_LLVM_fragment; + + SmallVector<uint64_t, 6> Buffer; + if (Error Err = upgradeDIExpression(Version, Elts, Buffer)) + return Err; MetadataList.assignValue( - GET_OR_DISTINCT(DIExpression, (Context, makeArrayRef(Record).slice(1))), - NextMetadataNo); + GET_OR_DISTINCT(DIExpression, (Context, Elts)), NextMetadataNo); NextMetadataNo++; break; } @@ -1551,15 +1671,17 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_IMPORTED_ENTITY: { - if (Record.size() != 6) + if (Record.size() != 6 && Record.size() != 7) return error("Invalid record"); IsDistinct = Record[0]; + bool HasFile = (Record.size() == 7); MetadataList.assignValue( GET_OR_DISTINCT(DIImportedEntity, (Context, Record[1], getMDOrNull(Record[2]), - getDITypeRefOrNull(Record[3]), Record[4], - getMDString(Record[5]))), + getDITypeRefOrNull(Record[3]), + HasFile ? getMDOrNull(Record[6]) : nullptr, + HasFile ? Record[4] : 0, getMDString(Record[5]))), NextMetadataNo); NextMetadataNo++; break; @@ -1611,7 +1733,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Error MetadataLoader::MetadataLoaderImpl::parseMetadataStrings( ArrayRef<uint64_t> Record, StringRef Blob, - std::function<void(StringRef)> CallBack) { + function_ref<void(StringRef)> CallBack) { // All the MDStrings in the block are emitted together in a single // record. The strings are concatenated and stored in a blob along with // their sizes. @@ -1808,8 +1930,8 @@ MetadataLoader::MetadataLoader(BitstreamCursor &Stream, Module &TheModule, BitcodeReaderValueList &ValueList, bool IsImporting, std::function<Type *(unsigned)> getTypeByID) - : Pimpl(llvm::make_unique<MetadataLoaderImpl>(Stream, TheModule, ValueList, - getTypeByID, IsImporting)) {} + : Pimpl(llvm::make_unique<MetadataLoaderImpl>( + Stream, TheModule, ValueList, std::move(getTypeByID), IsImporting)) {} Error MetadataLoader::parseMetadata(bool ModuleLevel) { return Pimpl->parseMetadata(ModuleLevel); @@ -1848,3 +1970,7 @@ bool MetadataLoader::isStrippingTBAA() { return Pimpl->isStrippingTBAA(); } unsigned MetadataLoader::size() const { return Pimpl->size(); } void MetadataLoader::shrinkTo(unsigned N) { return Pimpl->shrinkTo(N); } + +void MetadataLoader::upgradeDebugIntrinsics(Function &F) { + return Pimpl->upgradeDebugIntrinsics(F); +} diff --git a/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h index 442dfc9..f23dcc0 100644 --- a/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h +++ b/contrib/llvm/lib/Bitcode/Reader/MetadataLoader.h @@ -79,6 +79,9 @@ public: unsigned size() const; void shrinkTo(unsigned N); + + /// Perform bitcode upgrades on llvm.dbg.* calls. + void upgradeDebugIntrinsics(Function &F); }; } diff --git a/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp b/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp index 7152a51..f2a3439 100644 --- a/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp +++ b/contrib/llvm/lib/Bitcode/Reader/ValueList.cpp @@ -58,7 +58,7 @@ void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { if (Idx >= size()) resize(Idx + 1); - WeakVH &OldV = ValuePtrs[Idx]; + WeakTrackingVH &OldV = ValuePtrs[Idx]; if (!OldV) { OldV = V; return; @@ -73,7 +73,7 @@ void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { // If there was a forward reference to this value, replace it. Value *PrevVal = OldV; OldV->replaceAllUsesWith(V); - delete PrevVal; + PrevVal->deleteValue(); } } @@ -194,6 +194,6 @@ void BitcodeReaderValueList::resolveConstantForwardRefs() { // Update all ValueHandles, they should be the only users at this point. Placeholder->replaceAllUsesWith(RealVal); - delete Placeholder; + Placeholder->deleteValue(); } } diff --git a/contrib/llvm/lib/Bitcode/Reader/ValueList.h b/contrib/llvm/lib/Bitcode/Reader/ValueList.h index 3119d77..72775a3 100644 --- a/contrib/llvm/lib/Bitcode/Reader/ValueList.h +++ b/contrib/llvm/lib/Bitcode/Reader/ValueList.h @@ -20,7 +20,7 @@ namespace llvm { class Constant; class BitcodeReaderValueList { - std::vector<WeakVH> ValuePtrs; + std::vector<WeakTrackingVH> ValuePtrs; /// As we resolve forward-referenced constants, we add information about them /// to this vector. This allows us to resolve them in bulk instead of diff --git a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index ebb2022..dcffde1 100644 --- a/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -28,10 +28,13 @@ #include "llvm/IR/Operator.h" #include "llvm/IR/UseListOrder.h" #include "llvm/IR/ValueSymbolTable.h" +#include "llvm/MC/StringTableBuilder.h" +#include "llvm/Object/IRSymtab.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Program.h" #include "llvm/Support/SHA1.h" +#include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" #include <cctype> #include <map> @@ -76,21 +79,24 @@ protected: /// The stream created and owned by the client. BitstreamWriter &Stream; - /// Saves the offset of the VSTOffset record that must eventually be - /// backpatched with the offset of the actual VST. - uint64_t VSTOffsetPlaceholder = 0; + StringTableBuilder &StrtabBuilder; public: /// Constructs a BitcodeWriterBase object that writes to the provided /// \p Stream. - BitcodeWriterBase(BitstreamWriter &Stream) : Stream(Stream) {} + BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder) + : Stream(Stream), StrtabBuilder(StrtabBuilder) {} protected: - bool hasVSTOffsetPlaceholder() { return VSTOffsetPlaceholder != 0; } - void writeValueSymbolTableForwardDecl(); void writeBitcodeHeader(); + void writeModuleVersion(); }; +void BitcodeWriterBase::writeModuleVersion() { + // VERSION: [version#] + Stream.EmitRecord(bitc::MODULE_CODE_VERSION, ArrayRef<uint64_t>{2}); +} + /// Class to manage the bitcode writing for a module. class ModuleBitcodeWriter : public BitcodeWriterBase { /// Pointer to the buffer allocated by caller for bitcode writing. @@ -108,6 +114,16 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { /// True if a module hash record should be written. bool GenerateHash; + SHA1 Hasher; + + /// If non-null, when GenerateHash is true, the resulting hash is written + /// into ModHash. When GenerateHash is false, that specified value + /// is used as the hash instead of computing from the generated bitcode. + /// Can be used to produce the same module hash for a minimized bitcode + /// used just for the thin link as in the regular full bitcode that will + /// be used in the backend. + ModuleHash *ModHash; + /// The start bit of the identification block. uint64_t BitcodeStartBit; @@ -119,15 +135,22 @@ class ModuleBitcodeWriter : public BitcodeWriterBase { /// Tracks the last value id recorded in the GUIDToValueMap. unsigned GlobalValueId; + /// Saves the offset of the VSTOffset record that must eventually be + /// backpatched with the offset of the actual VST. + uint64_t VSTOffsetPlaceholder = 0; + public: /// Constructs a ModuleBitcodeWriter object for the given Module, /// writing to the provided \p Buffer. ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, + StringTableBuilder &StrtabBuilder, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - const ModuleSummaryIndex *Index, bool GenerateHash) - : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M), + const ModuleSummaryIndex *Index, bool GenerateHash, + ModuleHash *ModHash = nullptr) + : BitcodeWriterBase(Stream, StrtabBuilder), Buffer(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), Index(Index), - GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) { + GenerateHash(GenerateHash), ModHash(ModHash), + BitcodeStartBit(Stream.GetCurrentBitNo()) { // Assign ValueIds to any callee values in the index that came from // indirect call profiles and were recorded as a GUID not a Value* // (which would have been assigned an ID by the ValueEnumerator). @@ -138,14 +161,14 @@ public: return; for (const auto &GUIDSummaryLists : *Index) // Examine all summaries for this GUID. - for (auto &Summary : GUIDSummaryLists.second) + for (auto &Summary : GUIDSummaryLists.second.SummaryList) if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) // For each call in the function summary, see if the call // is to a GUID (which means it is for an indirect call, // otherwise we would have a Value for it). If so, synthesize // a value id. for (auto &CallEdge : FS->calls()) - if (CallEdge.first.isGUID()) + if (!CallEdge.first.getValue()) assignValueId(CallEdge.first.getGUID()); } @@ -155,10 +178,13 @@ public: private: uint64_t bitcodeStartBit() { return BitcodeStartBit; } + size_t addToStrtab(StringRef Str); + void writeAttributeGroupTable(); void writeAttributeTable(); void writeTypeTable(); void writeComdats(); + void writeValueSymbolTableForwardDecl(); void writeModuleInfo(); void writeValueAsMetadata(const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record); @@ -240,6 +266,7 @@ private: const GlobalObject &GO); void writeModuleMetadataKinds(); void writeOperandBundleTags(); + void writeSyncScopeNames(); void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal); void writeModuleConstants(); bool pushValueAndType(const Value *V, unsigned InstID, @@ -251,9 +278,9 @@ private: SmallVectorImpl<uint64_t> &Vals); void writeInstruction(const Instruction &I, unsigned InstID, SmallVectorImpl<unsigned> &Vals); - void writeValueSymbolTable( - const ValueSymbolTable &VST, bool IsModuleLevel = false, - DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex = nullptr); + void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST); + void writeGlobalValueSymbolTable( + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex); void writeUseList(UseListOrder &&Order); void writeUseListBlock(const Function *F); void @@ -285,11 +312,15 @@ private: } // Helper to get the valueId for the type of value recorded in VI. unsigned getValueId(ValueInfo VI) { - if (VI.isGUID()) + if (!VI.getValue()) return getValueId(VI.getGUID()); return VE.getValueID(VI.getValue()); } std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } + + unsigned getEncodedSyncScopeID(SyncScope::ID SSID) { + return unsigned(SSID); + } }; /// Class to manage the bitcode writing for a combined index. @@ -312,186 +343,75 @@ public: /// Constructs a IndexBitcodeWriter object for the given combined index, /// writing to the provided \p Buffer. When writing a subset of the index /// for a distributed backend, provide a \p ModuleToSummariesForIndex map. - IndexBitcodeWriter(BitstreamWriter &Stream, const ModuleSummaryIndex &Index, + IndexBitcodeWriter(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder, + const ModuleSummaryIndex &Index, const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex = nullptr) - : BitcodeWriterBase(Stream), Index(Index), + : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index), ModuleToSummariesForIndex(ModuleToSummariesForIndex) { // Assign unique value ids to all summaries to be written, for use // in writing out the call graph edges. Save the mapping from GUID // to the new global value id to use when writing those edges, which // are currently saved in the index in terms of GUID. - for (const auto &I : *this) + forEachSummary([&](GVInfo I) { GUIDToValueIdMap[I.first] = ++GlobalValueId; + }); } /// The below iterator returns the GUID and associated summary. typedef std::pair<GlobalValue::GUID, GlobalValueSummary *> GVInfo; - /// Iterator over the value GUID and summaries to be written to bitcode, - /// hides the details of whether they are being pulled from the entire - /// index or just those in a provided ModuleToSummariesForIndex map. - class iterator - : public llvm::iterator_facade_base<iterator, std::forward_iterator_tag, - GVInfo> { - /// Enables access to parent class. - const IndexBitcodeWriter &Writer; - - // Iterators used when writing only those summaries in a provided - // ModuleToSummariesForIndex map: - - /// Points to the last element in outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesBack; - /// Iterator on outer ModuleToSummariesForIndex map. - std::map<std::string, GVSummaryMapTy>::const_iterator ModuleSummariesIter; - /// Iterator on an inner global variable summary map. - GVSummaryMapTy::const_iterator ModuleGVSummariesIter; - - // Iterators used when writing all summaries in the index: - - /// Points to the last element in the Index outer GlobalValueMap. - const_gvsummary_iterator IndexSummariesBack; - /// Iterator on outer GlobalValueMap. - const_gvsummary_iterator IndexSummariesIter; - /// Iterator on an inner GlobalValueSummaryList. - GlobalValueSummaryList::const_iterator IndexGVSummariesIter; - - public: - /// Construct iterator from parent \p Writer and indicate if we are - /// constructing the end iterator. - iterator(const IndexBitcodeWriter &Writer, bool IsAtEnd) : Writer(Writer) { - // Set up the appropriate set of iterators given whether we are writing - // the full index or just a subset. - // Can't setup the Back or inner iterators if the corresponding map - // is empty. This will be handled specially in operator== as well. - if (Writer.ModuleToSummariesForIndex && - !Writer.ModuleToSummariesForIndex->empty()) { - for (ModuleSummariesBack = Writer.ModuleToSummariesForIndex->begin(); - std::next(ModuleSummariesBack) != - Writer.ModuleToSummariesForIndex->end(); - ModuleSummariesBack++) - ; - ModuleSummariesIter = !IsAtEnd - ? Writer.ModuleToSummariesForIndex->begin() - : ModuleSummariesBack; - ModuleGVSummariesIter = !IsAtEnd ? ModuleSummariesIter->second.begin() - : ModuleSummariesBack->second.end(); - } else if (!Writer.ModuleToSummariesForIndex && - Writer.Index.begin() != Writer.Index.end()) { - for (IndexSummariesBack = Writer.Index.begin(); - std::next(IndexSummariesBack) != Writer.Index.end(); - IndexSummariesBack++) - ; - IndexSummariesIter = - !IsAtEnd ? Writer.Index.begin() : IndexSummariesBack; - IndexGVSummariesIter = !IsAtEnd ? IndexSummariesIter->second.begin() - : IndexSummariesBack->second.end(); - } + /// Calls the callback for each value GUID and summary to be written to + /// bitcode. This hides the details of whether they are being pulled from the + /// entire index or just those in a provided ModuleToSummariesForIndex map. + template<typename Functor> + void forEachSummary(Functor Callback) { + if (ModuleToSummariesForIndex) { + for (auto &M : *ModuleToSummariesForIndex) + for (auto &Summary : M.second) + Callback(Summary); + } else { + for (auto &Summaries : Index) + for (auto &Summary : Summaries.second.SummaryList) + Callback({Summaries.first, Summary.get()}); } + } - /// Increment the appropriate set of iterators. - iterator &operator++() { - // First the inner iterator is incremented, then if it is at the end - // and there are more outer iterations to go, the inner is reset to - // the start of the next inner list. - if (Writer.ModuleToSummariesForIndex) { - ++ModuleGVSummariesIter; - if (ModuleGVSummariesIter == ModuleSummariesIter->second.end() && - ModuleSummariesIter != ModuleSummariesBack) { - ++ModuleSummariesIter; - ModuleGVSummariesIter = ModuleSummariesIter->second.begin(); - } - } else { - ++IndexGVSummariesIter; - if (IndexGVSummariesIter == IndexSummariesIter->second.end() && - IndexSummariesIter != IndexSummariesBack) { - ++IndexSummariesIter; - IndexGVSummariesIter = IndexSummariesIter->second.begin(); + /// Calls the callback for each entry in the modulePaths StringMap that + /// should be written to the module path string table. This hides the details + /// of whether they are being pulled from the entire index or just those in a + /// provided ModuleToSummariesForIndex map. + template <typename Functor> void forEachModule(Functor Callback) { + if (ModuleToSummariesForIndex) { + for (const auto &M : *ModuleToSummariesForIndex) { + const auto &MPI = Index.modulePaths().find(M.first); + if (MPI == Index.modulePaths().end()) { + // This should only happen if the bitcode file was empty, in which + // case we shouldn't be importing (the ModuleToSummariesForIndex + // would only include the module we are writing and index for). + assert(ModuleToSummariesForIndex->size() == 1); + continue; } + Callback(*MPI); } - return *this; - } - - /// Access the <GUID,GlobalValueSummary*> pair corresponding to the current - /// outer and inner iterator positions. - GVInfo operator*() { - if (Writer.ModuleToSummariesForIndex) - return std::make_pair(ModuleGVSummariesIter->first, - ModuleGVSummariesIter->second); - return std::make_pair(IndexSummariesIter->first, - IndexGVSummariesIter->get()); - } - - /// Checks if the iterators are equal, with special handling for empty - /// indexes. - bool operator==(const iterator &RHS) const { - if (Writer.ModuleToSummariesForIndex) { - // First ensure that both are writing the same subset. - if (Writer.ModuleToSummariesForIndex != - RHS.Writer.ModuleToSummariesForIndex) - return false; - // Already determined above that maps are the same, so if one is - // empty, they both are. - if (Writer.ModuleToSummariesForIndex->empty()) - return true; - // Ensure the ModuleGVSummariesIter are iterating over the same - // container before checking them below. - if (ModuleSummariesIter != RHS.ModuleSummariesIter) - return false; - return ModuleGVSummariesIter == RHS.ModuleGVSummariesIter; - } - // First ensure RHS also writing the full index, and that both are - // writing the same full index. - if (RHS.Writer.ModuleToSummariesForIndex || - &Writer.Index != &RHS.Writer.Index) - return false; - // Already determined above that maps are the same, so if one is - // empty, they both are. - if (Writer.Index.begin() == Writer.Index.end()) - return true; - // Ensure the IndexGVSummariesIter are iterating over the same - // container before checking them below. - if (IndexSummariesIter != RHS.IndexSummariesIter) - return false; - return IndexGVSummariesIter == RHS.IndexGVSummariesIter; + } else { + for (const auto &MPSE : Index.modulePaths()) + Callback(MPSE); } - }; - - /// Obtain the start iterator over the summaries to be written. - iterator begin() { return iterator(*this, /*IsAtEnd=*/false); } - /// Obtain the end iterator over the summaries to be written. - iterator end() { return iterator(*this, /*IsAtEnd=*/true); } + } /// Main entry point for writing a combined index to bitcode. void write(); private: - void writeIndex(); void writeModStrings(); - void writeCombinedValueSymbolTable(); void writeCombinedGlobalValueSummary(); - /// Indicates whether the provided \p ModulePath should be written into - /// the module string table, e.g. if full index written or if it is in - /// the provided subset. - bool doIncludeModule(StringRef ModulePath) { - return !ModuleToSummariesForIndex || - ModuleToSummariesForIndex->count(ModulePath); - } - - bool hasValueId(GlobalValue::GUID ValGUID) { - const auto &VMI = GUIDToValueIdMap.find(ValGUID); - return VMI != GUIDToValueIdMap.end(); - } - unsigned getValueId(GlobalValue::GUID ValGUID) { - const auto &VMI = GUIDToValueIdMap.find(ValGUID); - // If this GUID doesn't have an entry, assign one. - if (VMI == GUIDToValueIdMap.end()) { - GUIDToValueIdMap[ValGUID] = ++GlobalValueId; - return GlobalValueId; - } else { - return VMI->second; - } + Optional<unsigned> getValueId(GlobalValue::GUID ValGUID) { + auto VMI = GUIDToValueIdMap.find(ValGUID); + if (VMI == GUIDToValueIdMap.end()) + return None; + return VMI->second; } std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } }; @@ -570,14 +490,6 @@ static unsigned getEncodedOrdering(AtomicOrdering Ordering) { llvm_unreachable("Invalid ordering"); } -static unsigned getEncodedSynchScope(SynchronizationScope SynchScope) { - switch (SynchScope) { - case SingleThread: return bitc::SYNCHSCOPE_SINGLETHREAD; - case CrossThread: return bitc::SYNCHSCOPE_CROSSTHREAD; - } - llvm_unreachable("Invalid synch scope"); -} - static void writeStringRecord(BitstreamWriter &Stream, unsigned Code, StringRef Str, unsigned AbbrevToUse) { SmallVector<unsigned, 64> Vals; @@ -671,6 +583,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_RETURNS_TWICE; case Attribute::SExt: return bitc::ATTR_KIND_S_EXT; + case Attribute::Speculatable: + return bitc::ATTR_KIND_SPECULATABLE; case Attribute::StackAlignment: return bitc::ATTR_KIND_STACK_ALIGNMENT; case Attribute::StackProtect: @@ -709,63 +623,62 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { } void ModuleBitcodeWriter::writeAttributeGroupTable() { - const std::vector<AttributeSet> &AttrGrps = VE.getAttributeGroups(); + const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps = + VE.getAttributeGroups(); if (AttrGrps.empty()) return; Stream.EnterSubblock(bitc::PARAMATTR_GROUP_BLOCK_ID, 3); SmallVector<uint64_t, 64> Record; - for (unsigned i = 0, e = AttrGrps.size(); i != e; ++i) { - AttributeSet AS = AttrGrps[i]; - for (unsigned i = 0, e = AS.getNumSlots(); i != e; ++i) { - AttributeSet A = AS.getSlotAttributes(i); - - Record.push_back(VE.getAttributeGroupID(A)); - Record.push_back(AS.getSlotIndex(i)); - - for (AttributeSet::iterator I = AS.begin(0), E = AS.end(0); - I != E; ++I) { - Attribute Attr = *I; - if (Attr.isEnumAttribute()) { - Record.push_back(0); - Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); - } else if (Attr.isIntAttribute()) { - Record.push_back(1); - Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); - Record.push_back(Attr.getValueAsInt()); - } else { - StringRef Kind = Attr.getKindAsString(); - StringRef Val = Attr.getValueAsString(); - - Record.push_back(Val.empty() ? 3 : 4); - Record.append(Kind.begin(), Kind.end()); + for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) { + unsigned AttrListIndex = Pair.first; + AttributeSet AS = Pair.second; + Record.push_back(VE.getAttributeGroupID(Pair)); + Record.push_back(AttrListIndex); + + for (Attribute Attr : AS) { + if (Attr.isEnumAttribute()) { + Record.push_back(0); + Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); + } else if (Attr.isIntAttribute()) { + Record.push_back(1); + Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); + Record.push_back(Attr.getValueAsInt()); + } else { + StringRef Kind = Attr.getKindAsString(); + StringRef Val = Attr.getValueAsString(); + + Record.push_back(Val.empty() ? 3 : 4); + Record.append(Kind.begin(), Kind.end()); + Record.push_back(0); + if (!Val.empty()) { + Record.append(Val.begin(), Val.end()); Record.push_back(0); - if (!Val.empty()) { - Record.append(Val.begin(), Val.end()); - Record.push_back(0); - } } } - - Stream.EmitRecord(bitc::PARAMATTR_GRP_CODE_ENTRY, Record); - Record.clear(); } + + Stream.EmitRecord(bitc::PARAMATTR_GRP_CODE_ENTRY, Record); + Record.clear(); } Stream.ExitBlock(); } void ModuleBitcodeWriter::writeAttributeTable() { - const std::vector<AttributeSet> &Attrs = VE.getAttributes(); + const std::vector<AttributeList> &Attrs = VE.getAttributeLists(); if (Attrs.empty()) return; Stream.EnterSubblock(bitc::PARAMATTR_BLOCK_ID, 3); SmallVector<uint64_t, 64> Record; for (unsigned i = 0, e = Attrs.size(); i != e; ++i) { - const AttributeSet &A = Attrs[i]; - for (unsigned i = 0, e = A.getNumSlots(); i != e; ++i) - Record.push_back(VE.getAttributeGroupID(A.getSlotAttributes(i))); + AttributeList AL = Attrs[i]; + for (unsigned i = AL.index_begin(), e = AL.index_end(); i != e; ++i) { + AttributeSet AS = AL.getAttributes(i); + if (AS.hasAttributes()) + Record.push_back(VE.getAttributeGroupID({i, AS})); + } Stream.EmitRecord(bitc::PARAMATTR_CODE_ENTRY, Record); Record.clear(); @@ -972,7 +885,7 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { uint64_t RawFlags = 0; RawFlags |= Flags.NotEligibleToImport; // bool - RawFlags |= (Flags.LiveRoot << 1); + RawFlags |= (Flags.Live << 1); // Linkage don't need to be remapped at that time for the summary. Any future // change to the getEncodedLinkage() function will need to be taken into // account here as well. @@ -1035,16 +948,19 @@ static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) { llvm_unreachable("Invalid unnamed_addr"); } +size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) { + if (GenerateHash) + Hasher.update(Str); + return StrtabBuilder.add(Str); +} + void ModuleBitcodeWriter::writeComdats() { SmallVector<unsigned, 64> Vals; for (const Comdat *C : VE.getComdats()) { - // COMDAT: [selection_kind, name] + // COMDAT: [strtab offset, strtab size, selection_kind] + Vals.push_back(addToStrtab(C->getName())); + Vals.push_back(C->getName().size()); Vals.push_back(getEncodedComdatSelectionKind(*C)); - size_t Size = C->getName().size(); - assert(isUInt<32>(Size)); - Vals.push_back(Size); - for (char Chr : C->getName()) - Vals.push_back((unsigned char)Chr); Stream.EmitRecord(bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/0); Vals.clear(); } @@ -1053,7 +969,7 @@ void ModuleBitcodeWriter::writeComdats() { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched /// after the real VST is written. Saves the bit offset to backpatch. -void BitcodeWriterBase::writeValueSymbolTableForwardDecl() { +void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be @@ -1079,19 +995,18 @@ void BitcodeWriterBase::writeValueSymbolTableForwardDecl() { enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 }; /// Determine the encoding to use for the given string name and length. -static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { +static StringEncoding getStringEncoding(StringRef Str) { bool isChar6 = true; - for (const char *C = Str, *E = C + StrLen; C != E; ++C) { + for (char C : Str) { if (isChar6) - isChar6 = BitCodeAbbrevOp::isChar6(*C); - if ((unsigned char)*C & 128) + isChar6 = BitCodeAbbrevOp::isChar6(C); + if ((unsigned char)C & 128) // don't bother scanning the rest. return SE_Fixed8; } if (isChar6) return SE_Char6; - else - return SE_Fixed7; + return SE_Fixed7; } /// Emit top-level description of module, including target triple, inline asm, @@ -1156,6 +1071,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Add an abbrev for common globals with no visibility or thread localness. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(MaxGlobalType+1))); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2 @@ -1179,15 +1096,41 @@ void ModuleBitcodeWriter::writeModuleInfo() { SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } - // Emit the global variable information. SmallVector<unsigned, 64> Vals; + // Emit the module's source file name. + { + StringEncoding Bits = getStringEncoding(M.getSourceFileName()); + BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); + if (Bits == SE_Char6) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); + else if (Bits == SE_Fixed7) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); + + // MODULE_CODE_SOURCE_FILENAME: [namechar x N] + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(AbbrevOpToUse); + unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + for (const auto P : M.getSourceFileName()) + Vals.push_back((unsigned char)P); + + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); + Vals.clear(); + } + + // Emit the global variable information. for (const GlobalVariable &GV : M.globals()) { unsigned AbbrevToUse = 0; - // GLOBALVAR: [type, isconst, initid, + // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid, // linkage, alignment, section, visibility, threadlocal, // unnamed_addr, externally_initialized, dllstorageclass, - // comdat] + // comdat, attributes] + Vals.push_back(addToStrtab(GV.getName())); + Vals.push_back(GV.getName().size()); Vals.push_back(VE.getTypeID(GV.getValueType())); Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant()); Vals.push_back(GV.isDeclaration() ? 0 : @@ -1200,13 +1143,17 @@ void ModuleBitcodeWriter::writeModuleInfo() { GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None || GV.isExternallyInitialized() || GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass || - GV.hasComdat()) { + GV.hasComdat() || + GV.hasAttributes()) { Vals.push_back(getEncodedVisibility(GV)); Vals.push_back(getEncodedThreadLocalMode(GV)); Vals.push_back(getEncodedUnnamedAddr(GV)); Vals.push_back(GV.isExternallyInitialized()); Vals.push_back(getEncodedDLLStorageClass(GV)); Vals.push_back(GV.hasComdat() ? VE.getComdatID(GV.getComdat()) : 0); + + auto AL = GV.getAttributesAsList(AttributeList::FunctionIndex); + Vals.push_back(VE.getAttributeListID(AL)); } else { AbbrevToUse = SimpleGVarAbbrev; } @@ -1217,14 +1164,17 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the function proto information. for (const Function &F : M) { - // FUNCTION: [type, callingconv, isproto, linkage, paramattrs, alignment, - // section, visibility, gc, unnamed_addr, prologuedata, - // dllstorageclass, comdat, prefixdata, personalityfn] + // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto, + // linkage, paramattrs, alignment, section, visibility, gc, + // unnamed_addr, prologuedata, dllstorageclass, comdat, + // prefixdata, personalityfn] + Vals.push_back(addToStrtab(F.getName())); + Vals.push_back(F.getName().size()); Vals.push_back(VE.getTypeID(F.getFunctionType())); Vals.push_back(F.getCallingConv()); Vals.push_back(F.isDeclaration()); Vals.push_back(getEncodedLinkage(F)); - Vals.push_back(VE.getAttributeID(F.getAttributes())); + Vals.push_back(VE.getAttributeListID(F.getAttributes())); Vals.push_back(Log2_32(F.getAlignment())+1); Vals.push_back(F.hasSection() ? SectionMap[F.getSection()] : 0); Vals.push_back(getEncodedVisibility(F)); @@ -1246,8 +1196,10 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the alias information. for (const GlobalAlias &A : M.aliases()) { - // ALIAS: [alias type, aliasee val#, linkage, visibility, dllstorageclass, - // threadlocal, unnamed_addr] + // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage, + // visibility, dllstorageclass, threadlocal, unnamed_addr] + Vals.push_back(addToStrtab(A.getName())); + Vals.push_back(A.getName().size()); Vals.push_back(VE.getTypeID(A.getValueType())); Vals.push_back(A.getType()->getAddressSpace()); Vals.push_back(VE.getValueID(A.getAliasee())); @@ -1263,7 +1215,10 @@ void ModuleBitcodeWriter::writeModuleInfo() { // Emit the ifunc information. for (const GlobalIFunc &I : M.ifuncs()) { - // IFUNC: [ifunc type, address space, resolver val#, linkage, visibility] + // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver + // val#, linkage, visibility] + Vals.push_back(addToStrtab(I.getName())); + Vals.push_back(I.getName().size()); Vals.push_back(VE.getTypeID(I.getValueType())); Vals.push_back(I.getType()->getAddressSpace()); Vals.push_back(VE.getValueID(I.getResolver())); @@ -1273,34 +1228,6 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.clear(); } - // Emit the module's source file name. - { - StringEncoding Bits = getStringEncoding(M.getSourceFileName().data(), - M.getSourceFileName().size()); - BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); - if (Bits == SE_Char6) - AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); - else if (Bits == SE_Fixed7) - AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); - - // MODULE_CODE_SOURCE_FILENAME: [namechar x N] - auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(AbbrevOpToUse); - unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - - for (const auto P : M.getSourceFileName()) - Vals.push_back((unsigned char)P); - - // Emit the finished record. - Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); - Vals.clear(); - } - - // If we have a VST, write the VSTOFFSET record placeholder. - if (M.getValueSymbolTable().empty()) - return; writeValueSymbolTableForwardDecl(); } @@ -1326,6 +1253,8 @@ static uint64_t getOptimizationFlags(const Value *V) { Flags |= FastMathFlags::NoSignedZeros; if (FPMO->hasAllowReciprocal()) Flags |= FastMathFlags::AllowReciprocal; + if (FPMO->hasAllowContract()) + Flags |= FastMathFlags::AllowContract; } return Flags; @@ -1473,6 +1402,13 @@ void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N, Record.push_back(N->getFlags()); Record.push_back(VE.getMetadataOrNullID(N->getExtraData())); + // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means + // that there is no DWARF address space associated with DIDerivedType. + if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace()) + Record.push_back(*DWARFAddressSpace + 1); + else + Record.push_back(0); + Stream.EmitRecord(bitc::METADATA_DERIVED_TYPE, Record, Abbrev); Record.clear(); } @@ -1549,6 +1485,7 @@ void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, Record.push_back(N->getDWOId()); Record.push_back(VE.getMetadataOrNullID(N->getMacros().get())); Record.push_back(N->getSplitDebugInlining()); + Record.push_back(N->getDebugInfoForProfiling()); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -1578,6 +1515,7 @@ void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N, Record.push_back(VE.getMetadataOrNullID(N->getDeclaration())); Record.push_back(VE.getMetadataOrNullID(N->getVariables().get())); Record.push_back(N->getThisAdjustment()); + Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get())); Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev); Record.clear(); @@ -1613,9 +1551,7 @@ void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, unsigned Abbrev) { Record.push_back(N->isDistinct() | N->getExportSymbols() << 1); Record.push_back(VE.getMetadataOrNullID(N->getScope())); - Record.push_back(VE.getMetadataOrNullID(N->getFile())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); - Record.push_back(N->getLine()); Stream.EmitRecord(bitc::METADATA_NAMESPACE, Record, Abbrev); Record.clear(); @@ -1738,9 +1674,8 @@ void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { Record.reserve(N->getElements().size() + 1); - - const uint64_t HasOpFragmentFlag = 1 << 1; - Record.push_back((uint64_t)N->isDistinct() | HasOpFragmentFlag); + const uint64_t Version = 3 << 1; + Record.push_back((uint64_t)N->isDistinct() | Version); Record.append(N->elements_begin(), N->elements_end()); Stream.EmitRecord(bitc::METADATA_EXPRESSION, Record, Abbrev); @@ -1783,6 +1718,7 @@ void ModuleBitcodeWriter::writeDIImportedEntity( Record.push_back(VE.getMetadataOrNullID(N->getEntity())); Record.push_back(N->getLine()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); + Record.push_back(VE.getMetadataOrNullID(N->getRawFile())); Stream.EmitRecord(bitc::METADATA_IMPORTED_ENTITY, Record, Abbrev); Record.clear(); @@ -2104,6 +2040,24 @@ void ModuleBitcodeWriter::writeOperandBundleTags() { Stream.ExitBlock(); } +void ModuleBitcodeWriter::writeSyncScopeNames() { + SmallVector<StringRef, 8> SSNs; + M.getContext().getSyncScopeNames(SSNs); + if (SSNs.empty()) + return; + + Stream.EnterSubblock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID, 2); + + SmallVector<uint64_t, 64> Record; + for (auto SSN : SSNs) { + Record.append(SSN.begin(), SSN.end()); + Stream.EmitRecord(bitc::SYNC_SCOPE_NAME, Record, 0); + Record.clear(); + } + + Stream.ExitBlock(); +} + static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) { if ((int64_t)V >= 0) Vals.push_back(V << 1); @@ -2559,7 +2513,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(VE.getTypeID(SI.getCondition()->getType())); pushValue(SI.getCondition(), InstID, Vals); Vals.push_back(VE.getValueID(SI.getDefaultDest())); - for (SwitchInst::ConstCaseIt Case : SI.cases()) { + for (auto Case : SI.cases()) { Vals.push_back(VE.getValueID(Case.getCaseValue())); Vals.push_back(VE.getValueID(Case.getCaseSuccessor())); } @@ -2584,7 +2538,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_INVOKE; - Vals.push_back(VE.getAttributeID(II->getAttributes())); + Vals.push_back(VE.getAttributeListID(II->getAttributes())); Vals.push_back(II->getCallingConv() | 1 << 13); Vals.push_back(VE.getValueID(II->getNormalDest())); Vals.push_back(VE.getValueID(II->getUnwindDest())); @@ -2720,7 +2674,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(cast<LoadInst>(I).isVolatile()); if (cast<LoadInst>(I).isAtomic()) { Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering())); - Vals.push_back(getEncodedSynchScope(cast<LoadInst>(I).getSynchScope())); + Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID())); } break; case Instruction::Store: @@ -2734,7 +2688,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(cast<StoreInst>(I).isVolatile()); if (cast<StoreInst>(I).isAtomic()) { Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering())); - Vals.push_back(getEncodedSynchScope(cast<StoreInst>(I).getSynchScope())); + Vals.push_back( + getEncodedSyncScopeID(cast<StoreInst>(I).getSyncScopeID())); } break; case Instruction::AtomicCmpXchg: @@ -2746,7 +2701,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back( getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getSuccessOrdering())); Vals.push_back( - getEncodedSynchScope(cast<AtomicCmpXchgInst>(I).getSynchScope())); + getEncodedSyncScopeID(cast<AtomicCmpXchgInst>(I).getSyncScopeID())); Vals.push_back( getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getFailureOrdering())); Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak()); @@ -2760,12 +2715,12 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(cast<AtomicRMWInst>(I).isVolatile()); Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering())); Vals.push_back( - getEncodedSynchScope(cast<AtomicRMWInst>(I).getSynchScope())); + getEncodedSyncScopeID(cast<AtomicRMWInst>(I).getSyncScopeID())); break; case Instruction::Fence: Code = bitc::FUNC_CODE_INST_FENCE; Vals.push_back(getEncodedOrdering(cast<FenceInst>(I).getOrdering())); - Vals.push_back(getEncodedSynchScope(cast<FenceInst>(I).getSynchScope())); + Vals.push_back(getEncodedSyncScopeID(cast<FenceInst>(I).getSyncScopeID())); break; case Instruction::Call: { const CallInst &CI = cast<CallInst>(I); @@ -2776,7 +2731,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_CALL; - Vals.push_back(VE.getAttributeID(CI.getAttributes())); + Vals.push_back(VE.getAttributeListID(CI.getAttributes())); unsigned Flags = getOptimizationFlags(&I); Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV | @@ -2820,77 +2775,59 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.clear(); } -/// Emit names for globals/functions etc. \p IsModuleLevel is true when -/// we are writing the module-level VST, where we are including a function -/// bitcode index and need to backpatch the VST forward declaration record. -void ModuleBitcodeWriter::writeValueSymbolTable( - const ValueSymbolTable &VST, bool IsModuleLevel, - DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex) { - if (VST.empty()) { - // writeValueSymbolTableForwardDecl should have returned early as - // well. Ensure this handling remains in sync by asserting that - // the placeholder offset is not set. - assert(!IsModuleLevel || !hasVSTOffsetPlaceholder()); - return; - } +/// Write a GlobalValue VST to the module. The purpose of this data structure is +/// to allow clients to efficiently find the function body. +void ModuleBitcodeWriter::writeGlobalValueSymbolTable( + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) { + // Get the offset of the VST we are writing, and backpatch it into + // the VST forward declaration record. + uint64_t VSTOffset = Stream.GetCurrentBitNo(); + // The BitcodeStartBit was the stream offset of the identification block. + VSTOffset -= bitcodeStartBit(); + assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + // Note that we add 1 here because the offset is relative to one word + // before the start of the identification block, which was historically + // always the start of the regular bitcode header. + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1); + + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); + + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset + unsigned FnEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + for (const Function &F : M) { + uint64_t Record[2]; - if (IsModuleLevel && hasVSTOffsetPlaceholder()) { - // Get the offset of the VST we are writing, and backpatch it into - // the VST forward declaration record. - uint64_t VSTOffset = Stream.GetCurrentBitNo(); - // The BitcodeStartBit was the stream offset of the identification block. - VSTOffset -= bitcodeStartBit(); - assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + if (F.isDeclaration()) + continue; + + Record[0] = VE.getValueID(&F); + + // Save the word offset of the function (from the start of the + // actual bitcode written to the stream). + uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit(); + assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); // Note that we add 1 here because the offset is relative to one word // before the start of the identification block, which was historically // always the start of the regular bitcode header. - Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1); - } - - Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); + Record[1] = BitcodeIndex / 32 + 1; - // For the module-level VST, add abbrev Ids for the VST_CODE_FNENTRY - // records, which are not used in the per-function VSTs. - unsigned FnEntry8BitAbbrev; - unsigned FnEntry7BitAbbrev; - unsigned FnEntry6BitAbbrev; - unsigned GUIDEntryAbbrev; - if (IsModuleLevel && hasVSTOffsetPlaceholder()) { - // 8-bit fixed-width VST_CODE_FNENTRY function strings. - auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - FnEntry8BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + Stream.EmitRecord(bitc::VST_CODE_FNENTRY, Record, FnEntryAbbrev); + } - // 7-bit fixed width VST_CODE_FNENTRY function strings. - Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - FnEntry7BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + Stream.ExitBlock(); +} - // 6-bit char6 VST_CODE_FNENTRY function strings. - Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - FnEntry6BitAbbrev = Stream.EmitAbbrev(std::move(Abbv)); +/// Emit names for arguments, instructions and basic blocks in a function. +void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable( + const ValueSymbolTable &VST) { + if (VST.empty()) + return; - // FIXME: Change the name of this record as it is now used by - // the per-module index as well. - Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - GUIDEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - } + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); // FIXME: Set up the abbrev, we know how many values there are! // FIXME: We know if the type names can use 7-bit ascii. @@ -2898,51 +2835,18 @@ void ModuleBitcodeWriter::writeValueSymbolTable( for (const ValueName &Name : VST) { // Figure out the encoding to use for the name. - StringEncoding Bits = - getStringEncoding(Name.getKeyData(), Name.getKeyLength()); + StringEncoding Bits = getStringEncoding(Name.getKey()); unsigned AbbrevToUse = VST_ENTRY_8_ABBREV; NameVals.push_back(VE.getValueID(Name.getValue())); - Function *F = dyn_cast<Function>(Name.getValue()); - if (!F) { - // If value is an alias, need to get the aliased base object to - // see if it is a function. - auto *GA = dyn_cast<GlobalAlias>(Name.getValue()); - if (GA && GA->getBaseObject()) - F = dyn_cast<Function>(GA->getBaseObject()); - } - // VST_CODE_ENTRY: [valueid, namechar x N] - // VST_CODE_FNENTRY: [valueid, funcoffset, namechar x N] // VST_CODE_BBENTRY: [bbid, namechar x N] unsigned Code; if (isa<BasicBlock>(Name.getValue())) { Code = bitc::VST_CODE_BBENTRY; if (Bits == SE_Char6) AbbrevToUse = VST_BBENTRY_6_ABBREV; - } else if (F && !F->isDeclaration()) { - // Must be the module-level VST, where we pass in the Index and - // have a VSTOffsetPlaceholder. The function-level VST should not - // contain any Function symbols. - assert(FunctionToBitcodeIndex); - assert(hasVSTOffsetPlaceholder()); - - // Save the word offset of the function (from the start of the - // actual bitcode written to the stream). - uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - bitcodeStartBit(); - assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); - // Note that we add 1 here because the offset is relative to one word - // before the start of the identification block, which was historically - // always the start of the regular bitcode header. - NameVals.push_back(BitcodeIndex / 32 + 1); - - Code = bitc::VST_CODE_FNENTRY; - AbbrevToUse = FnEntry8BitAbbrev; - if (Bits == SE_Char6) - AbbrevToUse = FnEntry6BitAbbrev; - else if (Bits == SE_Fixed7) - AbbrevToUse = FnEntry7BitAbbrev; } else { Code = bitc::VST_CODE_ENTRY; if (Bits == SE_Char6) @@ -2958,47 +2862,7 @@ void ModuleBitcodeWriter::writeValueSymbolTable( Stream.EmitRecord(Code, NameVals, AbbrevToUse); NameVals.clear(); } - // Emit any GUID valueIDs created for indirect call edges into the - // module-level VST. - if (IsModuleLevel && hasVSTOffsetPlaceholder()) - for (const auto &GI : valueIds()) { - NameVals.push_back(GI.second); - NameVals.push_back(GI.first); - Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, - GUIDEntryAbbrev); - NameVals.clear(); - } - Stream.ExitBlock(); -} - -/// Emit function names and summary offsets for the combined index -/// used by ThinLTO. -void IndexBitcodeWriter::writeCombinedValueSymbolTable() { - assert(hasVSTOffsetPlaceholder() && "Expected non-zero VSTOffsetPlaceholder"); - // Get the offset of the VST we are writing, and backpatch it into - // the VST forward declaration record. - uint64_t VSTOffset = Stream.GetCurrentBitNo(); - assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); - Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); - Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); - - auto Abbv = std::make_shared<BitCodeAbbrev>(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid - unsigned EntryAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - - SmallVector<uint64_t, 64> NameVals; - for (const auto &GVI : valueIds()) { - // VST_CODE_COMBINED_ENTRY: [valueid, refguid] - NameVals.push_back(GVI.second); - NameVals.push_back(GVI.first); - - // Emit the finished record. - Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev); - NameVals.clear(); - } Stream.ExitBlock(); } @@ -3102,7 +2966,7 @@ void ModuleBitcodeWriter::writeFunction( // Emit names for all the instructions etc. if (auto *Symtab = F.getValueSymbolTable()) - writeValueSymbolTable(*Symtab); + writeFunctionLevelValueSymbolTable(*Symtab); if (NeedsMetadataAttachment) writeFunctionMetadataAttachment(F); @@ -3329,42 +3193,77 @@ void IndexBitcodeWriter::writeModStrings() { unsigned AbbrevHash = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<unsigned, 64> Vals; - for (const auto &MPSE : Index.modulePaths()) { - if (!doIncludeModule(MPSE.getKey())) - continue; - StringEncoding Bits = - getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size()); - unsigned AbbrevToUse = Abbrev8Bit; - if (Bits == SE_Char6) - AbbrevToUse = Abbrev6Bit; - else if (Bits == SE_Fixed7) - AbbrevToUse = Abbrev7Bit; + forEachModule( + [&](const StringMapEntry<std::pair<uint64_t, ModuleHash>> &MPSE) { + StringRef Key = MPSE.getKey(); + const auto &Value = MPSE.getValue(); + StringEncoding Bits = getStringEncoding(Key); + unsigned AbbrevToUse = Abbrev8Bit; + if (Bits == SE_Char6) + AbbrevToUse = Abbrev6Bit; + else if (Bits == SE_Fixed7) + AbbrevToUse = Abbrev7Bit; + + Vals.push_back(Value.first); + Vals.append(Key.begin(), Key.end()); + + // Emit the finished record. + Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse); + + // Emit an optional hash for the module now + const auto &Hash = Value.second; + if (llvm::any_of(Hash, [](uint32_t H) { return H; })) { + Vals.assign(Hash.begin(), Hash.end()); + // Emit the hash record. + Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash); + } - Vals.push_back(MPSE.getValue().first); + Vals.clear(); + }); + Stream.ExitBlock(); +} - for (const auto P : MPSE.getKey()) - Vals.push_back((unsigned char)P); +/// Write the function type metadata related records that need to appear before +/// a function summary entry (whether per-module or combined). +static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, + FunctionSummary *FS) { + if (!FS->type_tests().empty()) + Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); - // Emit the finished record. - Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse); + SmallVector<uint64_t, 64> Record; - Vals.clear(); - // Emit an optional hash for the module now - auto &Hash = MPSE.getValue().second; - bool AllZero = true; // Detect if the hash is empty, and do not generate it - for (auto Val : Hash) { - if (Val) - AllZero = false; - Vals.push_back(Val); + auto WriteVFuncIdVec = [&](uint64_t Ty, + ArrayRef<FunctionSummary::VFuncId> VFs) { + if (VFs.empty()) + return; + Record.clear(); + for (auto &VF : VFs) { + Record.push_back(VF.GUID); + Record.push_back(VF.Offset); } - if (!AllZero) { - // Emit the hash record. - Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash); + Stream.EmitRecord(Ty, Record); + }; + + WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS, + FS->type_test_assume_vcalls()); + WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS, + FS->type_checked_load_vcalls()); + + auto WriteConstVCallVec = [&](uint64_t Ty, + ArrayRef<FunctionSummary::ConstVCall> VCs) { + for (auto &VC : VCs) { + Record.clear(); + Record.push_back(VC.VFunc.GUID); + Record.push_back(VC.VFunc.Offset); + Record.insert(Record.end(), VC.Args.begin(), VC.Args.end()); + Stream.EmitRecord(Ty, Record); } + }; - Vals.clear(); - } - Stream.ExitBlock(); + WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL, + FS->type_test_assume_const_vcalls()); + WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL, + FS->type_checked_load_const_vcalls()); } // Helper to emit a single function summary record. @@ -3375,8 +3274,7 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( NameVals.push_back(ValueID); FunctionSummary *FS = cast<FunctionSummary>(Summary); - if (!FS->type_tests().empty()) - Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + writeFunctionTypeMetadataRecords(Stream, FS); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); @@ -3406,15 +3304,14 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( void ModuleBitcodeWriter::writeModuleLevelReferences( const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, unsigned FSModRefsAbbrev) { - auto Summaries = - Index->findGlobalValueSummaryList(GlobalValue::getGUID(V.getName())); - if (Summaries == Index->end()) { + auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName())); + if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might however // have a summary if the def was in module level asm). assert(V.isDeclaration()); return; } - auto *Summary = Summaries->second.front().get(); + auto *Summary = VI.getSummaryList()[0].get(); NameVals.push_back(VE.getValueID(&V)); GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); @@ -3439,7 +3336,15 @@ static const uint64_t INDEX_VERSION = 3; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { - Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); + // By default we compile with ThinLTO if the module has a summary, but the + // client can request full LTO with a module flag. + bool IsThinLTO = true; + if (auto *MD = + mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO"))) + IsThinLTO = MD->getZExtValue(); + Stream.EnterSubblock(IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID + : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID, + 4); Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); @@ -3448,6 +3353,11 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { return; } + for (const auto &GVI : valueIds()) { + Stream.EmitRecord(bitc::FS_VALUE_GUID, + ArrayRef<uint64_t>{GVI.second, GVI.first}); + } + // Abbrev for FS_PERMODULE. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); @@ -3498,15 +3408,14 @@ void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - auto Summaries = - Index->findGlobalValueSummaryList(GlobalValue::getGUID(F.getName())); - if (Summaries == Index->end()) { + ValueInfo VI = Index->getValueInfo(GlobalValue::getGUID(F.getName())); + if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might // however have a summary if the def was in module level asm). assert(F.isDeclaration()); continue; } - auto *Summary = Summaries->second.front().get(); + auto *Summary = VI.getSummaryList()[0].get(); writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), FSCallsAbbrev, FSCallsProfileAbbrev, F); } @@ -3540,6 +3449,11 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + for (const auto &GVI : valueIds()) { + Stream.EmitRecord(bitc::FS_VALUE_GUID, + ArrayRef<uint64_t>{GVI.second, GVI.first}); + } + // Abbrev for FS_COMBINED. auto Abbv = std::make_shared<BitCodeAbbrev>(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); @@ -3604,27 +3518,30 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.clear(); }; - for (const auto &I : *this) { + forEachSummary([&](GVInfo I) { GlobalValueSummary *S = I.second; assert(S); - assert(hasValueId(I.first)); - unsigned ValueId = getValueId(I.first); - SummaryToValueIdMap[S] = ValueId; + auto ValueId = getValueId(I.first); + assert(ValueId); + SummaryToValueIdMap[S] = *ValueId; if (auto *AS = dyn_cast<AliasSummary>(S)) { // Will process aliases as a post-pass because the reader wants all // global to be loaded first. Aliases.push_back(AS); - continue; + return; } if (auto *VS = dyn_cast<GlobalVarSummary>(S)) { - NameVals.push_back(ValueId); + NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(VS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); for (auto &RI : VS->refs()) { - NameVals.push_back(getValueId(RI.getGUID())); + auto RefValueId = getValueId(RI.getGUID()); + if (!RefValueId) + continue; + NameVals.push_back(*RefValueId); } // Emit the finished record. @@ -3632,22 +3549,28 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { FSModRefsAbbrev); NameVals.clear(); MaybeEmitOriginalName(*S); - continue; + return; } auto *FS = cast<FunctionSummary>(S); - if (!FS->type_tests().empty()) - Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests()); + writeFunctionTypeMetadataRecords(Stream, FS); - NameVals.push_back(ValueId); + NameVals.push_back(*ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); - NameVals.push_back(FS->refs().size()); + // Fill in below + NameVals.push_back(0); + unsigned Count = 0; for (auto &RI : FS->refs()) { - NameVals.push_back(getValueId(RI.getGUID())); + auto RefValueId = getValueId(RI.getGUID()); + if (!RefValueId) + continue; + NameVals.push_back(*RefValueId); + Count++; } + NameVals[4] = Count; bool HasProfileData = false; for (auto &EI : FS->calls()) { @@ -3659,9 +3582,20 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { for (auto &EI : FS->calls()) { // If this GUID doesn't have a value id, it doesn't have a function // summary and we don't need to record any calls to it. - if (!hasValueId(EI.first.getGUID())) - continue; - NameVals.push_back(getValueId(EI.first.getGUID())); + GlobalValue::GUID GUID = EI.first.getGUID(); + auto CallValueId = getValueId(GUID); + if (!CallValueId) { + // For SamplePGO, the indirect call targets for local functions will + // have its original name annotated in profile. We try to find the + // corresponding PGOFuncName as the GUID. + GUID = Index.getGUIDFromOriginalID(GUID); + if (GUID == 0) + continue; + CallValueId = getValueId(GUID); + if (!CallValueId) + continue; + } + NameVals.push_back(*CallValueId); if (HasProfileData) NameVals.push_back(static_cast<uint8_t>(EI.second.Hotness)); } @@ -3674,7 +3608,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EmitRecord(Code, NameVals, FSAbbrev); NameVals.clear(); MaybeEmitOriginalName(*S); - } + }); for (auto *AS : Aliases) { auto AliasValueId = SummaryToValueIdMap[AS]; @@ -3692,12 +3626,30 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { MaybeEmitOriginalName(*AS); } + if (!Index.cfiFunctionDefs().empty()) { + for (auto &S : Index.cfiFunctionDefs()) { + NameVals.push_back(StrtabBuilder.add(S)); + NameVals.push_back(S.size()); + } + Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DEFS, NameVals); + NameVals.clear(); + } + + if (!Index.cfiFunctionDecls().empty()) { + for (auto &S : Index.cfiFunctionDecls()) { + NameVals.push_back(StrtabBuilder.add(S)); + NameVals.push_back(S.size()); + } + Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DECLS, NameVals); + NameVals.clear(); + } + Stream.ExitBlock(); } /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the /// current llvm version, and a record for the epoch number. -void writeIdentificationBlock(BitstreamWriter &Stream) { +static void writeIdentificationBlock(BitstreamWriter &Stream) { Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5); // Write the "user readable" string identifying the bitcode producer @@ -3722,17 +3674,23 @@ void writeIdentificationBlock(BitstreamWriter &Stream) { void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { // Emit the module's hash. // MODULE_CODE_HASH: [5*i32] - SHA1 Hasher; - Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], - Buffer.size() - BlockStartPos)); - StringRef Hash = Hasher.result(); - uint32_t Vals[5]; - for (int Pos = 0; Pos < 20; Pos += 4) { - Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); - } + if (GenerateHash) { + uint32_t Vals[5]; + Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], + Buffer.size() - BlockStartPos)); + StringRef Hash = Hasher.result(); + for (int Pos = 0; Pos < 20; Pos += 4) { + Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos); + } - // Emit the finished record. - Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); + + if (ModHash) + // Save the written hash value. + std::copy(std::begin(Vals), std::end(Vals), std::begin(*ModHash)); + } else if (ModHash) + Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef<uint32_t>(*ModHash)); } void ModuleBitcodeWriter::write() { @@ -3741,10 +3699,7 @@ void ModuleBitcodeWriter::write() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); size_t BlockStartPos = Buffer.size(); - SmallVector<unsigned, 1> Vals; - unsigned CurVersion = 1; - Vals.push_back(CurVersion); - Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); + writeModuleVersion(); // Emit blockinfo, which defines the standard abbreviations etc. writeBlockInfo(); @@ -3778,6 +3733,7 @@ void ModuleBitcodeWriter::write() { writeUseListBlock(nullptr); writeOperandBundleTags(); + writeSyncScopeNames(); // Emit function bodies. DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex; @@ -3790,12 +3746,9 @@ void ModuleBitcodeWriter::write() { if (Index) writePerModuleGlobalValueSummary(); - writeValueSymbolTable(M.getValueSymbolTable(), - /* IsModuleLevel */ true, &FunctionToBitcodeIndex); + writeGlobalValueSymbolTable(FunctionToBitcodeIndex); - if (GenerateHash) { - writeModuleHash(BlockStartPos); - } + writeModuleHash(BlockStartPos); Stream.ExitBlock(); } @@ -3881,23 +3834,105 @@ BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer) writeBitcodeHeader(*Stream); } -BitcodeWriter::~BitcodeWriter() = default; +BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); } + +void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) { + Stream->EnterSubblock(Block, 3); + + auto Abbv = std::make_shared<BitCodeAbbrev>(); + Abbv->Add(BitCodeAbbrevOp(Record)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + auto AbbrevNo = Stream->EmitAbbrev(std::move(Abbv)); + + Stream->EmitRecordWithBlob(AbbrevNo, ArrayRef<uint64_t>{Record}, Blob); + + Stream->ExitBlock(); +} + +void BitcodeWriter::writeSymtab() { + assert(!WroteStrtab && !WroteSymtab); + + // If any module has module-level inline asm, we will require a registered asm + // parser for the target so that we can create an accurate symbol table for + // the module. + for (Module *M : Mods) { + if (M->getModuleInlineAsm().empty()) + continue; + + std::string Err; + const Triple TT(M->getTargetTriple()); + const Target *T = TargetRegistry::lookupTarget(TT.str(), Err); + if (!T || !T->hasMCAsmParser()) + return; + } + + WroteSymtab = true; + SmallVector<char, 0> Symtab; + // The irsymtab::build function may be unable to create a symbol table if the + // module is malformed (e.g. it contains an invalid alias). Writing a symbol + // table is not required for correctness, but we still want to be able to + // write malformed modules to bitcode files, so swallow the error. + if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) { + consumeError(std::move(E)); + return; + } + + writeBlob(bitc::SYMTAB_BLOCK_ID, bitc::SYMTAB_BLOB, + {Symtab.data(), Symtab.size()}); +} + +void BitcodeWriter::writeStrtab() { + assert(!WroteStrtab); + + std::vector<char> Strtab; + StrtabBuilder.finalizeInOrder(); + Strtab.resize(StrtabBuilder.getSize()); + StrtabBuilder.write((uint8_t *)Strtab.data()); + + writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB, + {Strtab.data(), Strtab.size()}); + + WroteStrtab = true; +} + +void BitcodeWriter::copyStrtab(StringRef Strtab) { + writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB, Strtab); + WroteStrtab = true; +} void BitcodeWriter::writeModule(const Module *M, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { - ModuleBitcodeWriter ModuleWriter( - M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash); + bool GenerateHash, ModuleHash *ModHash) { + assert(!WroteStrtab); + + // The Mods vector is used by irsymtab::build, which requires non-const + // Modules in case it needs to materialize metadata. But the bitcode writer + // requires that the module is materialized, so we can cast to non-const here, + // after checking that it is in fact materialized. + assert(M->isMaterialized()); + Mods.push_back(const_cast<Module *>(M)); + + ModuleBitcodeWriter ModuleWriter(M, Buffer, StrtabBuilder, *Stream, + ShouldPreserveUseListOrder, Index, + GenerateHash, ModHash); ModuleWriter.write(); } +void BitcodeWriter::writeIndex( + const ModuleSummaryIndex *Index, + const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { + IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, + ModuleToSummariesForIndex); + IndexWriter.write(); +} + /// WriteBitcodeToFile - Write the specified module to the specified output /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, const ModuleSummaryIndex *Index, - bool GenerateHash) { + bool GenerateHash, ModuleHash *ModHash) { SmallVector<char, 0> Buffer; Buffer.reserve(256*1024); @@ -3908,7 +3943,10 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); BitcodeWriter Writer(Buffer); - Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash); + Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash, + ModHash); + Writer.writeSymtab(); + Writer.writeStrtab(); if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) emitDarwinBCHeaderAndTrailer(Buffer, TT); @@ -3920,13 +3958,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, void IndexBitcodeWriter::write() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); - SmallVector<unsigned, 1> Vals; - unsigned CurVersion = 1; - Vals.push_back(CurVersion); - Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); - - // If we have a VST, write the VSTOFFSET record placeholder. - writeValueSymbolTableForwardDecl(); + writeModuleVersion(); // Write the module paths in the combined index. writeModStrings(); @@ -3934,10 +3966,6 @@ void IndexBitcodeWriter::write() { // Write the summary combined index records. writeCombinedGlobalValueSummary(); - // Need a special VST writer for the combined index (we don't have a - // real VST and real values when this is invoked). - writeCombinedValueSymbolTable(); - Stream.ExitBlock(); } @@ -3951,11 +3979,9 @@ void llvm::WriteIndexToFile( SmallVector<char, 0> Buffer; Buffer.reserve(256 * 1024); - BitstreamWriter Stream(Buffer); - writeBitcodeHeader(Stream); - - IndexBitcodeWriter IndexWriter(Stream, Index, ModuleToSummariesForIndex); - IndexWriter.write(); + BitcodeWriter Writer(Buffer); + Writer.writeIndex(&Index, ModuleToSummariesForIndex); + Writer.writeStrtab(); Out.write((char *)&Buffer.front(), Buffer.size()); } diff --git a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index 5d5bfab..bb626ba 100644 --- a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -314,10 +314,13 @@ ValueEnumerator::ValueEnumerator(const Module &M, // Remember what is the cutoff between globalvalue's and other constants. unsigned FirstConstant = Values.size(); - // Enumerate the global variable initializers. - for (const GlobalVariable &GV : M.globals()) + // Enumerate the global variable initializers and attributes. + for (const GlobalVariable &GV : M.globals()) { if (GV.hasInitializer()) EnumerateValue(GV.getInitializer()); + if (GV.hasAttributes()) + EnumerateAttributes(GV.getAttributesAsList(AttributeList::FunctionIndex)); + } // Enumerate the aliasees. for (const GlobalAlias &GA : M.aliases()) @@ -432,12 +435,14 @@ unsigned ValueEnumerator::getValueID(const Value *V) const { return I->second-1; } +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void ValueEnumerator::dump() const { print(dbgs(), ValueMap, "Default"); dbgs() << '\n'; print(dbgs(), MetadataMap, "MetaData"); dbgs() << '\n'; } +#endif void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map, const char *Name) const { @@ -452,7 +457,8 @@ void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map, OS << "Value: " << V->getName(); else OS << "Value: [null]\n"; - V->dump(); + V->print(errs()); + errs() << '\n'; OS << " Uses(" << std::distance(V->use_begin(),V->use_end()) << "):"; for (const Use &U : V->uses()) { @@ -549,7 +555,7 @@ void ValueEnumerator::EnumerateFunctionLocalMetadata( void ValueEnumerator::dropFunctionFromMetadata( MetadataMapType::value_type &FirstMD) { SmallVector<const MDNode *, 64> Worklist; - auto push = [this, &Worklist](MetadataMapType::value_type &MD) { + auto push = [&Worklist](MetadataMapType::value_type &MD) { auto &Entry = MD.second; // Nothing to do if this metadata isn't tagged. @@ -884,23 +890,26 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) { } } -void ValueEnumerator::EnumerateAttributes(AttributeSet PAL) { +void ValueEnumerator::EnumerateAttributes(AttributeList PAL) { if (PAL.isEmpty()) return; // null is always 0. // Do a lookup. - unsigned &Entry = AttributeMap[PAL]; + unsigned &Entry = AttributeListMap[PAL]; if (Entry == 0) { // Never saw this before, add it. - Attribute.push_back(PAL); - Entry = Attribute.size(); + AttributeLists.push_back(PAL); + Entry = AttributeLists.size(); } // Do lookups for all attribute groups. - for (unsigned i = 0, e = PAL.getNumSlots(); i != e; ++i) { - AttributeSet AS = PAL.getSlotAttributes(i); - unsigned &Entry = AttributeGroupMap[AS]; + for (unsigned i = PAL.index_begin(), e = PAL.index_end(); i != e; ++i) { + AttributeSet AS = PAL.getAttributes(i); + if (!AS.hasAttributes()) + continue; + IndexAndAttrSet Pair = {i, AS}; + unsigned &Entry = AttributeGroupMap[Pair]; if (Entry == 0) { - AttributeGroups.push_back(AS); + AttributeGroups.push_back(Pair); Entry = AttributeGroups.size(); } } diff --git a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h index a8d6cf9..e7ccc8d 100644 --- a/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h +++ b/contrib/llvm/lib/Bitcode/Writer/ValueEnumerator.h @@ -36,7 +36,7 @@ class LocalAsMetadata; class MDNode; class MDOperand; class NamedMDNode; -class AttributeSet; +class AttributeList; class ValueSymbolTable; class MDSymbolTable; class raw_ostream; @@ -48,6 +48,10 @@ public: // For each value, we remember its Value* and occurrence frequency. typedef std::vector<std::pair<const Value*, unsigned> > ValueList; + /// Attribute groups as encoded in bitcode are almost AttributeSets, but they + /// include the AttributeList index, so we have to track that in our map. + typedef std::pair<unsigned, AttributeSet> IndexAndAttrSet; + UseListOrderStack UseListOrders; private: @@ -102,13 +106,13 @@ private: bool ShouldPreserveUseListOrder; - typedef DenseMap<AttributeSet, unsigned> AttributeGroupMapType; + typedef DenseMap<IndexAndAttrSet, unsigned> AttributeGroupMapType; AttributeGroupMapType AttributeGroupMap; - std::vector<AttributeSet> AttributeGroups; + std::vector<IndexAndAttrSet> AttributeGroups; - typedef DenseMap<AttributeSet, unsigned> AttributeMapType; - AttributeMapType AttributeMap; - std::vector<AttributeSet> Attribute; + typedef DenseMap<AttributeList, unsigned> AttributeListMapType; + AttributeListMapType AttributeListMap; + std::vector<AttributeList> AttributeLists; /// GlobalBasicBlockIDs - This map memoizes the basic block ID's referenced by /// the "getGlobalBasicBlockID" method. @@ -166,16 +170,17 @@ public: unsigned getInstructionID(const Instruction *I) const; void setInstructionID(const Instruction *I); - unsigned getAttributeID(AttributeSet PAL) const { + unsigned getAttributeListID(AttributeList PAL) const { if (PAL.isEmpty()) return 0; // Null maps to zero. - AttributeMapType::const_iterator I = AttributeMap.find(PAL); - assert(I != AttributeMap.end() && "Attribute not in ValueEnumerator!"); + AttributeListMapType::const_iterator I = AttributeListMap.find(PAL); + assert(I != AttributeListMap.end() && "Attribute not in ValueEnumerator!"); return I->second; } - unsigned getAttributeGroupID(AttributeSet PAL) const { - if (PAL.isEmpty()) return 0; // Null maps to zero. - AttributeGroupMapType::const_iterator I = AttributeGroupMap.find(PAL); + unsigned getAttributeGroupID(IndexAndAttrSet Group) const { + if (!Group.second.hasAttributes()) + return 0; // Null maps to zero. + AttributeGroupMapType::const_iterator I = AttributeGroupMap.find(Group); assert(I != AttributeGroupMap.end() && "Attribute not in ValueEnumerator!"); return I->second; } @@ -206,10 +211,8 @@ public: const std::vector<const BasicBlock*> &getBasicBlocks() const { return BasicBlocks; } - const std::vector<AttributeSet> &getAttributes() const { - return Attribute; - } - const std::vector<AttributeSet> &getAttributeGroups() const { + const std::vector<AttributeList> &getAttributeLists() const { return AttributeLists; } + const std::vector<IndexAndAttrSet> &getAttributeGroups() const { return AttributeGroups; } @@ -283,7 +286,7 @@ private: void EnumerateValue(const Value *V); void EnumerateType(Type *T); void EnumerateOperandType(const Value *V); - void EnumerateAttributes(AttributeSet PAL); + void EnumerateAttributes(AttributeList PAL); void EnumerateValueSymbolTable(const ValueSymbolTable &ST); void EnumerateNamedMetadata(const Module &M); |
