diff options
Diffstat (limited to 'contrib/llvm/lib/ProfileData')
8 files changed, 198 insertions, 64 deletions
diff --git a/contrib/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/contrib/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index fcd4e24..6d907c7 100644 --- a/contrib/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/contrib/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -183,72 +183,101 @@ void FunctionRecordIterator::skipOtherFiles() { *this = FunctionRecordIterator(); } +Error CoverageMapping::loadFunctionRecord( + const CoverageMappingRecord &Record, + IndexedInstrProfReader &ProfileReader) { + StringRef OrigFuncName = Record.FunctionName; + if (Record.Filenames.empty()) + OrigFuncName = getFuncNameWithoutPrefix(OrigFuncName); + else + OrigFuncName = getFuncNameWithoutPrefix(OrigFuncName, Record.Filenames[0]); + + // Don't load records for functions we've already seen. + if (!FunctionNames.insert(OrigFuncName).second) + return Error::success(); + + CounterMappingContext Ctx(Record.Expressions); + + std::vector<uint64_t> Counts; + if (Error E = ProfileReader.getFunctionCounts(Record.FunctionName, + Record.FunctionHash, Counts)) { + instrprof_error IPE = InstrProfError::take(std::move(E)); + if (IPE == instrprof_error::hash_mismatch) { + MismatchedFunctionCount++; + return Error::success(); + } else if (IPE != instrprof_error::unknown_function) + return make_error<InstrProfError>(IPE); + Counts.assign(Record.MappingRegions.size(), 0); + } + Ctx.setCounts(Counts); + + assert(!Record.MappingRegions.empty() && "Function has no regions"); + + FunctionRecord Function(OrigFuncName, Record.Filenames); + for (const auto &Region : Record.MappingRegions) { + Expected<int64_t> ExecutionCount = Ctx.evaluate(Region.Count); + if (auto E = ExecutionCount.takeError()) { + llvm::consumeError(std::move(E)); + return Error::success(); + } + Function.pushRegion(Region, *ExecutionCount); + } + if (Function.CountedRegions.size() != Record.MappingRegions.size()) { + MismatchedFunctionCount++; + return Error::success(); + } + + Functions.push_back(std::move(Function)); + return Error::success(); +} + Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(CoverageMappingReader &CoverageReader, IndexedInstrProfReader &ProfileReader) { auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping()); - std::vector<uint64_t> Counts; - for (const auto &Record : CoverageReader) { - CounterMappingContext Ctx(Record.Expressions); - - Counts.clear(); - if (Error E = ProfileReader.getFunctionCounts( - Record.FunctionName, Record.FunctionHash, Counts)) { - instrprof_error IPE = InstrProfError::take(std::move(E)); - if (IPE == instrprof_error::hash_mismatch) { - Coverage->MismatchedFunctionCount++; - continue; - } else if (IPE != instrprof_error::unknown_function) - return make_error<InstrProfError>(IPE); - Counts.assign(Record.MappingRegions.size(), 0); - } - Ctx.setCounts(Counts); + for (const auto &Record : CoverageReader) + if (Error E = Coverage->loadFunctionRecord(Record, ProfileReader)) + return std::move(E); - assert(!Record.MappingRegions.empty() && "Function has no regions"); + return std::move(Coverage); +} - StringRef OrigFuncName = Record.FunctionName; - if (Record.Filenames.empty()) - OrigFuncName = getFuncNameWithoutPrefix(OrigFuncName); - else - OrigFuncName = - getFuncNameWithoutPrefix(OrigFuncName, Record.Filenames[0]); - FunctionRecord Function(OrigFuncName, Record.Filenames); - for (const auto &Region : Record.MappingRegions) { - Expected<int64_t> ExecutionCount = Ctx.evaluate(Region.Count); - if (auto E = ExecutionCount.takeError()) { - llvm::consumeError(std::move(E)); - break; - } - Function.pushRegion(Region, *ExecutionCount); - } - if (Function.CountedRegions.size() != Record.MappingRegions.size()) { - Coverage->MismatchedFunctionCount++; - continue; - } +Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load( + ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders, + IndexedInstrProfReader &ProfileReader) { + auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping()); - Coverage->Functions.push_back(std::move(Function)); - } + for (const auto &CoverageReader : CoverageReaders) + for (const auto &Record : *CoverageReader) + if (Error E = Coverage->loadFunctionRecord(Record, ProfileReader)) + return std::move(E); return std::move(Coverage); } Expected<std::unique_ptr<CoverageMapping>> -CoverageMapping::load(StringRef ObjectFilename, StringRef ProfileFilename, - StringRef Arch) { - auto CounterMappingBuff = MemoryBuffer::getFileOrSTDIN(ObjectFilename); - if (std::error_code EC = CounterMappingBuff.getError()) - return errorCodeToError(EC); - auto CoverageReaderOrErr = - BinaryCoverageReader::create(CounterMappingBuff.get(), Arch); - if (Error E = CoverageReaderOrErr.takeError()) - return std::move(E); - auto CoverageReader = std::move(CoverageReaderOrErr.get()); +CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames, + StringRef ProfileFilename, StringRef Arch) { auto ProfileReaderOrErr = IndexedInstrProfReader::create(ProfileFilename); if (Error E = ProfileReaderOrErr.takeError()) return std::move(E); auto ProfileReader = std::move(ProfileReaderOrErr.get()); - return load(*CoverageReader, *ProfileReader); + + SmallVector<std::unique_ptr<CoverageMappingReader>, 4> Readers; + SmallVector<std::unique_ptr<MemoryBuffer>, 4> Buffers; + for (StringRef ObjectFilename : ObjectFilenames) { + auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN(ObjectFilename); + if (std::error_code EC = CovMappingBufOrErr.getError()) + return errorCodeToError(EC); + auto CoverageReaderOrErr = + BinaryCoverageReader::create(CovMappingBufOrErr.get(), Arch); + if (Error E = CoverageReaderOrErr.takeError()) + return std::move(E); + Readers.push_back(std::move(CoverageReaderOrErr.get())); + Buffers.push_back(std::move(CovMappingBufOrErr.get())); + } + return load(Readers, *ProfileReader); } namespace { @@ -560,7 +589,7 @@ std::string getCoverageMapErrString(coveragemap_error Err) { // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. class CoverageMappingErrorCategoryType : public std::error_category { - const char *name() const LLVM_NOEXCEPT override { return "llvm.coveragemap"; } + const char *name() const noexcept override { return "llvm.coveragemap"; } std::string message(int IE) const override { return getCoverageMapErrString(static_cast<coveragemap_error>(IE)); } diff --git a/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index 1a4b4f5..a6c7031 100644 --- a/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -648,7 +648,7 @@ BinaryCoverageReader::create(std::unique_ptr<MemoryBuffer> &ObjectBuffer, StringRef Coverage; uint8_t BytesInAddress; support::endianness Endian; - Error E; + Error E = Error::success(); consumeError(std::move(E)); if (ObjectBuffer->getBuffer().startswith(TestingFormatMagic)) // This is a special format used for testing. diff --git a/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp index 8ff90d6..8235633 100644 --- a/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp +++ b/contrib/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp @@ -108,8 +108,16 @@ static void writeCounter(ArrayRef<CounterExpression> Expressions, Counter C, void CoverageMappingWriter::write(raw_ostream &OS) { // Sort the regions in an ascending order by the file id and the starting - // location. - std::stable_sort(MappingRegions.begin(), MappingRegions.end()); + // location. Sort by region kinds to ensure stable order for tests. + std::stable_sort( + MappingRegions.begin(), MappingRegions.end(), + [](const CounterMappingRegion &LHS, const CounterMappingRegion &RHS) { + if (LHS.FileID != RHS.FileID) + return LHS.FileID < RHS.FileID; + if (LHS.startLoc() != RHS.startLoc()) + return LHS.startLoc() < RHS.startLoc(); + return LHS.Kind < RHS.Kind; + }); // Write out the fileid -> filename mapping. encodeULEB128(VirtualFileMapping.size(), OS); diff --git a/contrib/llvm/lib/ProfileData/InstrProf.cpp b/contrib/llvm/lib/ProfileData/InstrProf.cpp index 6962f82..74acd9e 100644 --- a/contrib/llvm/lib/ProfileData/InstrProf.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProf.cpp @@ -14,6 +14,7 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Triple.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" @@ -69,6 +70,8 @@ std::string getInstrProfErrString(instrprof_error Err) { return "Failed to compress data (zlib)"; case instrprof_error::uncompress_failed: return "Failed to uncompress data (zlib)"; + case instrprof_error::empty_raw_profile: + return "Empty raw profile file"; } llvm_unreachable("A value of instrprof_error has no message."); } @@ -77,7 +80,7 @@ std::string getInstrProfErrString(instrprof_error Err) { // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. class InstrProfErrorCategoryType : public std::error_category { - const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; } + const char *name() const noexcept override { return "llvm.instrprof"; } std::string message(int IE) const override { return getInstrProfErrString(static_cast<instrprof_error>(IE)); } @@ -135,6 +138,9 @@ std::string getPGOFuncName(StringRef RawFuncName, // (when \c InLTO is true): LTO's internalization privatizes many global linkage // symbols. This happens after value profile annotation, but those internal // linkage functions should not have a source prefix. +// Additionally, for ThinLTO mode, exported internal functions are promoted +// and renamed. We need to ensure that the original internal PGO name is +// used when computing the GUID that is compared against the profiled GUIDs. // To differentiate compiler generated internal symbols from original ones, // PGOFuncName meta data are created and attached to the original internal // symbols in the value profile annotation step @@ -780,4 +786,72 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { F.setMetadata(getPGOFuncNameMetadataName(), N); } +bool needsComdatForCounter(const Function &F, const Module &M) { + if (F.hasComdat()) + return true; + + Triple TT(M.getTargetTriple()); + if (!TT.isOSBinFormatELF()) + return false; + + // See createPGOFuncNameVar for more details. To avoid link errors, profile + // counters for function with available_externally linkage needs to be changed + // to linkonce linkage. On ELF based systems, this leads to weak symbols to be + // created. Without using comdat, duplicate entries won't be removed by the + // linker leading to increased data segement size and raw profile size. Even + // worse, since the referenced counter from profile per-function data object + // will be resolved to the common strong definition, the profile counts for + // available_externally functions will end up being duplicated in raw profile + // data. This can result in distorted profile as the counts of those dups + // will be accumulated by the profile merger. + GlobalValue::LinkageTypes Linkage = F.getLinkage(); + if (Linkage != GlobalValue::ExternalWeakLinkage && + Linkage != GlobalValue::AvailableExternallyLinkage) + return false; + + return true; +} + +// Check if INSTR_PROF_RAW_VERSION_VAR is defined. +bool isIRPGOFlagSet(const Module *M) { + auto IRInstrVar = + M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); + if (!IRInstrVar || IRInstrVar->isDeclaration() || + IRInstrVar->hasLocalLinkage()) + return false; + + // Check if the flag is set. + if (!IRInstrVar->hasInitializer()) + return false; + + const Constant *InitVal = IRInstrVar->getInitializer(); + if (!InitVal) + return false; + + return (dyn_cast<ConstantInt>(InitVal)->getZExtValue() & + VARIANT_MASK_IR_PROF) != 0; +} + +// Check if we can safely rename this Comdat function. +bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { + if (F.getName().empty()) + return false; + if (!needsComdatForCounter(F, *(F.getParent()))) + return false; + // Unsafe to rename the address-taken function (which can be used in + // function comparison). + if (CheckAddressTaken && F.hasAddressTaken()) + return false; + // Only safe to do if this function may be discarded if it is not used + // in the compilation unit. + if (!GlobalValue::isDiscardableIfUnused(F.getLinkage())) + return false; + + // For AvailableExternallyLinkage functions. + if (!F.hasComdat()) { + assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage); + return true; + } + return true; +} } // end namespace llvm diff --git a/contrib/llvm/lib/ProfileData/InstrProfReader.cpp b/contrib/llvm/lib/ProfileData/InstrProfReader.cpp index 81c13b3..ad407f0 100644 --- a/contrib/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProfReader.cpp @@ -46,6 +46,9 @@ InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) return make_error<InstrProfError>(instrprof_error::too_large); + if (Buffer->getBufferSize() == 0) + return make_error<InstrProfError>(instrprof_error::empty_raw_profile); + std::unique_ptr<InstrProfReader> Result; // Create the reader. if (IndexedInstrProfReader::hasFormat(*Buffer)) @@ -286,7 +289,7 @@ Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) { if (CurrentPos + sizeof(RawInstrProf::Header) > End) return make_error<InstrProfError>(instrprof_error::malformed); // The writer ensures each profile is padded to start at an aligned address. - if (reinterpret_cast<size_t>(CurrentPos) % alignOf<uint64_t>()) + if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t)) return make_error<InstrProfError>(instrprof_error::malformed); // The magic should have the same byte order as in the previous header. uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos); diff --git a/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp b/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp index e25299e..029d756 100644 --- a/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -13,10 +13,18 @@ //===----------------------------------------------------------------------===// #include "llvm/ProfileData/InstrProfWriter.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/ProfileSummary.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/OnDiskHashTable.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <string> #include <tuple> +#include <utility> +#include <vector> using namespace llvm; @@ -29,6 +37,7 @@ struct PatchItem { }; namespace llvm { + // A wrapper class to abstract writer stream with support of bytes // back patching. class ProfOStream { @@ -40,6 +49,7 @@ public: uint64_t tell() { return OS.tell(); } void write(uint64_t V) { LE.write<uint64_t>(V); } + // \c patch can only be called when all data is written and flushed. // For raw_string_ostream, the patch is done on the target string // directly and it won't be reflected in the stream's internal buffer. @@ -65,6 +75,7 @@ public: } } } + // If \c OS is an instance of \c raw_fd_ostream, this field will be // true. Otherwise, \c OS will be an raw_string_ostream. bool IsFDOStream; @@ -139,7 +150,8 @@ public: } } }; -} + +} // end namespace llvm InstrProfWriter::InstrProfWriter(bool Sparse) : Sparse(Sparse), FunctionData(), ProfileKind(PF_Unknown), @@ -152,6 +164,7 @@ void InstrProfWriter::setValueProfDataEndianness( support::endianness Endianness) { InfoObj->ValueProfDataEndianness = Endianness; } + void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; } @@ -182,13 +195,20 @@ Error InstrProfWriter::addRecord(InstrProfRecord &&I, uint64_t Weight) { return Dest.takeError(); } +Error InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW) { + for (auto &I : IPW.FunctionData) + for (auto &Func : I.getValue()) + if (Error E = addRecord(std::move(Func.second), 1)) + return E; + return Error::success(); +} + bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { if (!Sparse) return true; for (const auto &Func : PD) { const InstrProfRecord &IPR = Func.second; - if (std::any_of(IPR.Counts.begin(), IPR.Counts.end(), - [](uint64_t Count) { return Count > 0; })) + if (any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) return true; } return false; @@ -261,7 +281,7 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) { // structure to be serialized out (to disk or buffer). std::unique_ptr<ProfileSummary> PS = ISB.getSummary(); setSummary(TheSummary.get(), *PS); - InfoObj->SummaryBuilder = 0; + InfoObj->SummaryBuilder = nullptr; // Now do the final patch: PatchItem PatchItems[] = { diff --git a/contrib/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/contrib/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp index f8c3717..9fb2ec1 100644 --- a/contrib/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/contrib/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -59,14 +59,14 @@ void SampleProfileSummaryBuilder::addRecord( void ProfileSummaryBuilder::computeDetailedSummary() { if (DetailedSummaryCutoffs.empty()) return; - auto Iter = CountFrequencies.begin(); - auto End = CountFrequencies.end(); std::sort(DetailedSummaryCutoffs.begin(), DetailedSummaryCutoffs.end()); + auto Iter = CountFrequencies.begin(); + const auto End = CountFrequencies.end(); uint32_t CountsSeen = 0; uint64_t CurrSum = 0, Count = 0; - for (uint32_t Cutoff : DetailedSummaryCutoffs) { + for (const uint32_t Cutoff : DetailedSummaryCutoffs) { assert(Cutoff <= 999999); APInt Temp(128, TotalCount); APInt N(128, Cutoff); diff --git a/contrib/llvm/lib/ProfileData/SampleProf.cpp b/contrib/llvm/lib/ProfileData/SampleProf.cpp index cb024611..5bcfff0 100644 --- a/contrib/llvm/lib/ProfileData/SampleProf.cpp +++ b/contrib/llvm/lib/ProfileData/SampleProf.cpp @@ -24,7 +24,7 @@ namespace { // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. class SampleProfErrorCategoryType : public std::error_category { - const char *name() const LLVM_NOEXCEPT override { return "llvm.sampleprof"; } + const char *name() const noexcept override { return "llvm.sampleprof"; } std::string message(int IE) const override { sampleprof_error E = static_cast<sampleprof_error>(IE); switch (E) { |
