diff options
Diffstat (limited to 'contrib/llvm/lib/ProfileData/InstrProf.cpp')
| -rw-r--r-- | contrib/llvm/lib/ProfileData/InstrProf.cpp | 471 |
1 files changed, 326 insertions, 145 deletions
diff --git a/contrib/llvm/lib/ProfileData/InstrProf.cpp b/contrib/llvm/lib/ProfileData/InstrProf.cpp index d677763..6962f82 100644 --- a/contrib/llvm/lib/ProfileData/InstrProf.cpp +++ b/contrib/llvm/lib/ProfileData/InstrProf.cpp @@ -17,55 +17,72 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/Compression.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" using namespace llvm; +static cl::opt<bool> StaticFuncFullModulePrefix( + "static-func-full-module-prefix", cl::init(false), + cl::desc("Use full module build paths in the profile counter names for " + "static functions.")); + namespace { +std::string getInstrProfErrString(instrprof_error Err) { + switch (Err) { + case instrprof_error::success: + return "Success"; + case instrprof_error::eof: + return "End of File"; + case instrprof_error::unrecognized_format: + return "Unrecognized instrumentation profile encoding format"; + case instrprof_error::bad_magic: + return "Invalid instrumentation profile data (bad magic)"; + case instrprof_error::bad_header: + return "Invalid instrumentation profile data (file header is corrupt)"; + case instrprof_error::unsupported_version: + return "Unsupported instrumentation profile format version"; + case instrprof_error::unsupported_hash_type: + return "Unsupported instrumentation profile hash type"; + case instrprof_error::too_large: + return "Too much profile data"; + case instrprof_error::truncated: + return "Truncated profile data"; + case instrprof_error::malformed: + return "Malformed instrumentation profile data"; + case instrprof_error::unknown_function: + return "No profile data available for function"; + case instrprof_error::hash_mismatch: + return "Function control flow change detected (hash mismatch)"; + case instrprof_error::count_mismatch: + return "Function basic block count change detected (counter mismatch)"; + case instrprof_error::counter_overflow: + return "Counter overflow"; + case instrprof_error::value_site_count_mismatch: + return "Function value site count change detected (counter mismatch)"; + case instrprof_error::compress_failed: + return "Failed to compress data (zlib)"; + case instrprof_error::uncompress_failed: + return "Failed to uncompress data (zlib)"; + } + llvm_unreachable("A value of instrprof_error has no message."); +} + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. class InstrProfErrorCategoryType : public std::error_category { const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; } std::string message(int IE) const override { - instrprof_error E = static_cast<instrprof_error>(IE); - switch (E) { - case instrprof_error::success: - return "Success"; - case instrprof_error::eof: - return "End of File"; - case instrprof_error::unrecognized_format: - return "Unrecognized instrumentation profile encoding format"; - case instrprof_error::bad_magic: - return "Invalid instrumentation profile data (bad magic)"; - case instrprof_error::bad_header: - return "Invalid instrumentation profile data (file header is corrupt)"; - case instrprof_error::unsupported_version: - return "Unsupported instrumentation profile format version"; - case instrprof_error::unsupported_hash_type: - return "Unsupported instrumentation profile hash type"; - case instrprof_error::too_large: - return "Too much profile data"; - case instrprof_error::truncated: - return "Truncated profile data"; - case instrprof_error::malformed: - return "Malformed instrumentation profile data"; - case instrprof_error::unknown_function: - return "No profile data available for function"; - case instrprof_error::hash_mismatch: - return "Function control flow change detected (hash mismatch)"; - case instrprof_error::count_mismatch: - return "Function basic block count change detected (counter mismatch)"; - case instrprof_error::counter_overflow: - return "Counter overflow"; - case instrprof_error::value_site_count_mismatch: - return "Function value site count change detected (counter mismatch)"; - } - llvm_unreachable("A value of instrprof_error has no message."); + return getInstrProfErrString(static_cast<instrprof_error>(IE)); } }; -} +} // end anonymous namespace static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory; @@ -75,34 +92,72 @@ const std::error_category &llvm::instrprof_category() { namespace llvm { +void SoftInstrProfErrors::addError(instrprof_error IE) { + if (IE == instrprof_error::success) + return; + + if (FirstError == instrprof_error::success) + FirstError = IE; + + switch (IE) { + case instrprof_error::hash_mismatch: + ++NumHashMismatches; + break; + case instrprof_error::count_mismatch: + ++NumCountMismatches; + break; + case instrprof_error::counter_overflow: + ++NumCounterOverflows; + break; + case instrprof_error::value_site_count_mismatch: + ++NumValueSiteCountMismatches; + break; + default: + llvm_unreachable("Not a soft error"); + } +} + +std::string InstrProfError::message() const { + return getInstrProfErrString(Err); +} + +char InstrProfError::ID = 0; + std::string getPGOFuncName(StringRef RawFuncName, GlobalValue::LinkageTypes Linkage, StringRef FileName, uint64_t Version LLVM_ATTRIBUTE_UNUSED) { + return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); +} + +// Return the PGOFuncName. This function has some special handling when called +// in LTO optimization. The following only applies when calling in LTO passes +// (when \c InLTO is true): LTO's internalization privatizes many global linkage +// symbols. This happens after value profile annotation, but those internal +// linkage functions should not have a source prefix. +// To differentiate compiler generated internal symbols from original ones, +// PGOFuncName meta data are created and attached to the original internal +// symbols in the value profile annotation step +// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta +// data, its original linkage must be non-internal. +std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { + if (!InLTO) { + StringRef FileName = (StaticFuncFullModulePrefix + ? F.getParent()->getName() + : sys::path::filename(F.getParent()->getName())); + return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); + } - // Function names may be prefixed with a binary '1' to indicate - // that the backend should not modify the symbols due to any platform - // naming convention. Do not include that '1' in the PGO profile name. - if (RawFuncName[0] == '\1') - RawFuncName = RawFuncName.substr(1); - - std::string FuncName = RawFuncName; - if (llvm::GlobalValue::isLocalLinkage(Linkage)) { - // For local symbols, prepend the main file name to distinguish them. - // Do not include the full path in the file name since there's no guarantee - // that it will stay the same, e.g., if the files are checked out from - // version control in different locations. - if (FileName.empty()) - FuncName = FuncName.insert(0, "<unknown>:"); - else - FuncName = FuncName.insert(0, FileName.str() + ":"); + // In LTO mode (when InLTO is true), first check if there is a meta data. + if (MDNode *MD = getPGOFuncNameMetadata(F)) { + StringRef S = cast<MDString>(MD->getOperand(0))->getString(); + return S.str(); } - return FuncName; -} -std::string getPGOFuncName(const Function &F, uint64_t Version) { - return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(), - Version); + // If there is no meta data, the function must be a global before the value + // profile annotation pass. Its current linkage may be internal if it is + // internalized in LTO mode. + return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { @@ -116,8 +171,8 @@ StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { // \p FuncName is the string used as profile lookup key for the function. A // symbol is created to hold the name. Return the legalized symbol name. -static std::string getPGOFuncNameVarName(StringRef FuncName, - GlobalValue::LinkageTypes Linkage) { +std::string getPGOFuncNameVarName(StringRef FuncName, + GlobalValue::LinkageTypes Linkage) { std::string VarName = getInstrProfNameVarPrefix(); VarName += FuncName; @@ -125,7 +180,7 @@ static std::string getPGOFuncNameVarName(StringRef FuncName, return VarName; // Now fix up illegal chars in local VarName that may upset the assembler. - const char *InvalidChars = "-:<>\"'"; + const char *InvalidChars = "-:<>/\"'"; size_t found = VarName.find_first_of(InvalidChars); while (found != std::string::npos) { VarName[found] = '_'; @@ -136,7 +191,7 @@ static std::string getPGOFuncNameVarName(StringRef FuncName, GlobalVariable *createPGOFuncNameVar(Module &M, GlobalValue::LinkageTypes Linkage, - StringRef FuncName) { + StringRef PGOFuncName) { // We generally want to match the function's linkage, but available_externally // and extern_weak both have the wrong semantics, and anything that doesn't @@ -149,10 +204,11 @@ GlobalVariable *createPGOFuncNameVar(Module &M, Linkage == GlobalValue::ExternalLinkage) Linkage = GlobalValue::PrivateLinkage; - auto *Value = ConstantDataArray::getString(M.getContext(), FuncName, false); + auto *Value = + ConstantDataArray::getString(M.getContext(), PGOFuncName, false); auto FuncNameVar = new GlobalVariable(M, Value->getType(), true, Linkage, Value, - getPGOFuncNameVarName(FuncName, Linkage)); + getPGOFuncNameVarName(PGOFuncName, Linkage)); // Hide the symbol so that we correctly get a copy for each executable. if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) @@ -161,63 +217,83 @@ GlobalVariable *createPGOFuncNameVar(Module &M, return FuncNameVar; } -GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) { - return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName); +GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { + return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); +} + +void InstrProfSymtab::create(Module &M, bool InLTO) { + for (Function &F : M) { + // Function may not have a name: like using asm("") to overwrite the name. + // Ignore in this case. + if (!F.hasName()) + continue; + const std::string &PGOFuncName = getPGOFuncName(F, InLTO); + addFuncName(PGOFuncName); + MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); + } + + finalizeSymtab(); } -int collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs, - bool doCompression, std::string &Result) { +Error collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs, + bool doCompression, std::string &Result) { + assert(NameStrs.size() && "No name data to emit"); + uint8_t Header[16], *P = Header; std::string UncompressedNameStrings = - join(NameStrs.begin(), NameStrs.end(), StringRef(" ")); + join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); + + assert(StringRef(UncompressedNameStrings) + .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && + "PGO name is invalid (contains separator token)"); unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); P += EncLen; - auto WriteStringToResult = [&](size_t CompressedLen, - const std::string &InputStr) { + auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { EncLen = encodeULEB128(CompressedLen, P); P += EncLen; char *HeaderStr = reinterpret_cast<char *>(&Header[0]); unsigned HeaderLen = P - &Header[0]; Result.append(HeaderStr, HeaderLen); Result += InputStr; - return 0; + return Error::success(); }; - if (!doCompression) + if (!doCompression) { return WriteStringToResult(0, UncompressedNameStrings); + } - SmallVector<char, 128> CompressedNameStrings; + SmallString<128> CompressedNameStrings; zlib::Status Success = zlib::compress(StringRef(UncompressedNameStrings), CompressedNameStrings, zlib::BestSizeCompression); if (Success != zlib::StatusOK) - return 1; + return make_error<InstrProfError>(instrprof_error::compress_failed); - return WriteStringToResult( - CompressedNameStrings.size(), - std::string(CompressedNameStrings.data(), CompressedNameStrings.size())); + return WriteStringToResult(CompressedNameStrings.size(), + CompressedNameStrings); } -StringRef getPGOFuncNameInitializer(GlobalVariable *NameVar) { +StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer()); StringRef NameStr = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); return NameStr; } -int collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars, - std::string &Result) { +Error collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars, + std::string &Result, bool doCompression) { std::vector<std::string> NameStrs; for (auto *NameVar : NameVars) { - NameStrs.push_back(getPGOFuncNameInitializer(NameVar)); + NameStrs.push_back(getPGOFuncNameVarInitializer(NameVar)); } - return collectPGOFuncNameStrings(NameStrs, zlib::isAvailable(), Result); + return collectPGOFuncNameStrings( + NameStrs, zlib::isAvailable() && doCompression, Result); } -int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { +Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { const uint8_t *P = reinterpret_cast<const uint8_t *>(NameStrings.data()); const uint8_t *EndP = reinterpret_cast<const uint8_t *>(NameStrings.data() + NameStrings.size()); @@ -235,7 +311,7 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { CompressedSize); if (zlib::uncompress(CompressedNameStrings, UncompressedNameStrings, UncompressedSize) != zlib::StatusOK) - return 1; + return make_error<InstrProfError>(instrprof_error::uncompress_failed); P += CompressedSize; NameStrings = StringRef(UncompressedNameStrings.data(), UncompressedNameStrings.size()); @@ -246,7 +322,7 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { } // Now parse the name strings. SmallVector<StringRef, 0> Names; - NameStrings.split(Names, ' '); + NameStrings.split(Names, getInstrProfNameSeparator()); for (StringRef &Name : Names) Symtab.addFuncName(Name); @@ -254,16 +330,16 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { P++; } Symtab.finalizeSymtab(); - return 0; + return Error::success(); } -instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, - uint64_t Weight) { +void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE, + InstrProfValueSiteRecord &Input, + uint64_t Weight) { this->sortByTargetValues(); Input.sortByTargetValues(); auto I = ValueData.begin(); auto IE = ValueData.end(); - instrprof_error Result = instrprof_error::success; for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE; ++J) { while (I != IE && I->Value < J->Value) @@ -272,92 +348,80 @@ instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, bool Overflowed; I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed); if (Overflowed) - Result = instrprof_error::counter_overflow; + SIPE.addError(instrprof_error::counter_overflow); ++I; continue; } ValueData.insert(I, *J); } - return Result; } -instrprof_error InstrProfValueSiteRecord::scale(uint64_t Weight) { - instrprof_error Result = instrprof_error::success; +void InstrProfValueSiteRecord::scale(SoftInstrProfErrors &SIPE, + uint64_t Weight) { for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) { bool Overflowed; I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed); if (Overflowed) - Result = instrprof_error::counter_overflow; + SIPE.addError(instrprof_error::counter_overflow); } - return Result; } // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. -instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind, - InstrProfRecord &Src, - uint64_t Weight) { +void InstrProfRecord::mergeValueProfData(uint32_t ValueKind, + InstrProfRecord &Src, + uint64_t Weight) { uint32_t ThisNumValueSites = getNumValueSites(ValueKind); uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); - if (ThisNumValueSites != OtherNumValueSites) - return instrprof_error::value_site_count_mismatch; + if (ThisNumValueSites != OtherNumValueSites) { + SIPE.addError(instrprof_error::value_site_count_mismatch); + return; + } std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = getValueSitesForKind(ValueKind); std::vector<InstrProfValueSiteRecord> &OtherSiteRecords = Src.getValueSitesForKind(ValueKind); - instrprof_error Result = instrprof_error::success; for (uint32_t I = 0; I < ThisNumValueSites; I++) - MergeResult(Result, ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight)); - return Result; + ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight); } -instrprof_error InstrProfRecord::merge(InstrProfRecord &Other, - uint64_t Weight) { +void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) { // If the number of counters doesn't match we either have bad data // or a hash collision. - if (Counts.size() != Other.Counts.size()) - return instrprof_error::count_mismatch; - - instrprof_error Result = instrprof_error::success; + if (Counts.size() != Other.Counts.size()) { + SIPE.addError(instrprof_error::count_mismatch); + return; + } for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { bool Overflowed; Counts[I] = SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); if (Overflowed) - Result = instrprof_error::counter_overflow; + SIPE.addError(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - MergeResult(Result, mergeValueProfData(Kind, Other, Weight)); - - return Result; + mergeValueProfData(Kind, Other, Weight); } -instrprof_error InstrProfRecord::scaleValueProfData(uint32_t ValueKind, - uint64_t Weight) { +void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) { uint32_t ThisNumValueSites = getNumValueSites(ValueKind); std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = getValueSitesForKind(ValueKind); - instrprof_error Result = instrprof_error::success; for (uint32_t I = 0; I < ThisNumValueSites; I++) - MergeResult(Result, ThisSiteRecords[I].scale(Weight)); - return Result; + ThisSiteRecords[I].scale(SIPE, Weight); } -instrprof_error InstrProfRecord::scale(uint64_t Weight) { - instrprof_error Result = instrprof_error::success; +void InstrProfRecord::scale(uint64_t Weight) { for (auto &Count : this->Counts) { bool Overflowed; Count = SaturatingMultiply(Count, Weight, &Overflowed); - if (Overflowed && Result == instrprof_error::success) { - Result = instrprof_error::counter_overflow; - } + if (Overflowed) + SIPE.addError(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - MergeResult(Result, scaleValueProfData(Kind, Weight)); - - return Result; + scaleValueProfData(Kind, Weight); } // Map indirect call target name hash to name string. @@ -371,8 +435,14 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, std::lower_bound(ValueMap->begin(), ValueMap->end(), Value, [](const std::pair<uint64_t, uint64_t> &LHS, uint64_t RHS) { return LHS.first < RHS; }); - if (Result != ValueMap->end()) + // Raw function pointer collected by value profiler may be from + // external functions that are not instrumented. They won't have + // mapping data to be used by the deserializer. Force the value to + // be 0 in this case. + if (Result != ValueMap->end() && Result->first == Value) Value = (uint64_t)Result->second; + else + Value = 0; break; } } @@ -388,7 +458,7 @@ void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, std::vector<InstrProfValueSiteRecord> &ValueSites = getValueSitesForKind(ValueKind); if (N == 0) - ValueSites.push_back(InstrProfValueSiteRecord()); + ValueSites.emplace_back(); else ValueSites.emplace_back(VData, VData + N); } @@ -422,10 +492,8 @@ uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, } void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, - uint32_t K, uint32_t S, - uint64_t (*Mapper)(uint32_t, uint64_t)) { - return reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite( - Dst, K, S, Mapper); + uint32_t K, uint32_t S) { + reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S); } ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { @@ -436,12 +504,12 @@ ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { } static ValueProfRecordClosure InstrProfRecordClosure = { - 0, + nullptr, getNumValueKindsInstrProf, getNumValueSitesInstrProf, getNumValueDataInstrProf, getNumValueDataForSiteInstrProf, - 0, + nullptr, getValueForSiteInstrProf, allocValueProfDataInstrProf}; @@ -526,45 +594,45 @@ static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { ValueProfData()); } -instrprof_error ValueProfData::checkIntegrity() { +Error ValueProfData::checkIntegrity() { if (NumValueKinds > IPVK_Last + 1) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); // Total size needs to be mulltiple of quadword size. if (TotalSize % sizeof(uint64_t)) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); ValueProfRecord *VR = getFirstValueProfRecord(this); for (uint32_t K = 0; K < this->NumValueKinds; K++) { if (VR->Kind > IPVK_Last) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); VR = getValueProfRecordNext(VR); if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); } - return instrprof_error::success; + return Error::success(); } -ErrorOr<std::unique_ptr<ValueProfData>> +Expected<std::unique_ptr<ValueProfData>> ValueProfData::getValueProfData(const unsigned char *D, const unsigned char *const BufferEnd, support::endianness Endianness) { using namespace support; if (D + sizeof(ValueProfData) > BufferEnd) - return instrprof_error::truncated; + return make_error<InstrProfError>(instrprof_error::truncated); const unsigned char *Header = D; uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness); if (D + TotalSize > BufferEnd) - return instrprof_error::too_large; + return make_error<InstrProfError>(instrprof_error::too_large); std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize); memcpy(VPD.get(), D, TotalSize); // Byte swap. VPD->swapBytesToHost(Endianness); - instrprof_error EC = VPD->checkIntegrity(); - if (EC != instrprof_error::success) - return EC; + Error E = VPD->checkIntegrity(); + if (E) + return std::move(E); return std::move(VPD); } @@ -599,4 +667,117 @@ void ValueProfData::swapBytesFromHost(support::endianness Endianness) { sys::swapByteOrder<uint32_t>(NumValueKinds); } +void annotateValueSite(Module &M, Instruction &Inst, + const InstrProfRecord &InstrProfR, + InstrProfValueKind ValueKind, uint32_t SiteIdx, + uint32_t MaxMDCount) { + uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); + if (!NV) + return; + + uint64_t Sum = 0; + std::unique_ptr<InstrProfValueData[]> VD = + InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); + + ArrayRef<InstrProfValueData> VDs(VD.get(), NV); + annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); +} + +void annotateValueSite(Module &M, Instruction &Inst, + ArrayRef<InstrProfValueData> VDs, + uint64_t Sum, InstrProfValueKind ValueKind, + uint32_t MaxMDCount) { + LLVMContext &Ctx = M.getContext(); + MDBuilder MDHelper(Ctx); + SmallVector<Metadata *, 3> Vals; + // Tag + Vals.push_back(MDHelper.createString("VP")); + // Value Kind + Vals.push_back(MDHelper.createConstant( + ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind))); + // Total Count + Vals.push_back( + MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum))); + + // Value Profile Data + uint32_t MDCount = MaxMDCount; + for (auto &VD : VDs) { + Vals.push_back(MDHelper.createConstant( + ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); + Vals.push_back(MDHelper.createConstant( + ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); + if (--MDCount == 0) + break; + } + Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); +} + +bool getValueProfDataFromInst(const Instruction &Inst, + InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, uint64_t &TotalC) { + MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); + if (!MD) + return false; + + unsigned NOps = MD->getNumOperands(); + + if (NOps < 5) + return false; + + // Operand 0 is a string tag "VP": + MDString *Tag = cast<MDString>(MD->getOperand(0)); + if (!Tag) + return false; + + if (!Tag->getString().equals("VP")) + return false; + + // Now check kind: + ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); + if (!KindInt) + return false; + if (KindInt->getZExtValue() != ValueKind) + return false; + + // Get total count + ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); + if (!TotalCInt) + return false; + TotalC = TotalCInt->getZExtValue(); + + ActualNumValueData = 0; + + for (unsigned I = 3; I < NOps; I += 2) { + if (ActualNumValueData >= MaxNumValueData) + break; + ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); + ConstantInt *Count = + mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1)); + if (!Value || !Count) + return false; + ValueData[ActualNumValueData].Value = Value->getZExtValue(); + ValueData[ActualNumValueData].Count = Count->getZExtValue(); + ActualNumValueData++; + } + return true; +} + +MDNode *getPGOFuncNameMetadata(const Function &F) { + return F.getMetadata(getPGOFuncNameMetadataName()); +} + +void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { + // Only for internal linkage functions. + if (PGOFuncName == F.getName()) + return; + // Don't create duplicated meta-data. + if (getPGOFuncNameMetadata(F)) + return; + LLVMContext &C = F.getContext(); + MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); + F.setMetadata(getPGOFuncNameMetadataName(), N); } + +} // end namespace llvm |
