diff options
Diffstat (limited to 'contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc')
| -rw-r--r-- | contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc | 498 |
1 files changed, 352 insertions, 146 deletions
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc index 49887b1..f511c99 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -24,8 +24,12 @@ // and atomically set Guard to -Guard. // - __sanitizer_cov_dump: dump the coverage data to disk. // For every module of the current process that has coverage data -// this will create a file module_name.PID.sancov. The file format is simple: -// it's just a sorted sequence of 4-byte offsets in the module. +// this will create a file module_name.PID.sancov. +// +// The file format is simple: the first 8 bytes is the magic, +// one of 0xC0BFFFFFFFFFFF64 and 0xC0BFFFFFFFFFFF32. The last byte of the +// magic defines the size of the following offsets. +// The rest of the data is the offsets in the module. // // Eventually, this coverage implementation should be obsoleted by a more // powerful general purpose Clang/LLVM coverage instrumentation. @@ -43,6 +47,9 @@ #include "sanitizer_symbolizer.h" #include "sanitizer_flags.h" +static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL; +static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL; + static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. static atomic_uintptr_t coverage_counter; @@ -56,7 +63,7 @@ static atomic_uintptr_t coverage_counter; // dump current memory layout to another file. static bool cov_sandboxed = false; -static int cov_fd = kInvalidFd; +static fd_t cov_fd = kInvalidFd; static unsigned int cov_max_block_size = 0; static bool coverage_enabled = false; static const char *coverage_dir; @@ -77,21 +84,34 @@ class CoverageData { uptr cache_size); void DumpCallerCalleePairs(); void DumpTrace(); + void DumpAsBitSet(); + void DumpCounters(); + void DumpOffsets(); + void DumpAll(); ALWAYS_INLINE void TraceBasicBlock(s32 *id); void InitializeGuardArray(s32 *guards); - void InitializeGuards(s32 *guards, uptr n, const char *module_name); + void InitializeGuards(s32 *guards, uptr n, const char *module_name, + uptr caller_pc); + void InitializeCounters(u8 *counters, uptr n); void ReinitializeGuards(); + uptr GetNumberOf8bitCounters(); + uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset); uptr *data(); uptr size(); private: + void DirectOpen(); + void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end); + // Maximal size pc array may ever grow. // We MmapNoReserve this space to ensure that the array is contiguous. - static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 24, 1 << 27); + static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64( + 1 << (SANITIZER_ANDROID ? 24 : (SANITIZER_WINDOWS ? 27 : 26)), + 1 << 27); // The amount file mapping for the pc array is grown by. static const uptr kPcArrayMmapSize = 64 * 1024; @@ -105,13 +125,27 @@ class CoverageData { // Current file mapped size of the pc array. uptr pc_array_mapped_size; // Descriptor of the file mapped pc array. - int pc_fd; + fd_t pc_fd; // Vector of coverage guard arrays, protected by mu. InternalMmapVectorNoCtor<s32*> guard_array_vec; - // Vector of module (compilation unit) names. - InternalMmapVectorNoCtor<const char*> comp_unit_name_vec; + struct NamedPcRange { + const char *copied_module_name; + uptr beg, end; // elements [beg,end) in pc_array. + }; + + // Vector of module and compilation unit pc ranges. + InternalMmapVectorNoCtor<NamedPcRange> comp_unit_name_vec; + InternalMmapVectorNoCtor<NamedPcRange> module_name_vec; + + struct CounterAndSize { + u8 *counters; + uptr n; + }; + + InternalMmapVectorNoCtor<CounterAndSize> counters_vec; + uptr num_8bit_counters; // Caller-Callee (cc) array, size and current index. static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24); @@ -133,8 +167,6 @@ class CoverageData { static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); StaticSpinMutex mu; - - void DirectOpen(); }; static CoverageData coverage_data; @@ -145,9 +177,9 @@ void CoverageData::DirectOpen() { InternalScopedString path(kMaxPathLength); internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw", coverage_dir, internal_getpid()); - pc_fd = OpenFile(path.data(), true); - if (internal_iserror(pc_fd)) { - Report(" Coverage: failed to open %s for writing\n", path.data()); + pc_fd = OpenFile(path.data(), RdWr); + if (pc_fd == kInvalidFd) { + Report("Coverage: failed to open %s for reading/writing\n", path.data()); Die(); } @@ -180,10 +212,13 @@ void CoverageData::Enable() { tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie( sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(), "CovInit::tr_event_array")); - Mprotect(reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]), - GetMmapGranularity()); + MprotectNoAccess( + reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]), + GetMmapGranularity()); tr_event_array_size = kTrEventArrayMaxSize; tr_event_pointer = tr_event_array; + + num_8bit_counters = 0; } void CoverageData::InitializeGuardArray(s32 *guards) { @@ -197,22 +232,22 @@ void CoverageData::InitializeGuardArray(s32 *guards) { void CoverageData::Disable() { if (pc_array) { - internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize); + UnmapOrDie(pc_array, sizeof(uptr) * kPcArrayMaxSize); pc_array = nullptr; } if (cc_array) { - internal_munmap(cc_array, sizeof(uptr *) * kCcArrayMaxSize); + UnmapOrDie(cc_array, sizeof(uptr *) * kCcArrayMaxSize); cc_array = nullptr; } if (tr_event_array) { - internal_munmap(tr_event_array, - sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + - GetMmapGranularity()); + UnmapOrDie(tr_event_array, + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + + GetMmapGranularity()); tr_event_array = nullptr; tr_event_pointer = nullptr; } if (pc_fd != kInvalidFd) { - internal_close(pc_fd); + CloseFile(pc_fd); pc_fd = kInvalidFd; } } @@ -231,8 +266,9 @@ void CoverageData::ReInit() { // In memory-mapped mode we must extend the new file to the known array // size. uptr size = atomic_load(&pc_array_size, memory_order_relaxed); + uptr npcs = size / sizeof(uptr); Enable(); - if (size) Extend(size); + if (size) Extend(npcs); if (coverage_enabled) CovUpdateMapping(coverage_dir); } else { Enable(); @@ -289,16 +325,69 @@ void CoverageData::Extend(uptr npcs) { atomic_store(&pc_array_size, size, memory_order_release); } +void CoverageData::InitializeCounters(u8 *counters, uptr n) { + if (!counters) return; + CHECK_EQ(reinterpret_cast<uptr>(counters) % 16, 0); + n = RoundUpTo(n, 16); // The compiler must ensure that counters is 16-aligned. + SpinMutexLock l(&mu); + counters_vec.push_back({counters, n}); + num_8bit_counters += n; +} + +void CoverageData::UpdateModuleNameVec(uptr caller_pc, uptr range_beg, + uptr range_end) { + auto sym = Symbolizer::GetOrInit(); + if (!sym) + return; + const char *module_name = sym->GetModuleNameForPc(caller_pc); + if (!module_name) return; + if (module_name_vec.empty() || + module_name_vec.back().copied_module_name != module_name) + module_name_vec.push_back({module_name, range_beg, range_end}); + else + module_name_vec.back().end = range_end; +} + void CoverageData::InitializeGuards(s32 *guards, uptr n, - const char *module_name) { + const char *comp_unit_name, + uptr caller_pc) { // The array 'guards' has n+1 elements, we use the element zero // to store 'n'. CHECK_LT(n, 1 << 30); guards[0] = static_cast<s32>(n); InitializeGuardArray(guards); SpinMutexLock l(&mu); - comp_unit_name_vec.push_back(module_name); + uptr range_end = atomic_load(&pc_array_index, memory_order_relaxed); + uptr range_beg = range_end - n; + comp_unit_name_vec.push_back({comp_unit_name, range_beg, range_end}); guard_array_vec.push_back(guards); + UpdateModuleNameVec(caller_pc, range_beg, range_end); +} + +static const uptr kBundleCounterBits = 16; + +// When coverage_order_pcs==true and SANITIZER_WORDSIZE==64 +// we insert the global counter into the first 16 bits of the PC. +uptr BundlePcAndCounter(uptr pc, uptr counter) { + if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) + return pc; + static const uptr kMaxCounter = (1 << kBundleCounterBits) - 1; + if (counter > kMaxCounter) + counter = kMaxCounter; + CHECK_EQ(0, pc >> (SANITIZER_WORDSIZE - kBundleCounterBits)); + return pc | (counter << (SANITIZER_WORDSIZE - kBundleCounterBits)); +} + +uptr UnbundlePc(uptr bundle) { + if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) + return bundle; + return (bundle << kBundleCounterBits) >> kBundleCounterBits; +} + +uptr UnbundleCounter(uptr bundle) { + if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) + return 0; + return bundle >> (SANITIZER_WORDSIZE - kBundleCounterBits); } // If guard is negative, atomically set it to -guard and store the PC in @@ -316,8 +405,8 @@ void CoverageData::Add(uptr pc, u32 *guard) { return; // May happen after fork when pc_array_index becomes 0. CHECK_LT(idx * sizeof(uptr), atomic_load(&pc_array_size, memory_order_acquire)); - pc_array[idx] = pc; - atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); + uptr counter = atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); + pc_array[idx] = BundlePcAndCounter(pc, counter); } // Registers a pair caller=>callee. @@ -354,6 +443,64 @@ void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[], } } +uptr CoverageData::GetNumberOf8bitCounters() { + return num_8bit_counters; +} + +// Map every 8bit counter to a 8-bit bitset and clear the counter. +uptr CoverageData::Update8bitCounterBitsetAndClearCounters(u8 *bitset) { + uptr num_new_bits = 0; + uptr cur = 0; + // For better speed we map 8 counters to 8 bytes of bitset at once. + static const uptr kBatchSize = 8; + CHECK_EQ(reinterpret_cast<uptr>(bitset) % kBatchSize, 0); + for (uptr i = 0, len = counters_vec.size(); i < len; i++) { + u8 *c = counters_vec[i].counters; + uptr n = counters_vec[i].n; + CHECK_EQ(n % 16, 0); + CHECK_EQ(cur % kBatchSize, 0); + CHECK_EQ(reinterpret_cast<uptr>(c) % kBatchSize, 0); + if (!bitset) { + internal_bzero_aligned16(c, n); + cur += n; + continue; + } + for (uptr j = 0; j < n; j += kBatchSize, cur += kBatchSize) { + CHECK_LT(cur, num_8bit_counters); + u64 *pc64 = reinterpret_cast<u64*>(c + j); + u64 *pb64 = reinterpret_cast<u64*>(bitset + cur); + u64 c64 = *pc64; + u64 old_bits_64 = *pb64; + u64 new_bits_64 = old_bits_64; + if (c64) { + *pc64 = 0; + for (uptr k = 0; k < kBatchSize; k++) { + u64 x = (c64 >> (8 * k)) & 0xff; + if (x) { + u64 bit = 0; + /**/ if (x >= 128) bit = 128; + else if (x >= 32) bit = 64; + else if (x >= 16) bit = 32; + else if (x >= 8) bit = 16; + else if (x >= 4) bit = 8; + else if (x >= 3) bit = 4; + else if (x >= 2) bit = 2; + else if (x >= 1) bit = 1; + u64 mask = bit << (8 * k); + if (!(new_bits_64 & mask)) { + num_new_bits++; + new_bits_64 |= mask; + } + } + } + *pb64 = new_bits_64; + } + } + } + CHECK_EQ(cur, num_8bit_counters); + return num_new_bits; +} + uptr *CoverageData::data() { return pc_array; } @@ -372,15 +519,15 @@ struct CovHeader { static void CovWritePacked(int pid, const char *module, const void *blob, unsigned int blob_size) { - if (cov_fd < 0) return; + if (cov_fd == kInvalidFd) return; unsigned module_name_length = internal_strlen(module); CovHeader header = {pid, module_name_length, blob_size}; if (cov_max_block_size == 0) { // Writing to a file. Just go ahead. - internal_write(cov_fd, &header, sizeof(header)); - internal_write(cov_fd, module, module_name_length); - internal_write(cov_fd, blob, blob_size); + WriteToFile(cov_fd, &header, sizeof(header)); + WriteToFile(cov_fd, module, module_name_length); + WriteToFile(cov_fd, blob, blob_size); } else { // Writing to a socket. We want to split the data into appropriately sized // blocks. @@ -403,8 +550,7 @@ static void CovWritePacked(int pid, const char *module, const void *blob, internal_memcpy(block_data_begin, blob_pos, payload_size); blob_pos += payload_size; ((CovHeader *)block.data())->data_length = payload_size; - internal_write(cov_fd, block.data(), - header_size_with_module + payload_size); + WriteToFile(cov_fd, block.data(), header_size_with_module + payload_size); } } } @@ -413,25 +559,25 @@ static void CovWritePacked(int pid, const char *module, const void *blob, // If packed = true and name == 0: <pid>.<sancov>.<packed>. // If packed = true and name != 0: <name>.<sancov>.<packed> (name is // user-supplied). -static int CovOpenFile(bool packed, const char *name, - const char *extension = "sancov") { - InternalScopedString path(kMaxPathLength); +static fd_t CovOpenFile(InternalScopedString *path, bool packed, + const char *name, const char *extension = "sancov") { + path->clear(); if (!packed) { CHECK(name); - path.append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(), + path->append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(), extension); } else { if (!name) - path.append("%s/%zd.%s.packed", coverage_dir, internal_getpid(), + path->append("%s/%zd.%s.packed", coverage_dir, internal_getpid(), extension); else - path.append("%s/%s.%s.packed", coverage_dir, name, extension); - } - uptr fd = OpenFile(path.data(), true); - if (internal_iserror(fd)) { - Report(" SanitizerCoverage: failed to open %s for writing\n", path.data()); - return -1; + path->append("%s/%s.%s.packed", coverage_dir, name, extension); } + error_t err; + fd_t fd = OpenFile(path->data(), WrOnly, &err); + if (fd == kInvalidFd) + Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n", + path->data(), err); return fd; } @@ -446,38 +592,40 @@ void CoverageData::DumpTrace() { for (uptr i = 0, n = size(); i < n; i++) { const char *module_name = "<unknown>"; uptr module_address = 0; - sym->GetModuleNameAndOffsetForPC(pc_array[i], &module_name, + sym->GetModuleNameAndOffsetForPC(UnbundlePc(pc_array[i]), &module_name, &module_address); out.append("%s 0x%zx\n", module_name, module_address); } - int fd = CovOpenFile(false, "trace-points"); - if (fd < 0) return; - internal_write(fd, out.data(), out.length()); - internal_close(fd); + InternalScopedString path(kMaxPathLength); + fd_t fd = CovOpenFile(&path, false, "trace-points"); + if (fd == kInvalidFd) return; + WriteToFile(fd, out.data(), out.length()); + CloseFile(fd); - fd = CovOpenFile(false, "trace-compunits"); - if (fd < 0) return; + fd = CovOpenFile(&path, false, "trace-compunits"); + if (fd == kInvalidFd) return; out.clear(); for (uptr i = 0; i < comp_unit_name_vec.size(); i++) - out.append("%s\n", comp_unit_name_vec[i]); - internal_write(fd, out.data(), out.length()); - internal_close(fd); + out.append("%s\n", comp_unit_name_vec[i].copied_module_name); + WriteToFile(fd, out.data(), out.length()); + CloseFile(fd); - fd = CovOpenFile(false, "trace-events"); - if (fd < 0) return; + fd = CovOpenFile(&path, false, "trace-events"); + if (fd == kInvalidFd) return; uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]); u8 *event_bytes = reinterpret_cast<u8*>(tr_event_array); // The trace file could be huge, and may not be written with a single syscall. while (bytes_to_write) { - uptr actually_written = internal_write(fd, event_bytes, bytes_to_write); - if (actually_written <= bytes_to_write) { + uptr actually_written; + if (WriteToFile(fd, event_bytes, bytes_to_write, &actually_written) && + actually_written <= bytes_to_write) { bytes_to_write -= actually_written; event_bytes += actually_written; } else { break; } } - internal_close(fd); + CloseFile(fd); VReport(1, " CovDump: Trace: %zd PCs written\n", size()); VReport(1, " CovDump: Trace: %zd Events written\n", max_idx); } @@ -514,10 +662,11 @@ void CoverageData::DumpCallerCalleePairs() { callee_module_address); } } - int fd = CovOpenFile(false, "caller-callee"); - if (fd < 0) return; - internal_write(fd, out.data(), out.length()); - internal_close(fd); + InternalScopedString path(kMaxPathLength); + fd_t fd = CovOpenFile(&path, false, "caller-callee"); + if (fd == kInvalidFd) return; + WriteToFile(fd, out.data(), out.length()); + CloseFile(fd); VReport(1, " CovDump: %zd caller-callee pairs written\n", total); } @@ -534,86 +683,123 @@ void CoverageData::TraceBasicBlock(s32 *id) { tr_event_pointer++; } -static void CovDumpAsBitSet() { +void CoverageData::DumpCounters() { + if (!common_flags()->coverage_counters) return; + uptr n = coverage_data.GetNumberOf8bitCounters(); + if (!n) return; + InternalScopedBuffer<u8> bitset(n); + coverage_data.Update8bitCounterBitsetAndClearCounters(bitset.data()); + InternalScopedString path(kMaxPathLength); + + for (uptr m = 0; m < module_name_vec.size(); m++) { + auto r = module_name_vec[m]; + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + const char *base_name = StripModuleName(r.copied_module_name); + fd_t fd = + CovOpenFile(&path, /* packed */ false, base_name, "counters-sancov"); + if (fd == kInvalidFd) return; + WriteToFile(fd, bitset.data() + r.beg, r.end - r.beg); + CloseFile(fd); + VReport(1, " CovDump: %zd counters written for '%s'\n", r.end - r.beg, + base_name); + } +} + +void CoverageData::DumpAsBitSet() { if (!common_flags()->coverage_bitset) return; - if (!coverage_data.size()) return; - int fd = CovOpenFile(/* packed */false, "combined", "bitset-sancov"); - if (fd < 0) return; - uptr n = coverage_data.size(); - uptr n_set_bits = 0; - InternalScopedBuffer<char> out(n); - for (uptr i = 0; i < n; i++) { - uptr pc = coverage_data.data()[i]; - out[i] = pc ? '1' : '0'; - if (pc) - n_set_bits++; + if (!size()) return; + InternalScopedBuffer<char> out(size()); + InternalScopedString path(kMaxPathLength); + for (uptr m = 0; m < module_name_vec.size(); m++) { + uptr n_set_bits = 0; + auto r = module_name_vec[m]; + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + for (uptr i = r.beg; i < r.end; i++) { + uptr pc = UnbundlePc(pc_array[i]); + out[i] = pc ? '1' : '0'; + if (pc) + n_set_bits++; + } + const char *base_name = StripModuleName(r.copied_module_name); + fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov"); + if (fd == kInvalidFd) return; + WriteToFile(fd, out.data() + r.beg, r.end - r.beg); + CloseFile(fd); + VReport(1, + " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n", + r.end - r.beg, base_name, n_set_bits); } - internal_write(fd, out.data(), n); - internal_close(fd); - VReport(1, " CovDump: bitset of %zd bits written, %zd bits are set\n", n, - n_set_bits); } -// Dump the coverage on disk. -static void CovDump() { - if (!coverage_enabled || common_flags()->coverage_direct) return; -#if !SANITIZER_WINDOWS - if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) - return; - CovDumpAsBitSet(); - coverage_data.DumpTrace(); +void CoverageData::DumpOffsets() { + auto sym = Symbolizer::GetOrInit(); if (!common_flags()->coverage_pcs) return; - uptr size = coverage_data.size(); - InternalMmapVector<u32> offsets(size); - uptr *vb = coverage_data.data(); - uptr *ve = vb + size; - SortArray(vb, size); - MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr mb, me, off, prot; - InternalScopedString module(kMaxPathLength); + CHECK_NE(sym, nullptr); + InternalMmapVector<uptr> offsets(0); InternalScopedString path(kMaxPathLength); - for (int i = 0; - proc_maps.Next(&mb, &me, &off, module.data(), module.size(), &prot); - i++) { - if ((prot & MemoryMappingLayout::kProtectionExecute) == 0) - continue; - while (vb < ve && *vb < mb) vb++; - if (vb >= ve) break; - if (*vb < me) { - offsets.clear(); - const uptr *old_vb = vb; - CHECK_LE(off, *vb); - for (; vb < ve && *vb < me; vb++) { - uptr diff = *vb - (i ? mb : 0) + off; - CHECK_LE(diff, 0xffffffffU); - offsets.push_back(static_cast<u32>(diff)); - } - const char *module_name = StripModuleName(module.data()); - if (cov_sandboxed) { - if (cov_fd >= 0) { - CovWritePacked(internal_getpid(), module_name, offsets.data(), - offsets.size() * sizeof(u32)); - VReport(1, " CovDump: %zd PCs written to packed file\n", vb - old_vb); - } - } else { - // One file per module per process. - path.clear(); - path.append("%s/%s.%zd.sancov", coverage_dir, module_name, - internal_getpid()); - int fd = CovOpenFile(false /* packed */, module_name); - if (fd > 0) { - internal_write(fd, offsets.data(), offsets.size() * sizeof(u32)); - internal_close(fd); - VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), - vb - old_vb); - } + for (uptr m = 0; m < module_name_vec.size(); m++) { + offsets.clear(); + uptr num_words_for_magic = SANITIZER_WORDSIZE == 64 ? 1 : 2; + for (uptr i = 0; i < num_words_for_magic; i++) + offsets.push_back(0); + auto r = module_name_vec[m]; + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + for (uptr i = r.beg; i < r.end; i++) { + uptr pc = UnbundlePc(pc_array[i]); + uptr counter = UnbundleCounter(pc_array[i]); + if (!pc) continue; // Not visited. + uptr offset = 0; + sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset); + offsets.push_back(BundlePcAndCounter(offset, counter)); + } + + CHECK_GE(offsets.size(), num_words_for_magic); + SortArray(offsets.data(), offsets.size()); + for (uptr i = 0; i < offsets.size(); i++) + offsets[i] = UnbundlePc(offsets[i]); + + uptr num_offsets = offsets.size() - num_words_for_magic; + u64 *magic_p = reinterpret_cast<u64*>(offsets.data()); + CHECK_EQ(*magic_p, 0ULL); + // FIXME: we may want to write 32-bit offsets even in 64-mode + // if all the offsets are small enough. + *magic_p = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32; + + const char *module_name = StripModuleName(r.copied_module_name); + if (cov_sandboxed) { + if (cov_fd != kInvalidFd) { + CovWritePacked(internal_getpid(), module_name, offsets.data(), + offsets.size() * sizeof(offsets[0])); + VReport(1, " CovDump: %zd PCs written to packed file\n", num_offsets); } + } else { + // One file per module per process. + fd_t fd = CovOpenFile(&path, false /* packed */, module_name); + if (fd == kInvalidFd) continue; + WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0])); + CloseFile(fd); + VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets); } } - if (cov_fd >= 0) - internal_close(cov_fd); - coverage_data.DumpCallerCalleePairs(); -#endif // !SANITIZER_WINDOWS + if (cov_fd != kInvalidFd) + CloseFile(cov_fd); +} + +void CoverageData::DumpAll() { + if (!coverage_enabled || common_flags()->coverage_direct) return; + if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) + return; + DumpAsBitSet(); + DumpCounters(); + DumpTrace(); + DumpOffsets(); + DumpCallerCalleePairs(); } void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { @@ -621,17 +807,21 @@ void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { if (!coverage_enabled) return; cov_sandboxed = args->coverage_sandboxed; if (!cov_sandboxed) return; - cov_fd = args->coverage_fd; cov_max_block_size = args->coverage_max_block_size; - if (cov_fd < 0) + if (args->coverage_fd >= 0) { + cov_fd = (fd_t)args->coverage_fd; + } else { + InternalScopedString path(kMaxPathLength); // Pre-open the file now. The sandbox won't allow us to do it later. - cov_fd = CovOpenFile(true /* packed */, 0); + cov_fd = CovOpenFile(&path, true /* packed */, 0); + } } -int MaybeOpenCovFile(const char *name) { +fd_t MaybeOpenCovFile(const char *name) { CHECK(name); - if (!coverage_enabled) return -1; - return CovOpenFile(true /* packed */, name); + if (!coverage_enabled) return kInvalidFd; + InternalScopedString path(kMaxPathLength); + return CovOpenFile(&path, true /* packed */, name); } void CovBeforeFork() { @@ -649,9 +839,7 @@ void InitializeCoverage(bool enabled, const char *dir) { coverage_dir = dir; coverage_data.Init(); if (enabled) coverage_data.Enable(); -#if !SANITIZER_WINDOWS if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump); -#endif } void ReInitializeCoverage(bool enabled, const char *dir) { @@ -674,7 +862,8 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) { } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) { atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard); - if (__sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) + if (static_cast<s32>( + __sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) < 0) __sanitizer_cov(guard); } SANITIZER_INTERFACE_ATTRIBUTE void @@ -687,10 +876,14 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { coverage_dir = common_flags()->coverage_dir; coverage_data.Init(); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); } +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { + coverage_data.DumpAll(); +} SANITIZER_INTERFACE_ATTRIBUTE void -__sanitizer_cov_module_init(s32 *guards, uptr npcs, const char *module_name) { - coverage_data.InitializeGuards(guards, npcs, module_name); +__sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters, + const char *comp_unit_name) { + coverage_data.InitializeGuards(guards, npcs, comp_unit_name, GET_CALLER_PC()); + coverage_data.InitializeCounters(counters, npcs); if (!common_flags()->coverage_direct) return; if (SANITIZER_ANDROID && coverage_enabled) { // dlopen/dlclose interceptors do not work on Android, so we rely on @@ -701,7 +894,7 @@ __sanitizer_cov_module_init(s32 *guards, uptr npcs, const char *module_name) { } SANITIZER_INTERFACE_ATTRIBUTE sptr __sanitizer_maybe_open_cov_file(const char *name) { - return MaybeOpenCovFile(name); + return (sptr)MaybeOpenCovFile(name); } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_total_unique_coverage() { @@ -728,4 +921,17 @@ uptr __sanitizer_get_coverage_guards(uptr **data) { *data = coverage_data.data(); return coverage_data.size(); } + +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_number_of_counters() { + return coverage_data.GetNumberOf8bitCounters(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) { + return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset); +} +// Default empty implementation (weak). Users should redefine it. +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_cmp() {} } // extern "C" |
