diff options
Diffstat (limited to 'llvm/pmu/pmu.cpp')
-rw-r--r-- | llvm/pmu/pmu.cpp | 491 |
1 files changed, 491 insertions, 0 deletions
diff --git a/llvm/pmu/pmu.cpp b/llvm/pmu/pmu.cpp new file mode 100644 index 0000000..640997f --- /dev/null +++ b/llvm/pmu/pmu.cpp @@ -0,0 +1,491 @@ +/* + * (C) 2018 by Computer System Laboratory, IIS, Academia Sinica, Taiwan. + * See COPYRIGHT in top-level directory. + */ + +#include <errno.h> +#include <sys/mman.h> +#include "pmu/pmu-global.h" +#include "pmu/pmu-events.h" + +/* + * Performance Monitoring Unit (PMU) tools. + */ +namespace pmu { + +static bool InitOnce; + +EventManager *EventMgr; /* Event manager. */ +GlobalConfig SysConfig; /* System-wide configuration. */ +EventID PreEvents[PMU_EVENT_MAX]; /* Pre-defined events. */ + + +/* Initialize system-wide configuration. */ +static void SetupGlobalConfig(PMUConfig &Config) +{ + /* Get page size. */ + SysConfig.PageSize = getpagesize(); + + /* Configure timeout and signal receiver for the timer. */ + SysConfig.SignalReceiver = Config.SignalReceiver; + if (SysConfig.SignalReceiver <= 0) + SysConfig.SignalReceiver = getpid(); + + SysConfig.Timeout = Config.Timeout; + if (SysConfig.Timeout == 0) + SysConfig.Timeout = PMU_TIMER_PERIOD; + + SysConfig.Timeout *= 1000; /* nanosecond */ + + /* Determine the Linux Perf version used by this tool and the kernel. + * We set the last few bytes of the perf_event_attr structure and see the + * size field returned from the kernel. */ + + SysConfig.PerfVersion = 0; + SysConfig.OSPerfVersion = 0; + + struct perf_event_attr attr; + perf_attr_init(&attr, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES); + attr.aux_watermark = 1; + int fd = sys_perf_event_open(&attr, 0, -1, -1, 0); + close(fd); + +#define CheckPerfVersion(_Ver) \ + do { \ + SysConfig.PerfVersion = _Ver; \ + if (attr.size == PERF_ATTR_SIZE_VER##_Ver) \ + SysConfig.OSPerfVersion = _Ver; \ + } while(0) + + CheckPerfVersion(1); + CheckPerfVersion(2); + CheckPerfVersion(3); + CheckPerfVersion(4); + CheckPerfVersion(5); + +#undef CheckPerfVersion +} + +/* Initialize pre-defined events. */ +static void SetupDefaultEvent() +{ + for (unsigned i = 0; i < PMU_EVENT_MAX; ++i) { + PreEvents[i].Type = -1; + PreEvents[i].Config = -1; + } + +#define SetupEvent(_Event,_Config) \ + PreEvents[_Event].Type = PERF_TYPE_HARDWARE; \ + PreEvents[_Event].Config = _Config; + + /* Basic events. */ + SetupEvent(PMU_CPU_CYCLES, PERF_COUNT_HW_CPU_CYCLES); + SetupEvent(PMU_REF_CPU_CYCLES, PERF_COUNT_HW_REF_CPU_CYCLES); + SetupEvent(PMU_INSTRUCTIONS, PERF_COUNT_HW_INSTRUCTIONS); + SetupEvent(PMU_LLC_REFERENCES, PERF_COUNT_HW_CACHE_REFERENCES); + SetupEvent(PMU_LLC_MISSES, PERF_COUNT_HW_CACHE_MISSES); + SetupEvent(PMU_BRANCH_INSTRUCTIONS, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); + SetupEvent(PMU_BRANCH_MISSES, PERF_COUNT_HW_BRANCH_MISSES); + +#undef SetEventCode +} + +/* Initialize the PMU module. */ +int PMU::Init(PMUConfig &Config) +{ + if (InitOnce == true) + return PMU_OK; + + /* Set the global configuration. */ + SetupGlobalConfig(Config); + + /* Initialize pre-defined event codes. */ + SetupDefaultEvent(); + + /* Allocate event manager. */ + EventMgr = new EventManager; + + /* Initialize target-specific events. */ +#if defined(__i386__) || defined(__x86_64__) + X86Init(); +#elif defined(__arm__) || defined (__aarch64__) + ARMInit(); +#elif defined(_ARCH_PPC) || defined(_ARCH_PPC64) + PPCInit(); +#endif + + Config.PerfVersion = SysConfig.PerfVersion; + Config.OSPerfVersion = SysConfig.OSPerfVersion; + + InitOnce = true; + return PMU_OK; +} + +/* Finalize the PMU module. */ +int PMU::Finalize(void) +{ + if (InitOnce == false) + return PMU_OK; + + delete EventMgr; + + InitOnce = false; + return PMU_OK; +} + +/* Stop the PMU module. */ +int PMU::Pause(void) +{ + EventMgr->Pause(); + return PMU_OK; +} + +/* Restart the PMU module. */ +int PMU::Resume(void) +{ + EventMgr->Resume(); + return PMU_OK; +} + +/* Start a counting/sampling/tracing event. */ +int PMU::Start(Handle Hndl) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + + if (perf_event_start(Event->getFD()) != 0) + return PMU_EEVENT; + + EventMgr->StartEvent(Event); + + return PMU_OK; +} + +/* Stop a counting/sampling/tracing event. */ +int PMU::Stop(Handle Hndl) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + + if (perf_event_stop(Event->getFD()) != 0) + return PMU_EEVENT; + + EventMgr->StopEvent(Event); + + return PMU_OK; +} + +/* Reset the hardware counter. */ +int PMU::Reset(Handle Hndl) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + + if (perf_event_reset(Event->getFD()) != 0) + return PMU_EEVENT; + return PMU_OK; +} + +/* Remove an event. */ +int PMU::Cleanup(Handle Hndl) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + + /* Do stop the event if the user hasn't called it. */ + if (Event->State != STATE_STOP) { + int EC = Stop(Hndl); + if (EC != PMU_OK) + return EC; + } + + /* At this point, this event has been removed from the sampling list and we + * no longer get overflow handling (if this is a sampling event). We are + * now able to release all resources. */ + + /* Stop all events in a group. */ + for (auto fd : Event->FD) + perf_event_stop(fd); + + /* Release allocated buffers. */ + if (Event->Data.Base) + munmap(Event->Data.Base, Event->Data.Size); + if (Event->Aux.Base) + munmap(Event->Aux.Base, Event->Aux.Size); + + for (auto fd : Event->FD) + close(fd); + + EventMgr->DeleteEvent(Event); + return PMU_OK; +} + +/* Start/stop a sampling/tracing event without acquiring a lock. + * Note that these two function should only be used within the overflow + * handler. Since the overflow handling is already in a locked section, + * acquiring a lock is not required. */ +int PMU::StartUnlocked(Handle Hndl) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + if (Event->Mode & MODE_COUNTER) + return PMU_EINVAL; + + if (perf_event_start(Event->getFD()) != 0) + return PMU_EEVENT; + + EventMgr->StartEvent(Event, false); + + return PMU_OK; +} + +int PMU::StopUnlocked(Handle Hndl) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + if (Event->Mode & MODE_COUNTER) + return PMU_EINVAL; + + if (perf_event_stop(Event->getFD()) != 0) + return PMU_EEVENT; + + EventMgr->StopEvent(Event, false); + + return PMU_OK; +} + +/* Open an event using the pre-defined event code. */ +int PMU::CreateEvent(unsigned EventCode, Handle &Hndl) +{ + int fd; + struct perf_event_attr Attr; + + Hndl = PMU_INVALID_HNDL; + + if (EventCode >= PMU_EVENT_MAX) + return PMU_EINVAL; + if (PreEvents[EventCode].Type == -1) + return PMU_ENOEVENT; + + perf_attr_init(&Attr, PreEvents[EventCode].Type, PreEvents[EventCode].Config); + fd = sys_perf_event_open(&Attr, 0, -1, -1, 0); + if (fd < 0) + return ErrorCode(errno); + + Hndl = EventMgr->AddEvent(fd); + if (Hndl == PMU_INVALID_HNDL) { + close(fd); + return PMU_ENOMEM; + } + return PMU_OK; +} + +/* Open an event using the raw event number and umask value. + * The raw event code is computed as (RawEvent | (Umask << 8)). */ +int PMU::CreateRawEvent(unsigned RawEvent, unsigned Umask, Handle &Hndl) +{ + int fd; + struct perf_event_attr Attr; + + Hndl = PMU_INVALID_HNDL; + + perf_attr_init(&Attr, PERF_TYPE_RAW, RawEvent | (Umask << 8)); + fd = sys_perf_event_open(&Attr, 0, -1, -1, 0); + if (fd < 0) + return ErrorCode(errno); + + Hndl = EventMgr->AddEvent(fd); + if (Hndl == PMU_INVALID_HNDL) { + close(fd); + return PMU_ENOMEM; + } + return PMU_OK; +} + +/* Open a sampling event, with the 1st EventCode as the interrupt event. + * The sample data will be recorded in a vector of type 'uint64_t'. + * The following vector shows the data format of sampling with N events: + * { pc, val1, val2, ..., valN, # 1st sample + * ... + * pc, val1, val2, ..., valN }; # nth sample + * + * Note that ownwership of the output vector is transferred to the user. + * It is the user's responsibility to free the resource of the vector. */ +int PMU::CreateSampleEvent(SampleConfig &Config, Handle &Hndl) +{ + unsigned i, NumEvents = Config.NumEvents; + unsigned NumPages = Config.NumPages; + uint64_t Period = Config.Period; + int fds[PMU_GROUP_EVENTS], EC = PMU_ENOMEM; + uint64_t DataSize; + void *Data; + + Hndl = PMU_INVALID_HNDL; + + if (NumPages == 0) + NumPages = PMU_SAMPLE_PAGES; + if (Period < 1e3) + Period = PMU_SAMPLE_PERIOD; + + if (NumEvents == 0 || NumEvents > PMU_GROUP_EVENTS || !isPowerOf2(NumPages)) + return PMU_EINVAL; + + /* Check event codes. */ + for (i = 0; i < NumEvents; ++i) { + unsigned EventCode = Config.EventCode[i]; + if (EventCode >= PMU_EVENT_MAX) + return PMU_EINVAL; + if (PreEvents[EventCode].Type == -1) + return PMU_ENOEVENT; + } + + /* Open the events. If more than one event is requested, set read_format + * to PERF_FORMAT_GROUP. */ + fds[0] = -1; + for (i = 0; i < NumEvents; ++i) { + struct perf_event_attr Attr; + unsigned EventCode = Config.EventCode[i]; + perf_attr_init(&Attr, PreEvents[EventCode].Type, PreEvents[EventCode].Config); + + Attr.disabled = !i; + if (i == 0) { + Attr.sample_period = Period; + Attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_READ; + Attr.read_format = (NumEvents > 1) ? PERF_FORMAT_GROUP : 0; + } + + fds[i] = sys_perf_event_open(&Attr, 0, -1, fds[0], 0); + if (fds[i] < 0) { + EC = ErrorCode(errno); + goto failed; + } + } + + /* Allocate buffer for the sampling data. */ + DataSize = (1 + NumPages) * SysConfig.PageSize; + Data = mmap(nullptr, DataSize, PROT_READ|PROT_WRITE, MAP_SHARED, fds[0], 0); + if (Data == MAP_FAILED) + goto failed; + + Hndl = EventMgr->AddSampleEvent(NumEvents, fds, DataSize, Data, + MODE_SAMPLE_IP | MODE_SAMPLE_READ, Config); + if (Hndl == PMU_INVALID_HNDL) { + munmap(Data, DataSize); + goto failed; + } + return PMU_OK; + +failed: + while (--i) + close(fds[i]); + return EC; +} + +/* Generate an IP histogram using EventCode as the interrupt event. + * The IP histogram will be recorded in a vector of type 'uint64_t' with + * the format: { pc1, pc2, pc3, ..., pcN }. + * Note that ownwership of the output vector is transferred to the user. + * It is the user's responsibility to free the resource of the vector. */ +int PMU::CreateSampleIP(Sample1Config &Config, Handle &Hndl) +{ + int fd; + unsigned EventCode = Config.EventCode; + unsigned NumPages = Config.NumPages; + uint64_t Period = Config.Period; + uint64_t DataSize; + void *Data; + + Hndl = PMU_INVALID_HNDL; + + if (NumPages == 0) + NumPages = PMU_SAMPLE_PAGES; + if (Period < 1e3) + Period = PMU_SAMPLE_PERIOD; + + if (!isPowerOf2(NumPages)) + return PMU_EINVAL; + + /* Check the events. */ + if (EventCode >= PMU_EVENT_MAX) + return PMU_EINVAL; + if (PreEvents[EventCode].Type == -1) + return PMU_ENOEVENT; + + struct perf_event_attr Attr; + perf_attr_init(&Attr, PreEvents[EventCode].Type, PreEvents[EventCode].Config); + + Attr.disabled = 1; + Attr.sample_period = Period; + Attr.sample_type = PERF_SAMPLE_IP; + + fd = sys_perf_event_open(&Attr, 0, -1, -1, 0); + if (fd < 0) + return ErrorCode(errno); + + /* Allocate buffer for the sampling data. */ + DataSize = (1 + NumPages) * SysConfig.PageSize; + Data = mmap(nullptr, DataSize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if (Data == MAP_FAILED) + goto failed; + + /* Set the sampling config. */ + SampleConfig SConfig; + SConfig.NumEvents = 1; + SConfig.EventCode[0] = Config.EventCode; + SConfig.NumPages = NumPages; + SConfig.Period = Period; + SConfig.Watermark = Config.Watermark; + SConfig.SampleHandler = Config.SampleHandler; + SConfig.Opaque = Config.Opaque; + + Hndl = EventMgr->AddSampleEvent(1, &fd, DataSize, Data, MODE_SAMPLE_IP, SConfig); + if (Hndl == PMU_INVALID_HNDL) { + munmap(Data, DataSize); + goto failed; + } + return PMU_OK; + +failed: + close(fd); + return PMU_ENOMEM; +} + +/* Read value from the hardware counter. */ +int PMU::ReadEvent(Handle Hndl, uint64_t &Value) +{ + auto Event = EventMgr->GetEvent(Hndl); + if (!Event) + return PMU_EINVAL; + + if (read(Event->getFD(), &Value, sizeof(uint64_t)) != sizeof(uint64_t)) + return PMU_EEVENT; + return PMU_OK; +} + +/* Convert error code to string. */ +const char *PMU::strerror(int ErrCode) +{ + switch (ErrCode) { + case PMU_OK: return "Success"; + case PMU_EINVAL: return "Invalid argument"; + case PMU_ENOMEM: return "Insufficient memory"; + case PMU_ENOEVENT: return "Pre-defined event not available"; + case PMU_EEVENT: return "Hardware event error"; + case PMU_EPERM: return "Permission denied"; + case PMU_EINTER: return "Internal error"; + case PMU_EDECODER: return "Decoder error"; + default: return "Unknown error"; + } +} + +} /* namespace pmu */ + +/* + * vim: ts=8 sts=4 sw=4 expandtab + */ |