MFP4:

- pmcstat(8) gprof output mode fixes:

  lib/libpmc/pmclog.{c,h}, sys/sys/pmclog.h:
  + Add a 'is_usermode' field to the PMCLOG_PCSAMPLE event
  + Add an 'entryaddr' field to the PMCLOG_PROCEXEC event,
    so that pmcstat(8) can determine where the runtime loader
    /libexec/ld-elf.so.1 is getting loaded.

  sys/kern/kern_exec.c:
  + Use a local struct to group the entry address of the image being
    exec()'ed and the process credential changed flag to the exec
    handling hook inside hwpmc(4).

  usr.sbin/pmcstat/*:
  + Support "-k kernelpath", "-D sampledir".
  + Implement the ELF bits of 'gmon.out' profile generation in a new
    file "pmcstat_log.c".  Move all log related functions to this
    file.
  + Move local definitions and prototypes to "pmcstat.h"

- Other bug fixes:
  + lib/libpmc/pmclog.c: correctly handle EOF in pmclog_read().
  + sys/dev/hwpmc_mod.c: unconditionally log a PROCEXIT event to all
    attached PMCs when a process exits.
  + sys/sys/pmc.h: correct a function prototype.
  + Improve usage checks in pmcstat(8).

Approved by:	re (blanket hwpmc)

5 files changed, 1604 insertions, 349 deletions

diff --git a/usr.sbin/pmcstat/Makefile b/usr.sbin/pmcstat/Makefile
index 8250990..102fe38 100644
--- a/usr.sbin/pmcstat/Makefile
+++ b/usr.sbin/pmcstat/Makefile
@@ -10,6 +10,6 @@ LDADD=	-lpmc -lm
 
 WARNS?=	6
 
-SRCS=	pmcstat.c
+SRCS=	pmcstat.c pmcstat.h pmcstat_log.c
 
 .include <bsd.prog.mk>
diff --git a/usr.sbin/pmcstat/pmcstat.8 b/usr.sbin/pmcstat/pmcstat.8
index 482d0f9..881c3b5 100644
--- a/usr.sbin/pmcstat/pmcstat.8
+++ b/usr.sbin/pmcstat/pmcstat.8
@@ -31,8 +31,8 @@
 .Nd "performance measurement with performance monitoring hardware"
 .Sh SYNOPSIS
 .Nm
-.Op Fl D Ar pathname
 .Op Fl C
+.Op Fl D Ar pathname
 .Op Fl E
 .Op Fl O Ar logfilename
 .Op Fl P Ar event-spec
@@ -42,6 +42,7 @@
 .Op Fl c Ar cpu
 .Op Fl d
 .Op Fl g
+.Op Fl k Ar kernelfile
 .Op Fl n Ar rate
 .Op Fl o Ar outputfile
 .Op Fl p Ar event-spec
@@ -154,6 +155,11 @@ The default is to measure events for the target process alone.
 .It Fl g
 Produce execution profiles in a format compatible with
 .Xr gprof 1 .
+.It Fl k Ar kernelfile
+Set the pathname of the kernel to argument
+.Ar kernelfile .
+The default is
+.Pa "/boot/kernel/kernel" .
 .It Fl n Ar rate
 Set the default sampling rate for subsequent sampling mode
 PMCs specified on the command line.
diff --git a/usr.sbin/pmcstat/pmcstat.c b/usr.sbin/pmcstat/pmcstat.c
index e4fc143..c32b55f 100644
--- a/usr.sbin/pmcstat/pmcstat.c
+++ b/usr.sbin/pmcstat/pmcstat.c
@@ -30,6 +30,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/event.h>
 #include <sys/queue.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/ttycom.h>
 #include <sys/wait.h>
@@ -44,13 +46,15 @@ __FBSDID("$FreeBSD$");
 #include <pmclog.h>
 #include <signal.h>
 #include <stdarg.h>
-#include <stdio.h>
 #include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sysexits.h>
 #include <unistd.h>
 
+#include "pmcstat.h"
+
 /*
  * A given invocation of pmcstat(8) can manage multiple PMCs of both
  * the system-wide and per-process variety.  Each of these could be in
@@ -68,90 +72,13 @@ __FBSDID("$FreeBSD$");
  * for a given executable into a single profile file.
  */
 
-/* Operation modes */
-
-#define	FLAG_HAS_PID			0x00000001
-#define	FLAG_HAS_WAIT_INTERVAL		0x00000002
-#define	FLAG_HAS_LOG_FILE		0x00000004
-#define	FLAG_HAS_PROCESS		0x00000008
-#define	FLAG_HAS_SAMPLING_PMCS		0x00000010
-#define	FLAG_HAS_COUNTING_PMCS		0x00000020
-#define	FLAG_HAS_PROCESS_PMCS		0x00000040
-#define	FLAG_HAS_SYSTEM_PMCS		0x00000080
-#define	FLAG_HAS_PIPE			0x00000100
-#define	FLAG_PROCESS_LOGFILE		0x00000200
-#define	FLAG_DO_GPROF			0x00000400
-#define	FLAG_DO_GPROF_MERGED		0x00000800
-
-#define	DEFAULT_SAMPLE_COUNT		65536
-#define	DEFAULT_WAIT_INTERVAL		5.0
-#define	DEFAULT_DISPLAY_HEIGHT		23
-#define	DEFAULT_BUFFER_SIZE		4096
-
-#define	WRITELOG_MAGIC			0xA55AA55A
-#define	PRINT_HEADER_PREFIX		"# "
-#define	READPIPEFD			0
-#define	WRITEPIPEFD			1
-#define	NPIPEFD				2
-
-enum pmcstat_state {
-	PMCSTAT_FINISHED = 0,
-	PMCSTAT_EXITING  = 1,
-	PMCSTAT_RUNNING  = 2
-};
-
-struct pmcstat_ev {
-	STAILQ_ENTRY(pmcstat_ev) ev_next;
-	char	       *ev_spec;  /* event specification */
-	char	       *ev_name;  /* (derived) event name */
-	enum pmc_mode	ev_mode;  /* desired mode */
-	int		ev_count; /* associated count if in sampling mode */
-	int		ev_cpu;	  /* specific cpu if requested */
-	int		ev_flags; /* PMC_F_* */
-	int		ev_cumulative;  /* show cumulative counts */
-	int		ev_fieldwidth;  /* print width */
-	int		ev_fieldskip;   /* #leading spaces */
-	pmc_value_t	ev_saved; /* saved value for incremental counts */
-	pmc_id_t	ev_pmcid; /* allocated ID */
-};
-
-struct pmcstat_args {
-	int	pa_required;
-	int	pa_flags;
-	pid_t	pa_pid;
-	FILE	*pa_outputfile;
-	FILE	*pa_logfile;
-	void	*pa_logparser;
-	char	*pa_outputdir;
-	double	pa_interval;
-	int	pa_argc;
-	char	**pa_argv;
-	STAILQ_HEAD(, pmcstat_ev) pa_head;
-} args;
+/* Globals */
 
 int	pmcstat_interrupt = 0;
 int	pmcstat_displayheight = DEFAULT_DISPLAY_HEIGHT;
 int	pmcstat_pipefd[NPIPEFD];
 int	pmcstat_kq;
 
-/* Function prototypes */
-void	pmcstat_cleanup(struct pmcstat_args *_a);
-int	pmcstat_close_log(struct pmcstat_args *_a);
-void	pmcstat_print_counters(struct pmcstat_args *_a);
-void	pmcstat_print_headers(struct pmcstat_args *_a);
-void	pmcstat_print_pmcs(struct pmcstat_args *_a);
-void	pmcstat_setup_process(struct pmcstat_args *_a);
-void	pmcstat_show_usage(void);
-void	pmcstat_start_pmcs(struct pmcstat_args *_a);
-void	pmcstat_start_process(struct pmcstat_args *_a);
-void	pmcstat_process_log(struct pmcstat_args *_a);
-int	pmcstat_print_log(struct pmcstat_args *_a);
-
-#define	PMCSTAT_PRINT_LOG(A,T,...) do {					\
-		fprintf((A)->pa_outputfile, T "\t" __VA_ARGS__);	\
-		fprintf((A)->pa_outputfile, "\n");			\
-	} while (0)
-
 /*
  * cleanup
  */
@@ -162,7 +89,7 @@ pmcstat_cleanup(struct pmcstat_args *a)
 	struct pmcstat_ev *ev, *tmp;
 
 	/* de-configure the log file if present. */
-	if (a->pa_flags & FLAG_HAS_LOG_FILE)
+	if (a->pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))
 		(void) pmc_configure_logfile(-1);
 
 	/* release allocated PMCs. */
@@ -181,6 +108,9 @@ pmcstat_cleanup(struct pmcstat_args *a)
 		pmclog_close(a->pa_logparser);
 		a->pa_logparser = NULL;
 	}
+
+	if (a->pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))
+		pmcstat_shutdown_logging();
 }
 
 void
@@ -208,7 +138,7 @@ pmcstat_print_headers(struct pmcstat_args *a)
 	struct pmcstat_ev *ev;
 	int c;
 
-	(void) fprintf(a->pa_outputfile, PRINT_HEADER_PREFIX);
+	(void) fprintf(a->pa_printfile, PRINT_HEADER_PREFIX);
 
 	STAILQ_FOREACH(ev, &a->pa_head, ev_next) {
 		if (PMC_IS_SAMPLING_MODE(ev->ev_mode))
@@ -217,16 +147,16 @@ pmcstat_print_headers(struct pmcstat_args *a)
 		c = PMC_IS_SYSTEM_MODE(ev->ev_mode) ? 's' : 'p';
 
 		if (ev->ev_fieldskip != 0) {
-			(void) fprintf(a->pa_outputfile, "%*s%c/%*s ",
+			(void) fprintf(a->pa_printfile, "%*s%c/%*s ",
 			    ev->ev_fieldskip, "", c,
 			    ev->ev_fieldwidth - ev->ev_fieldskip - 2,
 			    ev->ev_name);
 		} else
-			(void) fprintf(a->pa_outputfile, "%c/%*s ",
+			(void) fprintf(a->pa_printfile, "%c/%*s ",
 			    c, ev->ev_fieldwidth - 2, ev->ev_name);
 	}
 
-	(void) fflush(a->pa_outputfile);
+	(void) fflush(a->pa_printfile);
 }
 
 void
@@ -248,15 +178,17 @@ pmcstat_print_counters(struct pmcstat_args *a)
 			err(EX_OSERR, "ERROR: Cannot read pmc "
 			    "\"%s\"", ev->ev_name);
 
-		(void) fprintf(a->pa_outputfile, "%*ju ",
-		    ev->ev_fieldwidth + extra_width, (uintmax_t)
-		    ev->ev_cumulative ? value : (value - ev->ev_saved));
+		(void) fprintf(a->pa_printfile, "%*ju ",
+		    ev->ev_fieldwidth + extra_width,
+		    (uintmax_t) ev->ev_cumulative ? value :
+		    (value - ev->ev_saved));
+
 		if (ev->ev_cumulative == 0)
 			ev->ev_saved = value;
 		extra_width = 0;
 	}
 
-	(void) fflush(a->pa_outputfile);
+	(void) fflush(a->pa_printfile);
 }
 
 /*
@@ -268,15 +200,15 @@ pmcstat_print_pmcs(struct pmcstat_args *a)
 {
 	static int linecount = 0;
 
+	/* check if we need to print a header line */
 	if (++linecount > pmcstat_displayheight) {
-		(void) fprintf(a->pa_outputfile, "\n");
+		(void) fprintf(a->pa_printfile, "\n");
 		linecount = 1;
 	}
-
 	if (linecount == 1)
 		pmcstat_print_headers(a);
+	(void) fprintf(a->pa_printfile, "\n");
 
-	(void) fprintf(a->pa_outputfile, "\n");
 	pmcstat_print_counters(a);
 
 	return;
@@ -375,146 +307,6 @@ pmcstat_start_process(struct pmcstat_args *a)
 }
 
 
-/*
- * Process a log file in offline analysis mode.
- */
-
-void
-pmcstat_process_log(struct pmcstat_args *a)
-{
-	int runstate;
-
-	/*
-	 * If gprof style profiles haven't been asked for, just print the
-	 * log to the current output file.
-	 */
-	if ((a->pa_flags & (FLAG_DO_GPROF_MERGED|FLAG_DO_GPROF)) == 0) {
-		while ((runstate = pmcstat_print_log(a)) == PMCSTAT_RUNNING)
-			;
-		return;
-	}
-
-	/* convert the log to gprof compatible profiles */
-	assert(0);	/* To be implemented */
-}
-
-/*
- * Print log entries available in a configured parser.
- */
-
-int
-pmcstat_print_log(struct pmcstat_args *a)
-{
-	struct pmclog_ev ev;
-
-	while (pmclog_read(a->pa_logparser, &ev) == 0) {
-		assert(ev.pl_state == PMCLOG_OK);
-		switch (ev.pl_type) {
-		case PMCLOG_TYPE_CLOSELOG:
-			PMCSTAT_PRINT_LOG(a,"close",);
-			break;
-		case PMCLOG_TYPE_DROPNOTIFY:
-			PMCSTAT_PRINT_LOG(a,"drop",);
-			break;
-		case PMCLOG_TYPE_INITIALIZE:
-			PMCSTAT_PRINT_LOG(a,"init","0x%x \"%s\"",
-			    ev.pl_u.pl_i.pl_version,
-			    pmc_name_of_cputype(ev.pl_u.pl_i.pl_arch));
-			break;
-		case PMCLOG_TYPE_MAPPINGCHANGE:
-			PMCSTAT_PRINT_LOG(a,"mapping","%s %d %p %p \"%s\"",
-			    ev.pl_u.pl_m.pl_type == PMCLOG_MAPPING_INSERT ?
-			    	"insert" : "delete",
-			    ev.pl_u.pl_m.pl_pid,
-			    (void *) ev.pl_u.pl_m.pl_start,
-			    (void *) ev.pl_u.pl_m.pl_end,
-			    ev.pl_u.pl_m.pl_pathname);
-			break;
-		case PMCLOG_TYPE_PCSAMPLE:
-			PMCSTAT_PRINT_LOG(a,"sample","0x%x %d %p",
-			    ev.pl_u.pl_s.pl_pmcid,
-			    ev.pl_u.pl_s.pl_pid,
-			    (void *) ev.pl_u.pl_s.pl_pc);
-			break;
-		case PMCLOG_TYPE_PMCALLOCATE:
-			PMCSTAT_PRINT_LOG(a,"allocate","0x%x \"%s\" 0x%x",
-			    ev.pl_u.pl_a.pl_pmcid,
-			    ev.pl_u.pl_a.pl_evname,
-			    ev.pl_u.pl_a.pl_flags);
-			break;
-		case PMCLOG_TYPE_PMCATTACH:
-			PMCSTAT_PRINT_LOG(a,"attach","0x%x %d \"%s\"",
-			    ev.pl_u.pl_t.pl_pmcid,
-			    ev.pl_u.pl_t.pl_pid,
-			    ev.pl_u.pl_t.pl_pathname);
-			break;
-		case PMCLOG_TYPE_PMCDETACH:
-			PMCSTAT_PRINT_LOG(a,"detach","0x%x %d",
-			    ev.pl_u.pl_d.pl_pmcid,
-			    ev.pl_u.pl_d.pl_pid);
-			break;
-		case PMCLOG_TYPE_PROCCSW:
-			PMCSTAT_PRINT_LOG(a,"csw","0x%x %d %jd",
-			    ev.pl_u.pl_c.pl_pmcid,
-			    ev.pl_u.pl_c.pl_pid,
-			    ev.pl_u.pl_c.pl_value);
-			break;
-		case PMCLOG_TYPE_PROCEXEC:
-			PMCSTAT_PRINT_LOG(a,"exec","%d \"%s\"",
-			    ev.pl_u.pl_x.pl_pid,
-			    ev.pl_u.pl_x.pl_pathname);
-			break;
-		case PMCLOG_TYPE_PROCEXIT:
-			PMCSTAT_PRINT_LOG(a,"exitvalue","0x%x %d %jd",
-			    ev.pl_u.pl_e.pl_pmcid,
-			    ev.pl_u.pl_e.pl_pid,
-			    ev.pl_u.pl_e.pl_value);
-			break;
-		case PMCLOG_TYPE_PROCFORK:
-			PMCSTAT_PRINT_LOG(a,"fork","%d %d",
-			    ev.pl_u.pl_f.pl_oldpid,
-			    ev.pl_u.pl_f.pl_newpid);
-			break;
-		case PMCLOG_TYPE_USERDATA:
-			PMCSTAT_PRINT_LOG(a,"user","0x%x",
-			    ev.pl_u.pl_u.pl_userdata);
-			break;
-		case PMCLOG_TYPE_SYSEXIT:
-			PMCSTAT_PRINT_LOG(a,"exit","%d",
-			    ev.pl_u.pl_se.pl_pid);
-			break;
-		default:
-			fprintf(a->pa_outputfile, "unknown %d",
-			    ev.pl_type);
-		}
-	}
-
-	if (ev.pl_state == PMCLOG_EOF)
-		return PMCSTAT_FINISHED;
-	else if (ev.pl_state ==  PMCLOG_REQUIRE_DATA)
-		return PMCSTAT_RUNNING;
-
-	err(EX_DATAERR, "ERROR: event parsing failed "
-	    "(record %jd, offset 0x%jx)",
-	    (uintmax_t) ev.pl_count + 1, ev.pl_offset);
-	/*NOTREACHED*/
-}
-
-/*
- * Close a logfile, after first flushing all in-module queued data.
- */
-
-int
-pmcstat_close_log(struct pmcstat_args *a)
-{
-	if (pmc_flush_logfile() < 0 ||
-	    pmc_configure_logfile(-1) < 0)
-		err(EX_OSERR, "ERROR: logging failed");
-	a->pa_flags &= ~FLAG_HAS_LOG_FILE;
-	return a->pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING :
-	    PMCSTAT_FINISHED;
-}
-
 void
 pmcstat_show_usage(void)
 {
@@ -534,7 +326,6 @@ pmcstat_show_usage(void)
 	    "\t -c cpu\t\t set cpu for subsequent system-wide PMCs\n"
 	    "\t -d\t\t (toggle) track descendants\n"
 	    "\t -g\t\t produce gprof(1) compatible profiles\n"
-	    "\t -m\t\t merge gprof(1) profiles for executables\n"
 	    "\t -n rate\t set sampling rate\n"
 	    "\t -o file\t send print output to \"file\"\n"
 	    "\t -p spec\t allocate a process-private counting PMC\n"
@@ -556,7 +347,6 @@ main(int argc, char **argv)
 	int c, current_cpu, current_sampling_count;
 	int do_print, do_descendants;
 	int do_logproccsw, do_logprocexit;
-	int logfd;
 	int pipefd[2];
 	int use_cumulative_counts;
 	pid_t pid;
@@ -567,6 +357,7 @@ main(int argc, char **argv)
 	struct sigaction sa;
 	struct kevent kev;
 	struct winsize ws;
+	struct stat sb;
 
 	current_cpu 		= 0;
 	current_sampling_count  = DEFAULT_SAMPLE_COUNT;
@@ -577,15 +368,16 @@ main(int argc, char **argv)
 	args.pa_required	= 0;
 	args.pa_flags		= 0;
 	args.pa_pid		= (pid_t) -1;
-	args.pa_logfile		= NULL;
-	args.pa_outputdir	= NULL;
-	args.pa_outputfile	= stderr;
+	args.pa_logfd		= -1;
+	args.pa_samplesdir	= ".";
+	args.pa_kernel		= "/boot/kernel/kernel";
+	args.pa_printfile	= stderr;
 	args.pa_interval	= DEFAULT_WAIT_INTERVAL;
 	STAILQ_INIT(&args.pa_head);
 
 	ev = NULL;
 
-	while ((option = getopt(argc, argv, "CD:EO:P:R:S:Wc:dgmn:o:p:s:t:w:"))
+	while ((option = getopt(argc, argv, "CD:EO:P:R:S:Wc:dgk:n:o:p:s:t:w:"))
 	    != -1)
 		switch (option) {
 		case 'C':	/* cumulative values */
@@ -602,29 +394,37 @@ main(int argc, char **argv)
 			args.pa_required |= FLAG_HAS_SYSTEM_PMCS;
 			break;
 
+		case 'D':
+			if (stat(optarg, &sb) < 0)
+				err(EX_OSERR, "ERROR: Cannot stat \"%s\"",
+				    optarg);
+			if (!S_ISDIR(sb.st_mode))
+				errx(EX_USAGE, "ERROR: \"%s\" is not a "
+				    "directory", optarg);
+			args.pa_samplesdir = optarg;
+			args.pa_flags     |= FLAG_HAS_SAMPLESDIR;
+			args.pa_required  |= FLAG_DO_GPROF;
+			break;
+
 		case 'd':	/* toggle descendents */
 			do_descendants = !do_descendants;
 			args.pa_required |= FLAG_HAS_PROCESS_PMCS;
 			break;
 
-		case 'D':
-			args.pa_outputdir = optarg;
-			break;
-
 		case 'g':	/* produce gprof compatible profiles */
 			args.pa_flags |= FLAG_DO_GPROF;
-			args.pa_required |= FLAG_HAS_SAMPLING_PMCS;
 			break;
 
-		case 'm':	/* produce merged profiles */
-			args.pa_flags |= FLAG_DO_GPROF_MERGED;
-			args.pa_required |= FLAG_HAS_SAMPLING_PMCS;
+		case 'k':	/* pathname to the kernel */
+			args.pa_kernel = optarg;
+			args.pa_required |= FLAG_DO_GPROF;
+			args.pa_flags    |= FLAG_HAS_KERNELPATH;
 			break;
 
 		case 'E':	/* log process exit */
 			do_logprocexit = !do_logprocexit;
 			args.pa_required |= (FLAG_HAS_PROCESS_PMCS |
-			    FLAG_HAS_COUNTING_PMCS | FLAG_HAS_LOG_FILE);
+			    FLAG_HAS_COUNTING_PMCS | FLAG_HAS_OUTPUT_LOGFILE);
 			break;
 
 		case 'p':	/* process virtual counting PMC */
@@ -643,13 +443,14 @@ main(int argc, char **argv)
 
 			if (option == 'P' || option == 'p') {
 				args.pa_flags |= FLAG_HAS_PROCESS_PMCS;
-				args.pa_required |= (FLAG_HAS_PROCESS |
+				args.pa_required |= (FLAG_HAS_COMMANDLINE |
 				    FLAG_HAS_PID);
 			}
 
 			if (option == 'P' || option == 'S') {
 				args.pa_flags |= FLAG_HAS_SAMPLING_PMCS;
-				args.pa_required |= FLAG_HAS_LOG_FILE;
+				args.pa_required |= (FLAG_HAS_PIPE |
+				    FLAG_HAS_OUTPUT_LOGFILE);
 			}
 
 			if (option == 'p' || option == 's')
@@ -693,16 +494,6 @@ main(int argc, char **argv)
 
 			break;
 
-		case 'R':	/* read an existing log file */
-			if ((logfd = open(optarg, O_RDONLY, 0)) < 0)
-				err(EX_OSERR, "ERROR: Cannot open \"%s\" for "
-				    "reading", optarg);
-			if ((args.pa_logparser = pmclog_open(logfd))
-			    == NULL)
-				err(EX_OSERR, "ERROR: Cannot create parser");
-			args.pa_flags |= FLAG_PROCESS_LOGFILE;
-			break;
-
 		case 'n':	/* sampling count */
 			current_sampling_count = strtol(optarg, &end, 0);
 			if (*end != '\0' || current_sampling_count <= 0)
@@ -713,22 +504,30 @@ main(int argc, char **argv)
 			break;
 
 		case 'o':	/* outputfile */
-			if (args.pa_outputfile != NULL)
-				(void) fclose(args.pa_outputfile);
-			if ((args.pa_outputfile = fopen(optarg, "w")) == NULL)
+			if (args.pa_printfile != NULL)
+				(void) fclose(args.pa_printfile);
+			if ((args.pa_printfile = fopen(optarg, "w")) == NULL)
 				errx(EX_OSERR, "ERROR: cannot open \"%s\" for "
 				    "writing.", optarg);
-			args.pa_required |= FLAG_HAS_COUNTING_PMCS;
+			args.pa_flags |= FLAG_DO_PRINT;
 			break;
 
 		case 'O':	/* sampling output */
-			if (args.pa_logfile != NULL)
-				errx(EX_OSERR, "ERROR: option -O may only be "
+			if (args.pa_outputpath)
+				errx(EX_USAGE, "ERROR: option -O may only be "
 				    "specified once.");
-			if ((args.pa_logfile = fopen(optarg, "w")) == NULL)
-				errx(EX_OSERR, "ERROR: cannot open \"%s\" for "
-				    "writing.", optarg);
-			args.pa_flags |= FLAG_HAS_LOG_FILE;
+			args.pa_outputpath = optarg;
+			args.pa_flags |= FLAG_HAS_OUTPUT_LOGFILE;
+			break;
+
+		case 'R':	/* read an existing log file */
+			if (args.pa_logparser != NULL)
+				errx(EX_USAGE, "ERROR: option -R may only be "
+				    "specified once.");
+			args.pa_inputpath = optarg;
+			if (args.pa_printfile == stderr)
+				args.pa_printfile = stdout;
+			args.pa_flags |= FLAG_READ_LOGFILE;
 			break;
 
 		case 't':	/* target pid */
@@ -748,13 +547,14 @@ main(int argc, char **argv)
 				errx(EX_USAGE, "ERROR: Illegal wait interval "
 				    "value \"%s\".", optarg);
 			args.pa_flags |= FLAG_HAS_WAIT_INTERVAL;
+			args.pa_required |= FLAG_HAS_COUNTING_PMCS;
 			args.pa_interval = interval;
 			break;
 
 		case 'W':	/* toggle LOG_CSW */
 			do_logproccsw = !do_logproccsw;
 			args.pa_required |= (FLAG_HAS_PROCESS_PMCS |
-			    FLAG_HAS_COUNTING_PMCS | FLAG_HAS_LOG_FILE);
+			    FLAG_HAS_COUNTING_PMCS | FLAG_HAS_OUTPUT_LOGFILE);
 			break;
 
 		case '?':
@@ -767,16 +567,21 @@ main(int argc, char **argv)
 	args.pa_argc = (argc -= optind);
 	args.pa_argv = (argv += optind);
 
-	if (argc)
-		args.pa_flags |= FLAG_HAS_PROCESS;
+	if (argc)	/* command line present */
+		args.pa_flags |= FLAG_HAS_COMMANDLINE;
 
 	/*
 	 * Check invocation syntax.
 	 */
 
-	if (args.pa_flags & FLAG_PROCESS_LOGFILE) {
+	/* disallow -O and -R together */
+	if (args.pa_outputpath && args.pa_inputpath)
+		errx(EX_USAGE, "ERROR: options -O and -R are mutually "
+		    "exclusive.");
+
+	if (args.pa_flags & FLAG_READ_LOGFILE) {
 		errmsg = NULL;
-		if (args.pa_flags & FLAG_HAS_PROCESS)
+		if (args.pa_flags & FLAG_HAS_COMMANDLINE)
 			errmsg = "a command line specification";
 		else if (args.pa_flags & FLAG_HAS_PID)
 			errmsg = "option -t";
@@ -785,10 +590,9 @@ main(int argc, char **argv)
 		if (errmsg)
 			errx(EX_USAGE, "ERROR: option -R may not be used with "
 			    "%s.", errmsg);
-	} else if (STAILQ_EMPTY(&args.pa_head)) {
-		warnx("ERROR: At least one PMC event must be specified");
+	} else if (STAILQ_EMPTY(&args.pa_head))
+		/* All other uses require a PMC spec. */
 		pmcstat_show_usage();
-	}
 
 	/* check for -t pid without a process PMC spec */
 	if ((args.pa_required & FLAG_HAS_PID) &&
@@ -798,20 +602,20 @@ main(int argc, char **argv)
 
 	/* check for process-mode options without a command or -t pid */
 	if ((args.pa_required & FLAG_HAS_PROCESS_PMCS) &&
-	    (args.pa_flags & (FLAG_HAS_PROCESS | FLAG_HAS_PID)) == 0)
-		errx(EX_USAGE, "ERROR: options -d,-E,-p,-P,-W require a "
-		    "command line or target process.");
+	    (args.pa_flags & (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID)) == 0)
+		errx(EX_USAGE, "ERROR: options -d, -E, -p, -P, and -W require "
+		    "a command line or target process.");
 
 	/* check for -p | -P without a target process of some sort */
-	if ((args.pa_required & (FLAG_HAS_PROCESS | FLAG_HAS_PID)) &&
-	    (args.pa_flags & (FLAG_HAS_PROCESS | FLAG_HAS_PID)) == 0)
-		errx(EX_USAGE, "ERROR: the -P or -p options require a "
+	if ((args.pa_required & (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID)) &&
+	    (args.pa_flags & (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID)) == 0)
+		errx(EX_USAGE, "ERROR: options -P and -p require a "
 		    "target process or a command line.");
 
 	/* check for process-mode options without a process-mode PMC */
 	if ((args.pa_required & FLAG_HAS_PROCESS_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_PROCESS_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -d,-E,-W require a "
+		errx(EX_USAGE, "ERROR: options -d, -E, and -W require a "
 		    "process mode PMC to be specified.");
 
 	/* check for -c cpu and not system mode PMCs */
@@ -823,30 +627,70 @@ main(int argc, char **argv)
 	/* check for counting mode options without a counting PMC */
 	if ((args.pa_required & FLAG_HAS_COUNTING_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_COUNTING_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -C,-o,-W require at least one "
-		    "counting mode PMC to be specified.");
+		errx(EX_USAGE, "ERROR: options -C, -o and -W require at least "
+		    "one counting mode PMC to be specified.");
 
 	/* check for sampling mode options without a sampling PMC spec */
 	if ((args.pa_required & FLAG_HAS_SAMPLING_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -n,-O require at least one "
-		    "sampling mode PMC to be specified.");
+		errx(EX_USAGE, "ERROR: options -n and -O require at least "
+		    "one sampling mode PMC to be specified.");
 
-	if ((args.pa_flags & (FLAG_HAS_PID | FLAG_HAS_PROCESS)) ==
-	    (FLAG_HAS_PID | FLAG_HAS_PROCESS))
+	if ((args.pa_flags & (FLAG_HAS_PID | FLAG_HAS_COMMANDLINE)) ==
+	    (FLAG_HAS_PID | FLAG_HAS_COMMANDLINE))
 		errx(EX_USAGE,
 		    "ERROR: option -t cannot be specified with a command "
 		    "line.");
 
+	/* check if -g is being used correctly */
+	if ((args.pa_flags & FLAG_DO_GPROF) &&
+	    !(args.pa_flags & (FLAG_HAS_SAMPLING_PMCS|FLAG_READ_LOGFILE)))
+		errx(EX_USAGE, "ERROR: option -g requires sampling PMCs or -R "
+		    "to be specified.");
+
 	/* check if -O was spuriously specified */
-	if ((args.pa_flags & FLAG_HAS_LOG_FILE) &&
-	    (args.pa_required & FLAG_HAS_LOG_FILE) == 0)
+	if ((args.pa_flags & FLAG_HAS_OUTPUT_LOGFILE) &&
+	    (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0)
 		errx(EX_USAGE,
 		    "ERROR: option -O is used only with options "
-		    "-E,-P,-S and -W.");
+		    "-E, -P, -S and -W.");
+
+	/* -D dir and -k kernel path require -g */
+	if ((args.pa_flags & FLAG_HAS_KERNELPATH) &&
+	    ((args.pa_flags & FLAG_DO_GPROF) == 0))
+	    errx(EX_USAGE, "ERROR: option -k is only used with -g.");
+
+	if ((args.pa_flags & FLAG_HAS_SAMPLESDIR) &&
+	    ((args.pa_flags & FLAG_DO_GPROF) == 0))
+	    errx(EX_USAGE, "ERROR: option -D is only used with -g.");
+
+	/*
+	 * Disallow textual output of sampling PMCs if counting PMCs
+	 * have also been asked for, mostly because the combined output
+	 * is difficult to make sense of.
+	 */
+	if ((args.pa_flags & FLAG_HAS_COUNTING_PMCS) &&
+	    (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) &&
+	    ((args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0))
+		errx(EX_USAGE, "ERROR: option -O is required if counting and "
+		    "sampling PMCs are specified together.");
 
 	/* if we've been asked to process a log file, do that and exit */
-	if (args.pa_flags & FLAG_PROCESS_LOGFILE) {
+	if (args.pa_flags & FLAG_READ_LOGFILE) {
+		/*
+		 * Print the log in textual form if we haven't been
+		 * asked to generate gmon.out files.
+		 */
+		if ((args.pa_flags & FLAG_DO_GPROF) == 0)
+			args.pa_flags |= FLAG_DO_PRINT;
+
+		pmcstat_initialize_logging(&args);
+		if ((args.pa_logfd = pmcstat_open(args.pa_inputpath,
+		    PMCSTAT_OPEN_FOR_READ)) < 0)
+			err(EX_OSERR, "ERROR: Cannot open \"%s\" for "
+			    "reading", args.pa_inputpath);
+		if ((args.pa_logparser = pmclog_open(args.pa_logfd)) == NULL)
+			err(EX_OSERR, "ERROR: Cannot create parser");
 		pmcstat_process_log(&args);
 		exit(EX_OK);
 	}
@@ -864,11 +708,52 @@ main(int argc, char **argv)
 		err(EX_OSERR, "ERROR: Cannot determine the number of PMCs "
 		    "on CPU %d", 0);
 
+	/* Allocate a kqueue */
+	if ((pmcstat_kq = kqueue()) < 0)
+		err(EX_OSERR, "ERROR: Cannot allocate kqueue");
+
+	/*
+	 * Configure the specified log file or setup a default log
+	 * consumer via a pipe.
+	 */
+	if (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) {
+		if (args.pa_outputpath) {
+			if ((args.pa_logfd = pmcstat_open(args.pa_outputpath,
+			    PMCSTAT_OPEN_FOR_WRITE)) < 0)
+				err(EX_OSERR, "ERROR: Cannot open \"%s\" for "
+				    "writing", args.pa_outputpath);
+		} else {
+			/*
+			 * process the log on the fly by reading it in
+			 * through a pipe.
+			 */
+			if (pipe(pipefd) < 0)
+				err(EX_OSERR, "ERROR: pipe(2) failed");
+
+			if (fcntl(pipefd[READPIPEFD], F_SETFL, O_NONBLOCK) < 0)
+				err(EX_OSERR, "ERROR: fcntl(2) failed");
+
+			EV_SET(&kev, pipefd[READPIPEFD], EVFILT_READ, EV_ADD,
+			    0, 0, NULL);
+
+			if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
+				err(EX_OSERR, "ERROR: Cannot register kevent");
+
+			args.pa_logfd = pipefd[WRITEPIPEFD];
+
+			args.pa_flags |= (FLAG_HAS_PIPE | FLAG_DO_PRINT);
+			args.pa_logparser = pmclog_open(pipefd[READPIPEFD]);
+		}
+
+		if (pmc_configure_logfile(args.pa_logfd) < 0)
+			err(EX_OSERR, "ERROR: Cannot configure log file");
+	}
+
 	/*
 	 * Allocate PMCs.
 	 */
 
-	STAILQ_FOREACH(ev, &args.pa_head, ev_next)
+	STAILQ_FOREACH(ev, &args.pa_head, ev_next) {
 	    if (pmc_allocate(ev->ev_spec, ev->ev_mode,
 		    ev->ev_flags, ev->ev_cpu, &ev->ev_pmcid) < 0)
 		    err(EX_OSERR, "ERROR: Cannot allocate %s-mode pmc with "
@@ -876,6 +761,12 @@ main(int argc, char **argv)
 			PMC_IS_SYSTEM_MODE(ev->ev_mode) ? "system" : "process",
 			ev->ev_spec);
 
+	    if (PMC_IS_SAMPLING_MODE(ev->ev_mode) &&
+		pmc_set(ev->ev_pmcid, ev->ev_count) < 0)
+		    err(EX_OSERR, "ERROR: Cannot set sampling count "
+			"for PMC \"%s\"", ev->ev_name);
+	}
+
 	/* compute printout widths */
 	STAILQ_FOREACH(ev, &args.pa_head, ev_next) {
 		int counter_width;
@@ -896,18 +787,14 @@ main(int argc, char **argv)
 		}
 	}
 
-	/* Allocate a kqueue */
-	if ((pmcstat_kq = kqueue()) < 0)
-		err(EX_OSERR, "ERROR: Cannot allocate kqueue");
-
 	/*
 	 * If our output is being set to a terminal, register a handler
 	 * for window size changes.
 	 */
 
-	if (isatty(fileno(args.pa_outputfile))) {
+	if (isatty(fileno(args.pa_printfile))) {
 
-		if (ioctl(fileno(args.pa_outputfile), TIOCGWINSZ, &ws) < 0)
+		if (ioctl(fileno(args.pa_printfile), TIOCGWINSZ, &ws) < 0)
 			err(EX_OSERR, "ERROR: Cannot determine window size");
 
 		pmcstat_displayheight = ws.ws_row - 1;
@@ -937,41 +824,9 @@ main(int argc, char **argv)
 	if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
 		err(EX_OSERR, "ERROR: Cannot register kevent for SIGCHLD");
 
-	/*
-	 * Configure the specified log file or setup a default log
-	 * consumer via a pipe.
-	 */
-	if (args.pa_required & FLAG_HAS_LOG_FILE) {
-
-		if (args.pa_logfile == NULL) {
-			if (pipe(pipefd) < 0)
-				err(EX_OSERR, "ERROR: pipe(2) failed");
-
-			EV_SET(&kev, pipefd[READPIPEFD], EVFILT_READ, EV_ADD,
-			    0, 0, NULL);
-
-			if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
-				err(EX_OSERR, "ERROR: Cannot register kevent");
-
-			logfd = pipefd[WRITEPIPEFD];
-
-			args.pa_flags |= (FLAG_HAS_PIPE | FLAG_HAS_LOG_FILE);
-			args.pa_logparser = pmclog_open(pipefd[READPIPEFD]);
-		} else
-			logfd = fileno(args.pa_logfile);
-
-		if (pmc_configure_logfile(logfd) < 0)
-			err(EX_OSERR, "ERROR: Cannot configure log file");
-
-		STAILQ_FOREACH(ev, &args.pa_head, ev_next)
-		    if (PMC_IS_SAMPLING_MODE(ev->ev_mode) &&
-			pmc_set(ev->ev_pmcid, ev->ev_count) < 0)
-			    err(EX_OSERR, "ERROR: Cannot set sampling count "
-				"for PMC \"%s\"", ev->ev_name);
-	}
-
-	/* setup a timer for any counting mode PMCs */
-	if (args.pa_flags & FLAG_HAS_COUNTING_PMCS) {
+	/* setup a timer if we have counting mode PMCs needing to be printed */
+	if ((args.pa_flags & FLAG_HAS_COUNTING_PMCS) &&
+	    (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
 		EV_SET(&kev, 0, EVFILT_TIMER, EV_ADD, 0,
 		    args.pa_interval * 1000, NULL);
 
@@ -981,16 +836,21 @@ main(int argc, char **argv)
 	}
 
 	/* attach PMCs to the target process, starting it if specified */
-	if (args.pa_flags & FLAG_HAS_PROCESS)
+	if (args.pa_flags & (FLAG_HAS_PID | FLAG_HAS_COMMANDLINE))
 		pmcstat_setup_process(&args);
 
 	/* start the pmcs */
 	pmcstat_start_pmcs(&args);
 
 	/* start the (commandline) process if needed */
-	if (args.pa_flags & FLAG_HAS_PROCESS)
+	if (args.pa_flags & FLAG_HAS_COMMANDLINE)
 		pmcstat_start_process(&args);
 
+	/* initialize logging if printing the configured log */
+	if ((args.pa_flags & FLAG_DO_PRINT) &&
+	    (args.pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE)))
+		pmcstat_initialize_logging(&args);
+
 	/* Handle SIGINT using the kqueue loop */
 	sa.sa_handler = SIG_IGN;
 	sa.sa_flags   = 0;
@@ -1018,7 +878,8 @@ main(int argc, char **argv)
 
 		switch (kev.filter) {
 		case EVFILT_PROC:  /* target has exited */
-			if (args.pa_flags & FLAG_HAS_LOG_FILE)
+			if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE |
+				FLAG_HAS_PIPE))
 				runstate = pmcstat_close_log(&args);
 			break;
 
@@ -1042,20 +903,24 @@ main(int argc, char **argv)
 				 * of its targets, or if logfile
 				 * writes encounter an error.
 				 */
-				if (args.pa_flags & FLAG_HAS_LOG_FILE)
+				if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE |
+				    FLAG_HAS_PIPE)) {
 					runstate = pmcstat_close_log(&args);
+					if (args.pa_flags &
+					    (FLAG_DO_PRINT|FLAG_DO_GPROF))
+						pmcstat_process_log(&args);
+				}
 				do_print = 1; /* print PMCs at exit */
 				runstate = PMCSTAT_FINISHED;
 			} else if (kev.ident == SIGINT) {
-				/* pass the signal on to the child process */
-				if ((args.pa_flags & FLAG_HAS_PROCESS) &&
-				    (args.pa_flags & FLAG_HAS_PID) == 0)
+				/* Kill the child process if we started it */
+				if (args.pa_flags & FLAG_HAS_COMMANDLINE)
 					if (kill(args.pa_pid, SIGINT) != 0)
 						err(EX_OSERR, "ERROR: cannot "
 						    "signal child process");
 				runstate = PMCSTAT_FINISHED;
 			} else if (kev.ident == SIGWINCH) {
-				if (ioctl(fileno(args.pa_outputfile),
+				if (ioctl(fileno(args.pa_printfile),
 					TIOCGWINSZ, &ws) < 0)
 				    err(EX_OSERR, "ERROR: Cannot determine "
 					"window size");
@@ -1071,17 +936,19 @@ main(int argc, char **argv)
 
 		}
 
-		if (do_print) {
+		if (do_print &&
+		    (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
 			pmcstat_print_pmcs(&args);
-			if (runstate == PMCSTAT_FINISHED) /* final newline */
-				(void) fprintf(args.pa_outputfile, "\n");
+			if (runstate == PMCSTAT_FINISHED && /* final newline */
+			    (args.pa_flags & FLAG_DO_PRINT) == 0)
+				(void) fprintf(args.pa_printfile, "\n");
 			do_print = 0;
 		}
 
 	} while (runstate != PMCSTAT_FINISHED);
 
 	/* flush any pending log entries */
-	if (args.pa_flags & FLAG_HAS_LOG_FILE)
+	if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE))
 		pmc_flush_logfile();
 
 	pmcstat_cleanup(&args);
diff --git a/usr.sbin/pmcstat/pmcstat.h b/usr.sbin/pmcstat/pmcstat.h
new file mode 100644
index 0000000..393e3fa
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcstat.h
@@ -0,0 +1,127 @@
+/*-
+ * Copyright (c) 2005, Joseph Koshy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_H_
+#define	_PMCSTAT_H_
+
+#define	FLAG_HAS_PID			0x00000001	/* explicit pid */
+#define	FLAG_HAS_WAIT_INTERVAL		0x00000002	/* -w secs */
+#define	FLAG_HAS_OUTPUT_LOGFILE		0x00000004	/* -O file or pipe */
+#define	FLAG_HAS_COMMANDLINE		0x00000008	/* command */
+#define	FLAG_HAS_SAMPLING_PMCS		0x00000010	/* -S or -P */
+#define	FLAG_HAS_COUNTING_PMCS		0x00000020	/* -s or -p */
+#define	FLAG_HAS_PROCESS_PMCS		0x00000040	/* -P or -p */
+#define	FLAG_HAS_SYSTEM_PMCS		0x00000080	/* -S or -s */
+#define	FLAG_HAS_PIPE			0x00000100	/* implicit log */
+#define	FLAG_READ_LOGFILE		0x00000200	/* -R file */
+#define	FLAG_DO_GPROF			0x00000400	/* -g */
+#define	FLAG_HAS_SAMPLESDIR		0x00000800	/* -D dir */
+#define	FLAG_HAS_KERNELPATH		0x00001000	/* -k kernel */
+#define	FLAG_DO_PRINT			0x00002000	/* -o */
+
+#define	DEFAULT_SAMPLE_COUNT		65536
+#define	DEFAULT_WAIT_INTERVAL		5.0
+#define	DEFAULT_DISPLAY_HEIGHT		23
+#define	DEFAULT_BUFFER_SIZE		4096
+
+#define	PRINT_HEADER_PREFIX		"# "
+#define	READPIPEFD			0
+#define	WRITEPIPEFD			1
+#define	NPIPEFD				2
+
+#define	PMCSTAT_OPEN_FOR_READ		0
+#define	PMCSTAT_OPEN_FOR_WRITE		1
+#define	PMCSTAT_DEFAULT_NW_HOST		"localhost"
+#define	PMCSTAT_DEFAULT_NW_PORT		"9000"
+#define	PMCSTAT_NHASH			256
+#define	PMCSTAT_HASH_MASK		0xFF
+
+#define	PMCSTAT_LDD_COMMAND		"/usr/bin/ldd"
+
+#define	PMCSTAT_PRINT_ENTRY(A,T,...) do {				\
+		fprintf((A)->pa_printfile, "%-8s", T);			\
+		fprintf((A)->pa_printfile, " "  __VA_ARGS__);		\
+		fprintf((A)->pa_printfile, "\n");			\
+	} while (0)
+
+enum pmcstat_state {
+	PMCSTAT_FINISHED = 0,
+	PMCSTAT_EXITING  = 1,
+	PMCSTAT_RUNNING  = 2
+};
+
+struct pmcstat_ev {
+	STAILQ_ENTRY(pmcstat_ev) ev_next;
+	char	       *ev_spec;  /* event specification */
+	char	       *ev_name;  /* (derived) event name */
+	enum pmc_mode	ev_mode;  /* desired mode */
+	int		ev_count; /* associated count if in sampling mode */
+	int		ev_cpu;	  /* specific cpu if requested */
+	int		ev_flags; /* PMC_F_* */
+	int		ev_cumulative;  /* show cumulative counts */
+	int		ev_fieldwidth;  /* print width */
+	int		ev_fieldskip;   /* #leading spaces */
+	pmc_value_t	ev_saved; /* saved value for incremental counts */
+	pmc_id_t	ev_pmcid; /* allocated ID */
+};
+
+struct pmcstat_args {
+	int	pa_flags;		/* argument flags */
+	int	pa_required;		/* required features */
+	pid_t	pa_pid;			/* attached to pid */
+	FILE	*pa_printfile;		/* where to send printed output */
+	int	pa_logfd;		/* output log file */
+	char	*pa_inputpath;		/* path to input log */
+	char	*pa_outputpath;		/* path to output log */
+	void	*pa_logparser;		/* log file parser */
+	const char	*pa_kernel;	/* pathname of the kernel */
+	const char	*pa_samplesdir;	/* directory for profile files */
+	double	pa_interval;		/* printing interval in seconds */
+	int	pa_argc;
+	char	**pa_argv;
+	STAILQ_HEAD(, pmcstat_ev) pa_head;
+} args;
+
+/* Function prototypes */
+void	pmcstat_cleanup(struct pmcstat_args *_a);
+int	pmcstat_close_log(struct pmcstat_args *_a);
+void	pmcstat_initialize_logging(struct pmcstat_args *_a);
+int	pmcstat_open(const char *_p, int _mode);
+void	pmcstat_print_counters(struct pmcstat_args *_a);
+void	pmcstat_print_headers(struct pmcstat_args *_a);
+void	pmcstat_print_pmcs(struct pmcstat_args *_a);
+void	pmcstat_setup_process(struct pmcstat_args *_a);
+void	pmcstat_show_usage(void);
+void	pmcstat_shutdown_logging(void);
+void	pmcstat_start_pmcs(struct pmcstat_args *_a);
+void	pmcstat_start_process(struct pmcstat_args *_a);
+void	pmcstat_process_log(struct pmcstat_args *_a);
+int	pmcstat_print_log(struct pmcstat_args *_a);
+int	pmcstat_convert_log(struct pmcstat_args *_a);
+
+#endif	/* _PMCSTAT_H_ */
diff --git a/usr.sbin/pmcstat/pmcstat_log.c b/usr.sbin/pmcstat/pmcstat_log.c
new file mode 100644
index 0000000..bf0231d
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcstat_log.c
@@ -0,0 +1,1255 @@
+/*-
+ * Copyright (c) 2005, Joseph Koshy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Transform a hwpmc(4) log into human readable form and into gprof(1)
+ * compatible profiles.
+ */
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/gmon.h>
+#include <sys/imgact_aout.h>
+#include <sys/imgact_elf.h>
+#include <sys/mman.h>
+#include <sys/pmc.h>
+#include <sys/queue.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#include <netinet/in.h>
+
+#include <assert.h>
+#include <err.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <limits.h>
+#include <pmc.h>
+#include <pmclog.h>
+#include <sysexits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "pmcstat.h"
+
+#define	min(A,B)		((A) < (B) ? (A) : (B))
+#define	max(A,B)		((A) > (B) ? (A) : (B))
+
+/*
+ * A simple implementation to intern strings.  Each interned string is
+ * assigned a unique address, so that subsequent string compares can
+ * be done by a simple pointer comparision.
+ */
+
+struct pmcstat_string {
+	LIST_ENTRY(pmcstat_string)	ps_next;	/* hash link */
+	int		ps_len;
+	int		ps_hash;
+	const char	*ps_string;
+};
+
+static LIST_HEAD(,pmcstat_string)	pmcstat_string_hash[PMCSTAT_NHASH];
+
+/*
+ * 'pmcstat_pmcs' is a mapping for PMC ids to their human-readable
+ * names.
+ */
+
+struct pmcstat_pmcrecord {
+	LIST_ENTRY(pmcstat_pmcrecord)	pr_next;
+	pmc_id_t	pr_pmcid;
+	const char 	*pr_pmcname;
+};
+
+static LIST_HEAD(,pmcstat_pmcrecord)	pmcstat_pmcs =
+	LIST_HEAD_INITIALIZER(&pmcstat_pmcs);
+
+struct pmcstat_gmonfile {
+	LIST_ENTRY(pmcstat_gmonfile)	pgf_next; /* list of entries */
+	pmc_id_t	pgf_pmcid;	/* id of the associated pmc */
+	size_t		pgf_nsamples;	/* number of samples in this gmon.out */
+	const char	*pgf_name;	/* name of gmon.out file */
+	size_t		pgf_ndatabytes;	/* number of bytes mapped */
+	void		*pgf_gmondata;	/* pointer to mmap'ed data */
+};
+
+static TAILQ_HEAD(,pmcstat_gmonfile)	pmcstat_gmonfiles =
+	TAILQ_HEAD_INITIALIZER(pmcstat_gmonfiles);
+
+#define	GM_TO_BUCKETS(GM)	((uint16_t *) ((char *) (GM) + sizeof(*(GM))))
+
+/*
+ * A 'pmcstat_image' structure describes an executable program on
+ * disk.  'pi_internedpath' is a cookie representing the pathname of
+ * the executable.  'pi_start' and 'pi_end' are the least and greatest
+ * virtual addresses for the text segments in the executable.
+ * 'pi_gmonlist' contains a linked list of gmon.out files associated
+ * with this image.
+ */
+
+enum pmcstat_image_type {
+	PMCSTAT_IMAGE_UNKNOWN = 0,
+	PMCSTAT_IMAGE_ELF,
+	PMCSTAT_IMAGE_AOUT
+};
+
+struct pmcstat_image {
+	LIST_ENTRY(pmcstat_image) pi_next;	/* hash link */
+	TAILQ_ENTRY(pmcstat_image) pi_lru;	/* LRU list */
+	const char	*pi_internedpath;	/* cookie */
+	const char	*pi_samplename;		/* sample file name */
+
+	enum pmcstat_image_type pi_type;	/* executable type */
+	uintfptr_t	pi_start;		/* start address (inclusive) */
+	uintfptr_t	pi_end;			/* end address (exclusive) */
+	int		pi_isdynamic;		/* whether a dynamic object */
+
+	LIST_HEAD(,pmcstat_gmonfile) pi_gmlist;
+};
+
+static LIST_HEAD(,pmcstat_image)	pmcstat_image_hash[PMCSTAT_NHASH];
+static TAILQ_HEAD(,pmcstat_image)	pmcstat_image_lru =
+	TAILQ_HEAD_INITIALIZER(pmcstat_image_lru);
+
+struct pmcstat_pcmap {
+	TAILQ_ENTRY(pmcstat_pcmap) ppm_next;
+	uintfptr_t	ppm_lowpc;
+	uintfptr_t	ppm_highpc;
+	struct pmcstat_image *ppm_image;
+};
+
+/*
+ * A 'pmcstat_process' structure tracks processes.
+ */
+
+struct pmcstat_process {
+	LIST_ENTRY(pmcstat_process) pp_next;	/* hash-next */
+	pid_t			pp_pid;		/* associated pid */
+	int			pp_isactive;	/* whether active */
+	TAILQ_HEAD(,pmcstat_pcmap) pp_map;	/* address range map */
+};
+
+static LIST_HEAD(,pmcstat_process) pmcstat_process_hash[PMCSTAT_NHASH];
+
+static struct pmcstat_process *pmcstat_kernproc; /* kernel 'process' */
+
+/*
+ * Prototypes
+ */
+
+static void	pmcstat_gmon_create_file(struct pmcstat_gmonfile *_pgf,
+    struct pmcstat_image *_image);
+static const char *pmcstat_gmon_create_name(const char *_sd,
+    struct pmcstat_image *_img, pmc_id_t _pmcid);
+static void	pmcstat_gmon_map_file(struct pmcstat_gmonfile *_pgf);
+static void	pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *_pgf);
+
+static struct pmcstat_image *pmcstat_image_from_path(const char *_path);
+static enum pmcstat_image_type pmcstat_image_get_type(const char *_p);
+static void	pmcstat_image_get_elf_params(struct pmcstat_image *_image,
+    uintfptr_t *_minp, uintfptr_t *_maxp, int *_isdyn);
+static void	pmcstat_image_increment_bucket(struct pmcstat_pcmap *_pcm,
+    uintfptr_t _pc, pmc_id_t _pmcid, struct pmcstat_args *_a);
+static void	pmcstat_image_link(struct pmcstat_process *_pp,
+    struct pmcstat_image *_i, uintfptr_t _lpc, uintfptr_t _hpc);
+
+static void	pmcstat_pmcid_add(pmc_id_t _pmcid, const char *_name,
+    struct pmcstat_args *_a);
+static const char *pmcstat_pmcid_to_name(pmc_id_t _pmcid);
+
+static void	pmcstat_process_add_elf_image(struct pmcstat_process *_pp,
+    const char *_path);
+static struct pmcstat_process *pmcstat_process_lookup(pid_t _pid, int _allocate);
+static struct pmcstat_pcmap *pmcstat_process_find_map(
+    struct pmcstat_process *_p, uintfptr_t _pc);
+static void	pmcstat_process_new_image(struct pmcstat_process *_pp,
+    const char *_path);
+
+static int	pmcstat_string_compute_hash(const char *_string);
+static const char *pmcstat_string_intern(const char *_s);
+static struct pmcstat_string *pmcstat_string_lookup(const char *_s);
+
+
+/*
+ * Create a gmon.out file and size it.
+ */
+
+static void
+pmcstat_gmon_create_file(struct pmcstat_gmonfile *pgf,
+    struct pmcstat_image *image)
+{
+	int fd;
+	size_t count;
+	struct gmonhdr gm;
+	char buffer[DEFAULT_BUFFER_SIZE];
+
+	if ((fd = open(pgf->pgf_name, O_RDWR|O_NOFOLLOW|O_CREAT,
+		 S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0)
+		err(EX_OSERR, "ERROR: Cannot open \"%s\"", pgf->pgf_name);
+
+	gm.lpc = image->pi_start;
+	gm.hpc = image->pi_end;
+	gm.ncnt = pgf->pgf_nsamples;
+	gm.version = GMONVERSION;
+	gm.profrate = 0;		/* use ticks */
+	gm.histcounter_type = 0;	/* compatibility with moncontrol() */
+	gm.spare[0] = gm.spare[1] = 0;
+
+	/* Write out the gmon header */
+	if (write(fd, &gm, sizeof(gm)) < 0)
+		goto error;
+
+	/* Zero fill the samples[] array */
+	(void) memset(buffer, 0, sizeof(buffer));
+
+	count = pgf->pgf_ndatabytes - sizeof(struct gmonhdr);
+	while (count > sizeof(buffer)) {
+		if (write(fd, &buffer, sizeof(buffer)) < 0)
+			goto error;
+		count -= sizeof(buffer);
+	}
+
+	if (write(fd, &buffer, count) < 0)
+		goto error;
+
+	(void) close(fd);
+
+	return;
+
+ error:
+	err(EX_OSERR, "ERROR: Cannot write \"%s\"", pgf->pgf_name);
+}
+
+const char *
+pmcstat_gmon_create_name(const char *samplesdir, struct pmcstat_image *image,
+    pmc_id_t pmcid)
+{
+	const char *pmcname;
+	char fullpath[PATH_MAX];
+
+	pmcname = pmcstat_pmcid_to_name(pmcid);
+
+	(void) snprintf(fullpath, sizeof(fullpath),
+	    "%s/%s/%s", samplesdir, pmcname, image->pi_samplename);
+
+	return pmcstat_string_intern(fullpath);
+}
+
+
+static void
+pmcstat_gmon_map_file(struct pmcstat_gmonfile *pgf)
+{
+	int fd;
+
+	/* the gmon.out file must already exist */
+	if ((fd = open(pgf->pgf_name, O_RDWR | O_NOFOLLOW, 0)) < 0)
+		err(EX_OSERR, "ERROR: cannot open \"%s\"",
+		    pgf->pgf_name);
+
+	pgf->pgf_gmondata = mmap(NULL, pgf->pgf_ndatabytes,
+	    PROT_READ|PROT_WRITE, MAP_NOSYNC|MAP_SHARED, fd, 0);
+
+	if (pgf->pgf_gmondata == MAP_FAILED)
+		/* XXX unmap a few files and try again? */
+		err(EX_OSERR, "ERROR: cannot map \"%s\"", pgf->pgf_name);
+
+	(void) close(fd);
+}
+
+/*
+ * Unmap the data mapped from a gmon.out file.
+ */
+
+static void
+pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *pgf)
+{
+	(void) msync(pgf->pgf_gmondata, pgf->pgf_ndatabytes,
+	    MS_SYNC);
+	(void) munmap(pgf->pgf_gmondata, pgf->pgf_ndatabytes);
+	pgf->pgf_gmondata = NULL;
+}
+
+static void
+pmcstat_image_get_elf_params(struct pmcstat_image *image, uintfptr_t *minp,
+    uintfptr_t *maxp, int *is_dynamic)
+{
+	int fd, i;
+	struct stat st;
+	void *mapbase;
+	uintfptr_t minva, maxva;
+	const Elf_Ehdr *h;
+	const Elf_Phdr *ph;
+	const Elf_Shdr *sh;
+	const char *path;
+
+	minva = ~(uintfptr_t) 0;
+	maxva = (uintfptr_t) 0;
+	path = image->pi_internedpath;
+
+	if ((fd = open(path, O_RDONLY, 0)) < 0)
+		err(EX_OSERR, "ERROR: Cannot open \"%s\"", path);
+
+	if (fstat(fd, &st) < 0)
+		err(EX_OSERR, "ERROR: Cannot stat \"%s\"", path);
+
+	if ((mapbase = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) ==
+	    MAP_FAILED)
+		err(EX_OSERR, "ERROR: Cannot mmap \"%s\"", path);
+
+	(void) close(fd);
+
+	h = (const Elf_Ehdr *) mapbase;
+	if (!IS_ELF(*h))
+		err(EX_SOFTWARE, "ERROR: \"%s\" not an ELF file", path);
+
+	sh = (const Elf_Shdr *)((const char *) mapbase + h->e_shoff);
+
+	if (h->e_type == ET_EXEC || h->e_type == ET_DYN) {
+		/*
+		 * Some kind of shared object: find the min,max va for
+		 * its executable sections.
+		 */
+		for (i = 0; i < h->e_shnum; i++)
+			if (sh[i].sh_flags & SHF_EXECINSTR) { /* code */
+				minva = min(minva, sh[i].sh_addr);
+				maxva = max(maxva, sh[i].sh_addr +
+				    sh[i].sh_size);
+			}
+	} else
+		err(EX_DATAERR, "ERROR: Unknown file type for \"%s\"",
+		    image->pi_internedpath);
+
+	*is_dynamic = 0;
+	if (h->e_type == ET_EXEC) {
+		ph = (const Elf_Phdr *)((const char *) mapbase + h->e_phoff);
+		for (i = 0; i < h->e_phnum; i++) {
+			switch (ph[i].p_type) {
+			case PT_DYNAMIC:
+				*is_dynamic = 1;
+				break;
+			}
+		}
+	}
+
+	if (munmap(mapbase, st.st_size) < 0)
+		err(EX_OSERR, "ERROR: Cannot unmap \"%s\"", path);
+
+	*minp = minva;
+	*maxp = maxva;
+
+}
+
+/*
+ * Locate an image descriptor given an interned path.
+ */
+
+static struct pmcstat_image *
+pmcstat_image_from_path(const char *internedpath)
+{
+	int count, hash, nlen;
+	struct pmcstat_image *pi;
+	char *sn;
+	char name[NAME_MAX];
+
+	hash = pmcstat_string_compute_hash(internedpath);
+
+	/* look for an existing entry */
+	LIST_FOREACH(pi, &pmcstat_image_hash[hash], pi_next)
+	    if (pi->pi_internedpath == internedpath) {
+		    /* move descriptor to the head of the lru list */
+		    TAILQ_REMOVE(&pmcstat_image_lru, pi, pi_lru);
+		    TAILQ_INSERT_HEAD(&pmcstat_image_lru, pi, pi_lru);
+		    return pi;
+	    }
+
+	/*
+	 * allocate a new entry and place at the head of the hash and
+	 * LRU lists
+	 */
+	pi = malloc(sizeof(*pi));
+	if (pi == NULL)
+		return NULL;
+
+	pi->pi_type = PMCSTAT_IMAGE_UNKNOWN;
+	pi->pi_internedpath = internedpath;
+	pi->pi_start = ~0;
+	pi->pi_end = 0;
+
+	/* look for a suitable name for the sample files */
+	if ((sn = basename(internedpath)) == NULL)
+		err(EX_OSERR, "ERROR: Cannot process \"%s\"", internedpath);
+
+	nlen = strlen(sn);
+	nlen = min(nlen, (int) sizeof(name) - 6);	/* ".gmon\0" */
+
+	snprintf(name, sizeof(name), "%.*s.gmon",
+	    nlen, sn);
+
+	if (pmcstat_string_lookup(name) == NULL)
+		pi->pi_samplename = pmcstat_string_intern(name);
+	else {
+		nlen = strlen(sn);
+		nlen = min(nlen, (int) sizeof(name)-10); /* "~ddd.gmon\0" */
+		count = 0;
+		do {
+			count++;
+			snprintf(name, sizeof(name), "%.*s~%3.3d",
+			    nlen, sn, count);
+			if (pmcstat_string_lookup(name) == NULL) {
+				pi->pi_samplename = pmcstat_string_intern(name);
+				count = 0;
+			}
+		} while (count > 0);
+	}
+
+	LIST_INIT(&pi->pi_gmlist);
+
+	LIST_INSERT_HEAD(&pmcstat_image_hash[hash], pi, pi_next);
+	TAILQ_INSERT_HEAD(&pmcstat_image_lru, pi, pi_lru);
+
+	return pi;
+}
+
+/*
+ * Given an open file, determine its file type.
+ */
+
+static enum pmcstat_image_type
+pmcstat_image_get_type(const char *path)
+{
+	int fd;
+	Elf_Ehdr *eh;
+	struct exec *ex;
+	ssize_t nbytes;
+	char buffer[DEFAULT_BUFFER_SIZE];
+
+	if ((fd = open(path, O_RDONLY)) < 0)
+		err(EX_OSERR, "ERROR: Cannot open \"%s\"", path);
+
+	if ((nbytes = pread(fd, buffer, sizeof(buffer), 0)) < 0)
+		err(EX_OSERR, "ERROR: Cannot read \"%s\"", path);
+
+	(void) close(fd);
+
+	/* check if its an ELF file */
+	if ((unsigned) nbytes >= sizeof(Elf_Ehdr)) {
+		eh = (Elf_Ehdr *) buffer;
+		if (IS_ELF(*eh))
+			return PMCSTAT_IMAGE_ELF;
+	}
+
+	/* Look for an A.OUT header */
+	if ((unsigned) nbytes >= sizeof(struct exec)) {
+		ex = (struct exec *) buffer;
+		if (!N_BADMAG(*ex))
+			return PMCSTAT_IMAGE_AOUT;
+	}
+
+	return PMCSTAT_IMAGE_UNKNOWN;
+}
+
+/*
+ * Increment the bucket in the gmon.out file corresponding to 'pmcid'
+ * and 'pc'.
+ */
+
+static void
+pmcstat_image_increment_bucket(struct pmcstat_pcmap *map, uintfptr_t pc,
+    pmc_id_t pmcid, struct pmcstat_args *a)
+{
+	struct pmcstat_image *image;
+	struct pmcstat_gmonfile *pgf;
+	uintfptr_t bucket;
+	HISTCOUNTER *hc;
+
+	assert(pc >= map->ppm_lowpc && pc < map->ppm_highpc);
+
+	/*
+	 * Find the gmon file corresponding to 'pmcid', creating it if
+	 * needed.
+	 */
+
+	image = map->ppm_image;
+
+	LIST_FOREACH(pgf, &image->pi_gmlist, pgf_next)
+	    if (pgf->pgf_pmcid == pmcid)
+		    break;
+
+	/* If we don't have a gmon.out file for this PMCid, create one */
+	if (pgf == NULL) {
+		if ((pgf = calloc(1, sizeof(*pgf))) == NULL)
+			err(EX_OSERR, "ERROR:");
+
+		pgf->pgf_gmondata = NULL;	/* mark as unmapped */
+		pgf->pgf_name = pmcstat_gmon_create_name(a->pa_samplesdir,
+		    image, pmcid);
+		pgf->pgf_pmcid = pmcid;
+		pgf->pgf_nsamples = (image->pi_end - image->pi_start) /
+		    FUNCTION_ALIGNMENT;	/* see <machine/profile.h> */
+		pgf->pgf_ndatabytes = sizeof(struct gmonhdr) +
+		    pgf->pgf_nsamples * sizeof(HISTCOUNTER);
+
+		pmcstat_gmon_create_file(pgf, image);
+
+		LIST_INSERT_HEAD(&image->pi_gmlist, pgf, pgf_next);
+	}
+
+	/*
+	 * Map the gmon file in if needed.  It may have been mapped
+	 * out under memory pressure.
+	 */
+	if (pgf->pgf_gmondata == NULL)
+		pmcstat_gmon_map_file(pgf);
+
+	bucket = (pc - map->ppm_lowpc) / FUNCTION_ALIGNMENT;
+
+	assert(bucket < pgf->pgf_nsamples);
+
+	hc = (HISTCOUNTER *) ((char *) pgf->pgf_gmondata +
+	    sizeof(struct gmonhdr));
+	hc[bucket]++;
+
+}
+
+/*
+ * Record the fact that PC values from 'lowpc' to 'highpc' come from
+ * image 'image'.
+ */
+
+static void
+pmcstat_image_link(struct pmcstat_process *pp, struct pmcstat_image *image,
+    uintfptr_t lowpc, uintfptr_t highpc)
+{
+	struct pmcstat_pcmap *pcm, *pcmnew;
+
+	if ((pcmnew = malloc(sizeof(*pcmnew))) == NULL)
+		err(EX_OSERR, "ERROR: ");
+
+	pcmnew->ppm_lowpc  = lowpc;
+	pcmnew->ppm_highpc = highpc;
+	pcmnew->ppm_image  = image;
+
+	TAILQ_FOREACH(pcm, &pp->pp_map, ppm_next)
+	    if (pcm->ppm_lowpc < lowpc)
+		    break;
+
+	if (pcm == NULL)
+		TAILQ_INSERT_TAIL(&pp->pp_map, pcmnew, ppm_next);
+	else
+		TAILQ_INSERT_BEFORE(pcm, pcmnew, ppm_next);
+}
+
+/*
+ * Add a {pmcid,name} mapping.
+ */
+
+static void
+pmcstat_pmcid_add(pmc_id_t pmcid, const char *name, struct pmcstat_args *a)
+{
+	struct pmcstat_pmcrecord *pr;
+	struct stat st;
+	char fullpath[PATH_MAX];
+
+	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
+	    if (pr->pr_pmcid == pmcid) {
+		    pr->pr_pmcname = name;
+		    return;
+	    }
+
+	if ((pr = malloc(sizeof(*pr))) == NULL)
+		err(EX_OSERR, "ERROR: Cannot allocate pmc record");
+
+	pr->pr_pmcid = pmcid;
+	pr->pr_pmcname = name;
+	LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next);
+
+	(void) snprintf(fullpath, sizeof(fullpath), "%s/%s", a->pa_samplesdir,
+	    name);
+
+	/* If the path name exists, it should be a directory */
+	if (stat(fullpath, &st) == 0 && S_ISDIR(st.st_mode))
+		return;
+
+	if (mkdir(fullpath, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) < 0)
+		err(EX_OSERR, "ERROR: Cannot create directory \"%s\"",
+		    fullpath);
+}
+
+/*
+ * Given a pmcid in use, find its human-readable name, or a 
+ */
+
+static const char *
+pmcstat_pmcid_to_name(pmc_id_t pmcid)
+{
+	struct pmcstat_pmcrecord *pr;
+	char fullpath[PATH_MAX];
+
+	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
+	    if (pr->pr_pmcid == pmcid)
+		    return pr->pr_pmcname;
+
+	/* create a default name and add this entry */
+	if ((pr = malloc(sizeof(*pr))) == NULL)
+		err(EX_OSERR, "ERROR: ");
+	pr->pr_pmcid = pmcid;
+
+	(void) snprintf(fullpath, sizeof(fullpath), "%X", (unsigned int) pmcid);
+	pr->pr_pmcname = pmcstat_string_intern(fullpath);
+
+	LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next);
+
+	return pr->pr_pmcname;
+}
+
+/*
+ * Associate an ELF image with a process.  Argument 'path' names the
+ * executable while 'fd' is an already open descriptor to it.
+ */
+
+static void
+pmcstat_process_add_elf_image(struct pmcstat_process *pp, const char *path)
+{
+	int isdynamic;
+	size_t linelen;
+	FILE *rf;
+	char *line;
+	uintfptr_t minva, maxva;
+	uintmax_t libstart;
+	struct pmcstat_image *image;
+	char libpath[PATH_MAX];
+	char command[PATH_MAX + sizeof(PMCSTAT_LDD_COMMAND) + 1];
+
+	minva = ~ (uintfptr_t) 0;
+	maxva = (uintfptr_t) 0;
+	isdynamic = 0;
+
+	if ((image = pmcstat_image_from_path(path)) == NULL)
+		return;
+
+	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) {
+
+		pmcstat_image_get_elf_params(image, &minva, &maxva,
+		    &isdynamic);
+
+		image->pi_type = PMCSTAT_IMAGE_ELF;
+		image->pi_start = minva;
+		image->pi_end = maxva;
+		image->pi_isdynamic = isdynamic;
+	}
+
+	/* create a map entry for the base executable */
+	pmcstat_image_link(pp, image, minva, maxva);
+
+	if (image->pi_isdynamic) {
+
+		(void) snprintf(command, sizeof(command), "%s %s",
+		    PMCSTAT_LDD_COMMAND, path);
+
+		if ((rf = popen(command, "r")) == NULL)
+			err(EX_OSERR, "ERROR: Cannot create pipe");
+
+		(void) fgetln(rf, &linelen);
+
+		while (!feof(rf) && !ferror(rf)) {
+
+			if ((line = fgetln(rf, &linelen)) == NULL)
+				continue;
+			line[linelen-1] = '\0';
+
+			if (sscanf(line, "%s %jx",
+				libpath, &libstart) != 2)
+				continue;
+
+			image = pmcstat_image_from_path(
+				pmcstat_string_intern(libpath));
+			if (image == NULL)
+				err(EX_OSERR, "ERROR: Cannot process "
+				    "\"%s\"", libpath);
+
+			if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) {
+
+				pmcstat_image_get_elf_params(image,
+				    &minva, &maxva, &isdynamic);
+
+				image->pi_type = PMCSTAT_IMAGE_ELF;
+				image->pi_start = minva;
+				image->pi_end = maxva;
+				image->pi_isdynamic = isdynamic;
+			}
+
+			pmcstat_image_link(pp, image, libstart + image->pi_start,
+			    libstart + image->pi_end);
+		}
+
+		(void) pclose(rf);
+
+	}
+}
+
+/*
+ * Find the process descriptor corresponding to a PID.  If 'allocate'
+ * is zero, we return a NULL if a pid descriptor could not be found or
+ * a process descriptor process.  If 'allocate' is non-zero, then we
+ * will attempt to allocate a fresh process descriptor.  Zombie
+ * process descriptors are only removed if a fresh allocation for the
+ * same PID is requested.
+ */
+
+static struct pmcstat_process *
+pmcstat_process_lookup(pid_t pid, int allocate)
+{
+	uint32_t hash;
+	struct pmcstat_pcmap *ppm, *ppmtmp;
+	struct pmcstat_process *pp, *pptmp;
+
+	hash = (uint32_t) pid & PMCSTAT_HASH_MASK;	/* simplicity wins */
+
+	LIST_FOREACH_SAFE(pp, &pmcstat_process_hash[hash], pp_next, pptmp)
+	    if (pp->pp_pid == pid) {
+		    /* Found a descriptor, check and process zombies */
+		    if (allocate && !pp->pp_isactive) {
+			    /* remove maps */
+			    TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next,
+				ppmtmp) {
+				    TAILQ_REMOVE(&pp->pp_map, ppm, ppm_next);
+				    free(ppm);
+			    }
+			    /* remove process entry */
+			    LIST_REMOVE(pp, pp_next);
+			    free(pp);
+			    break;
+		    }
+		    return pp;
+	    }
+
+	if (!allocate)
+		return NULL;
+
+	if ((pp = malloc(sizeof(*pp))) == NULL)
+		err(EX_OSERR, "ERROR: Cannot allocate pid descriptor");
+
+	pp->pp_pid = pid;
+	pp->pp_isactive = 1;
+
+	TAILQ_INIT(&pp->pp_map);
+
+	LIST_INSERT_HEAD(&pmcstat_process_hash[hash], pp, pp_next);
+	return pp;
+}
+
+/*
+ * Find the map entry associated with process 'p' at PC value 'pc'.
+ */
+
+static struct pmcstat_pcmap *
+pmcstat_process_find_map(struct pmcstat_process *p, uintfptr_t pc)
+{
+	struct pmcstat_pcmap *ppm;
+
+	TAILQ_FOREACH(ppm, &p->pp_map, ppm_next)
+	    if (pc >= ppm->ppm_lowpc && pc < ppm->ppm_highpc)
+		    return ppm;
+
+	return NULL;
+}
+
+/*
+ * Associate an image and a process.
+ */
+
+static void
+pmcstat_process_new_image(struct pmcstat_process *pp, const char *path)
+{
+	enum pmcstat_image_type filetype;
+	struct pmcstat_image *image;
+
+	if ((image = pmcstat_image_from_path(path)) == NULL)
+		return;
+
+	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
+		filetype = pmcstat_image_get_type(path);
+	else
+		filetype = image->pi_type;
+
+	switch (filetype) {
+	case PMCSTAT_IMAGE_ELF:
+		pmcstat_process_add_elf_image(pp, path);
+		break;
+
+	case PMCSTAT_IMAGE_AOUT:
+		break;
+
+	default:
+		err(EX_SOFTWARE, "ERROR: Unsupported executable type \"%s\"",
+		    path);
+	}
+}
+
+
+
+/*
+ * Compute a 'hash' value for a string.
+ */
+
+static int
+pmcstat_string_compute_hash(const char *s)
+{
+	int hash;
+
+	for (hash = 0; *s; s++)
+		hash ^= *s;
+
+	return hash & PMCSTAT_HASH_MASK;
+}
+
+/*
+ * Intern a copy of string 's', and return a pointer to it.
+ */
+
+static const char *
+pmcstat_string_intern(const char *s)
+{
+	struct pmcstat_string *ps;
+	int hash, len;
+
+	hash = pmcstat_string_compute_hash(s);
+	len  = strlen(s);
+
+	if ((ps = pmcstat_string_lookup(s)) != NULL)
+		return ps->ps_string;
+
+	if ((ps = malloc(sizeof(*ps))) == NULL)
+		err(EX_OSERR, "ERROR: Could not intern string");
+	ps->ps_len = len;
+	ps->ps_hash = hash;
+	ps->ps_string = strdup(s);
+	LIST_INSERT_HEAD(&pmcstat_string_hash[hash], ps, ps_next);
+	return ps->ps_string;
+}
+
+static struct pmcstat_string *
+pmcstat_string_lookup(const char *s)
+{
+	struct pmcstat_string *ps;
+	int hash, len;
+
+	hash = pmcstat_string_compute_hash(s);
+	len = strlen(s);
+
+	LIST_FOREACH(ps, &pmcstat_string_hash[hash], ps_next)
+	    if (ps->ps_len == len && ps->ps_hash == hash &&
+		strcmp(ps->ps_string, s) == 0)
+		    return ps;
+	return NULL;
+}
+
+/*
+ * Public Interfaces.
+ */
+
+/*
+ * Close a logfile, after first flushing all in-module queued data.
+ */
+
+int
+pmcstat_close_log(struct pmcstat_args *a)
+{
+	if (pmc_flush_logfile() < 0 ||
+	    pmc_configure_logfile(-1) < 0)
+		err(EX_OSERR, "ERROR: logging failed");
+	a->pa_flags &= ~(FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE);
+	return a->pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING :
+	    PMCSTAT_FINISHED;
+}
+
+
+int
+pmcstat_convert_log(struct pmcstat_args *a)
+{
+	uintfptr_t pc;
+	struct pmcstat_process *pp, *ppnew;
+	struct pmcstat_pcmap *ppm, *ppmtmp;
+	struct pmclog_ev ev;
+	const char *image_path;
+
+	while (pmclog_read(a->pa_logparser, &ev) == 0) {
+		assert(ev.pl_state == PMCLOG_OK);
+
+		switch (ev.pl_type) {
+		case PMCLOG_TYPE_MAPPINGCHANGE:
+			/*
+			 * Introduce an address range mapping for a
+			 * process.
+			 */
+			break;
+
+		case PMCLOG_TYPE_PCSAMPLE:
+
+			/*
+			 * We bring in the gmon file for the image
+			 * currently associated with the PMC & pid
+			 * pair and increment the appropriate entry
+			 * bin inside this.
+			 */
+			pc = ev.pl_u.pl_s.pl_pc;
+			pp = pmcstat_process_lookup(ev.pl_u.pl_s.pl_pid, 1);
+			if ((ppm = pmcstat_process_find_map(pp, pc)) == NULL &&
+			    (ppm = pmcstat_process_find_map(pmcstat_kernproc,
+				pc)) == NULL) {
+				printf("!%d unknown %jx\n", pp->pp_pid,
+				    (uintmax_t) pc);
+				break; /* unknown process,offset pair */
+			}
+
+			pmcstat_image_increment_bucket(ppm, pc,
+			    ev.pl_u.pl_s.pl_pmcid, a);
+
+			break;
+
+		case PMCLOG_TYPE_PMCALLOCATE:
+			/*
+			 * Record the association pmc id between this
+			 * PMC and its name.
+			 */
+			pmcstat_pmcid_add(ev.pl_u.pl_a.pl_pmcid,
+			    pmcstat_string_intern(ev.pl_u.pl_a.pl_evname), a);
+			break;
+
+		case PMCLOG_TYPE_PROCEXEC:
+
+			/*
+			 * Change the executable image associated with
+			 * a process.
+			 */
+			pp = pmcstat_process_lookup(ev.pl_u.pl_x.pl_pid, 1);
+
+			/* delete the current process map */
+			TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next, ppmtmp) {
+				TAILQ_REMOVE(&pp->pp_map, ppm, ppm_next);
+				free(ppm);
+			}
+
+			/* locate the descriptor for the new 'base' image */
+			image_path = pmcstat_string_intern(
+				ev.pl_u.pl_x.pl_pathname);
+
+			/* link to the new image */
+			pmcstat_process_new_image(pp, image_path);
+			break;
+
+		case PMCLOG_TYPE_PROCEXIT:
+
+			/*
+			 * Due to the way the log is generated, the
+			 * last few samples corresponding to a process
+			 * may appear in the log after the process
+			 * exit event is recorded.  Thus we keep the
+			 * process' descriptor and associated data
+			 * structures around, but mark the process as
+			 * having exited.
+			 */
+			pp = pmcstat_process_lookup(ev.pl_u.pl_e.pl_pid, 0);
+			if (pp == NULL)
+				break;
+			pp->pp_isactive = 0;	/* make a zombie */
+			break;
+
+		case PMCLOG_TYPE_SYSEXIT:
+			pp = pmcstat_process_lookup(ev.pl_u.pl_se.pl_pid, 0);
+			if (pp == NULL)
+				break;
+			pp->pp_isactive = 0;	/* make a zombie */
+			break;
+
+		case PMCLOG_TYPE_PROCFORK:
+
+			/*
+			 * If we had been tracking 'oldpid', then clone
+			 * its pid descriptor.
+			 */
+			pp = pmcstat_process_lookup(ev.pl_u.pl_f.pl_oldpid, 0);
+			if (pp == NULL)
+				break;
+
+			ppnew =
+			    pmcstat_process_lookup(ev.pl_u.pl_f.pl_newpid, 1);
+
+			/* copy the old process' address maps */
+			TAILQ_FOREACH(ppm, &pp->pp_map, ppm_next)
+			    pmcstat_image_link(ppnew, ppm->ppm_image,
+				ppm->ppm_lowpc, ppm->ppm_highpc);
+			break;
+
+		default:	/* other types of entries are not relevant */
+			break;
+		}
+	}
+
+	if (ev.pl_state == PMCLOG_EOF)
+		return PMCSTAT_FINISHED;
+	else if (ev.pl_state == PMCLOG_REQUIRE_DATA)
+		return PMCSTAT_RUNNING;
+
+	err(EX_DATAERR, "ERROR: event parsing failed (record %jd, "
+	    "offset 0x%jx)", (uintmax_t) ev.pl_count + 1, ev.pl_offset);
+}
+
+
+/*
+ * Open a log file, for reading or writing.
+ *
+ * The function returns the fd of a successfully opened log or -1 in
+ * case of failure.
+ */
+
+int
+pmcstat_open(const char *path, int mode)
+{
+	int fd;
+
+	/*
+	 * If 'path' is "-" then open one of stdin or stdout depending
+	 * on the value of 'mode'.  Otherwise, treat 'path' as a file
+	 * name and open that.
+	 */
+	if (path[0] == '-' && path[1] == '\0')
+		fd = (mode == PMCSTAT_OPEN_FOR_READ) ? 0 : 1;
+	else
+		fd = open(path, mode == PMCSTAT_OPEN_FOR_READ ?
+		    O_RDONLY : (O_WRONLY|O_CREAT|O_TRUNC),
+		    S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
+
+	return fd;
+}
+
+/*
+ * Print log entries as text.
+ */
+
+int
+pmcstat_print_log(struct pmcstat_args *a)
+{
+	struct pmclog_ev ev;
+
+	while (pmclog_read(a->pa_logparser, &ev) == 0) {
+		assert(ev.pl_state == PMCLOG_OK);
+		switch (ev.pl_type) {
+		case PMCLOG_TYPE_CLOSELOG:
+			PMCSTAT_PRINT_ENTRY(a,"closelog",);
+			break;
+		case PMCLOG_TYPE_DROPNOTIFY:
+			PMCSTAT_PRINT_ENTRY(a,"drop",);
+			break;
+		case PMCLOG_TYPE_INITIALIZE:
+			PMCSTAT_PRINT_ENTRY(a,"initlog","0x%x \"%s\"",
+			    ev.pl_u.pl_i.pl_version,
+			    pmc_name_of_cputype(ev.pl_u.pl_i.pl_arch));
+			break;
+		case PMCLOG_TYPE_MAPPINGCHANGE:
+			PMCSTAT_PRINT_ENTRY(a,"mapping","%s %d %p %p \"%s\"",
+			    ev.pl_u.pl_m.pl_type == PMCLOG_MAPPING_INSERT ?
+			    	"insert" : "delete",
+			    ev.pl_u.pl_m.pl_pid,
+			    (void *) ev.pl_u.pl_m.pl_start,
+			    (void *) ev.pl_u.pl_m.pl_end,
+			    ev.pl_u.pl_m.pl_pathname);
+			break;
+		case PMCLOG_TYPE_PCSAMPLE:
+			PMCSTAT_PRINT_ENTRY(a,"sample","0x%x %d %p %c",
+			    ev.pl_u.pl_s.pl_pmcid,
+			    ev.pl_u.pl_s.pl_pid,
+			    (void *) ev.pl_u.pl_s.pl_pc,
+			    ev.pl_u.pl_s.pl_usermode ? 'u' : 's');
+			break;
+		case PMCLOG_TYPE_PMCALLOCATE:
+			PMCSTAT_PRINT_ENTRY(a,"allocate","0x%x \"%s\" 0x%x",
+			    ev.pl_u.pl_a.pl_pmcid,
+			    ev.pl_u.pl_a.pl_evname,
+			    ev.pl_u.pl_a.pl_flags);
+			break;
+		case PMCLOG_TYPE_PMCATTACH:
+			PMCSTAT_PRINT_ENTRY(a,"attach","0x%x %d \"%s\"",
+			    ev.pl_u.pl_t.pl_pmcid,
+			    ev.pl_u.pl_t.pl_pid,
+			    ev.pl_u.pl_t.pl_pathname);
+			break;
+		case PMCLOG_TYPE_PMCDETACH:
+			PMCSTAT_PRINT_ENTRY(a,"detach","0x%x %d",
+			    ev.pl_u.pl_d.pl_pmcid,
+			    ev.pl_u.pl_d.pl_pid);
+			break;
+		case PMCLOG_TYPE_PROCCSW:
+			PMCSTAT_PRINT_ENTRY(a,"cswval","0x%x %d %jd",
+			    ev.pl_u.pl_c.pl_pmcid,
+			    ev.pl_u.pl_c.pl_pid,
+			    ev.pl_u.pl_c.pl_value);
+			break;
+		case PMCLOG_TYPE_PROCEXEC:
+			PMCSTAT_PRINT_ENTRY(a,"exec","0x%x %d %p \"%s\"",
+			    ev.pl_u.pl_x.pl_pmcid,
+			    ev.pl_u.pl_x.pl_pid,
+			    (void *) ev.pl_u.pl_x.pl_entryaddr,
+			    ev.pl_u.pl_x.pl_pathname);
+			break;
+		case PMCLOG_TYPE_PROCEXIT:
+			PMCSTAT_PRINT_ENTRY(a,"exitval","0x%x %d %jd",
+			    ev.pl_u.pl_e.pl_pmcid,
+			    ev.pl_u.pl_e.pl_pid,
+			    ev.pl_u.pl_e.pl_value);
+			break;
+		case PMCLOG_TYPE_PROCFORK:
+			PMCSTAT_PRINT_ENTRY(a,"fork","%d %d",
+			    ev.pl_u.pl_f.pl_oldpid,
+			    ev.pl_u.pl_f.pl_newpid);
+			break;
+		case PMCLOG_TYPE_USERDATA:
+			PMCSTAT_PRINT_ENTRY(a,"userdata","0x%x",
+			    ev.pl_u.pl_u.pl_userdata);
+			break;
+		case PMCLOG_TYPE_SYSEXIT:
+			PMCSTAT_PRINT_ENTRY(a,"exit","%d",
+			    ev.pl_u.pl_se.pl_pid);
+			break;
+		default:
+			fprintf(a->pa_printfile, "unknown %d",
+			    ev.pl_type);
+		}
+	}
+
+	if (ev.pl_state == PMCLOG_EOF)
+		return PMCSTAT_FINISHED;
+	else if (ev.pl_state ==  PMCLOG_REQUIRE_DATA)
+		return PMCSTAT_RUNNING;
+
+	err(EX_DATAERR, "ERROR: event parsing failed "
+	    "(record %jd, offset 0x%jx)",
+	    (uintmax_t) ev.pl_count + 1, ev.pl_offset);
+	/*NOTREACHED*/
+}
+
+/*
+ * Process a log file in offline analysis mode.
+ */
+
+void
+pmcstat_process_log(struct pmcstat_args *a)
+{
+
+	/*
+	 * If gprof style profiles haven't been asked for, just print the
+	 * log to the current output file.
+	 */
+	if (a->pa_flags & FLAG_DO_PRINT)
+		pmcstat_print_log(a);
+	else
+		/* convert the log to gprof compatible profiles */
+		pmcstat_convert_log(a);
+
+	return;
+}
+
+void
+pmcstat_initialize_logging(struct pmcstat_args *a)
+{
+	int i, isdynamic;
+	const char *kernpath;
+	struct pmcstat_image *img;
+	uintfptr_t minva, maxva;
+
+	/* use a convenient format for 'ldd' output */
+	if (setenv("LD_TRACE_LOADED_OBJECTS_FMT1","%p %x\n",1) != 0)
+		goto error;
+
+	/* Initialize hash tables */
+	for (i = 0; i < PMCSTAT_NHASH; i++) {
+		LIST_INIT(&pmcstat_image_hash[i]);
+		LIST_INIT(&pmcstat_process_hash[i]);
+		LIST_INIT(&pmcstat_string_hash[i]);
+	}
+
+	/* create a fake 'process' entry for the kernel with pid == -1 */
+	if ((pmcstat_kernproc = pmcstat_process_lookup((pid_t) -1, 1)) == NULL)
+		goto error;
+
+	if ((kernpath = pmcstat_string_intern(a->pa_kernel)) == NULL)
+		goto error;
+
+	img = pmcstat_image_from_path(kernpath);
+
+	pmcstat_image_get_elf_params(img, &minva, &maxva, &isdynamic);
+	img->pi_type = PMCSTAT_IMAGE_ELF;
+	img->pi_start = minva;
+	img->pi_end = maxva;
+
+	pmcstat_image_link(pmcstat_kernproc, img, minva, maxva);
+
+	return;
+
+ error:
+	err(EX_OSERR, "ERROR: Cannot initialize logging");
+}
+
+void
+pmcstat_shutdown_logging(void)
+{
+	int i;
+	struct pmcstat_gmonfile *pgf, *pgftmp;
+	struct pmcstat_image *pi, *pitmp;
+	struct pmcstat_process *pp, *pptmp;
+	struct pmcstat_string *ps, *pstmp;
+
+	for (i = 0; i < PMCSTAT_NHASH; i++) {
+		LIST_FOREACH_SAFE(pi, &pmcstat_image_hash[i], pi_next, pitmp) {
+			/* flush gmon.out data to disk */
+			LIST_FOREACH_SAFE(pgf, &pi->pi_gmlist, pgf_next,
+			    pgftmp) {
+			    pmcstat_gmon_unmap_file(pgf);
+			    LIST_REMOVE(pgf, pgf_next);
+			    free(pgf);
+			}
+
+			LIST_REMOVE(pi, pi_next);
+			free(pi);
+		}
+		LIST_FOREACH_SAFE(pp, &pmcstat_process_hash[i], pp_next,
+		    pptmp) {
+			LIST_REMOVE(pp, pp_next);
+			free(pp);
+		}
+		LIST_FOREACH_SAFE(ps, &pmcstat_string_hash[i], ps_next,
+		    pstmp) {
+			LIST_REMOVE(ps, ps_next);
+			free(ps);
+		}
+	}
+}