- Reorganize code in 'plugin' to share log processing.

- Kcachegrind (calltree) support with assembly/source
code mapping and call count estimator (-F).
- Top mode for calltree and callgraph plugin (-T).

MFC after:	1 month

15 files changed, 3670 insertions, 1220 deletions

diff --git a/usr.sbin/pmcstat/Makefile b/usr.sbin/pmcstat/Makefile
index c115c42..a29b016 100644
--- a/usr.sbin/pmcstat/Makefile
+++ b/usr.sbin/pmcstat/Makefile
@@ -6,8 +6,9 @@ PROG=	pmcstat
 MAN=	pmcstat.8
 
 DPADD=	${LIBELF} ${LIBKVM} ${LIBPMC} ${LIBM}
-LDADD=	-lelf -lkvm -lpmc -lm
+LDADD=	-lelf -lkvm -lpmc -lm -lncurses
 
-SRCS=	pmcstat.c pmcstat.h pmcstat_log.c
+SRCS=	pmcstat.c pmcstat.h pmcstat_log.c \
+pmcpl_callgraph.c pmcpl_gprof.c pmcpl_annotate.c pmcpl_calltree.c
 
 .include <bsd.prog.mk>
diff --git a/usr.sbin/pmcstat/pmcpl_annotate.c b/usr.sbin/pmcstat/pmcpl_annotate.c
new file mode 100644
index 0000000..802983c
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_annotate.c
@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Transform a hwpmc(4) log into human readable form, and into
+ * gprof(1) compatible profiles.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/gmon.h>
+#include <sys/imgact_aout.h>
+#include <sys/imgact_elf.h>
+#include <sys/mman.h>
+#include <sys/pmc.h>
+#include <sys/queue.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#include <netinet/in.h>
+
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <gelf.h>
+#include <libgen.h>
+#include <limits.h>
+#include <netdb.h>
+#include <pmc.h>
+#include <pmclog.h>
+#include <sysexits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "pmcstat.h"
+#include "pmcstat_log.h"
+#include "pmcpl_annotate.h"
+
+/*
+ * Record a callchain.
+ */
+
+void
+pmcpl_annotate_process(struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu)
+{
+	struct pmcstat_pcmap *map;
+	struct pmcstat_symbol *sym;
+	uintfptr_t newpc;
+	struct pmcstat_image *image;
+
+	(void) pmcr; (void) nsamples; (void) usermode; (void) cpu;
+
+	map = pmcstat_process_find_map(usermode ? pp : pmcstat_kernproc, cc[0]);
+	if (map == NULL) {
+		/* Unknown offset. */
+		pmcstat_stats.ps_samples_unknown_offset++;
+		return;
+	}
+
+	assert(cc[0] >= map->ppm_lowpc && cc[0] < map->ppm_highpc);
+
+	image = map->ppm_image;
+	newpc = cc[0] - (map->ppm_lowpc +
+		(image->pi_vaddr - image->pi_start));
+	sym = pmcstat_symbol_search(image, newpc);
+	if (sym == NULL)
+		return;
+
+	fprintf(args.pa_graphfile, "%p %s 0x%jx 0x%jx\n",
+		(void *)cc[0],
+		pmcstat_string_unintern(sym->ps_name),
+		(uintmax_t)(sym->ps_start +
+		image->pi_vaddr), (uintmax_t)(sym->ps_end +
+		image->pi_vaddr));
+}
diff --git a/usr.sbin/pmcstat/pmcpl_annotate.h b/usr.sbin/pmcstat/pmcpl_annotate.h
new file mode 100644
index 0000000..482bcd4
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_annotate.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_PL_ANNOTATE_H_
+#define	_PMCSTAT_PL_ANNOTATE_H_
+
+/* Function prototypes */
+void pmcpl_annotate_process(
+    struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu);
+
+#endif	/* _PMCSTAT_PL_ANNOTATE_H_ */
diff --git a/usr.sbin/pmcstat/pmcpl_callgraph.c b/usr.sbin/pmcstat/pmcpl_callgraph.c
new file mode 100644
index 0000000..d6f1a9d
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_callgraph.c
@@ -0,0 +1,682 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Transform a hwpmc(4) log into human readable form, and into
+ * gprof(1) compatible profiles.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/gmon.h>
+#include <sys/imgact_aout.h>
+#include <sys/imgact_elf.h>
+#include <sys/mman.h>
+#include <sys/pmc.h>
+#include <sys/queue.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#include <netinet/in.h>
+
+#include <assert.h>
+#include <curses.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <gelf.h>
+#include <libgen.h>
+#include <limits.h>
+#include <netdb.h>
+#include <pmc.h>
+#include <pmclog.h>
+#include <sysexits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "pmcstat.h"
+#include "pmcstat_log.h"
+#include "pmcstat_top.h"
+#include "pmcpl_callgraph.h"
+
+/* Get the sample value in percent related to nsamples. */
+#define PMCPL_CG_COUNTP(a) \
+	((a)->pcg_count * 100.0 / nsamples)
+
+/*
+ * The toplevel CG nodes (i.e., with rank == 0) are placed in a hash table.
+ */
+
+struct pmcstat_cgnode_hash_list pmcstat_cgnode_hash[PMCSTAT_NHASH];
+int pmcstat_cgnode_hash_count;
+
+static pmcstat_interned_string pmcstat_previous_filename_printed;
+
+static struct pmcstat_cgnode *
+pmcstat_cgnode_allocate(struct pmcstat_image *image, uintfptr_t pc)
+{
+	struct pmcstat_cgnode *cg;
+
+	if ((cg = malloc(sizeof(*cg))) == NULL)
+		err(EX_OSERR, "ERROR: Cannot allocate callgraph node");
+
+	cg->pcg_image = image;
+	cg->pcg_func = pc;
+
+	cg->pcg_count = 0;
+	cg->pcg_nchildren = 0;
+	LIST_INIT(&cg->pcg_children);
+
+	return (cg);
+}
+
+/*
+ * Free a node and its children.
+ */
+static void
+pmcstat_cgnode_free(struct pmcstat_cgnode *cg)
+{
+	struct pmcstat_cgnode *cgc, *cgtmp;
+
+	LIST_FOREACH_SAFE(cgc, &cg->pcg_children, pcg_sibling, cgtmp)
+		pmcstat_cgnode_free(cgc);
+	free(cg);
+}
+
+/*
+ * Look for a callgraph node associated with pmc `pmcid' in the global
+ * hash table that corresponds to the given `pc' value in the process
+ * `pp'.
+ */
+static struct pmcstat_cgnode *
+pmcstat_cgnode_hash_lookup_pc(struct pmcstat_process *pp, pmc_id_t pmcid,
+    uintfptr_t pc, int usermode)
+{
+	struct pmcstat_pcmap *ppm;
+	struct pmcstat_symbol *sym;
+	struct pmcstat_image *image;
+	struct pmcstat_cgnode *cg;
+	struct pmcstat_cgnode_hash *h;
+	uintfptr_t loadaddress;
+	unsigned int i, hash;
+
+	ppm = pmcstat_process_find_map(usermode ? pp : pmcstat_kernproc, pc);
+	if (ppm == NULL)
+		return (NULL);
+
+	image = ppm->ppm_image;
+
+	loadaddress = ppm->ppm_lowpc + image->pi_vaddr - image->pi_start;
+	pc -= loadaddress;	/* Convert to an offset in the image. */
+
+	/*
+	 * Try determine the function at this offset.  If we can't
+	 * find a function round leave the `pc' value alone.
+	 */
+	if ((sym = pmcstat_symbol_search(image, pc)) != NULL)
+		pc = sym->ps_start;
+
+	for (hash = i = 0; i < sizeof(uintfptr_t); i++)
+		hash += (pc >> i) & 0xFF;
+
+	hash &= PMCSTAT_HASH_MASK;
+
+	cg = NULL;
+	LIST_FOREACH(h, &pmcstat_cgnode_hash[hash], pch_next)
+	{
+		if (h->pch_pmcid != pmcid)
+			continue;
+
+		cg = h->pch_cgnode;
+
+		assert(cg != NULL);
+
+		if (cg->pcg_image == image && cg->pcg_func == pc)
+			return (cg);
+	}
+
+	/*
+	 * We haven't seen this (pmcid, pc) tuple yet, so allocate a
+	 * new callgraph node and a new hash table entry for it.
+	 */
+	cg = pmcstat_cgnode_allocate(image, pc);
+	if ((h = malloc(sizeof(*h))) == NULL)
+		err(EX_OSERR, "ERROR: Could not allocate callgraph node");
+
+	h->pch_pmcid = pmcid;
+	h->pch_cgnode = cg;
+	LIST_INSERT_HEAD(&pmcstat_cgnode_hash[hash], h, pch_next);
+
+	pmcstat_cgnode_hash_count++;
+
+	return (cg);
+}
+
+/*
+ * Compare two callgraph nodes for sorting.
+ */
+static int
+pmcstat_cgnode_compare(const void *a, const void *b)
+{
+	const struct pmcstat_cgnode *const *pcg1, *const *pcg2, *cg1, *cg2;
+
+	pcg1 = (const struct pmcstat_cgnode *const *) a;
+	cg1 = *pcg1;
+	pcg2 = (const struct pmcstat_cgnode *const *) b;
+	cg2 = *pcg2;
+
+	/* Sort in reverse order */
+	if (cg1->pcg_count < cg2->pcg_count)
+		return (1);
+	if (cg1->pcg_count > cg2->pcg_count)
+		return (-1);
+	return (0);
+}
+
+/*
+ * Find (allocating if a needed) a callgraph node in the given
+ * parent with the same (image, pcoffset) pair.
+ */
+
+static struct pmcstat_cgnode *
+pmcstat_cgnode_find(struct pmcstat_cgnode *parent, struct pmcstat_image *image,
+    uintfptr_t pcoffset)
+{
+	struct pmcstat_cgnode *child;
+
+	LIST_FOREACH(child, &parent->pcg_children, pcg_sibling) {
+		if (child->pcg_image == image &&
+		    child->pcg_func == pcoffset)
+			return (child);
+	}
+
+	/*
+	 * Allocate a new structure.
+	 */
+
+	child = pmcstat_cgnode_allocate(image, pcoffset);
+
+	/*
+	 * Link it into the parent.
+	 */
+	LIST_INSERT_HEAD(&parent->pcg_children, child, pcg_sibling);
+	parent->pcg_nchildren++;
+
+	return (child);
+}
+
+/*
+ * Print one callgraph node.  The output format is:
+ *
+ * indentation %(parent's samples) #nsamples function@object
+ */
+static void
+pmcstat_cgnode_print(struct pmcstat_cgnode *cg, int depth, uint32_t total)
+{
+	uint32_t n;
+	const char *space;
+	struct pmcstat_symbol *sym;
+	struct pmcstat_cgnode **sortbuffer, **cgn, *pcg;
+
+	space = " ";
+
+	if (depth > 0)
+		(void) fprintf(args.pa_graphfile, "%*s", depth, space);
+
+	if (cg->pcg_count == total)
+		(void) fprintf(args.pa_graphfile, "100.0%% ");
+	else
+		(void) fprintf(args.pa_graphfile, "%05.2f%% ",
+		    100.0 * cg->pcg_count / total);
+
+	n = fprintf(args.pa_graphfile, " [%u] ", cg->pcg_count);
+
+	/* #samples is a 12 character wide field. */
+	if (n < 12)
+		(void) fprintf(args.pa_graphfile, "%*s", 12 - n, space);
+
+	if (depth > 0)
+		(void) fprintf(args.pa_graphfile, "%*s", depth, space);
+
+	sym = pmcstat_symbol_search(cg->pcg_image, cg->pcg_func);
+	if (sym)
+		(void) fprintf(args.pa_graphfile, "%s",
+		    pmcstat_string_unintern(sym->ps_name));
+	else
+		(void) fprintf(args.pa_graphfile, "%p",
+		    (void *) (cg->pcg_image->pi_vaddr + cg->pcg_func));
+
+	if (pmcstat_previous_filename_printed !=
+	    cg->pcg_image->pi_fullpath) {
+		pmcstat_previous_filename_printed = cg->pcg_image->pi_fullpath;
+		(void) fprintf(args.pa_graphfile, " @ %s\n",
+		    pmcstat_string_unintern(
+		    pmcstat_previous_filename_printed));
+	} else
+		(void) fprintf(args.pa_graphfile, "\n");
+
+	if (cg->pcg_nchildren == 0)
+		return;
+
+	if ((sortbuffer = (struct pmcstat_cgnode **)
+		malloc(sizeof(struct pmcstat_cgnode *) *
+		    cg->pcg_nchildren)) == NULL)
+		err(EX_OSERR, "ERROR: Cannot print callgraph");
+	cgn = sortbuffer;
+
+	LIST_FOREACH(pcg, &cg->pcg_children, pcg_sibling)
+	    *cgn++ = pcg;
+
+	assert(cgn - sortbuffer == (int) cg->pcg_nchildren);
+
+	qsort(sortbuffer, cg->pcg_nchildren, sizeof(struct pmcstat_cgnode *),
+	    pmcstat_cgnode_compare);
+
+	for (cgn = sortbuffer, n = 0; n < cg->pcg_nchildren; n++, cgn++)
+		pmcstat_cgnode_print(*cgn, depth+1, cg->pcg_count);
+
+	free(sortbuffer);
+}
+
+/*
+ * Record a callchain.
+ */
+
+void
+pmcpl_cg_process(struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu)
+{
+	uintfptr_t pc, loadaddress;
+	uint32_t n;
+	struct pmcstat_image *image;
+	struct pmcstat_pcmap *ppm;
+	struct pmcstat_symbol *sym;
+	struct pmcstat_cgnode *parent, *child;
+	struct pmcstat_process *km;
+	pmc_id_t pmcid;
+
+	(void) cpu;
+
+	/*
+	 * Find the callgraph node recorded in the global hash table
+	 * for this (pmcid, pc).
+	 */
+
+	pc = cc[0];
+	pmcid = pmcr->pr_pmcid;
+	parent = pmcstat_cgnode_hash_lookup_pc(pp, pmcid, pc, usermode);
+	if (parent == NULL) {
+		pmcstat_stats.ps_callchain_dubious_frames++;
+		return;
+	}
+
+	parent->pcg_count++;
+
+	/*
+	 * For each return address in the call chain record, subject
+	 * to the maximum depth desired.
+	 * - Find the image associated with the sample.  Stop if there
+	 *   there is no valid image at that address.
+	 * - Find the function that overlaps the return address.
+	 * - If found: use the start address of the function.
+	 *   If not found (say an object's symbol table is not present or
+	 *   is incomplete), round down to th gprof bucket granularity.
+	 * - Convert return virtual address to an offset in the image.
+	 * - Look for a child with the same {offset,image} tuple,
+	 *   inserting one if needed.
+	 * - Increment the count of occurrences of the child.
+	 */
+	km = pmcstat_kernproc;
+
+	for (n = 1; n < (uint32_t) args.pa_graphdepth && n < nsamples; n++,
+	    parent = child) {
+		pc = cc[n];
+
+		ppm = pmcstat_process_find_map(usermode ? pp : km, pc);
+		if (ppm == NULL) {
+			/* Detect full frame capture (kernel + user). */
+			if (!usermode) {
+				ppm = pmcstat_process_find_map(pp, pc);
+				if (ppm != NULL)
+					km = pp;
+			}
+		}
+		if (ppm == NULL)
+			return;
+
+		image = ppm->ppm_image;
+		loadaddress = ppm->ppm_lowpc + image->pi_vaddr -
+		    image->pi_start;
+		pc -= loadaddress;
+
+		if ((sym = pmcstat_symbol_search(image, pc)) != NULL)
+			pc = sym->ps_start;
+
+		child = pmcstat_cgnode_find(parent, image, pc);
+		child->pcg_count++;
+	}
+}
+
+/*
+ * Printing a callgraph for a PMC.
+ */
+static void
+pmcstat_callgraph_print_for_pmcid(struct pmcstat_pmcrecord *pmcr)
+{
+	int n, nentries;
+	uint32_t nsamples;
+	pmc_id_t pmcid;
+	struct pmcstat_cgnode **sortbuffer, **cgn;
+	struct pmcstat_cgnode_hash *pch;
+
+	/*
+	 * We pull out all callgraph nodes in the top-level hash table
+	 * with a matching PMC id.  We then sort these based on the
+	 * frequency of occurrence.  Each callgraph node is then
+	 * printed.
+	 */
+
+	nsamples = 0;
+	pmcid = pmcr->pr_pmcid;
+	if ((sortbuffer = (struct pmcstat_cgnode **)
+	    malloc(sizeof(struct pmcstat_cgnode *) *
+	    pmcstat_cgnode_hash_count)) == NULL)
+		err(EX_OSERR, "ERROR: Cannot sort callgraph");
+	cgn = sortbuffer;
+
+	for (n = 0; n < PMCSTAT_NHASH; n++)
+		LIST_FOREACH(pch, &pmcstat_cgnode_hash[n], pch_next)
+		    if (pch->pch_pmcid == pmcid) {
+			    nsamples += pch->pch_cgnode->pcg_count;
+			    *cgn++ = pch->pch_cgnode;
+		    }
+
+	nentries = cgn - sortbuffer;
+	assert(nentries <= pmcstat_cgnode_hash_count);
+
+	if (nentries == 0) {
+		free(sortbuffer);
+		return;
+	}
+
+	qsort(sortbuffer, nentries, sizeof(struct pmcstat_cgnode *),
+	    pmcstat_cgnode_compare);
+
+	(void) fprintf(args.pa_graphfile,
+	    "@ %s [%u samples]\n\n",
+	    pmcstat_string_unintern(pmcr->pr_pmcname),
+	    nsamples);
+
+	for (cgn = sortbuffer, n = 0; n < nentries; n++, cgn++) {
+		pmcstat_previous_filename_printed = NULL;
+		pmcstat_cgnode_print(*cgn, 0, nsamples);
+		(void) fprintf(args.pa_graphfile, "\n");
+	}
+
+	free(sortbuffer);
+}
+
+/*
+ * Print out callgraphs.
+ */
+
+static void
+pmcstat_callgraph_print(void)
+{
+	struct pmcstat_pmcrecord *pmcr;
+
+	LIST_FOREACH(pmcr, &pmcstat_pmcs, pr_next)
+	    pmcstat_callgraph_print_for_pmcid(pmcr);
+}
+
+static void
+pmcstat_cgnode_topprint(struct pmcstat_cgnode *cg,
+    int depth, uint32_t nsamples)
+{
+	int v_attrs, vs_len, ns_len, width, len, n, nchildren;
+	float v;
+	char ns[30], vs[10];
+	struct pmcstat_symbol *sym;
+	struct pmcstat_cgnode **sortbuffer, **cgn, *pcg;
+
+	(void) depth;
+
+	/* Format value. */
+	v = PMCPL_CG_COUNTP(cg);
+	snprintf(vs, sizeof(vs), "%.1f", v);
+	v_attrs = PMCSTAT_ATTRPERCENT(v);
+
+	/* Format name. */
+	sym = pmcstat_symbol_search(cg->pcg_image, cg->pcg_func);
+	if (sym != NULL) {
+		snprintf(ns, sizeof(ns), "%s",
+		    pmcstat_string_unintern(sym->ps_name));
+	} else
+		snprintf(ns, sizeof(ns), "%p",
+		    (void *)cg->pcg_func);
+
+	PMCSTAT_ATTRON(v_attrs);
+	PMCSTAT_PRINTW("%5.5s", vs);
+	PMCSTAT_ATTROFF(v_attrs);
+	PMCSTAT_PRINTW(" %-10.10s %-20.20s",
+	    pmcstat_string_unintern(cg->pcg_image->pi_name),
+	    ns);
+
+	nchildren = cg->pcg_nchildren;
+	if (nchildren == 0) {
+		PMCSTAT_PRINTW("\n");
+		return;
+	}
+
+	width = pmcstat_displaywidth - 40;
+
+	if ((sortbuffer = (struct pmcstat_cgnode **)
+		malloc(sizeof(struct pmcstat_cgnode *) *
+		    nchildren)) == NULL)
+		err(EX_OSERR, "ERROR: Cannot print callgraph");
+	cgn = sortbuffer;
+
+	LIST_FOREACH(pcg, &cg->pcg_children, pcg_sibling)
+	    *cgn++ = pcg;
+
+	assert(cgn - sortbuffer == (int)nchildren);
+
+	qsort(sortbuffer, nchildren, sizeof(struct pmcstat_cgnode *),
+	    pmcstat_cgnode_compare);
+
+	/* Count how many callers. */
+	for (cgn = sortbuffer, n = 0; n < nchildren; n++, cgn++) {
+		pcg = *cgn;
+
+		v = PMCPL_CG_COUNTP(pcg);
+		if (v < pmcstat_threshold)
+			break;
+	}
+	nchildren = n;
+
+	for (cgn = sortbuffer, n = 0; n < nchildren; n++, cgn++) {
+		pcg = *cgn;
+
+		/* Format value. */
+		if (nchildren > 1) {
+			v = PMCPL_CG_COUNTP(pcg);
+			vs_len = snprintf(vs, sizeof(vs), ":%.1f", v);
+			v_attrs = PMCSTAT_ATTRPERCENT(v);
+		} else
+			vs_len = 0;
+
+		/* Format name. */
+		sym = pmcstat_symbol_search(pcg->pcg_image, pcg->pcg_func);
+		if (sym != NULL) {
+			ns_len = snprintf(ns, sizeof(ns), "%s",
+			    pmcstat_string_unintern(sym->ps_name));
+		} else
+			ns_len = snprintf(ns, sizeof(ns), "%p",
+			    (void *)pcg->pcg_func);
+
+		len = ns_len + vs_len + 1;
+		if (width - len < 0) {
+			PMCSTAT_PRINTW("...");
+			break;
+		}
+		width -= len;
+
+		PMCSTAT_PRINTW(" %s", ns);
+		if (nchildren > 1) {
+			PMCSTAT_ATTRON(v_attrs);
+			PMCSTAT_PRINTW("%s", vs);
+			PMCSTAT_ATTROFF(v_attrs);
+		}
+	}
+	PMCSTAT_PRINTW("\n");
+	free(sortbuffer);
+}
+
+/*
+ * Top mode display.
+ */
+
+void
+pmcpl_cg_topdisplay(void)
+{
+	int n, nentries;
+	uint32_t nsamples;
+	struct pmcstat_cgnode **sortbuffer, **cgn;
+	struct pmcstat_cgnode_hash *pch;
+	struct pmcstat_pmcrecord *pmcr;
+
+	pmcr = pmcstat_pmcindex_to_pmcr(pmcstat_pmcinfilter);
+
+	/*
+	 * We pull out all callgraph nodes in the top-level hash table
+	 * with a matching PMC index.  We then sort these based on the
+	 * frequency of occurrence.  Each callgraph node is then
+	 * printed.
+	 */
+
+	nsamples = 0;
+
+	if ((sortbuffer = (struct pmcstat_cgnode **)
+	    malloc(sizeof(struct pmcstat_cgnode *) *
+	    pmcstat_cgnode_hash_count)) == NULL)
+		err(EX_OSERR, "ERROR: Cannot sort callgraph");
+	cgn = sortbuffer;
+
+	for (n = 0; n < PMCSTAT_NHASH; n++)
+		LIST_FOREACH(pch, &pmcstat_cgnode_hash[n], pch_next)
+		    if (pmcr == NULL || pch->pch_pmcid == pmcr->pr_pmcid) {
+			    nsamples += pch->pch_cgnode->pcg_count;
+			    *cgn++ = pch->pch_cgnode;
+		    }
+
+	nentries = cgn - sortbuffer;
+	assert(nentries <= pmcstat_cgnode_hash_count);
+
+	if (nentries == 0) {
+		free(sortbuffer);
+		return;
+	}
+
+	qsort(sortbuffer, nentries, sizeof(struct pmcstat_cgnode *),
+	    pmcstat_cgnode_compare);
+
+	PMCSTAT_PRINTW("%5.5s %-10.10s %-20.20s %s\n",
+	    "%SAMP", "IMAGE", "FUNCTION", "CALLERS");
+
+	nentries = min(pmcstat_displayheight - 2, nentries);
+
+	for (cgn = sortbuffer, n = 0; n < nentries; n++, cgn++) {
+		if (PMCPL_CG_COUNTP(*cgn) < pmcstat_threshold)
+			break;
+		pmcstat_cgnode_topprint(*cgn, 0, nsamples);
+	}
+
+	free(sortbuffer);
+}
+
+/*
+ * Handle top mode keypress.
+ */
+
+int
+pmcpl_cg_topkeypress(int c, WINDOW *w)
+{
+
+	(void) c; (void) w;
+
+	return 0;
+}
+
+int
+pmcpl_cg_init(void)
+{
+	int i;
+
+	pmcstat_cgnode_hash_count = 0;
+	pmcstat_previous_filename_printed = NULL;
+
+	for (i = 0; i < PMCSTAT_NHASH; i++) {
+		LIST_INIT(&pmcstat_cgnode_hash[i]);
+	}
+
+	return (0);
+}
+
+void
+pmcpl_cg_shutdown(FILE *mf)
+{
+	int i;
+	struct pmcstat_cgnode_hash *pch, *pchtmp;
+
+	(void) mf;
+
+	if (args.pa_flags & FLAG_DO_CALLGRAPHS)
+		pmcstat_callgraph_print();
+
+	/*
+	 * Free memory.
+	 */
+	for (i = 0; i < PMCSTAT_NHASH; i++) {
+		LIST_FOREACH_SAFE(pch, &pmcstat_cgnode_hash[i], pch_next,
+		    pchtmp) {
+			pmcstat_cgnode_free(pch->pch_cgnode);
+			LIST_REMOVE(pch, pch_next);
+			free(pch);
+		}
+	}
+}
+
diff --git a/usr.sbin/pmcstat/pmcpl_callgraph.h b/usr.sbin/pmcstat/pmcpl_callgraph.h
new file mode 100644
index 0000000..aaf0e1b
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_callgraph.h
@@ -0,0 +1,67 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_PL_CALLGRAPH_H_
+#define	_PMCSTAT_PL_CALLGRAPH_H_
+
+/*
+ * Each call graph node is tracked by a pmcstat_cgnode struct.
+ */
+
+struct pmcstat_cgnode {
+	struct pmcstat_image	*pcg_image;
+	uintfptr_t		pcg_func;
+	uint32_t		pcg_count;
+	uint32_t		pcg_nchildren;
+	LIST_ENTRY(pmcstat_cgnode) pcg_sibling;
+	LIST_HEAD(,pmcstat_cgnode) pcg_children;
+};
+
+struct pmcstat_cgnode_hash {
+	struct pmcstat_cgnode  *pch_cgnode;
+	pmc_id_t		pch_pmcid;
+	LIST_ENTRY(pmcstat_cgnode_hash) pch_next;
+};
+extern LIST_HEAD(pmcstat_cgnode_hash_list, pmcstat_cgnode_hash) pmcstat_cgnode_hash[PMCSTAT_NHASH];
+extern int pmcstat_cgnode_hash_count;
+
+/* Function prototypes */
+int pmcpl_cg_init(void);
+void pmcpl_cg_shutdown(FILE *mf);
+void pmcpl_cg_process(
+    struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu);
+int pmcpl_cg_topkeypress(int c, WINDOW *w);
+void pmcpl_cg_topdisplay(void);
+void pmcpl_cg_configure(char *opt);
+
+#endif	/* _PMCSTAT_PL_CALLGRAPH_H_ */
diff --git a/usr.sbin/pmcstat/pmcpl_calltree.c b/usr.sbin/pmcstat/pmcpl_calltree.c
new file mode 100644
index 0000000..498092d
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_calltree.c
@@ -0,0 +1,1000 @@
+/*-
+ * Copyright (c) 2009, Fabien Thomas
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Process hwpmc(4) samples as calltree.
+ *
+ * Output file format compatible with Kcachegrind (kdesdk).
+ * Handle top mode with a sorted tree display.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/queue.h>
+
+#include <assert.h>
+#include <curses.h>
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <pmc.h>
+#include <pmclog.h>
+#include <sysexits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sysexits.h>
+
+#include "pmcstat.h"
+#include "pmcstat_log.h"
+#include "pmcstat_top.h"
+#include "pmcpl_calltree.h"
+
+#define PMCPL_CT_GROWSIZE	4
+
+static pmcstat_interned_string pmcpl_ct_prevfn;
+
+static int pmcstat_skiplink = 0;
+
+struct pmcpl_ct_node;
+
+/* Get the sample value for PMC a. */
+#define PMCPL_CT_SAMPLE(a, b) \
+	((a) < (b)->npmcs ? (b)->sb[a] : 0)
+
+/* Get the sample value in percent related to rsamples. */
+#define PMCPL_CT_SAMPLEP(a, b) \
+	(PMCPL_CT_SAMPLE(a, b) * 100.0 / rsamples->sb[a])
+
+struct pmcpl_ct_sample {
+	int		npmcs;		/* Max pmc index available. */
+	unsigned	*sb;		/* Sample buffer for 0..npmcs. */
+};
+
+struct pmcpl_ct_arc {
+	struct pmcpl_ct_sample	pcta_samples;
+	struct pmcpl_ct_sample	pcta_callid;
+	unsigned		pcta_call;
+	struct pmcpl_ct_node	*pcta_child;
+};
+
+struct pmcpl_ct_instr {
+	uintfptr_t		pctf_func;
+	struct pmcpl_ct_sample	pctf_samples;
+};
+
+/*
+ * Each calltree node is tracked by a pmcpl_ct_node struct.
+ */
+struct pmcpl_ct_node {
+#define PMCPL_PCT_TAG	0x00000001	/* Loop detection. */
+	uint32_t		pct_flags;
+	struct pmcstat_image	*pct_image;
+	uintfptr_t		pct_func;
+	struct pmcpl_ct_sample	pct_samples;
+
+	int			pct_narc;
+	int			pct_arc_c;
+	struct pmcpl_ct_arc 	*pct_arc;
+
+	/* TODO: optimize for large number of items. */
+	int			pct_ninstr;
+	int			pct_instr_c;
+	struct pmcpl_ct_instr	*pct_instr;
+};
+
+struct pmcpl_ct_node_hash {
+	struct pmcpl_ct_node  *pch_ctnode;
+	LIST_ENTRY(pmcpl_ct_node_hash) pch_next;
+};
+
+struct pmcpl_ct_sample pmcpl_ct_callid;
+
+#define PMCPL_CT_MAXCOL		PMC_CALLCHAIN_DEPTH_MAX	
+#define PMCPL_CT_MAXLINE	256
+struct pmcpl_ct_node  *pmcpl_ct_topscreen[PMCPL_CT_MAXCOL][PMCPL_CT_MAXLINE];
+
+/*
+ * All nodes indexed by function/image name are placed in a hash table.
+ */
+static LIST_HEAD(,pmcpl_ct_node_hash) pmcpl_ct_node_hash[PMCSTAT_NHASH];
+
+/*
+ * Root node for the graph.
+ */
+static struct pmcpl_ct_node *pmcpl_ct_root;
+
+/*
+ * Prototypes
+ */
+
+/*
+ * Initialize a samples.
+ */
+
+static void
+pmcpl_ct_samples_init(struct pmcpl_ct_sample *samples)
+{
+
+	samples->npmcs = 0;
+	samples->sb = NULL;
+}
+
+/*
+ * Free a samples.
+ */
+
+static void
+pmcpl_ct_samples_free(struct pmcpl_ct_sample *samples)
+{
+
+	samples->npmcs = 0;
+	free(samples->sb);
+	samples->sb = NULL;
+}
+
+/*
+ * Grow a sample block to store pmcstat_npmcs PMCs.
+ */
+
+static void
+pmcpl_ct_samples_grow(struct pmcpl_ct_sample *samples)
+{
+	int npmcs;
+
+	/* Enough storage. */
+	if (pmcstat_npmcs <= samples->npmcs)
+                return;
+
+	npmcs = samples->npmcs +
+	    max(pmcstat_npmcs - samples->npmcs, PMCPL_CT_GROWSIZE);
+	samples->sb = realloc(samples->sb, npmcs * sizeof(unsigned));
+	if (samples->sb == NULL)
+		errx(EX_SOFTWARE, "ERROR: out of memory");
+	bzero((char *)samples->sb + samples->npmcs * sizeof(unsigned),
+	    (npmcs - samples->npmcs) * sizeof(unsigned));
+	samples->npmcs = npmcs;
+}
+
+/*
+ * Compute the sum of all root arcs.
+ */
+
+static void
+pmcpl_ct_samples_root(struct pmcpl_ct_sample *samples)
+{
+	int i, pmcin;
+
+	pmcpl_ct_samples_init(samples);
+	pmcpl_ct_samples_grow(samples);
+
+	for (i = 0; i < pmcpl_ct_root->pct_narc; i++)
+		for (pmcin = 0; pmcin < pmcstat_npmcs; pmcin++)
+			samples->sb[pmcin] += PMCPL_CT_SAMPLE(pmcin,
+			    &pmcpl_ct_root->pct_arc[i].pcta_samples);
+}
+
+/*
+ * Grow the arc table.
+ */
+
+static void
+pmcpl_ct_arc_grow(int cursize, int *maxsize, struct pmcpl_ct_arc **items)
+{
+	int nmaxsize;
+
+	if (cursize < *maxsize)
+		return;
+
+	nmaxsize = *maxsize + max(cursize + 1 - *maxsize, PMCPL_CT_GROWSIZE);
+	*items = realloc(*items, nmaxsize * sizeof(struct pmcpl_ct_arc));
+	if (*items == NULL)
+		errx(EX_SOFTWARE, "ERROR: out of memory");
+	bzero((char *)*items + *maxsize * sizeof(struct pmcpl_ct_arc),
+	    (nmaxsize - *maxsize) * sizeof(struct pmcpl_ct_arc));
+	*maxsize = nmaxsize;
+}
+
+/*
+ * Compare two arc by samples value.
+ */
+static int
+pmcpl_ct_arc_compare(void *thunk, const void *a, const void *b)
+{
+	const struct pmcpl_ct_arc *ct1, *ct2;
+	int pmcin = *(int *)thunk;
+
+	ct1 = (const struct pmcpl_ct_arc *) a;
+	ct2 = (const struct pmcpl_ct_arc *) b;
+
+	/* Sort in reverse order */
+	if (PMCPL_CT_SAMPLE(pmcin, &ct1->pcta_samples) <
+	    PMCPL_CT_SAMPLE(pmcin, &ct2->pcta_samples))
+		return (1);
+	if (PMCPL_CT_SAMPLE(pmcin, &ct1->pcta_samples) >
+	    PMCPL_CT_SAMPLE(pmcin, &ct2->pcta_samples))
+		return (-1);
+	return (0);
+}
+
+/*
+ * Grow the instr table.
+ */
+
+static void
+pmcpl_ct_instr_grow(int cursize, int *maxsize, struct pmcpl_ct_instr **items)
+{
+	int nmaxsize;
+
+	if (cursize < *maxsize)
+		return;
+
+	nmaxsize = *maxsize + max(cursize + 1 - *maxsize, PMCPL_CT_GROWSIZE);
+	*items = realloc(*items, nmaxsize * sizeof(struct pmcpl_ct_instr));
+	if (*items == NULL)
+		errx(EX_SOFTWARE, "ERROR: out of memory");
+	bzero((char *)*items + *maxsize * sizeof(struct pmcpl_ct_instr),
+	    (nmaxsize - *maxsize) * sizeof(struct pmcpl_ct_instr));
+	*maxsize = nmaxsize;
+}
+
+/*
+ * Add a new instruction sample to given node.
+ */
+
+static void
+pmcpl_ct_instr_add(struct pmcpl_ct_node *ct, int pmcin, uintfptr_t pc)
+{
+	int i;
+	struct pmcpl_ct_instr *in;
+
+	for (i = 0; i<ct->pct_ninstr; i++) {
+		if (ct->pct_instr[i].pctf_func == pc) {
+			in = &ct->pct_instr[i];
+			pmcpl_ct_samples_grow(&in->pctf_samples);
+			in->pctf_samples.sb[pmcin]++;
+			return;
+		}
+	}
+
+	pmcpl_ct_instr_grow(ct->pct_ninstr, &ct->pct_instr_c, &ct->pct_instr);
+	in = &ct->pct_instr[ct->pct_ninstr];
+	in->pctf_func = pc;
+	pmcpl_ct_samples_init(&in->pctf_samples);
+	pmcpl_ct_samples_grow(&in->pctf_samples);
+	in->pctf_samples.sb[pmcin] = 1;
+	ct->pct_ninstr++;
+}
+
+/*
+ * Allocate a new node.
+ */
+
+static struct pmcpl_ct_node *
+pmcpl_ct_node_allocate(struct pmcstat_image *image, uintfptr_t pc)
+{
+	struct pmcpl_ct_node *ct;
+
+	if ((ct = malloc(sizeof(*ct))) == NULL)
+		err(EX_OSERR, "ERROR: Cannot allocate callgraph node");
+
+	ct->pct_flags	= 0;
+	ct->pct_image 	= image;
+	ct->pct_func	= pc;
+
+	pmcpl_ct_samples_init(&ct->pct_samples);
+
+	ct->pct_narc	= 0;
+	ct->pct_arc_c	= 0;
+	ct->pct_arc	= NULL;
+
+	ct->pct_ninstr	= 0;
+	ct->pct_instr_c	= 0;
+	ct->pct_instr	= NULL;
+
+	return (ct);
+}
+
+/*
+ * Free a node.
+ */
+
+static void
+pmcpl_ct_node_free(struct pmcpl_ct_node *ct)
+{
+	int i;
+
+	for (i = 0; i < ct->pct_narc; i++) {
+		pmcpl_ct_samples_free(&ct->pct_arc[i].pcta_samples);
+		pmcpl_ct_samples_free(&ct->pct_arc[i].pcta_callid);
+	}
+
+	pmcpl_ct_samples_free(&ct->pct_samples);
+	free(ct->pct_arc);
+	free(ct->pct_instr);
+	free(ct);
+}
+
+/*
+ * Clear the graph tag on each node.
+ */
+static void
+pmcpl_ct_node_cleartag(void)
+{
+	int i;
+	struct pmcpl_ct_node_hash *pch;
+
+	for (i = 0; i < PMCSTAT_NHASH; i++)
+		LIST_FOREACH(pch, &pmcpl_ct_node_hash[i], pch_next)
+			pch->pch_ctnode->pct_flags &= ~PMCPL_PCT_TAG;
+
+	pmcpl_ct_root->pct_flags &= ~PMCPL_PCT_TAG;
+}
+
+/*
+ * Print the callchain line by line with maximum cost at top.
+ */ 
+
+static int
+pmcpl_ct_node_dumptop(int pmcin, struct pmcpl_ct_node *ct,
+    struct pmcpl_ct_sample *rsamples, int x, int *y)
+{
+	int i;
+
+	if (ct->pct_flags & PMCPL_PCT_TAG)
+		return 0;
+
+	ct->pct_flags |= PMCPL_PCT_TAG;
+
+	if (x >= PMCPL_CT_MAXCOL) {
+		pmcpl_ct_topscreen[x][*y] = NULL;
+		return 1;
+	}
+	pmcpl_ct_topscreen[x][*y] = ct;
+
+	/*
+	 * This is a terminal node
+	 */
+	if (ct->pct_narc == 0) {
+		pmcpl_ct_topscreen[x+1][*y] = NULL;
+		if (*y >= PMCPL_CT_MAXLINE ||
+		    *y >= pmcstat_displaywidth)
+			return 1;
+		*y = *y + 1;
+		for (i=0; i < x; i++)
+			pmcpl_ct_topscreen[i][*y] =
+			    pmcpl_ct_topscreen[i][*y - 1];
+		return 0;
+	}
+
+	/*
+	 * Quicksort the arcs.
+	 */
+	qsort_r(ct->pct_arc, ct->pct_narc, sizeof(struct pmcpl_ct_arc),
+	    &pmcin, pmcpl_ct_arc_compare);
+
+	for (i = 0; i < ct->pct_narc; i++) {
+		if (PMCPL_CT_SAMPLEP(pmcin,
+		    &ct->pct_arc[i].pcta_samples) > pmcstat_threshold) {
+			if (pmcpl_ct_node_dumptop(pmcin,
+			        ct->pct_arc[i].pcta_child,
+			        rsamples, x+1, y))
+				return 1;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Format and display given PMC index.
+ */
+
+static void
+pmcpl_ct_node_printtop(struct pmcpl_ct_sample *rsamples, int pmcin, int maxy)
+{
+	int v_attrs, ns_len, vs_len, is_len, width, indentwidth, x, y;
+	float v;
+	char ns[30], vs[10], is[20];
+	struct pmcpl_ct_node *ct;
+	struct pmcstat_symbol *sym;
+	const char *space = " ";
+
+	for (y = 0; y < maxy; y++) {
+		/* Output image. */
+		ct = pmcpl_ct_topscreen[0][y];
+		snprintf(is, sizeof(is), "%-10.10s",
+		    pmcstat_string_unintern(ct->pct_image->pi_name));
+		PMCSTAT_PRINTW("%s ", is);
+		width = indentwidth = 11;
+
+		for (x = 0; pmcpl_ct_topscreen[x][y] !=NULL; x++) {
+
+			ct = pmcpl_ct_topscreen[x][y];
+
+			ns[0] = '\0'; ns_len = 0;
+			vs[0] = '\0'; vs_len = 0;
+			is[0] = '\0'; is_len = 0;
+
+			/* Format value. */
+			v = PMCPL_CT_SAMPLEP(pmcin, &ct->pct_samples);
+			if (v > pmcstat_threshold)
+				vs_len  = snprintf(vs, sizeof(vs), "(%.1f%%)", v);
+			v_attrs = PMCSTAT_ATTRPERCENT(v);
+
+			if (pmcstat_skiplink && v <= pmcstat_threshold) {
+				PMCSTAT_PRINTW(". ");
+				width += 2;
+				continue;
+			}
+			sym = pmcstat_symbol_search(ct->pct_image, ct->pct_func);
+			if (sym != NULL) {
+				ns_len = snprintf(ns, sizeof(ns), "%s",
+				    pmcstat_string_unintern(sym->ps_name));
+			} else
+				ns_len = snprintf(ns, sizeof(ns), "%p",
+				    (void *)ct->pct_func);
+
+			/* Format image. */
+			if (x > 0 && pmcpl_ct_topscreen[x-1][y]->pct_image != ct->pct_image)
+				is_len = snprintf(is, sizeof(is), "@%s",
+				    pmcstat_string_unintern(ct->pct_image->pi_name));
+
+			/* Check for line wrap. */
+			width += ns_len + is_len + vs_len + 1;
+			if (width >= pmcstat_displaywidth) {
+				PMCSTAT_PRINTW("\n%*s", indentwidth, space);
+				width = indentwidth + ns_len + is_len + vs_len;
+			}
+
+			PMCSTAT_ATTRON(v_attrs);
+			PMCSTAT_PRINTW("%s%s%s ", ns, is, vs);
+			PMCSTAT_ATTROFF(v_attrs);
+		}
+		PMCSTAT_PRINTW("\n");
+	}
+}
+
+/*
+ * Output top mode snapshot.
+ */
+
+void
+pmcpl_ct_topdisplay(void)
+{
+	int i, x, y, pmcin;
+	struct pmcpl_ct_sample rsamples;
+
+	pmcpl_ct_samples_root(&rsamples);
+
+	PMCSTAT_PRINTW("%-10.10s %s\n", "IMAGE", "CALLTREE");
+
+	for (pmcin = 0; pmcin < pmcstat_npmcs; pmcin++) {
+		/* Filter PMCs. */
+		if (pmcstat_pmcinfilter != pmcin)
+			continue;
+
+		pmcpl_ct_node_cleartag();
+
+		/* Quicksort the arcs. */
+		qsort_r(pmcpl_ct_root->pct_arc,
+		    pmcpl_ct_root->pct_narc,
+		    sizeof(struct pmcpl_ct_arc),
+		    &pmcin, pmcpl_ct_arc_compare);
+
+		x = y = 0;
+		for (i = 0; i < pmcpl_ct_root->pct_narc; i++) {
+			if (pmcpl_ct_node_dumptop(pmcin,
+			        pmcpl_ct_root->pct_arc[i].pcta_child,
+			        &rsamples, x, &y)) {
+				break;
+			}
+		}
+
+		pmcpl_ct_node_printtop(&rsamples, pmcin, y);
+	}
+	pmcpl_ct_samples_free(&rsamples);
+}
+
+/*
+ * Handle top mode keypress.
+ */
+
+int
+pmcpl_ct_topkeypress(int c, WINDOW *w)
+{
+
+	switch (c) {
+	case 'f':
+		pmcstat_skiplink = !pmcstat_skiplink;
+		wprintw(w, "skip empty link %s", pmcstat_skiplink ? "on" : "off");
+		break;
+	}
+
+	return 0;
+}
+
+/*
+ * Look for a callgraph node associated with pmc `pmcid' in the global
+ * hash table that corresponds to the given `pc' value in the process map
+ * `ppm'.
+ */
+
+static struct pmcpl_ct_node *
+pmcpl_ct_node_hash_lookup_pc(struct pmcpl_ct_node *parent,
+    struct pmcstat_pcmap *ppm, uintfptr_t pc, int pmcin)
+{
+	struct pmcstat_symbol *sym;
+	struct pmcstat_image *image;
+	struct pmcpl_ct_node *ct;
+	struct pmcpl_ct_node_hash *h;
+	struct pmcpl_ct_arc *arc;
+	uintfptr_t loadaddress;
+	int i;
+	unsigned int hash;
+
+	assert(parent != NULL);
+
+	image = ppm->ppm_image;
+
+	loadaddress = ppm->ppm_lowpc + image->pi_vaddr - image->pi_start;
+	pc -= loadaddress;	/* Convert to an offset in the image. */
+
+	/*
+	 * Try determine the function at this offset.  If we can't
+	 * find a function round leave the `pc' value alone.
+	 */
+	if ((sym = pmcstat_symbol_search(image, pc)) != NULL)
+		pc = sym->ps_start;
+
+	for (hash = i = 0; i < (int)sizeof(uintfptr_t); i++)
+		hash += (pc >> i) & 0xFF;
+
+	hash &= PMCSTAT_HASH_MASK;
+
+	ct = NULL;
+	LIST_FOREACH(h, &pmcpl_ct_node_hash[hash], pch_next) {
+		ct = h->pch_ctnode;
+
+		assert(ct != NULL);
+
+		if (ct->pct_image == image && ct->pct_func == pc) {
+			/*
+			 * Find related arc in parent node and
+			 * increment the sample count.
+			 */
+			for (i = 0; i < parent->pct_narc; i++) {
+				if (parent->pct_arc[i].pcta_child == ct) {
+					arc = &parent->pct_arc[i];
+					pmcpl_ct_samples_grow(&arc->pcta_samples);
+					arc->pcta_samples.sb[pmcin]++;
+					/* Estimate call count. */
+					pmcpl_ct_samples_grow(&arc->pcta_callid);
+					if (pmcpl_ct_callid.sb[pmcin] -
+					    arc->pcta_callid.sb[pmcin] > 1)
+						arc->pcta_call++;
+					arc->pcta_callid.sb[pmcin] =
+					    pmcpl_ct_callid.sb[pmcin];
+					return (ct);
+				}
+			}
+
+			/*
+			 * No arc found for us, add ourself to the parent.
+			 */
+			pmcpl_ct_arc_grow(parent->pct_narc,
+			    &parent->pct_arc_c, &parent->pct_arc);
+			arc = &parent->pct_arc[parent->pct_narc];
+			pmcpl_ct_samples_grow(&arc->pcta_samples);
+			arc->pcta_samples.sb[pmcin] = 1;
+			arc->pcta_call = 1;
+			pmcpl_ct_samples_grow(&arc->pcta_callid);
+			arc->pcta_callid.sb[pmcin] = pmcpl_ct_callid.sb[pmcin];
+			arc->pcta_child = ct;
+			parent->pct_narc++;
+			return (ct);
+		}
+	}
+
+	/*
+	 * We haven't seen this (pmcid, pc) tuple yet, so allocate a
+	 * new callgraph node and a new hash table entry for it.
+	 */
+	ct = pmcpl_ct_node_allocate(image, pc);
+	if ((h = malloc(sizeof(*h))) == NULL)
+		err(EX_OSERR, "ERROR: Could not allocate callgraph node");
+
+	h->pch_ctnode = ct;
+	LIST_INSERT_HEAD(&pmcpl_ct_node_hash[hash], h, pch_next);
+
+	pmcpl_ct_arc_grow(parent->pct_narc,
+	    &parent->pct_arc_c, &parent->pct_arc);
+	arc = &parent->pct_arc[parent->pct_narc];
+	pmcpl_ct_samples_grow(&arc->pcta_samples);
+	arc->pcta_samples.sb[pmcin] = 1;
+	arc->pcta_call = 1;
+	pmcpl_ct_samples_grow(&arc->pcta_callid);
+	arc->pcta_callid.sb[pmcin] = pmcpl_ct_callid.sb[pmcin];
+	arc->pcta_child = ct;
+	parent->pct_narc++;
+	return (ct);
+}
+
+/*
+ * Record a callchain.
+ */
+
+void
+pmcpl_ct_process(struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu)
+{
+	int n, pmcin;
+	struct pmcstat_pcmap *ppm[PMC_CALLCHAIN_DEPTH_MAX];
+	struct pmcstat_process *km;
+	struct pmcpl_ct_node *parent, *child;
+
+	(void) cpu;
+
+	assert(nsamples>0 && nsamples<=PMC_CALLCHAIN_DEPTH_MAX);
+
+	/* Get the PMC index. */
+	pmcin = pmcr->pr_pmcin;
+
+	/*
+	 * Validate mapping for the callchain.
+	 * Go from bottom to first invalid entry.
+	 */
+	km = pmcstat_kernproc;
+	for (n = 0; n < (int)nsamples; n++) {
+		ppm[n] = pmcstat_process_find_map(usermode ?
+		    pp : km, cc[n]);
+		if (ppm[n] == NULL) {
+			/* Detect full frame capture (kernel + user). */
+			if (!usermode) {
+				ppm[n] = pmcstat_process_find_map(pp, cc[n]);
+				if (ppm[n] != NULL)
+					km = pp;
+			}
+		}
+		if (ppm[n] == NULL)
+			break;
+	}
+	if (n-- == 0) {
+		pmcstat_stats.ps_callchain_dubious_frames++;
+		return;
+	}
+
+	/* Increase the call generation counter. */
+	pmcpl_ct_samples_grow(&pmcpl_ct_callid);
+	pmcpl_ct_callid.sb[pmcin]++;
+
+	/*
+	 * Iterate remaining addresses.
+	 */
+	for (parent = pmcpl_ct_root, child = NULL; n >= 0; n--) {
+		child = pmcpl_ct_node_hash_lookup_pc(parent, ppm[n], cc[n],
+		    pmcin);
+		if (child == NULL) {
+			pmcstat_stats.ps_callchain_dubious_frames++;
+			continue;
+		}
+		parent = child;
+	}
+
+	/*
+	 * Increment the sample count for this PMC.
+	 */
+	if (child != NULL) {
+		pmcpl_ct_samples_grow(&child->pct_samples);
+		child->pct_samples.sb[pmcin]++;
+
+		/* Update per instruction sample if required. */
+		if (args.pa_ctdumpinstr)
+			pmcpl_ct_instr_add(child, pmcin, cc[0] -
+			    (ppm[0]->ppm_lowpc + ppm[0]->ppm_image->pi_vaddr -
+			     ppm[0]->ppm_image->pi_start));
+	}
+}
+
+/*
+ * Print node self cost.
+ */
+
+static void
+pmcpl_ct_node_printself(struct pmcpl_ct_node *ct)
+{
+	int i, j, line;
+	uintptr_t addr;
+	struct pmcstat_symbol *sym;
+	char sourcefile[PATH_MAX];
+	char funcname[PATH_MAX];
+
+	/*
+	 * Object binary.
+	 */
+#ifdef PMCPL_CT_OPTIMIZEFN
+	if (pmcpl_ct_prevfn != ct->pct_image->pi_fullpath) {
+#endif
+		pmcpl_ct_prevfn = ct->pct_image->pi_fullpath;
+		fprintf(args.pa_graphfile, "ob=%s\n",
+		    pmcstat_string_unintern(pmcpl_ct_prevfn));
+#ifdef PMCPL_CT_OPTIMIZEFN
+	}
+#endif
+
+	/*
+	 * Function name.
+	 */
+	if (pmcstat_image_addr2line(ct->pct_image, ct->pct_func,
+	    sourcefile, sizeof(sourcefile), &line,
+	    funcname, sizeof(funcname))) {
+		fprintf(args.pa_graphfile, "fn=%s\n",
+		    funcname);
+	} else {
+		sym = pmcstat_symbol_search(ct->pct_image, ct->pct_func);
+		if (sym != NULL)
+			fprintf(args.pa_graphfile, "fn=%s\n",
+			    pmcstat_string_unintern(sym->ps_name));
+		else
+			fprintf(args.pa_graphfile, "fn=%p\n",
+			    (void *)(ct->pct_image->pi_vaddr + ct->pct_func));
+	}
+
+	/*
+	 * Self cost.
+	 */
+	if (ct->pct_ninstr > 0) {
+		for (i = 0; i < ct->pct_ninstr; i++) {
+			addr = ct->pct_image->pi_vaddr +
+			    ct->pct_instr[i].pctf_func;
+			line = 0;
+			if (pmcstat_image_addr2line(ct->pct_image, addr,
+			    sourcefile, sizeof(sourcefile), &line,
+			    funcname, sizeof(funcname)))
+				fprintf(args.pa_graphfile, "fl=%s\n", sourcefile);
+			fprintf(args.pa_graphfile, "%p %u", (void *)addr, line);
+			for (j = 0; j<pmcstat_npmcs; j++)
+				fprintf(args.pa_graphfile, " %u",
+				    PMCPL_CT_SAMPLE(j,
+				    &ct->pct_instr[i].pctf_samples));
+			fprintf(args.pa_graphfile, "\n");
+		}
+	} else {
+		addr = ct->pct_image->pi_vaddr + ct->pct_func;
+		line = 0;
+		if (pmcstat_image_addr2line(ct->pct_image, addr,
+		    sourcefile, sizeof(sourcefile), &line,
+		    funcname, sizeof(funcname)))
+			fprintf(args.pa_graphfile, "fl=%s\n", sourcefile);
+		fprintf(args.pa_graphfile, "* *");
+		for (i = 0; i<pmcstat_npmcs ; i++)
+			fprintf(args.pa_graphfile, " %u",
+			    PMCPL_CT_SAMPLE(i, &ct->pct_samples));
+		fprintf(args.pa_graphfile, "\n");
+	}
+}
+
+/*
+ * Print node child cost.
+ */
+
+static void
+pmcpl_ct_node_printchild(struct pmcpl_ct_node *ct)
+{
+	int i, j, line;
+	uintptr_t addr;
+	struct pmcstat_symbol *sym;
+	struct pmcpl_ct_node *child;
+	char sourcefile[PATH_MAX];
+	char funcname[PATH_MAX];
+
+	/*
+	 * Child cost.
+	 * TODO: attach child cost to the real position in the funtion.
+	 * TODO: cfn=<fn> / call <ncall> addr(<fn>) / addr(call <fn>) <arccost>
+	 */
+	for (i=0 ; i<ct->pct_narc; i++) {
+		child = ct->pct_arc[i].pcta_child;
+
+		/* Object binary. */
+#ifdef PMCPL_CT_OPTIMIZEFN
+		if (pmcpl_ct_prevfn != child->pct_image->pi_fullpath) {
+#endif
+			pmcpl_ct_prevfn = child->pct_image->pi_fullpath;
+			fprintf(args.pa_graphfile, "cob=%s\n",
+			    pmcstat_string_unintern(pmcpl_ct_prevfn));
+#if PMCPL_CT_OPTIMIZEFN
+		}
+#endif
+		/* Child function name. */
+		addr = child->pct_image->pi_vaddr + child->pct_func;
+		/* Child function source file. */
+		if (pmcstat_image_addr2line(child->pct_image, addr,
+		    sourcefile, sizeof(sourcefile), &line,
+		    funcname, sizeof(funcname))) {
+			fprintf(args.pa_graphfile, "cfn=%s\n", funcname);
+			fprintf(args.pa_graphfile, "cfl=%s\n", sourcefile);
+		} else {
+			sym = pmcstat_symbol_search(child->pct_image,
+			    child->pct_func);
+			if (sym != NULL)
+				fprintf(args.pa_graphfile, "cfn=%s\n",
+				    pmcstat_string_unintern(sym->ps_name));
+			else
+				fprintf(args.pa_graphfile, "cfn=%p\n", (void *)addr);
+		}
+
+		/* Child function address, line and call count. */
+		fprintf(args.pa_graphfile, "calls=%u %p %u\n",
+		    ct->pct_arc[i].pcta_call, (void *)addr, line);
+
+		if (ct->pct_image != NULL) {
+			/* Call address, line, sample. */
+			addr = ct->pct_image->pi_vaddr + ct->pct_func;
+			line = 0;
+			pmcstat_image_addr2line(ct->pct_image, addr, sourcefile,
+			    sizeof(sourcefile), &line,
+			    funcname, sizeof(funcname));
+			fprintf(args.pa_graphfile, "%p %u", (void *)addr, line);
+		}
+		else
+			fprintf(args.pa_graphfile, "* *");
+		for (j = 0; j<pmcstat_npmcs; j++)
+			fprintf(args.pa_graphfile, " %u",
+			    PMCPL_CT_SAMPLE(j, &ct->pct_arc[i].pcta_samples));
+		fprintf(args.pa_graphfile, "\n");
+	}
+}
+
+/*
+ * Clean the PMC name for Kcachegrind formula
+ */
+
+static void
+pmcpl_ct_fixup_pmcname(char *s)
+{
+	char *p;
+
+	for (p = s; *p; p++)
+		if (!isalnum(*p))
+			*p = '_';
+}
+
+/*
+ * Print a calltree (KCachegrind) for all PMCs.
+ */
+
+static void
+pmcpl_ct_print(void)
+{
+	int n, i;
+	struct pmcpl_ct_node_hash *pch;
+	struct pmcpl_ct_sample rsamples;
+	char name[40];
+
+	pmcpl_ct_samples_root(&rsamples);
+	pmcpl_ct_prevfn = NULL;
+
+	fprintf(args.pa_graphfile,
+		"version: 1\n"
+		"creator: pmcstat\n"
+		"positions: instr line\n"
+		"events:");
+	for (i=0; i<pmcstat_npmcs; i++) {
+		snprintf(name, sizeof(name), "%s_%d",
+		    pmcstat_pmcindex_to_name(i), i);
+		pmcpl_ct_fixup_pmcname(name);
+		fprintf(args.pa_graphfile, " %s", name);
+	}
+	fprintf(args.pa_graphfile, "\nsummary:");
+	for (i=0; i<pmcstat_npmcs ; i++)
+		fprintf(args.pa_graphfile, " %u",
+		    PMCPL_CT_SAMPLE(i, &rsamples));
+	fprintf(args.pa_graphfile, "\n\n");
+
+	/*
+	 * Fake root node
+	 */
+	fprintf(args.pa_graphfile, "ob=FreeBSD\n");
+	fprintf(args.pa_graphfile, "fn=ROOT\n");
+	fprintf(args.pa_graphfile, "* *");
+	for (i = 0; i<pmcstat_npmcs ; i++)
+		fprintf(args.pa_graphfile, " 0");
+	fprintf(args.pa_graphfile, "\n");
+	pmcpl_ct_node_printchild(pmcpl_ct_root);
+
+	for (n = 0; n < PMCSTAT_NHASH; n++)
+		LIST_FOREACH(pch, &pmcpl_ct_node_hash[n], pch_next) {
+			pmcpl_ct_node_printself(pch->pch_ctnode);
+			pmcpl_ct_node_printchild(pch->pch_ctnode);
+	}
+
+	pmcpl_ct_samples_free(&rsamples);
+}
+
+int
+pmcpl_ct_configure(char *opt)
+{
+
+	if (strncmp(opt, "skiplink=", 9) == 0) {
+		pmcstat_skiplink = atoi(opt+9);
+	} else
+		return (0);
+
+	return (1);
+}
+
+int
+pmcpl_ct_init(void)
+{
+	int i;
+
+	pmcpl_ct_prevfn = NULL;
+	pmcpl_ct_root = pmcpl_ct_node_allocate(NULL, 0);
+
+	for (i = 0; i < PMCSTAT_NHASH; i++)
+		LIST_INIT(&pmcpl_ct_node_hash[i]);
+
+	pmcpl_ct_samples_init(&pmcpl_ct_callid);
+
+	return (0);
+}
+
+void
+pmcpl_ct_shutdown(FILE *mf)
+{
+	int i;
+	struct pmcpl_ct_node_hash *pch, *pchtmp;
+
+	(void) mf;
+
+	if (args.pa_flags & FLAG_DO_CALLGRAPHS)
+		pmcpl_ct_print();
+
+	/*
+	 * Free memory.
+	 */
+
+	for (i = 0; i < PMCSTAT_NHASH; i++) {
+		LIST_FOREACH_SAFE(pch, &pmcpl_ct_node_hash[i], pch_next,
+		    pchtmp) {
+			pmcpl_ct_node_free(pch->pch_ctnode);
+			free(pch);
+		}
+	}
+
+	pmcpl_ct_node_free(pmcpl_ct_root);
+	pmcpl_ct_root = NULL;
+
+	pmcpl_ct_samples_free(&pmcpl_ct_callid);
+}
+
diff --git a/usr.sbin/pmcstat/pmcpl_calltree.h b/usr.sbin/pmcstat/pmcpl_calltree.h
new file mode 100644
index 0000000..f54957f
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_calltree.h
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2009, Fabien Thomas
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_PL_CALLTREE_H_
+#define	_PMCSTAT_PL_CALLTREE_H_
+
+/* Function prototypes */
+int pmcpl_ct_init(void);
+void pmcpl_ct_shutdown(FILE *mf);
+void pmcpl_ct_process(
+    struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu);
+int pmcpl_ct_topkeypress(int c, WINDOW *w);
+void pmcpl_ct_topdisplay(void);
+int pmcpl_ct_configure(char *opt);
+
+#endif	/* _PMCSTAT_PL_CALLTREE_H_ */
diff --git a/usr.sbin/pmcstat/pmcpl_gprof.c b/usr.sbin/pmcstat/pmcpl_gprof.c
new file mode 100644
index 0000000..9327eb9
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_gprof.c
@@ -0,0 +1,533 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * Copyright (c) 2009, Fabien Thomas
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Transform a hwpmc(4) log into human readable form, and into
+ * gprof(1) compatible profiles.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/gmon.h>
+#include <sys/imgact_aout.h>
+#include <sys/imgact_elf.h>
+#include <sys/mman.h>
+#include <sys/pmc.h>
+#include <sys/queue.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#include <netinet/in.h>
+
+#include <assert.h>
+#include <curses.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <gelf.h>
+#include <libgen.h>
+#include <limits.h>
+#include <netdb.h>
+#include <pmc.h>
+#include <pmclog.h>
+#include <sysexits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "pmcstat.h"
+#include "pmcstat_log.h"
+#include "pmcpl_callgraph.h"
+#include "pmcpl_gprof.h"
+
+/*
+ * struct pmcstat_gmonfile tracks a given 'gmon.out' file.  These
+ * files are mmap()'ed in as needed.
+ */
+
+struct pmcstat_gmonfile {
+	LIST_ENTRY(pmcstat_gmonfile)	pgf_next; /* list of entries */
+	int		pgf_overflow;	/* whether a count overflowed */
+	pmc_id_t	pgf_pmcid;	/* id of the associated pmc */
+	size_t		pgf_nbuckets;	/* #buckets in this gmon.out */
+	unsigned int	pgf_nsamples;	/* #samples in this gmon.out */
+	pmcstat_interned_string pgf_name;	/* pathname of gmon.out file */
+	size_t		pgf_ndatabytes;	/* number of bytes mapped */
+	void		*pgf_gmondata;	/* pointer to mmap'ed data */
+	FILE		*pgf_file;	/* used when writing gmon arcs */
+};
+
+/*
+ * Prototypes
+ */
+
+static void	pmcstat_gmon_create_file(struct pmcstat_gmonfile *_pgf,
+    struct pmcstat_image *_image);
+static pmcstat_interned_string pmcstat_gmon_create_name(const char *_sd,
+    struct pmcstat_image *_img, pmc_id_t _pmcid);
+static void	pmcstat_gmon_map_file(struct pmcstat_gmonfile *_pgf);
+static void	pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *_pgf);
+
+static struct pmcstat_gmonfile *pmcstat_image_find_gmonfile(struct
+    pmcstat_image *_i, pmc_id_t _id);
+
+/*
+ * Create a gmon.out file and size it.
+ */
+
+static void
+pmcstat_gmon_create_file(struct pmcstat_gmonfile *pgf,
+    struct pmcstat_image *image)
+{
+	int fd;
+	size_t count;
+	struct gmonhdr gm;
+	const char *pathname;
+	char buffer[DEFAULT_BUFFER_SIZE];
+
+	pathname = pmcstat_string_unintern(pgf->pgf_name);
+	if ((fd = open(pathname, O_RDWR|O_NOFOLLOW|O_CREAT,
+		 S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0)
+		err(EX_OSERR, "ERROR: Cannot open \"%s\"", pathname);
+
+	gm.lpc = image->pi_start;
+	gm.hpc = image->pi_end;
+	gm.ncnt = (pgf->pgf_nbuckets * sizeof(HISTCOUNTER)) +
+	    sizeof(struct gmonhdr);
+	gm.version = GMONVERSION;
+	gm.profrate = 0;		/* use ticks */
+	gm.histcounter_type = 0;	/* compatibility with moncontrol() */
+	gm.spare[0] = gm.spare[1] = 0;
+
+	/* Write out the gmon header */
+	if (write(fd, &gm, sizeof(gm)) < 0)
+		goto error;
+
+	/* Zero fill the samples[] array */
+	(void) memset(buffer, 0, sizeof(buffer));
+
+	count = pgf->pgf_ndatabytes - sizeof(struct gmonhdr);
+	while (count > sizeof(buffer)) {
+		if (write(fd, &buffer, sizeof(buffer)) < 0)
+			goto error;
+		count -= sizeof(buffer);
+	}
+
+	if (write(fd, &buffer, count) < 0)
+		goto error;
+
+	(void) close(fd);
+
+	return;
+
+ error:
+	err(EX_OSERR, "ERROR: Cannot write \"%s\"", pathname);
+}
+
+/*
+ * Determine the full pathname of a gmon.out file for a given
+ * (image,pmcid) combination.  Return the interned string.
+ */
+
+pmcstat_interned_string
+pmcstat_gmon_create_name(const char *samplesdir, struct pmcstat_image *image,
+    pmc_id_t pmcid)
+{
+	const char *pmcname;
+	char fullpath[PATH_MAX];
+
+	pmcname = pmcstat_pmcid_to_name(pmcid);
+
+	(void) snprintf(fullpath, sizeof(fullpath),
+	    "%s/%s/%s", samplesdir, pmcname,
+	    pmcstat_string_unintern(image->pi_samplename));
+
+	return (pmcstat_string_intern(fullpath));
+}
+
+
+/*
+ * Mmap in a gmon.out file for processing.
+ */
+
+static void
+pmcstat_gmon_map_file(struct pmcstat_gmonfile *pgf)
+{
+	int fd;
+	const char *pathname;
+
+	pathname = pmcstat_string_unintern(pgf->pgf_name);
+
+	/* the gmon.out file must already exist */
+	if ((fd = open(pathname, O_RDWR | O_NOFOLLOW, 0)) < 0)
+		err(EX_OSERR, "ERROR: cannot open \"%s\"", pathname);
+
+	pgf->pgf_gmondata = mmap(NULL, pgf->pgf_ndatabytes,
+	    PROT_READ|PROT_WRITE, MAP_NOSYNC|MAP_SHARED, fd, 0);
+
+	if (pgf->pgf_gmondata == MAP_FAILED)
+		err(EX_OSERR, "ERROR: cannot map \"%s\"", pathname);
+
+	(void) close(fd);
+}
+
+/*
+ * Unmap a gmon.out file after sync'ing its data to disk.
+ */
+
+static void
+pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *pgf)
+{
+	(void) msync(pgf->pgf_gmondata, pgf->pgf_ndatabytes,
+	    MS_SYNC);
+	(void) munmap(pgf->pgf_gmondata, pgf->pgf_ndatabytes);
+	pgf->pgf_gmondata = NULL;
+}
+
+static void
+pmcstat_gmon_append_arc(struct pmcstat_image *image, pmc_id_t pmcid,
+    uintptr_t rawfrom, uintptr_t rawto, uint32_t count)
+{
+	struct rawarc arc;	/* from <sys/gmon.h> */
+	const char *pathname;
+	struct pmcstat_gmonfile *pgf;
+
+	if ((pgf = pmcstat_image_find_gmonfile(image, pmcid)) == NULL)
+		return;
+
+	if (pgf->pgf_file == NULL) {
+		pathname = pmcstat_string_unintern(pgf->pgf_name);
+		if ((pgf->pgf_file = fopen(pathname, "a")) == NULL)
+			return;
+	}
+
+	arc.raw_frompc = rawfrom + image->pi_vaddr;
+	arc.raw_selfpc = rawto + image->pi_vaddr;
+	arc.raw_count = count;
+
+	(void) fwrite(&arc, sizeof(arc), 1, pgf->pgf_file);
+
+}
+
+static struct pmcstat_gmonfile *
+pmcstat_image_find_gmonfile(struct pmcstat_image *image, pmc_id_t pmcid)
+{
+	struct pmcstat_gmonfile *pgf;
+	LIST_FOREACH(pgf, &image->pi_gmlist, pgf_next)
+	    if (pgf->pgf_pmcid == pmcid)
+		    return (pgf);
+	return (NULL);
+}
+
+static void
+pmcstat_cgnode_do_gmon_arcs(struct pmcstat_cgnode *cg, pmc_id_t pmcid)
+{
+	struct pmcstat_cgnode *cgc;
+
+	/*
+	 * Look for child nodes that belong to the same image.
+	 */
+
+	LIST_FOREACH(cgc, &cg->pcg_children, pcg_sibling) {
+		if (cgc->pcg_image == cg->pcg_image)
+			pmcstat_gmon_append_arc(cg->pcg_image, pmcid,
+			    cgc->pcg_func, cg->pcg_func, cgc->pcg_count);
+		if (cgc->pcg_nchildren > 0)
+			pmcstat_cgnode_do_gmon_arcs(cgc, pmcid);
+	}
+}
+
+static void
+pmcstat_callgraph_do_gmon_arcs_for_pmcid(pmc_id_t pmcid)
+{
+	int n;
+	struct pmcstat_cgnode_hash *pch;
+
+	for (n = 0; n < PMCSTAT_NHASH; n++)
+		LIST_FOREACH(pch, &pmcstat_cgnode_hash[n], pch_next)
+			if (pch->pch_pmcid == pmcid &&
+			    pch->pch_cgnode->pcg_nchildren > 1)
+				pmcstat_cgnode_do_gmon_arcs(pch->pch_cgnode,
+				    pmcid);
+}
+
+
+static void
+pmcstat_callgraph_do_gmon_arcs(void)
+{
+	struct pmcstat_pmcrecord *pmcr;
+
+	LIST_FOREACH(pmcr, &pmcstat_pmcs, pr_next)
+		pmcstat_callgraph_do_gmon_arcs_for_pmcid(pmcr->pr_pmcid);
+}
+
+void
+pmcpl_gmon_initimage(struct pmcstat_image *pi)
+{
+	int count, nlen;
+	char *sn;
+	char name[NAME_MAX];
+
+	/*
+	 * Look for a suitable name for the sample files associated
+	 * with this image: if `basename(path)`+".gmon" is available,
+	 * we use that, otherwise we try iterating through
+	 * `basename(path)`+ "~" + NNN + ".gmon" till we get a free
+	 * entry.
+	 */
+	if ((sn = basename(pmcstat_string_unintern(pi->pi_execpath))) == NULL)
+		err(EX_OSERR, "ERROR: Cannot process \"%s\"",
+		    pmcstat_string_unintern(pi->pi_execpath));
+
+	nlen = strlen(sn);
+	nlen = min(nlen, (int) (sizeof(name) - sizeof(".gmon")));
+
+	snprintf(name, sizeof(name), "%.*s.gmon", nlen, sn);
+
+	/* try use the unabridged name first */
+	if (pmcstat_string_lookup(name) == NULL)
+		pi->pi_samplename = pmcstat_string_intern(name);
+	else {
+		/*
+		 * Otherwise use a prefix from the original name and
+		 * upto 3 digits.
+		 */
+		nlen = strlen(sn);
+		nlen = min(nlen, (int) (sizeof(name)-sizeof("~NNN.gmon")));
+		count = 0;
+		do {
+			if (++count > 999)
+				errx(EX_CANTCREAT, "ERROR: cannot create a "
+				    "gmon file for \"%s\"", name);
+			snprintf(name, sizeof(name), "%.*s~%3.3d.gmon",
+			    nlen, sn, count);
+			if (pmcstat_string_lookup(name) == NULL) {
+				pi->pi_samplename =
+				    pmcstat_string_intern(name);
+				count = 0;
+			}
+		} while (count > 0);
+	}
+
+	LIST_INIT(&pi->pi_gmlist);
+}
+
+void
+pmcpl_gmon_shutdownimage(struct pmcstat_image *pi)
+{
+	struct pmcstat_gmonfile *pgf, *pgftmp;
+
+	LIST_FOREACH_SAFE(pgf, &pi->pi_gmlist, pgf_next, pgftmp) {
+		if (pgf->pgf_file)
+			(void) fclose(pgf->pgf_file);
+		LIST_REMOVE(pgf, pgf_next);
+		free(pgf);
+	}
+}
+
+void
+pmcpl_gmon_newpmc(pmcstat_interned_string ps, struct pmcstat_pmcrecord *pr)
+{
+	struct stat st;
+	char fullpath[PATH_MAX];
+
+	(void) pr;
+
+	/*
+	 * Create the appropriate directory to hold gmon.out files.
+	 */
+
+	(void) snprintf(fullpath, sizeof(fullpath), "%s/%s", args.pa_samplesdir,
+	    pmcstat_string_unintern(ps));
+
+	/* If the path name exists, it should be a directory */
+	if (stat(fullpath, &st) == 0 && S_ISDIR(st.st_mode))
+		return;
+
+	if (mkdir(fullpath, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) < 0)
+		err(EX_OSERR, "ERROR: Cannot create directory \"%s\"",
+		    fullpath);
+}
+
+/*
+ * Increment the bucket in the gmon.out file corresponding to 'pmcid'
+ * and 'pc'.
+ */
+
+void
+pmcpl_gmon_process(struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu)
+{
+	struct pmcstat_pcmap *map;
+	struct pmcstat_image *image;
+	struct pmcstat_gmonfile *pgf;
+	uintfptr_t bucket;
+	HISTCOUNTER *hc;
+	pmc_id_t pmcid;
+
+	(void) nsamples; (void) usermode; (void) cpu;
+
+	map = pmcstat_process_find_map(usermode ? pp : pmcstat_kernproc, cc[0]);
+	if (map == NULL) {
+		/* Unknown offset. */
+		pmcstat_stats.ps_samples_unknown_offset++;
+		return;
+	}
+
+	assert(cc[0] >= map->ppm_lowpc && cc[0] < map->ppm_highpc);
+
+	image = map->ppm_image;
+	pmcid = pmcr->pr_pmcid;
+
+	/*
+	 * If this is the first time we are seeing a sample for
+	 * this executable image, try determine its parameters.
+	 */
+	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
+		pmcstat_image_determine_type(image);
+
+	assert(image->pi_type != PMCSTAT_IMAGE_UNKNOWN);
+
+	/* Ignore samples in images that we know nothing about. */
+	if (image->pi_type == PMCSTAT_IMAGE_INDETERMINABLE) {
+		pmcstat_stats.ps_samples_indeterminable++;
+		return;
+	}
+
+	/*
+	 * Find the gmon file corresponding to 'pmcid', creating it if
+	 * needed.
+	 */
+	pgf = pmcstat_image_find_gmonfile(image, pmcid);
+	if (pgf == NULL) {
+		if ((pgf = calloc(1, sizeof(*pgf))) == NULL)
+			err(EX_OSERR, "ERROR:");
+
+		pgf->pgf_gmondata = NULL;	/* mark as unmapped */
+		pgf->pgf_name = pmcstat_gmon_create_name(args.pa_samplesdir,
+		    image, pmcid);
+		pgf->pgf_pmcid = pmcid;
+		assert(image->pi_end > image->pi_start);
+		pgf->pgf_nbuckets = (image->pi_end - image->pi_start) /
+		    FUNCTION_ALIGNMENT;	/* see <machine/profile.h> */
+		pgf->pgf_ndatabytes = sizeof(struct gmonhdr) +
+		    pgf->pgf_nbuckets * sizeof(HISTCOUNTER);
+		pgf->pgf_nsamples = 0;
+		pgf->pgf_file = NULL;
+
+		pmcstat_gmon_create_file(pgf, image);
+
+		LIST_INSERT_HEAD(&image->pi_gmlist, pgf, pgf_next);
+	}
+
+	/*
+	 * Map the gmon file in if needed.  It may have been mapped
+	 * out under memory pressure.
+	 */
+	if (pgf->pgf_gmondata == NULL)
+		pmcstat_gmon_map_file(pgf);
+
+	assert(pgf->pgf_gmondata != NULL);
+
+	/*
+	 *
+	 */
+
+	bucket = (cc[0] - map->ppm_lowpc) / FUNCTION_ALIGNMENT;
+
+	assert(bucket < pgf->pgf_nbuckets);
+
+	hc = (HISTCOUNTER *) ((uintptr_t) pgf->pgf_gmondata +
+	    sizeof(struct gmonhdr));
+
+	/* saturating add */
+	if (hc[bucket] < 0xFFFFU)  /* XXX tie this to sizeof(HISTCOUNTER) */
+		hc[bucket]++;
+	else /* mark that an overflow occurred */
+		pgf->pgf_overflow = 1;
+
+	pgf->pgf_nsamples++;
+}
+
+/*
+ * Shutdown module.
+ */
+
+void
+pmcpl_gmon_shutdown(FILE *mf)
+{
+	int i;
+	struct pmcstat_gmonfile *pgf;
+	struct pmcstat_image *pi;
+
+	/*
+	 * Sync back all gprof flat profile data.
+	 */
+	for (i = 0; i < PMCSTAT_NHASH; i++) {
+		LIST_FOREACH(pi, &pmcstat_image_hash[i], pi_next) {
+			if (mf)
+				(void) fprintf(mf, " \"%s\" => \"%s\"",
+				    pmcstat_string_unintern(pi->pi_execpath),
+				    pmcstat_string_unintern(
+				    pi->pi_samplename));
+
+			/* flush gmon.out data to disk */
+			LIST_FOREACH(pgf, &pi->pi_gmlist, pgf_next) {
+				pmcstat_gmon_unmap_file(pgf);
+				if (mf)
+					(void) fprintf(mf, " %s/%d",
+					    pmcstat_pmcid_to_name(
+					    pgf->pgf_pmcid),
+					    pgf->pgf_nsamples);
+				if (pgf->pgf_overflow && args.pa_verbosity >= 1)
+					warnx("WARNING: profile \"%s\" "
+					    "overflowed.",
+					    pmcstat_string_unintern(
+					        pgf->pgf_name));
+			}
+
+			if (mf)
+				(void) fprintf(mf, "\n");
+		}
+	}
+
+	/*
+	 * Compute arcs and add these to the gprof files.
+	 */
+	if (args.pa_flags & FLAG_DO_GPROF && args.pa_graphdepth > 1)
+		pmcstat_callgraph_do_gmon_arcs();
+}
diff --git a/usr.sbin/pmcstat/pmcpl_gprof.h b/usr.sbin/pmcstat/pmcpl_gprof.h
new file mode 100644
index 0000000..069082f
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcpl_gprof.h
@@ -0,0 +1,47 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * Copyright (c) 2009, Fabien Thomas
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_PL_GPROF_H_
+#define	_PMCSTAT_PL_GPROF_H_
+
+/* Function prototypes */
+void pmcpl_gmon_shutdown(FILE *mf);
+void pmcpl_gmon_process(
+    struct pmcstat_process *pp, struct pmcstat_pmcrecord *pmcr,
+    uint32_t nsamples, uintfptr_t *cc, int usermode, uint32_t cpu);
+void pmcpl_gmon_initimage(struct pmcstat_image *pi);
+void pmcpl_gmon_shutdownimage(struct pmcstat_image *pi);
+void pmcpl_gmon_newpmc(pmcstat_interned_string ps,
+	struct pmcstat_pmcrecord *pr);
+
+#endif	/* _PMCSTAT_PL_GPROF_H_ */
diff --git a/usr.sbin/pmcstat/pmcstat.8 b/usr.sbin/pmcstat/pmcstat.8
index a4e6f1f..309eb3e 100644
--- a/usr.sbin/pmcstat/pmcstat.8
+++ b/usr.sbin/pmcstat/pmcstat.8
@@ -36,6 +36,7 @@
 .Op Fl C
 .Op Fl D Ar pathname
 .Op Fl E
+.Op Fl F Ar pathname
 .Op Fl G Ar pathname
 .Op Fl M Ar mapfilename
 .Op Fl N
@@ -43,9 +44,11 @@
 .Op Fl P Ar event-spec
 .Op Fl R Ar logfilename
 .Op Fl S Ar event-spec
+.Op Fl T
 .Op Fl W
 .Op Fl c Ar cpu-spec
 .Op Fl d
+.Op Fl f Ar pluginopt
 .Op Fl g
 .Op Fl k Ar kerneldir
 .Op Fl m Ar pathname
@@ -129,6 +132,16 @@ complex pipeline of processes when used in conjunction with the
 .Fl d
 option.
 The default is to not to enable per-process tracking.
+.It Fl F Ar pathname
+Print calltree (Kcachegrind) information to file
+.Ar pathname .
+If argument
+.Ar pathname
+is a
+.Dq Li -
+this information is sent to the output file specified by the
+.Fl o
+option.
 .It Fl G Ar pathname
 Print callchain information to file
 .Ar pathname .
@@ -195,6 +208,12 @@ Perform offline analysis using sampling data in file
 Allocate a system mode sampling PMC measuring hardware events
 specified in
 .Ar event-spec .
+.It Fl T
+Use a top like mode for sampling PMCs. The following hotkeys
+can be used: 'c+a' switch to accumulative mode, 'c+d' switch
+to delta mode, 'm' merge PMCs, 'n' change view, 'p' show next
+PMC, ' ' pause, 'q' quit. calltree only: 'f' cost under threshold
+is seen as a dot.
 .It Fl W
 Toggle logging the incremental counts seen by the threads of a
 tracked process each time they are scheduled on a CPU.
@@ -218,6 +237,12 @@ Toggle between process mode PMCs measuring events for the target
 process' current and future children or only measuring events for
 the target process.
 The default is to measure events for the target process alone.
+.It Fl f Ar pluginopt
+Pass option string to the active plugin.
+.br
+threshold=<float> do not display cost under specified value (Top).
+.br
+skiplink=0|1 replace node with cost under threshold by a dot (Top).
 .It Fl g
 Produce profiles in a format compatible with
 .Xr gprof 1 .
@@ -286,7 +311,8 @@ regular expression for selecting processes based on their command names.
 .It Fl v
 Increase verbosity.
 .It Fl w Ar secs
-Print the values of all counting mode PMCs every
+Print the values of all counting mode PMCs or sampling mode PMCs
+for top mode every
 .Ar secs
 seconds.
 The argument
diff --git a/usr.sbin/pmcstat/pmcstat.c b/usr.sbin/pmcstat/pmcstat.c
index 6496ddb..a73d293 100644
--- a/usr.sbin/pmcstat/pmcstat.c
+++ b/usr.sbin/pmcstat/pmcstat.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/wait.h>
 
 #include <assert.h>
+#include <curses.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -106,13 +107,15 @@ __FBSDID("$FreeBSD$");
 
 int	pmcstat_interrupt = 0;
 int	pmcstat_displayheight = DEFAULT_DISPLAY_HEIGHT;
+int	pmcstat_displaywidth  = DEFAULT_DISPLAY_WIDTH;
 int	pmcstat_sockpair[NSOCKPAIRFD];
 int	pmcstat_kq;
 kvm_t	*pmcstat_kvm;
 struct kinfo_proc *pmcstat_plist;
+struct pmcstat_args args;
 
 void
-pmcstat_attach_pmcs(struct pmcstat_args *a)
+pmcstat_attach_pmcs(void)
 {
 	struct pmcstat_ev *ev;
 	struct pmcstat_target *pt;
@@ -120,10 +123,10 @@ pmcstat_attach_pmcs(struct pmcstat_args *a)
 
 	/* Attach all process PMCs to target processes. */
 	count = 0;
-	STAILQ_FOREACH(ev, &a->pa_events, ev_next) {
+	STAILQ_FOREACH(ev, &args.pa_events, ev_next) {
 		if (PMC_IS_SYSTEM_MODE(ev->ev_mode))
 			continue;
-		SLIST_FOREACH(pt, &a->pa_targets, pt_next)
+		SLIST_FOREACH(pt, &args.pa_targets, pt_next)
 			if (pmc_attach(ev->ev_pmcid, pt->pt_pid) == 0)
 				count++;
 			else if (errno != ESRCH)
@@ -138,12 +141,12 @@ pmcstat_attach_pmcs(struct pmcstat_args *a)
 
 
 void
-pmcstat_cleanup(struct pmcstat_args *a)
+pmcstat_cleanup(void)
 {
 	struct pmcstat_ev *ev, *tmp;
 
 	/* release allocated PMCs. */
-	STAILQ_FOREACH_SAFE(ev, &a->pa_events, ev_next, tmp)
+	STAILQ_FOREACH_SAFE(ev, &args.pa_events, ev_next, tmp)
 	    if (ev->ev_pmcid != PMC_ID_INVALID) {
 		if (pmc_stop(ev->ev_pmcid) < 0)
 			err(EX_OSERR, "ERROR: cannot stop pmc 0x%x "
@@ -153,25 +156,25 @@ pmcstat_cleanup(struct pmcstat_args *a)
 			    "0x%x \"%s\"", ev->ev_pmcid, ev->ev_name);
 		free(ev->ev_name);
 		free(ev->ev_spec);
-		STAILQ_REMOVE(&a->pa_events, ev, pmcstat_ev, ev_next);
+		STAILQ_REMOVE(&args.pa_events, ev, pmcstat_ev, ev_next);
 		free(ev);
 	    }
 
 	/* de-configure the log file if present. */
-	if (a->pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))
+	if (args.pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))
 		(void) pmc_configure_logfile(-1);
 
-	if (a->pa_logparser) {
-		pmclog_close(a->pa_logparser);
-		a->pa_logparser = NULL;
+	if (args.pa_logparser) {
+		pmclog_close(args.pa_logparser);
+		args.pa_logparser = NULL;
 	}
 
-	if (a->pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))
-		pmcstat_shutdown_logging(a);
+	if (args.pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))
+		pmcstat_shutdown_logging();
 }
 
 void
-pmcstat_clone_event_descriptor(struct pmcstat_args *a, struct pmcstat_ev *ev,
+pmcstat_clone_event_descriptor(struct pmcstat_ev *ev,
     uint32_t cpumask)
 {
 	int cpu;
@@ -194,14 +197,14 @@ pmcstat_clone_event_descriptor(struct pmcstat_args *a, struct pmcstat_ev *ev,
 		ev_clone->ev_saved = ev->ev_saved;
 		ev_clone->ev_spec  = strdup(ev->ev_spec);
 
-		STAILQ_INSERT_TAIL(&a->pa_events, ev_clone, ev_next);
+		STAILQ_INSERT_TAIL(&args.pa_events, ev_clone, ev_next);
 
 		cpumask &= ~(1 << cpu);
 	}
 }
 
 void
-pmcstat_create_process(struct pmcstat_args *a)
+pmcstat_create_process(void)
 {
 	char token;
 	pid_t pid;
@@ -229,10 +232,10 @@ pmcstat_create_process(struct pmcstat_args *a)
 		(void) close(pmcstat_sockpair[CHILDSOCKET]);
 
 		/* exec() the program requested */
-		execvp(*a->pa_argv, a->pa_argv);
+		execvp(*args.pa_argv, args.pa_argv);
 		/* and if that fails, notify the parent */
 		kill(getppid(), SIGCHLD);
-		err(EX_OSERR, "ERROR: execvp \"%s\" failed", *a->pa_argv);
+		err(EX_OSERR, "ERROR: execvp \"%s\" failed", *args.pa_argv);
 		/*NOTREACHED*/
 
 	default:	/* parent */
@@ -250,7 +253,7 @@ pmcstat_create_process(struct pmcstat_args *a)
 		errx(EX_SOFTWARE, "ERROR: Out of memory.");
 
 	pt->pt_pid = pid;
-	SLIST_INSERT_HEAD(&a->pa_targets, pt, pt_next);
+	SLIST_INSERT_HEAD(&args.pa_targets, pt, pt_next);
 
 	/* Wait for the child to signal that its ready to go. */
 	if (read(pmcstat_sockpair[PARENTSOCKET], &token, 1) < 0)
@@ -260,7 +263,7 @@ pmcstat_create_process(struct pmcstat_args *a)
 }
 
 void
-pmcstat_find_targets(struct pmcstat_args *a, const char *spec)
+pmcstat_find_targets(const char *spec)
 {
 	int n, nproc, pid, rv;
 	struct pmcstat_target *pt;
@@ -275,7 +278,7 @@ pmcstat_find_targets(struct pmcstat_args *a, const char *spec)
 		if ((pt = malloc(sizeof(*pt))) == NULL)
 			goto outofmemory;
 		pt->pt_pid = pid;
-		SLIST_INSERT_HEAD(&a->pa_targets, pt, pt_next);
+		SLIST_INSERT_HEAD(&args.pa_targets, pt, pt_next);
 		return;
 	}
 
@@ -302,7 +305,7 @@ pmcstat_find_targets(struct pmcstat_args *a, const char *spec)
 			if ((pt = malloc(sizeof(*pt))) == NULL)
 				goto outofmemory;
 			pt->pt_pid = kp->ki_pid;
-			SLIST_INSERT_HEAD(&a->pa_targets, pt, pt_next);
+			SLIST_INSERT_HEAD(&args.pa_targets, pt, pt_next);
 		} else if (rv != REG_NOMATCH) {
 			regerror(rv, &reg, errbuf, sizeof(errbuf));
 			errx(EX_SOFTWARE, "ERROR: Regex evalation failed: %s",
@@ -343,17 +346,17 @@ pmcstat_get_cpumask(const char *cpuspec)
 }
 
 void
-pmcstat_kill_process(struct pmcstat_args *a)
+pmcstat_kill_process(void)
 {
 	struct pmcstat_target *pt;
 
-	assert(a->pa_flags & FLAG_HAS_COMMANDLINE);
+	assert(args.pa_flags & FLAG_HAS_COMMANDLINE);
 
 	/*
 	 * If a command line was specified, it would be the very first
 	 * in the list, before any other processes specified by -t.
 	 */
-	pt = SLIST_FIRST(&a->pa_targets);
+	pt = SLIST_FIRST(&args.pa_targets);
 	assert(pt != NULL);
 
 	if (kill(pt->pt_pid, SIGINT) != 0)
@@ -361,7 +364,7 @@ pmcstat_kill_process(struct pmcstat_args *a)
 }
 
 void
-pmcstat_start_pmcs(struct pmcstat_args *a)
+pmcstat_start_pmcs(void)
 {
 	struct pmcstat_ev *ev;
 
@@ -372,7 +375,7 @@ pmcstat_start_pmcs(struct pmcstat_args *a)
 	    if (pmc_start(ev->ev_pmcid) < 0) {
 	        warn("ERROR: Cannot start pmc 0x%x \"%s\"",
 		    ev->ev_pmcid, ev->ev_name);
-		pmcstat_cleanup(a);
+		pmcstat_cleanup();
 		exit(EX_OSERR);
 	    }
 	}
@@ -380,37 +383,37 @@ pmcstat_start_pmcs(struct pmcstat_args *a)
 }
 
 void
-pmcstat_print_headers(struct pmcstat_args *a)
+pmcstat_print_headers(void)
 {
 	struct pmcstat_ev *ev;
 	int c, w;
 
-	(void) fprintf(a->pa_printfile, PRINT_HEADER_PREFIX);
+	(void) fprintf(args.pa_printfile, PRINT_HEADER_PREFIX);
 
-	STAILQ_FOREACH(ev, &a->pa_events, ev_next) {
+	STAILQ_FOREACH(ev, &args.pa_events, ev_next) {
 		if (PMC_IS_SAMPLING_MODE(ev->ev_mode))
 			continue;
 
 		c = PMC_IS_SYSTEM_MODE(ev->ev_mode) ? 's' : 'p';
 
 		if (ev->ev_fieldskip != 0)
-			(void) fprintf(a->pa_printfile, "%*s",
+			(void) fprintf(args.pa_printfile, "%*s",
 			    ev->ev_fieldskip, "");
 		w = ev->ev_fieldwidth - ev->ev_fieldskip - 2;
 
 		if (c == 's')
-			(void) fprintf(a->pa_printfile, "s/%02d/%-*s ",
+			(void) fprintf(args.pa_printfile, "s/%02d/%-*s ",
 			    ev->ev_cpu, w-3, ev->ev_name);
 		else
-			(void) fprintf(a->pa_printfile, "p/%*s ", w,
+			(void) fprintf(args.pa_printfile, "p/%*s ", w,
 			    ev->ev_name);
 	}
 
-	(void) fflush(a->pa_printfile);
+	(void) fflush(args.pa_printfile);
 }
 
 void
-pmcstat_print_counters(struct pmcstat_args *a)
+pmcstat_print_counters(void)
 {
 	int extra_width;
 	struct pmcstat_ev *ev;
@@ -418,7 +421,7 @@ pmcstat_print_counters(struct pmcstat_args *a)
 
 	extra_width = sizeof(PRINT_HEADER_PREFIX) - 1;
 
-	STAILQ_FOREACH(ev, &a->pa_events, ev_next) {
+	STAILQ_FOREACH(ev, &args.pa_events, ev_next) {
 
 		/* skip sampling mode counters */
 		if (PMC_IS_SAMPLING_MODE(ev->ev_mode))
@@ -428,7 +431,7 @@ pmcstat_print_counters(struct pmcstat_args *a)
 			err(EX_OSERR, "ERROR: Cannot read pmc "
 			    "\"%s\"", ev->ev_name);
 
-		(void) fprintf(a->pa_printfile, "%*ju ",
+		(void) fprintf(args.pa_printfile, "%*ju ",
 		    ev->ev_fieldwidth + extra_width,
 		    (uintmax_t) ev->ev_cumulative ? value :
 		    (value - ev->ev_saved));
@@ -438,7 +441,7 @@ pmcstat_print_counters(struct pmcstat_args *a)
 		extra_width = 0;
 	}
 
-	(void) fflush(a->pa_printfile);
+	(void) fflush(args.pa_printfile);
 }
 
 /*
@@ -446,20 +449,20 @@ pmcstat_print_counters(struct pmcstat_args *a)
  */
 
 void
-pmcstat_print_pmcs(struct pmcstat_args *a)
+pmcstat_print_pmcs(void)
 {
 	static int linecount = 0;
 
 	/* check if we need to print a header line */
 	if (++linecount > pmcstat_displayheight) {
-		(void) fprintf(a->pa_printfile, "\n");
+		(void) fprintf(args.pa_printfile, "\n");
 		linecount = 1;
 	}
 	if (linecount == 1)
-		pmcstat_print_headers(a);
-	(void) fprintf(a->pa_printfile, "\n");
+		pmcstat_print_headers();
+	(void) fprintf(args.pa_printfile, "\n");
 
-	pmcstat_print_counters(a);
+	pmcstat_print_counters();
 
 	return;
 }
@@ -493,6 +496,8 @@ pmcstat_show_usage(void)
 	    "\t -C\t\t (toggle) show cumulative counts\n"
 	    "\t -D path\t create profiles in directory \"path\"\n"
 	    "\t -E\t\t (toggle) show counts at process exit\n"
+	    "\t -F file\t write a system-wide callgraph (Kcachegrind format)"
+		" to \"file\"\n"
 	    "\t -G file\t write a system-wide callgraph to \"file\"\n"
 	    "\t -M file\t print executable/gmon file map to \"file\"\n"
 	    "\t -N\t\t (toggle) capture callchains\n"
@@ -500,9 +505,11 @@ pmcstat_show_usage(void)
 	    "\t -P spec\t allocate a process-private sampling PMC\n"
 	    "\t -R file\t read events from \"file\"\n"
 	    "\t -S spec\t allocate a system-wide sampling PMC\n"
+	    "\t -T\t\t start in top mode\n"
 	    "\t -W\t\t (toggle) show counts per context switch\n"
 	    "\t -c cpu-list\t set cpus for subsequent system-wide PMCs\n"
 	    "\t -d\t\t (toggle) track descendants\n"
+	    "\t -f spec\t pass \"spec\" to as plugin option\n"
 	    "\t -g\t\t produce gprof(1) compatible profiles\n"
 	    "\t -k dir\t\t set the path to the kernel\n"
 	    "\t -n rate\t set sampling rate\n"
@@ -520,6 +527,24 @@ pmcstat_show_usage(void)
 }
 
 /*
+ * At exit handler for top mode
+ */
+
+void
+pmcstat_topexit(void)
+{
+	if (!args.pa_toptty)
+		return;
+
+	/*
+	 * Shutdown ncurses.
+	 */
+	clrtoeol();
+	refresh();
+	endwin();
+}
+
+/*
  * Main
  */
 
@@ -535,6 +560,7 @@ main(int argc, char **argv)
 	int graphdepth;
 	int pipefd[2];
 	int use_cumulative_counts;
+	short cf, cb;
 	uint32_t cpumask;
 	char *end, *tmp;
 	const char *errmsg, *graphfilename;
@@ -570,6 +596,13 @@ main(int argc, char **argv)
 	args.pa_mapfilename	= NULL;
 	args.pa_inputpath	= NULL;
 	args.pa_outputpath	= NULL;
+	args.pa_pplugin		= PMCSTAT_PL_NONE;
+	args.pa_plugin		= PMCSTAT_PL_NONE;
+	args.pa_ctdumpinstr	= 1;
+	args.pa_topmode		= PMCSTAT_TOP_DELTA;
+	args.pa_toptty		= 0;
+	args.pa_topcolor	= 0;
+	args.pa_mergepmc	= 0;
 	STAILQ_INIT(&args.pa_events);
 	SLIST_INIT(&args.pa_targets);
 	bzero(&ds_start, sizeof(ds_start));
@@ -594,7 +627,7 @@ main(int argc, char **argv)
 	}
 
 	while ((option = getopt(argc, argv,
-	    "CD:EG:M:NO:P:R:S:Wc:dgk:m:n:o:p:qr:s:t:vw:z:")) != -1)
+	    "CD:EF:G:M:NO:P:R:S:TWc:df:gk:m:n:o:p:qr:s:t:vw:z:")) != -1)
 		switch (option) {
 		case 'C':	/* cumulative values */
 			use_cumulative_counts = !use_cumulative_counts;
@@ -628,13 +661,28 @@ main(int argc, char **argv)
 			args.pa_required |= FLAG_HAS_PROCESS_PMCS;
 			break;
 
+		case 'F':	/* produce a system-wide calltree */
+			args.pa_flags |= FLAG_DO_CALLGRAPHS;
+			args.pa_plugin = PMCSTAT_PL_CALLTREE;
+			graphfilename = optarg;
+			break;
+
+		case 'f':	/* plugins options */
+			if (args.pa_plugin == PMCSTAT_PL_NONE)
+				err(EX_USAGE, "ERROR: Need -g/-G/-m/-T.");
+			pmcstat_pluginconfigure_log(optarg);
+			break;
+
 		case 'G':	/* produce a system-wide callgraph */
 			args.pa_flags |= FLAG_DO_CALLGRAPHS;
+			args.pa_plugin = PMCSTAT_PL_CALLGRAPH;
 			graphfilename = optarg;
 			break;
 
 		case 'g':	/* produce gprof compatible profiles */
 			args.pa_flags |= FLAG_DO_GPROF;
+			args.pa_pplugin = PMCSTAT_PL_CALLGRAPH;
+			args.pa_plugin	= PMCSTAT_PL_GPROF;
 			break;
 
 		case 'k':	/* pathname to the kernel */
@@ -645,8 +693,9 @@ main(int argc, char **argv)
 			break;
 
 		case 'm':
-			args.pa_flags |= FLAG_WANTS_MAPPINGS;
-			graphfilename = optarg;
+			args.pa_flags |= FLAG_DO_ANNOTATE;
+			args.pa_plugin = PMCSTAT_PL_ANNOTATE;
+			graphfilename  = optarg;
 			break;
 
 		case 'E':	/* log process exit */
@@ -732,7 +781,7 @@ main(int argc, char **argv)
 			STAILQ_INSERT_TAIL(&args.pa_events, ev, ev_next);
 
 			if (option == 's' || option == 'S')
-				pmcstat_clone_event_descriptor(&args, ev,
+				pmcstat_clone_event_descriptor(ev,
 				    cpumask & ~(1 << ev->ev_cpu));
 
 			break;
@@ -782,12 +831,21 @@ main(int argc, char **argv)
 			break;
 
 		case 't':	/* target pid or process name */
-			pmcstat_find_targets(&args, optarg);
+			pmcstat_find_targets(optarg);
 
 			args.pa_flags |= FLAG_HAS_TARGET;
 			args.pa_required |= FLAG_HAS_PROCESS_PMCS;
 			break;
 
+		case 'T':	/* top mode */
+			args.pa_flags |= FLAG_DO_TOP;
+			args.pa_plugin = PMCSTAT_PL_CALLGRAPH;
+			args.pa_ctdumpinstr = 0;
+			args.pa_mergepmc = 1;
+			if (args.pa_printfile == stderr)
+				args.pa_printfile = stdout;
+			break;
+
 		case 'v':	/* verbose */
 			args.pa_verbosity++;
 			break;
@@ -798,7 +856,6 @@ main(int argc, char **argv)
 				errx(EX_USAGE, "ERROR: Illegal wait interval "
 				    "value \"%s\".", optarg);
 			args.pa_flags |= FLAG_HAS_WAIT_INTERVAL;
-			args.pa_required |= FLAG_HAS_COUNTING_PMCS;
 			args.pa_interval = interval;
 			break;
 
@@ -833,7 +890,7 @@ main(int argc, char **argv)
 		args.pa_flags |= FLAG_HAS_COMMANDLINE;
 
 	if (args.pa_flags & (FLAG_DO_GPROF | FLAG_DO_CALLGRAPHS |
-	    FLAG_WANTS_MAPPINGS))
+	    FLAG_DO_ANNOTATE | FLAG_DO_TOP))
 		args.pa_flags |= FLAG_DO_ANALYSIS;
 
 	/*
@@ -846,11 +903,11 @@ main(int argc, char **argv)
 		    "exclusive.");
 
 	/* -m option is allowed with -R only. */
-	if (args.pa_flags & FLAG_WANTS_MAPPINGS && args.pa_inputpath == NULL)
+	if (args.pa_flags & FLAG_DO_ANNOTATE && args.pa_inputpath == NULL)
 		errx(EX_USAGE, "ERROR: option -m requires an input file");
 
 	/* -m option is not allowed combined with -g or -G. */
-	if (args.pa_flags & FLAG_WANTS_MAPPINGS &&
+	if (args.pa_flags & FLAG_DO_ANNOTATE &&
 	    args.pa_flags & (FLAG_DO_GPROF | FLAG_DO_CALLGRAPHS))
 		errx(EX_USAGE, "ERROR: option -m and -g | -G are mutually "
 		    "exclusive");
@@ -904,7 +961,7 @@ main(int argc, char **argv)
 	/* check for counting mode options without a counting PMC */
 	if ((args.pa_required & FLAG_HAS_COUNTING_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_COUNTING_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -C, -W, -o and -w require at "
+		errx(EX_USAGE, "ERROR: options -C, -W and -o require at "
 		    "least one counting mode PMC to be specified.");
 
 	/* check for sampling mode options without a sampling PMC spec */
@@ -913,10 +970,10 @@ main(int argc, char **argv)
 		errx(EX_USAGE, "ERROR: options -N, -n and -O require at "
 		    "least one sampling mode PMC to be specified.");
 
-	/* check if -g/-G are being used correctly */
+	/* check if -g/-G/-m/-T are being used correctly */
 	if ((args.pa_flags & FLAG_DO_ANALYSIS) &&
 	    !(args.pa_flags & (FLAG_HAS_SAMPLING_PMCS|FLAG_READ_LOGFILE)))
-		errx(EX_USAGE, "ERROR: options -g/-G require sampling PMCs "
+		errx(EX_USAGE, "ERROR: options -g/-G/-m/-T require sampling PMCs "
 		    "or -R to be specified.");
 
 	/* check if -O was spuriously specified */
@@ -926,11 +983,11 @@ main(int argc, char **argv)
 		    "ERROR: option -O is used only with options "
 		    "-E, -P, -S and -W.");
 
-	/* -k kernel path require -g/-G or -R */
+	/* -k kernel path require -g/-G/-m/-T or -R */
 	if ((args.pa_flags & FLAG_HAS_KERNELPATH) &&
 	    (args.pa_flags & FLAG_DO_ANALYSIS) == 0 &&
 	    (args.pa_flags & FLAG_READ_LOGFILE) == 0)
-	    errx(EX_USAGE, "ERROR: option -k is only used with -g/-R.");
+	    errx(EX_USAGE, "ERROR: option -k is only used with -g/-R/-m/-T.");
 
 	/* -D only applies to gprof output mode (-g) */
 	if ((args.pa_flags & FLAG_HAS_SAMPLESDIR) &&
@@ -943,6 +1000,11 @@ main(int argc, char **argv)
 	    (args.pa_flags & FLAG_READ_LOGFILE) == 0)
 	    errx(EX_USAGE, "ERROR: option -M is only used with -g/-R.");
 
+	/* -T is incompatible with -R (replay logfile is a TODO) */
+	if ((args.pa_flags & FLAG_DO_TOP) &&
+	    (args.pa_flags & FLAG_READ_LOGFILE))
+		errx(EX_USAGE, "ERROR: option -T is incompatible with -R.");
+
 	/*
 	 * Disallow textual output of sampling PMCs if counting PMCs
 	 * have also been asked for, mostly because the combined output
@@ -996,7 +1058,7 @@ main(int argc, char **argv)
 				    "for writing", graphfilename);
 		}
 	}
-	if (args.pa_flags & FLAG_WANTS_MAPPINGS) {
+	if (args.pa_flags & FLAG_DO_ANNOTATE) {
 		args.pa_graphfile = fopen(graphfilename, "w");
 		if (args.pa_graphfile == NULL)
 			err(EX_OSERR, "ERROR: cannot open \"%s\" for writing",
@@ -1012,13 +1074,13 @@ main(int argc, char **argv)
 		if ((args.pa_flags & FLAG_DO_ANALYSIS) == 0)
 			args.pa_flags |= FLAG_DO_PRINT;
 
-		pmcstat_initialize_logging(&args);
+		pmcstat_initialize_logging();
 		args.pa_logfd = pmcstat_open_log(args.pa_inputpath,
 		    PMCSTAT_OPEN_FOR_READ);
 		if ((args.pa_logparser = pmclog_open(args.pa_logfd)) == NULL)
 			err(EX_OSERR, "ERROR: Cannot create parser");
-		pmcstat_process_log(&args);
-		pmcstat_shutdown_logging(&args);
+		pmcstat_process_log();
+		pmcstat_shutdown_logging();
 		exit(EX_OK);
 	}
 
@@ -1062,7 +1124,9 @@ main(int argc, char **argv)
 
 			args.pa_logfd = pipefd[WRITEPIPEFD];
 
-			args.pa_flags |= (FLAG_HAS_PIPE | FLAG_DO_PRINT);
+			args.pa_flags |= FLAG_HAS_PIPE;
+			if ((args.pa_flags & FLAG_DO_TOP) == 0)
+				args.pa_flags |= FLAG_DO_PRINT;
 			args.pa_logparser = pmclog_open(pipefd[READPIPEFD]);
 		}
 
@@ -1126,12 +1190,24 @@ main(int argc, char **argv)
 			err(EX_OSERR, "ERROR: Cannot determine window size");
 
 		pmcstat_displayheight = ws.ws_row - 1;
+		pmcstat_displaywidth  = ws.ws_col - 1;
 
 		EV_SET(&kev, SIGWINCH, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL);
 
 		if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
 			err(EX_OSERR, "ERROR: Cannot register kevent for "
 			    "SIGWINCH");
+
+		args.pa_toptty = 1;
+	}
+
+	/*
+	 * Listen to key input in top mode.
+	 */
+	if (args.pa_flags & FLAG_DO_TOP) {
+		EV_SET(&kev, fileno(stdin), EVFILT_READ, EV_ADD, 0, 0, NULL);
+		if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
+			err(EX_OSERR, "ERROR: Cannot register kevent");
 	}
 
 	EV_SET(&kev, SIGINT, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL);
@@ -1152,9 +1228,13 @@ main(int argc, char **argv)
 	if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
 		err(EX_OSERR, "ERROR: Cannot register kevent for SIGCHLD");
 
-	/* setup a timer if we have counting mode PMCs needing to be printed */
-	if ((args.pa_flags & FLAG_HAS_COUNTING_PMCS) &&
-	    (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
+	/* 
+	 * Setup a timer if we have counting mode PMCs needing to be printed or
+	 * top mode plugin is active.
+	 */
+	if (((args.pa_flags & FLAG_HAS_COUNTING_PMCS) &&
+	     (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) ||
+	    (args.pa_flags & FLAG_DO_TOP)) {
 		EV_SET(&kev, 0, EVFILT_TIMER, EV_ADD, 0,
 		    args.pa_interval * 1000, NULL);
 
@@ -1165,7 +1245,7 @@ main(int argc, char **argv)
 
 	/* attach PMCs to the target process, starting it if specified */
 	if (args.pa_flags & FLAG_HAS_COMMANDLINE)
-		pmcstat_create_process(&args);
+		pmcstat_create_process();
 
 	if (check_driver_stats && pmc_get_driver_stats(&ds_start) < 0)
 		err(EX_OSERR, "ERROR: Cannot retrieve driver statistics");
@@ -1176,7 +1256,7 @@ main(int argc, char **argv)
 			errx(EX_DATAERR, "ERROR: No matching target "
 			    "processes.");
 		if (args.pa_flags & FLAG_HAS_PROCESS_PMCS)
-			pmcstat_attach_pmcs(&args);
+			pmcstat_attach_pmcs();
 
 		if (pmcstat_kvm) {
 			kvm_close(pmcstat_kvm);
@@ -1185,16 +1265,16 @@ main(int argc, char **argv)
 	}
 
 	/* start the pmcs */
-	pmcstat_start_pmcs(&args);
+	pmcstat_start_pmcs();
 
 	/* start the (commandline) process if needed */
 	if (args.pa_flags & FLAG_HAS_COMMANDLINE)
 		pmcstat_start_process();
 
 	/* initialize logging if printing the configured log */
-	if ((args.pa_flags & FLAG_DO_PRINT) &&
+	if ((args.pa_flags & (FLAG_DO_PRINT | FLAG_DO_TOP)) &&
 	    (args.pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE)))
-		pmcstat_initialize_logging(&args);
+		pmcstat_initialize_logging();
 
 	/* Handle SIGINT using the kqueue loop */
 	sa.sa_handler = SIG_IGN;
@@ -1205,6 +1285,37 @@ main(int argc, char **argv)
 		err(EX_OSERR, "ERROR: Cannot install signal handler");
 
 	/*
+	 * Setup the top mode display.
+	 */
+	if (args.pa_flags & FLAG_DO_TOP) {
+		args.pa_flags &= ~FLAG_DO_PRINT;
+
+		if (args.pa_toptty) {
+			/*
+			 * Init ncurses.
+			 */
+			initscr();
+			if(has_colors() == TRUE) {
+				args.pa_topcolor = 1;
+				start_color();
+				use_default_colors();
+				pair_content(0, &cf, &cb);
+				init_pair(1, COLOR_RED, cb);
+				init_pair(2, COLOR_YELLOW, cb);
+				init_pair(3, COLOR_GREEN, cb);
+			}
+			cbreak();
+			noecho();
+			nonl();
+			nodelay(stdscr, 1);
+			intrflush(stdscr, FALSE);
+			keypad(stdscr, TRUE);
+			clear();
+			atexit(pmcstat_topexit);
+		}
+	}
+
+	/*
 	 * loop till either the target process (if any) exits, or we
 	 * are killed by a SIGINT.
 	 */
@@ -1225,14 +1336,18 @@ main(int argc, char **argv)
 		case EVFILT_PROC:  /* target has exited */
 			if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE |
 				FLAG_HAS_PIPE))
-				runstate = pmcstat_close_log(&args);
+				runstate = pmcstat_close_log();
 			else
 				runstate = PMCSTAT_FINISHED;
 			do_print = 1;
 			break;
 
 		case EVFILT_READ:  /* log file data is present */
-			runstate = pmcstat_process_log(&args);
+			if (kev.ident == (unsigned)fileno(stdin)) {
+				if (pmcstat_keypress_log())
+					runstate = pmcstat_close_log();
+			} else
+				runstate = pmcstat_process_log();
 			break;
 
 		case EVFILT_SIGNAL:
@@ -1253,17 +1368,17 @@ main(int argc, char **argv)
 				 */
 				if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE |
 				    FLAG_HAS_PIPE)) {
-					runstate = pmcstat_close_log(&args);
+					runstate = pmcstat_close_log();
 					if (args.pa_flags &
 					    (FLAG_DO_PRINT|FLAG_DO_ANALYSIS))
-						pmcstat_process_log(&args);
+						pmcstat_process_log();
 				}
 				do_print = 1; /* print PMCs at exit */
 				runstate = PMCSTAT_FINISHED;
 			} else if (kev.ident == SIGINT) {
 				/* Kill the child process if we started it */
 				if (args.pa_flags & FLAG_HAS_COMMANDLINE)
-					pmcstat_kill_process(&args);
+					pmcstat_kill_process();
 				/* Close the pipe to self, if present. */
 				if (args.pa_flags & FLAG_HAS_PIPE)
 					(void) close(pipefd[READPIPEFD]);
@@ -1274,6 +1389,7 @@ main(int argc, char **argv)
 				    err(EX_OSERR, "ERROR: Cannot determine "
 					"window size");
 				pmcstat_displayheight = ws.ws_row - 1;
+				pmcstat_displaywidth  = ws.ws_col - 1;
 			} else
 				assert(0);
 
@@ -1285,22 +1401,30 @@ main(int argc, char **argv)
 
 		}
 
-		if (do_print &&
-		    (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
-			pmcstat_print_pmcs(&args);
-			if (runstate == PMCSTAT_FINISHED && /* final newline */
-			    (args.pa_flags & FLAG_DO_PRINT) == 0)
-				(void) fprintf(args.pa_printfile, "\n");
+		if (do_print) {
+			if ((args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
+				pmcstat_print_pmcs();
+				if (runstate == PMCSTAT_FINISHED && /* final newline */
+				    (args.pa_flags & FLAG_DO_PRINT) == 0)
+					(void) fprintf(args.pa_printfile, "\n");
+			}
+			if (args.pa_flags & FLAG_DO_TOP)
+				pmcstat_display_log();
 			do_print = 0;
 		}
 
 	} while (runstate != PMCSTAT_FINISHED);
 
+	if ((args.pa_flags & FLAG_DO_TOP) && args.pa_toptty) {
+		pmcstat_topexit();
+		args.pa_toptty = 0;
+	}
+
 	/* flush any pending log entries */
 	if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE))
 		pmc_flush_logfile();
 
-	pmcstat_cleanup(&args);
+	pmcstat_cleanup();
 
 	free(args.pa_kernel);
 
diff --git a/usr.sbin/pmcstat/pmcstat.h b/usr.sbin/pmcstat/pmcstat.h
index 5d6f5f2..1bb7862 100644
--- a/usr.sbin/pmcstat/pmcstat.h
+++ b/usr.sbin/pmcstat/pmcstat.h
@@ -47,13 +47,15 @@
 #define	FLAG_HAS_SAMPLESDIR		0x00000800	/* -D dir */
 #define	FLAG_HAS_KERNELPATH		0x00001000	/* -k kernel */
 #define	FLAG_DO_PRINT			0x00002000	/* -o */
-#define	FLAG_DO_CALLGRAPHS		0x00004000	/* -G */
-#define	FLAG_DO_ANALYSIS		0x00008000	/* -g or -G */
-#define	FLAG_WANTS_MAPPINGS		0x00010000	/* -m */
+#define	FLAG_DO_CALLGRAPHS		0x00004000	/* -G or -F */
+#define	FLAG_DO_ANNOTATE		0x00008000	/* -m */
+#define	FLAG_DO_TOP			0x00010000	/* -T */
+#define	FLAG_DO_ANALYSIS		0x00020000	/* -g or -G or -m or -T */
 
 #define	DEFAULT_SAMPLE_COUNT		65536
 #define	DEFAULT_WAIT_INTERVAL		5.0
-#define	DEFAULT_DISPLAY_HEIGHT		23
+#define	DEFAULT_DISPLAY_HEIGHT		256		/* file virtual height */
+#define	DEFAULT_DISPLAY_WIDTH		1024		/* file virtual width */
 #define	DEFAULT_BUFFER_SIZE		4096
 #define	DEFAULT_CALLGRAPH_DEPTH		4
 
@@ -75,12 +77,24 @@
 
 #define	PMCSTAT_LDD_COMMAND		"/usr/bin/ldd"
 
-#define	PMCSTAT_PRINT_ENTRY(A,T,...) do {				\
-		(void) fprintf((A)->pa_printfile, "%-9s", T);		\
-		(void) fprintf((A)->pa_printfile, " "  __VA_ARGS__);	\
-		(void) fprintf((A)->pa_printfile, "\n");		\
+#define	PMCSTAT_PRINT_ENTRY(T,...) do {					\
+		(void) fprintf(args.pa_printfile, "%-9s", T);		\
+		(void) fprintf(args.pa_printfile, " "  __VA_ARGS__);	\
+		(void) fprintf(args.pa_printfile, "\n");		\
 	} while (0)
 
+#define PMCSTAT_PL_NONE		0
+#define PMCSTAT_PL_CALLGRAPH	1
+#define PMCSTAT_PL_GPROF	2
+#define PMCSTAT_PL_ANNOTATE	3
+#define PMCSTAT_PL_CALLTREE	4
+
+#define PMCSTAT_TOP_DELTA 	0
+#define PMCSTAT_TOP_ACCUM	1
+
+#define	min(A,B)		((A) < (B) ? (A) : (B))
+#define	max(A,B)		((A) > (B) ? (A) : (B))
+
 enum pmcstat_state {
 	PMCSTAT_FINISHED = 0,
 	PMCSTAT_EXITING  = 1,
@@ -110,6 +124,8 @@ struct pmcstat_target {
 struct pmcstat_args {
 	int	pa_flags;		/* argument flags */
 	int	pa_required;		/* required features */
+	int	pa_pplugin;		/* pre-processing plugin */
+	int	pa_plugin;		/* analysis plugin */
 	int	pa_verbosity;		/* verbosity level */
 	FILE	*pa_printfile;		/* where to send printed output */
 	int	pa_logfd;		/* output log file */
@@ -124,31 +140,44 @@ struct pmcstat_args {
 	int	pa_graphdepth;		/* print depth for callgraphs */
 	double	pa_interval;		/* printing interval in seconds */
 	uint32_t pa_cpumask;		/* filter for CPUs analysed */
+	int	pa_ctdumpinstr;		/* dump instructions with calltree */
+	int	pa_topmode;		/* delta or accumulative */
+	int	pa_toptty;		/* output to tty or file */
+	int	pa_topcolor;		/* terminal support color */
+	int	pa_mergepmc;		/* merge PMC with same name */
 	int	pa_argc;
 	char	**pa_argv;
 	STAILQ_HEAD(, pmcstat_ev) pa_events;
 	SLIST_HEAD(, pmcstat_target) pa_targets;
-} args;
+};
+
+extern int pmcstat_displayheight;	/* current terminal height */
+extern int pmcstat_displaywidth;	/* current terminal width */
+extern struct pmcstat_args args;	/* command line args */
 
 /* Function prototypes */
-void	pmcstat_attach_pmcs(struct pmcstat_args *_a);
-void	pmcstat_cleanup(struct pmcstat_args *_a);
-void	pmcstat_clone_event_descriptor(struct pmcstat_args *_a,
+void	pmcstat_attach_pmcs(void);
+void	pmcstat_cleanup(void);
+void	pmcstat_clone_event_descriptor(
     struct pmcstat_ev *_ev, uint32_t _cpumask);
-int	pmcstat_close_log(struct pmcstat_args *_a);
-void	pmcstat_create_process(struct pmcstat_args *_a);
-void	pmcstat_find_targets(struct pmcstat_args *_a, const char *_arg);
-void	pmcstat_initialize_logging(struct pmcstat_args *_a);
-void	pmcstat_kill_process(struct pmcstat_args *_a);
+int	pmcstat_close_log(void);
+void	pmcstat_create_process(void);
+void	pmcstat_find_targets(const char *_arg);
+void	pmcstat_initialize_logging(void);
+void	pmcstat_kill_process(void);
 int	pmcstat_open_log(const char *_p, int _mode);
-void	pmcstat_print_counters(struct pmcstat_args *_a);
-void	pmcstat_print_headers(struct pmcstat_args *_a);
-void	pmcstat_print_pmcs(struct pmcstat_args *_a);
+void	pmcstat_print_counters(void);
+void	pmcstat_print_headers(void);
+void	pmcstat_print_pmcs(void);
 void	pmcstat_show_usage(void);
-void	pmcstat_shutdown_logging(struct pmcstat_args *_a);
-void	pmcstat_start_pmcs(struct pmcstat_args *_a);
+void	pmcstat_shutdown_logging(void);
+void	pmcstat_start_pmcs(void);
 void	pmcstat_start_process(void);
-int	pmcstat_process_log(struct pmcstat_args *_a);
+int	pmcstat_process_log(void);
+int	pmcstat_keypress_log(void);
+void	pmcstat_display_log(void);
+void	pmcstat_pluginconfigure_log(char *_opt);
 uint32_t pmcstat_get_cpumask(const char *_a);
+void    pmcstat_topexit(void);
 
 #endif	/* _PMCSTAT_H_ */
diff --git a/usr.sbin/pmcstat/pmcstat_log.c b/usr.sbin/pmcstat/pmcstat_log.c
index a403852..5811af3 100644
--- a/usr.sbin/pmcstat/pmcstat_log.c
+++ b/usr.sbin/pmcstat/pmcstat_log.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in.h>
 
 #include <assert.h>
+#include <curses.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -68,9 +69,8 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 
 #include "pmcstat.h"
-
-#define	min(A,B)		((A) < (B) ? (A) : (B))
-#define	max(A,B)		((A) > (B) ? (A) : (B))
+#include "pmcstat_log.h"
+#include "pmcstat_top.h"
 
 #define	PMCSTAT_ALLOCATE		1
 
@@ -81,6 +81,7 @@ __FBSDID("$FreeBSD$");
  * pmcstat_shutdown_logging()		orderly shutdown, called last
  * pmcstat_open_log()			open an eventlog for processing
  * pmcstat_process_log()		print/convert an event log
+ * pmcstat_display_log()		top mode display for the log
  * pmcstat_close_log()			finish processing an event log
  *
  * IMPLEMENTATION NOTES
@@ -127,236 +128,125 @@ __FBSDID("$FreeBSD$");
  * also given a 'rank' that reflects its depth in the call stack.
  */
 
-typedef const void *pmcstat_interned_string;
-
-/*
- * 'pmcstat_pmcrecord' is a mapping from PMC ids to human-readable
- * names.
- */
-
-struct pmcstat_pmcrecord {
-	LIST_ENTRY(pmcstat_pmcrecord)	pr_next;
-	pmc_id_t			pr_pmcid;
-	pmcstat_interned_string	pr_pmcname;
-};
-
-static LIST_HEAD(,pmcstat_pmcrecord)	pmcstat_pmcs =
-	LIST_HEAD_INITIALIZER(pmcstat_pmcs);
-
-
-/*
- * struct pmcstat_gmonfile tracks a given 'gmon.out' file.  These
- * files are mmap()'ed in as needed.
- */
-
-struct pmcstat_gmonfile {
-	LIST_ENTRY(pmcstat_gmonfile)	pgf_next; /* list of entries */
-	int		pgf_overflow;	/* whether a count overflowed */
-	pmc_id_t	pgf_pmcid;	/* id of the associated pmc */
-	size_t		pgf_nbuckets;	/* #buckets in this gmon.out */
-	unsigned int	pgf_nsamples;	/* #samples in this gmon.out */
-	pmcstat_interned_string pgf_name;	/* pathname of gmon.out file */
-	size_t		pgf_ndatabytes;	/* number of bytes mapped */
-	void		*pgf_gmondata;	/* pointer to mmap'ed data */
-	FILE		*pgf_file;	/* used when writing gmon arcs */
-};
-
-/*
- * A 'pmcstat_image' structure describes an executable program on
- * disk.  'pi_execpath' is a cookie representing the pathname of
- * the executable.  'pi_start' and 'pi_end' are the least and greatest
- * virtual addresses for the text segments in the executable.
- * 'pi_gmonlist' contains a linked list of gmon.out files associated
- * with this image.
- */
-
-enum pmcstat_image_type {
-	PMCSTAT_IMAGE_UNKNOWN = 0,	/* never looked at the image */
-	PMCSTAT_IMAGE_INDETERMINABLE,	/* can't tell what the image is */
-	PMCSTAT_IMAGE_ELF32,		/* ELF 32 bit object */
-	PMCSTAT_IMAGE_ELF64,		/* ELF 64 bit object */
-	PMCSTAT_IMAGE_AOUT		/* AOUT object */
-};
-
-struct pmcstat_image {
-	LIST_ENTRY(pmcstat_image) pi_next;	/* hash link */
-	TAILQ_ENTRY(pmcstat_image) pi_lru;	/* LRU list */
-	pmcstat_interned_string	pi_execpath;    /* cookie */
-	pmcstat_interned_string pi_samplename;  /* sample path name */
-	pmcstat_interned_string pi_fullpath;    /* path to FS object */
-
-	enum pmcstat_image_type pi_type;	/* executable type */
-
-	/*
-	 * Executables have pi_start and pi_end; these are zero
-	 * for shared libraries.
-	 */
-	uintfptr_t	pi_start;	/* start address (inclusive) */
-	uintfptr_t	pi_end;		/* end address (exclusive) */
-	uintfptr_t	pi_entry;	/* entry address */
-	uintfptr_t	pi_vaddr;	/* virtual address where loaded */
-	int		pi_isdynamic;	/* whether a dynamic object */
-	int		pi_iskernelmodule;
-	pmcstat_interned_string pi_dynlinkerpath; /* path in .interp */
-
-	/* All symbols associated with this object. */
-	struct pmcstat_symbol *pi_symbols;
-	size_t		pi_symcount;
-
-	/*
-	 * An image can be associated with one or more gmon.out files;
-	 * one per PMC.
-	 */
-	LIST_HEAD(,pmcstat_gmonfile) pi_gmlist;
-};
+struct pmcstat_pmcs pmcstat_pmcs = LIST_HEAD_INITIALIZER(pmcstat_pmcs);
 
 /*
  * All image descriptors are kept in a hash table.
  */
-static LIST_HEAD(,pmcstat_image)	pmcstat_image_hash[PMCSTAT_NHASH];
-
-/*
- * A 'pmcstat_pcmap' structure maps a virtual address range to an
- * underlying 'pmcstat_image' descriptor.
- */
-struct pmcstat_pcmap {
-	TAILQ_ENTRY(pmcstat_pcmap) ppm_next;
-	uintfptr_t	ppm_lowpc;
-	uintfptr_t	ppm_highpc;
-	struct pmcstat_image *ppm_image;
-};
+struct pmcstat_image_hash_list pmcstat_image_hash[PMCSTAT_NHASH];
 
 /*
- * A 'pmcstat_process' structure models processes.  Each process is
- * associated with a set of pmcstat_pcmap structures that map
- * addresses inside it to executable objects.  This set is implemented
- * as a list, kept sorted in ascending order of mapped addresses.
- *
- * 'pp_pid' holds the pid of the process.  When a process exits, the
- * 'pp_isactive' field is set to zero, but the process structure is
- * not immediately reclaimed because there may still be samples in the
- * log for this process.
+ * All process descriptors are kept in a hash table.
  */
+struct pmcstat_process_hash_list pmcstat_process_hash[PMCSTAT_NHASH];
 
-struct pmcstat_process {
-	LIST_ENTRY(pmcstat_process) pp_next;	/* hash-next */
-	pid_t			pp_pid;		/* associated pid */
-	int			pp_isactive;	/* whether active */
-	uintfptr_t		pp_entryaddr;	/* entry address */
-	TAILQ_HEAD(,pmcstat_pcmap) pp_map;	/* address range map */
-};
+struct pmcstat_stats pmcstat_stats; /* statistics */
 
-/*
- * All process descriptors are kept in a hash table.
- */
-static LIST_HEAD(,pmcstat_process) pmcstat_process_hash[PMCSTAT_NHASH];
+struct pmcstat_process *pmcstat_kernproc; /* kernel 'process' */
 
-static struct pmcstat_process *pmcstat_kernproc; /* kernel 'process' */
+#include "pmcpl_gprof.h"
+#include "pmcpl_callgraph.h"
+#include "pmcpl_annotate.h"
+#include "pmcpl_calltree.h"
 
-/*
- * Each function symbol tracked by pmcstat(8).
- */
+struct pmc_plugins  {
+	const char 	*pl_name;	/* name */
 
-struct pmcstat_symbol {
-	pmcstat_interned_string ps_name;
-	uint64_t	ps_start;
-	uint64_t	ps_end;
-};
+	/* configure */
+	int (*pl_configure)(char *opt);
 
-/*
- * Each call graph node is tracked by a pmcstat_cgnode struct.
- */
+	/* init and shutdown */
+	int (*pl_init)(void);
+	void (*pl_shutdown)(FILE *mf);
 
-struct pmcstat_cgnode {
-	struct pmcstat_image	*pcg_image;
-	uintfptr_t		pcg_func;
-	uint32_t		pcg_count;
-	uint32_t		pcg_nchildren;
-	LIST_ENTRY(pmcstat_cgnode) pcg_sibling;
-	LIST_HEAD(,pmcstat_cgnode) pcg_children;
-};
+	/* sample processing */
+	void (*pl_process)(struct pmcstat_process *pp,
+	    struct pmcstat_pmcrecord *pmcr, uint32_t nsamples,
+	    uintfptr_t *cc, int usermode, uint32_t cpu);
 
-struct pmcstat_cgnode_hash {
-	struct pmcstat_cgnode  *pch_cgnode;
-	uint32_t		pch_pmcid;
-	LIST_ENTRY(pmcstat_cgnode_hash) pch_next;
-};
+	/* image */
+	void (*pl_initimage)(struct pmcstat_image *pi);
+	void (*pl_shutdownimage)(struct pmcstat_image *pi);
 
-static int pmcstat_cgnode_hash_count;
-static pmcstat_interned_string pmcstat_previous_filename_printed;
+	/* pmc */
+	void (*pl_newpmc)(pmcstat_interned_string ps,
+		struct pmcstat_pmcrecord *pr);
+	
+	/* top display */
+	void (*pl_topdisplay)(void);
 
-/*
- * The toplevel CG nodes (i.e., with rank == 0) are placed in a hash table.
- */
+	/* top keypress */
+	int (*pl_topkeypress)(int c, WINDOW *w);
 
-static LIST_HEAD(,pmcstat_cgnode_hash) pmcstat_cgnode_hash[PMCSTAT_NHASH];
+} plugins[] = {
+	{
+		.pl_name		= "none",
+	},
+	{
+		.pl_name		= "callgraph",
+		.pl_init		= pmcpl_cg_init,
+		.pl_shutdown		= pmcpl_cg_shutdown,
+		.pl_process		= pmcpl_cg_process,
+		.pl_topkeypress		= pmcpl_cg_topkeypress,
+		.pl_topdisplay		= pmcpl_cg_topdisplay
+	},
+	{
+		.pl_name		= "gprof",
+		.pl_shutdown		= pmcpl_gmon_shutdown,
+		.pl_process		= pmcpl_gmon_process,
+		.pl_initimage		= pmcpl_gmon_initimage,
+		.pl_shutdownimage	= pmcpl_gmon_shutdownimage,
+		.pl_newpmc		= pmcpl_gmon_newpmc
+	},
+	{
+		.pl_name		= "annotate",
+		.pl_process		= pmcpl_annotate_process
+	},
+	{
+		.pl_name		= "calltree",
+		.pl_configure		= pmcpl_ct_configure,
+		.pl_init		= pmcpl_ct_init,
+		.pl_shutdown		= pmcpl_ct_shutdown,
+		.pl_process		= pmcpl_ct_process,
+		.pl_topkeypress		= pmcpl_ct_topkeypress,
+		.pl_topdisplay		= pmcpl_ct_topdisplay
+	},
+	{
+		.pl_name		= NULL
+	}
+};
 
-/* Misc. statistics */
-static struct pmcstat_stats {
-	int ps_exec_aout;	/* # a.out executables seen */
-	int ps_exec_elf;	/* # elf executables seen */
-	int ps_exec_errors;	/* # errors processing executables */
-	int ps_exec_indeterminable; /* # unknown executables seen */
-	int ps_samples_total;	/* total number of samples processed */
-	int ps_samples_skipped; /* #samples filtered out for any reason */
-	int ps_samples_unknown_offset;	/* #samples of rank 0 not in a map */
-	int ps_samples_indeterminable;	/* #samples in indeterminable images */
-	int ps_callchain_dubious_frames;/* #dubious frame pointers seen */
-} pmcstat_stats;
+int pmcstat_mergepmc;
 
+int pmcstat_pmcinfilter = 0; /* PMC filter for top mode. */
+float pmcstat_threshold = 0.5; /* Cost filter for top mode. */
 
 /*
  * Prototypes
  */
 
-static void	pmcstat_gmon_create_file(struct pmcstat_gmonfile *_pgf,
-    struct pmcstat_image *_image);
-static pmcstat_interned_string pmcstat_gmon_create_name(const char *_sd,
-    struct pmcstat_image *_img, pmc_id_t _pmcid);
-static void	pmcstat_gmon_map_file(struct pmcstat_gmonfile *_pgf);
-static void	pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *_pgf);
-
-static void pmcstat_image_determine_type(struct pmcstat_image *_image,
-    struct pmcstat_args *_a);
-static struct pmcstat_gmonfile *pmcstat_image_find_gmonfile(struct
-    pmcstat_image *_i, pmc_id_t _id);
 static struct pmcstat_image *pmcstat_image_from_path(pmcstat_interned_string
     _path, int _iskernelmodule);
-static void pmcstat_image_get_aout_params(struct pmcstat_image *_image,
-    struct pmcstat_args *_a);
-static void pmcstat_image_get_elf_params(struct pmcstat_image *_image,
-    struct pmcstat_args *_a);
-static void	pmcstat_image_increment_bucket(struct pmcstat_pcmap *_pcm,
-    uintfptr_t _pc, pmc_id_t _pmcid, struct pmcstat_args *_a);
+static void pmcstat_image_get_aout_params(struct pmcstat_image *_image);
+static void pmcstat_image_get_elf_params(struct pmcstat_image *_image);
 static void	pmcstat_image_link(struct pmcstat_process *_pp,
     struct pmcstat_image *_i, uintfptr_t _lpc);
 
 static void	pmcstat_pmcid_add(pmc_id_t _pmcid,
-    pmcstat_interned_string _name, struct pmcstat_args *_a);
-static const char *pmcstat_pmcid_to_name(pmc_id_t _pmcid);
+    pmcstat_interned_string _name);
 
 static void	pmcstat_process_aout_exec(struct pmcstat_process *_pp,
-    struct pmcstat_image *_image, uintfptr_t _entryaddr,
-    struct pmcstat_args *_a);
+    struct pmcstat_image *_image, uintfptr_t _entryaddr);
 static void	pmcstat_process_elf_exec(struct pmcstat_process *_pp,
-    struct pmcstat_image *_image, uintfptr_t _entryaddr,
-    struct pmcstat_args *_a);
+    struct pmcstat_image *_image, uintfptr_t _entryaddr);
 static void	pmcstat_process_exec(struct pmcstat_process *_pp,
-    pmcstat_interned_string _path, uintfptr_t _entryaddr,
-    struct pmcstat_args *_ao);
+    pmcstat_interned_string _path, uintfptr_t _entryaddr);
 static struct pmcstat_process *pmcstat_process_lookup(pid_t _pid,
     int _allocate);
-static struct pmcstat_pcmap *pmcstat_process_find_map(
-    struct pmcstat_process *_p, uintfptr_t _pc);
-
 static int	pmcstat_string_compute_hash(const char *_string);
 static void pmcstat_string_initialize(void);
-static pmcstat_interned_string pmcstat_string_intern(const char *_s);
-static pmcstat_interned_string pmcstat_string_lookup(const char *_s);
 static int	pmcstat_string_lookup_hash(pmcstat_interned_string _is);
 static void pmcstat_string_shutdown(void);
-static const char *pmcstat_string_unintern(pmcstat_interned_string _is);
-
 
 /*
  * A simple implementation of interned strings.  Each interned string
@@ -375,6 +265,16 @@ struct pmcstat_string {
 static LIST_HEAD(,pmcstat_string)	pmcstat_string_hash[PMCSTAT_NHASH];
 
 /*
+ * PMC count.
+ */
+int pmcstat_npmcs;
+
+/*
+ * PMC Top mode pause state.
+ */
+int pmcstat_pause;
+
+/*
  * Compute a 'hash' value for a string.
  */
 
@@ -394,7 +294,7 @@ pmcstat_string_compute_hash(const char *s)
  * interned structure.
  */
 
-static pmcstat_interned_string
+pmcstat_interned_string
 pmcstat_string_intern(const char *s)
 {
 	struct pmcstat_string *ps;
@@ -416,7 +316,7 @@ pmcstat_string_intern(const char *s)
 	return ((pmcstat_interned_string) ps);
 }
 
-static const char *
+const char *
 pmcstat_string_unintern(pmcstat_interned_string str)
 {
 	const char *s;
@@ -425,7 +325,7 @@ pmcstat_string_unintern(pmcstat_interned_string str)
 	return (s);
 }
 
-static pmcstat_interned_string
+pmcstat_interned_string
 pmcstat_string_lookup(const char *s)
 {
 	struct pmcstat_string *ps;
@@ -483,163 +383,13 @@ pmcstat_string_shutdown(void)
 }
 
 /*
- * Create a gmon.out file and size it.
- */
-
-static void
-pmcstat_gmon_create_file(struct pmcstat_gmonfile *pgf,
-    struct pmcstat_image *image)
-{
-	int fd;
-	size_t count;
-	struct gmonhdr gm;
-	const char *pathname;
-	char buffer[DEFAULT_BUFFER_SIZE];
-
-	pathname = pmcstat_string_unintern(pgf->pgf_name);
-	if ((fd = open(pathname, O_RDWR|O_NOFOLLOW|O_CREAT,
-		 S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0)
-		err(EX_OSERR, "ERROR: Cannot open \"%s\"", pathname);
-
-	gm.lpc = image->pi_start;
-	gm.hpc = image->pi_end;
-	gm.ncnt = (pgf->pgf_nbuckets * sizeof(HISTCOUNTER)) +
-	    sizeof(struct gmonhdr);
-	gm.version = GMONVERSION;
-	gm.profrate = 0;		/* use ticks */
-	gm.histcounter_type = 0;	/* compatibility with moncontrol() */
-	gm.spare[0] = gm.spare[1] = 0;
-
-	/* Write out the gmon header */
-	if (write(fd, &gm, sizeof(gm)) < 0)
-		goto error;
-
-	/* Zero fill the samples[] array */
-	(void) memset(buffer, 0, sizeof(buffer));
-
-	count = pgf->pgf_ndatabytes - sizeof(struct gmonhdr);
-	while (count > sizeof(buffer)) {
-		if (write(fd, &buffer, sizeof(buffer)) < 0)
-			goto error;
-		count -= sizeof(buffer);
-	}
-
-	if (write(fd, &buffer, count) < 0)
-		goto error;
-
-	(void) close(fd);
-
-	return;
-
- error:
-	err(EX_OSERR, "ERROR: Cannot write \"%s\"", pathname);
-}
-
-/*
- * Determine the full pathname of a gmon.out file for a given
- * (image,pmcid) combination.  Return the interned string.
- */
-
-pmcstat_interned_string
-pmcstat_gmon_create_name(const char *samplesdir, struct pmcstat_image *image,
-    pmc_id_t pmcid)
-{
-	const char *pmcname;
-	char fullpath[PATH_MAX];
-
-	pmcname = pmcstat_pmcid_to_name(pmcid);
-
-	(void) snprintf(fullpath, sizeof(fullpath),
-	    "%s/%s/%s", samplesdir, pmcname,
-	    pmcstat_string_unintern(image->pi_samplename));
-
-	return (pmcstat_string_intern(fullpath));
-}
-
-
-/*
- * Mmap in a gmon.out file for processing.
- */
-
-static void
-pmcstat_gmon_map_file(struct pmcstat_gmonfile *pgf)
-{
-	int fd;
-	const char *pathname;
-
-	pathname = pmcstat_string_unintern(pgf->pgf_name);
-
-	/* the gmon.out file must already exist */
-	if ((fd = open(pathname, O_RDWR | O_NOFOLLOW, 0)) < 0)
-		err(EX_OSERR, "ERROR: cannot open \"%s\"", pathname);
-
-	pgf->pgf_gmondata = mmap(NULL, pgf->pgf_ndatabytes,
-	    PROT_READ|PROT_WRITE, MAP_NOSYNC|MAP_SHARED, fd, 0);
-
-	if (pgf->pgf_gmondata == MAP_FAILED)
-		err(EX_OSERR, "ERROR: cannot map \"%s\"", pathname);
-
-	(void) close(fd);
-}
-
-/*
- * Unmap a gmon.out file after sync'ing its data to disk.
- */
-
-static void
-pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *pgf)
-{
-	(void) msync(pgf->pgf_gmondata, pgf->pgf_ndatabytes,
-	    MS_SYNC);
-	(void) munmap(pgf->pgf_gmondata, pgf->pgf_ndatabytes);
-	pgf->pgf_gmondata = NULL;
-}
-
-static void
-pmcstat_gmon_append_arc(struct pmcstat_image *image, pmc_id_t pmcid,
-    uintptr_t rawfrom, uintptr_t rawto, uint32_t count)
-{
-	struct rawarc arc;	/* from <sys/gmon.h> */
-	const char *pathname;
-	struct pmcstat_gmonfile *pgf;
-
-	if ((pgf = pmcstat_image_find_gmonfile(image, pmcid)) == NULL)
-		return;
-
-	if (pgf->pgf_file == NULL) {
-		pathname = pmcstat_string_unintern(pgf->pgf_name);
-		if ((pgf->pgf_file = fopen(pathname, "a")) == NULL)
-			return;
-	}
-
-	arc.raw_frompc = rawfrom + image->pi_vaddr;
-	arc.raw_selfpc = rawto + image->pi_vaddr;
-	arc.raw_count = count;
-
-	(void) fwrite(&arc, sizeof(arc), 1, pgf->pgf_file);
-
-}
-
-static struct pmcstat_gmonfile *
-pmcstat_image_find_gmonfile(struct pmcstat_image *image, pmc_id_t pmcid)
-{
-	struct pmcstat_gmonfile *pgf;
-	LIST_FOREACH(pgf, &image->pi_gmlist, pgf_next)
-	    if (pgf->pgf_pmcid == pmcid)
-		    return (pgf);
-	return (NULL);
-}
-
-
-/*
  * Determine whether a given executable image is an A.OUT object, and
  * if so, fill in its parameters from the text file.
  * Sets image->pi_type.
  */
 
 static void
-pmcstat_image_get_aout_params(struct pmcstat_image *image,
-    struct pmcstat_args *a)
+pmcstat_image_get_aout_params(struct pmcstat_image *image)
 {
 	int fd;
 	ssize_t nbytes;
@@ -655,7 +405,7 @@ pmcstat_image_get_aout_params(struct pmcstat_image *image,
 		    "unsupported \"%s\"", path);
 
 	(void) snprintf(buffer, sizeof(buffer), "%s%s",
-	    a->pa_fsroot, path);
+	    args.pa_fsroot, path);
 
 	if ((fd = open(buffer, O_RDONLY, 0)) < 0 ||
 	    (nbytes = read(fd, &ex, sizeof(ex))) < 0) {
@@ -702,7 +452,7 @@ pmcstat_symbol_compare(const void *a, const void *b)
  * Map an address to a symbol in an image.
  */
 
-static struct pmcstat_symbol *
+struct pmcstat_symbol *
 pmcstat_symbol_search(struct pmcstat_image *image, uintfptr_t addr)
 {
 	struct pmcstat_symbol sym;
@@ -825,12 +575,12 @@ pmcstat_image_add_symbols(struct pmcstat_image *image, Elf *e,
  */
 
 static void
-pmcstat_image_get_elf_params(struct pmcstat_image *image,
-    struct pmcstat_args *a)
+pmcstat_image_get_elf_params(struct pmcstat_image *image)
 {
 	int fd;
 	size_t i, nph, nsh;
 	const char *path, *elfbase;
+	char *p, *endp;
 	uintfptr_t minva, maxva;
 	Elf *e;
 	Elf_Scn *scn;
@@ -858,10 +608,10 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image,
 	 */
 	if (image->pi_iskernelmodule)
 		(void) snprintf(buffer, sizeof(buffer), "%s%s/%s",
-		    a->pa_fsroot, a->pa_kernel, path);
+		    args.pa_fsroot, args.pa_kernel, path);
 	else
 		(void) snprintf(buffer, sizeof(buffer), "%s%s",
-		    a->pa_fsroot, path);
+		    args.pa_fsroot, path);
 
 	e = NULL;
 	if ((fd = open(buffer, O_RDONLY, 0)) < 0 ||
@@ -960,6 +710,14 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image,
 	image->pi_type  = image_type;
 	image->pi_fullpath = pmcstat_string_intern(buffer);
 
+	/* Build display name
+	 */
+	endp = buffer;
+	for (p = buffer; *p; p++)
+		if (*p == '/')
+			endp = p+1;
+	image->pi_name = pmcstat_string_intern(endp);
+
  done:
 	(void) elf_end(e);
 	if (fd >= 0)
@@ -972,17 +730,16 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image,
  * If no handler claims the image, set its type to 'INDETERMINABLE'.
  */
 
-static void
-pmcstat_image_determine_type(struct pmcstat_image *image,
-    struct pmcstat_args *a)
+void
+pmcstat_image_determine_type(struct pmcstat_image *image)
 {
 	assert(image->pi_type == PMCSTAT_IMAGE_UNKNOWN);
 
 	/* Try each kind of handler in turn */
 	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
-		pmcstat_image_get_elf_params(image, a);
+		pmcstat_image_get_elf_params(image);
 	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
-		pmcstat_image_get_aout_params(image, a);
+		pmcstat_image_get_aout_params(image);
 
 	/*
 	 * Otherwise, remember that we tried to determine
@@ -1006,10 +763,8 @@ static struct pmcstat_image *
 pmcstat_image_from_path(pmcstat_interned_string internedpath,
     int iskernelmodule)
 {
-	int count, hash, nlen;
+	int hash;
 	struct pmcstat_image *pi;
-	char *sn;
-	char name[NAME_MAX];
 
 	hash = pmcstat_string_lookup_hash(internedpath);
 
@@ -1038,50 +793,12 @@ pmcstat_image_from_path(pmcstat_interned_string internedpath,
 	pi->pi_dynlinkerpath = NULL;
 	pi->pi_symbols = NULL;
 	pi->pi_symcount = 0;
+	pi->pi_addr2line = NULL;
 
-	/*
-	 * Look for a suitable name for the sample files associated
-	 * with this image: if `basename(path)`+".gmon" is available,
-	 * we use that, otherwise we try iterating through
-	 * `basename(path)`+ "~" + NNN + ".gmon" till we get a free
-	 * entry.
-	 */
-	if ((sn = basename(pmcstat_string_unintern(internedpath))) == NULL)
-		err(EX_OSERR, "ERROR: Cannot process \"%s\"",
-		    pmcstat_string_unintern(internedpath));
-
-	nlen = strlen(sn);
-	nlen = min(nlen, (int) (sizeof(name) - sizeof(".gmon")));
-
-	snprintf(name, sizeof(name), "%.*s.gmon", nlen, sn);
-
-	/* try use the unabridged name first */
-	if (pmcstat_string_lookup(name) == NULL)
-		pi->pi_samplename = pmcstat_string_intern(name);
-	else {
-		/*
-		 * Otherwise use a prefix from the original name and
-		 * upto 3 digits.
-		 */
-		nlen = strlen(sn);
-		nlen = min(nlen, (int) (sizeof(name)-sizeof("~NNN.gmon")));
-		count = 0;
-		do {
-			if (++count > 999)
-				errx(EX_CANTCREAT, "ERROR: cannot create a "
-				    "gmon file for \"%s\"", name);
-			snprintf(name, sizeof(name), "%.*s~%3.3d.gmon",
-			    nlen, sn, count);
-			if (pmcstat_string_lookup(name) == NULL) {
-				pi->pi_samplename =
-				    pmcstat_string_intern(name);
-				count = 0;
-			}
-		} while (count > 0);
-	}
-
-
-	LIST_INIT(&pi->pi_gmlist);
+	if (plugins[args.pa_pplugin].pl_initimage != NULL)
+		plugins[args.pa_pplugin].pl_initimage(pi);
+	if (plugins[args.pa_plugin].pl_initimage != NULL)
+		plugins[args.pa_plugin].pl_initimage(pi);
 
 	LIST_INSERT_HEAD(&pmcstat_image_hash[hash], pi, pi_next);
 
@@ -1089,94 +806,6 @@ pmcstat_image_from_path(pmcstat_interned_string internedpath,
 }
 
 /*
- * Increment the bucket in the gmon.out file corresponding to 'pmcid'
- * and 'pc'.
- */
-
-static void
-pmcstat_image_increment_bucket(struct pmcstat_pcmap *map, uintfptr_t pc,
-    pmc_id_t pmcid, struct pmcstat_args *a)
-{
-	struct pmcstat_image *image;
-	struct pmcstat_gmonfile *pgf;
-	uintfptr_t bucket;
-	HISTCOUNTER *hc;
-
-	assert(pc >= map->ppm_lowpc && pc < map->ppm_highpc);
-
-	image = map->ppm_image;
-
-	/*
-	 * If this is the first time we are seeing a sample for
-	 * this executable image, try determine its parameters.
-	 */
-	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
-		pmcstat_image_determine_type(image, a);
-
-	assert(image->pi_type != PMCSTAT_IMAGE_UNKNOWN);
-
-	/* Ignore samples in images that we know nothing about. */
-	if (image->pi_type == PMCSTAT_IMAGE_INDETERMINABLE) {
-		pmcstat_stats.ps_samples_indeterminable++;
-		return;
-	}
-
-	/*
-	 * Find the gmon file corresponding to 'pmcid', creating it if
-	 * needed.
-	 */
-	pgf = pmcstat_image_find_gmonfile(image, pmcid);
-	if (pgf == NULL) {
-		if ((pgf = calloc(1, sizeof(*pgf))) == NULL)
-			err(EX_OSERR, "ERROR:");
-
-		pgf->pgf_gmondata = NULL;	/* mark as unmapped */
-		pgf->pgf_name = pmcstat_gmon_create_name(a->pa_samplesdir,
-		    image, pmcid);
-		pgf->pgf_pmcid = pmcid;
-		assert(image->pi_end > image->pi_start);
-		pgf->pgf_nbuckets = (image->pi_end - image->pi_start) /
-		    FUNCTION_ALIGNMENT;	/* see <machine/profile.h> */
-		pgf->pgf_ndatabytes = sizeof(struct gmonhdr) +
-		    pgf->pgf_nbuckets * sizeof(HISTCOUNTER);
-		pgf->pgf_nsamples = 0;
-		pgf->pgf_file = NULL;
-
-		pmcstat_gmon_create_file(pgf, image);
-
-		LIST_INSERT_HEAD(&image->pi_gmlist, pgf, pgf_next);
-	}
-
-	/*
-	 * Map the gmon file in if needed.  It may have been mapped
-	 * out under memory pressure.
-	 */
-	if (pgf->pgf_gmondata == NULL)
-		pmcstat_gmon_map_file(pgf);
-
-	assert(pgf->pgf_gmondata != NULL);
-
-	/*
-	 *
-	 */
-
-	bucket = (pc - map->ppm_lowpc) / FUNCTION_ALIGNMENT;
-
-	assert(bucket < pgf->pgf_nbuckets);
-
-	hc = (HISTCOUNTER *) ((uintptr_t) pgf->pgf_gmondata +
-	    sizeof(struct gmonhdr));
-
-	/* saturating add */
-	if (hc[bucket] < 0xFFFFU)  /* XXX tie this to sizeof(HISTCOUNTER) */
-		hc[bucket]++;
-	else /* mark that an overflow occurred */
-		pgf->pgf_overflow = 1;
-
-	pgf->pgf_nsamples++;
-}
-
-/*
  * Record the fact that PC values from 'start' to 'end' come from
  * image 'image'.
  */
@@ -1284,72 +913,181 @@ pmcstat_image_unmap(struct pmcstat_process *pp, uintfptr_t start,
 }
 
 /*
+ * Resolve file name and line number for the given address.
+ */
+int
+pmcstat_image_addr2line(struct pmcstat_image *image, uintfptr_t addr,
+    char *sourcefile, size_t sourcefile_len, unsigned *sourceline,
+    char *funcname, size_t funcname_len)
+{
+	static int addr2line_warn = 0;
+
+	char *sep, cmdline[PATH_MAX], imagepath[PATH_MAX];
+	int fd;
+
+	if (image->pi_addr2line == NULL) {
+		snprintf(imagepath, sizeof(imagepath), "%s.symbols",
+		    pmcstat_string_unintern(image->pi_fullpath));
+		fd = open(imagepath, O_RDONLY);
+		if (fd < 0) {
+			snprintf(imagepath, sizeof(imagepath), "%s",
+			    pmcstat_string_unintern(image->pi_fullpath));
+		} else
+			close(fd);
+		snprintf(cmdline, sizeof(cmdline), "addr2line -Cfe \"%s\"",
+		    imagepath);
+		image->pi_addr2line = popen(cmdline, "r+");
+		if (image->pi_addr2line == NULL) {
+			if (!addr2line_warn) {
+				addr2line_warn = 1;
+				warnx("WARNING: addr2line is needed"
+				    "for source code information.");
+			}
+			return (0);
+		}
+	}
+
+	if (feof(image->pi_addr2line) || ferror(image->pi_addr2line)) {
+		warnx("WARNING: addr2line pipe error");
+		pclose(image->pi_addr2line);
+		image->pi_addr2line = NULL;
+		return (0);
+	}
+
+	fprintf(image->pi_addr2line, "%p\n", (void *)addr);
+
+	if (fgets(funcname, funcname_len, image->pi_addr2line) == NULL) {
+		warnx("WARNING: addr2line function name read error");
+		return (0);
+	}
+	sep = strchr(funcname, '\n');
+	if (sep != NULL)
+		*sep = '\0';
+
+	if (fgets(sourcefile, sourcefile_len, image->pi_addr2line) == NULL) {
+		warnx("WARNING: addr2line source file read error");
+		return (0);
+	}
+	sep = strchr(sourcefile, ':');
+	if (sep == NULL) {
+		warnx("WARNING: addr2line source line separator missing");
+		return (0);
+	}
+	*sep = '\0';
+	*sourceline = atoi(sep+1);
+	if (*sourceline == 0)
+		return (0);
+
+	return (1);
+}
+
+/*
  * Add a {pmcid,name} mapping.
  */
 
 static void
-pmcstat_pmcid_add(pmc_id_t pmcid, pmcstat_interned_string ps,
-    struct pmcstat_args *a)
+pmcstat_pmcid_add(pmc_id_t pmcid, pmcstat_interned_string ps)
 {
-	struct pmcstat_pmcrecord *pr;
-	struct stat st;
-	char fullpath[PATH_MAX];
+	struct pmcstat_pmcrecord *pr, *prm;
 
 	/* Replace an existing name for the PMC. */
+	prm = NULL;
 	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
-	    if (pr->pr_pmcid == pmcid) {
-		    pr->pr_pmcname = ps;
-		    return;
-	    }
+		if (pr->pr_pmcid == pmcid) {
+			pr->pr_pmcname = ps;
+			return;
+		} else if (pr->pr_pmcname == ps)
+			prm = pr;
 
 	/*
-	 * Otherwise, allocate a new descriptor and create the
-	 * appropriate directory to hold gmon.out files.
+	 * Otherwise, allocate a new descriptor and call the
+	 * plugins hook.
 	 */
 	if ((pr = malloc(sizeof(*pr))) == NULL)
 		err(EX_OSERR, "ERROR: Cannot allocate pmc record");
 
 	pr->pr_pmcid = pmcid;
 	pr->pr_pmcname = ps;
-	LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next);
+	pr->pr_pmcin = pmcstat_npmcs++;
+	pr->pr_merge = prm == NULL ? pr : prm;
 
-	(void) snprintf(fullpath, sizeof(fullpath), "%s/%s", a->pa_samplesdir,
-	    pmcstat_string_unintern(ps));
-
-	/* If the path name exists, it should be a directory */
-	if (stat(fullpath, &st) == 0 && S_ISDIR(st.st_mode))
-		return;
+	LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next);
 
-	if (mkdir(fullpath, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) < 0)
-		err(EX_OSERR, "ERROR: Cannot create directory \"%s\"",
-		    fullpath);
+	if (plugins[args.pa_pplugin].pl_newpmc != NULL)
+		plugins[args.pa_pplugin].pl_newpmc(ps, pr);
+	if (plugins[args.pa_plugin].pl_newpmc != NULL)
+		plugins[args.pa_plugin].pl_newpmc(ps, pr);
 }
 
 /*
  * Given a pmcid in use, find its human-readable name.
  */
 
-static const char *
+const char *
 pmcstat_pmcid_to_name(pmc_id_t pmcid)
 {
 	struct pmcstat_pmcrecord *pr;
-	char fullpath[PATH_MAX];
 
 	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
 	    if (pr->pr_pmcid == pmcid)
 		    return (pmcstat_string_unintern(pr->pr_pmcname));
 
-	/* create a default name and add this entry */
-	if ((pr = malloc(sizeof(*pr))) == NULL)
-		err(EX_OSERR, "ERROR: ");
-	pr->pr_pmcid = pmcid;
+	err(EX_SOFTWARE, "ERROR: cannot find pmcid");
+	return NULL;
+}
 
-	(void) snprintf(fullpath, sizeof(fullpath), "%X", (unsigned int) pmcid);
-	pr->pr_pmcname = pmcstat_string_intern(fullpath);
+/*
+ * Convert PMC index to name.
+ */
 
-	LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next);
+const char *
+pmcstat_pmcindex_to_name(int pmcin)
+{
+	struct pmcstat_pmcrecord *pr;
+
+	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
+		if (pr->pr_pmcin == pmcin)
+			return pmcstat_string_unintern(pr->pr_pmcname);
+
+	err(EX_SOFTWARE, "ERROR: cannot find pmcid name");
+	return NULL;
+}
+
+/*
+ * Return PMC record with given index.
+ */
+
+struct pmcstat_pmcrecord *
+pmcstat_pmcindex_to_pmcr(int pmcin)
+{
+	struct pmcstat_pmcrecord *pr;
+
+	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next)
+		if (pr->pr_pmcin == pmcin)
+			return pr;
 
-	return (pmcstat_string_unintern(pr->pr_pmcname));
+	err(EX_SOFTWARE, "ERROR: invalid pmcindex");
+	return NULL;
+}
+
+/*
+ * Get PMC record by id, apply merge policy.
+ */
+
+static struct pmcstat_pmcrecord *
+pmcstat_lookup_pmcid(pmc_id_t pmcid)
+{
+	struct pmcstat_pmcrecord *pr;
+
+	LIST_FOREACH(pr, &pmcstat_pmcs, pr_next) {
+		if (pr->pr_pmcid == pmcid) {
+			if (pmcstat_mergepmc)
+				return pr->pr_merge;
+			return pr;
+		}
+	}
+
+	return NULL;
 }
 
 /*
@@ -1358,13 +1096,11 @@ pmcstat_pmcid_to_name(pmc_id_t pmcid)
 
 static void
 pmcstat_process_aout_exec(struct pmcstat_process *pp,
-    struct pmcstat_image *image, uintfptr_t entryaddr,
-    struct pmcstat_args *a)
+    struct pmcstat_image *image, uintfptr_t entryaddr)
 {
 	(void) pp;
 	(void) image;
 	(void) entryaddr;
-	(void) a;
 	/* TODO Implement a.out handling */
 }
 
@@ -1374,8 +1110,7 @@ pmcstat_process_aout_exec(struct pmcstat_process *pp,
 
 static void
 pmcstat_process_elf_exec(struct pmcstat_process *pp,
-    struct pmcstat_image *image, uintfptr_t entryaddr,
-    struct pmcstat_args *a)
+    struct pmcstat_image *image, uintfptr_t entryaddr)
 {
 	uintmax_t libstart;
 	struct pmcstat_image *rtldimage;
@@ -1414,8 +1149,7 @@ pmcstat_process_elf_exec(struct pmcstat_process *pp,
 		 * this we can figure out the address where the
 		 * runtime loader's file object had been mapped to.
 		 */
-		rtldimage = pmcstat_image_from_path(image->pi_dynlinkerpath,
-		    0);
+		rtldimage = pmcstat_image_from_path(image->pi_dynlinkerpath, 0);
 		if (rtldimage == NULL) {
 			warnx("WARNING: Cannot find image for \"%s\".",
 			    pmcstat_string_unintern(image->pi_dynlinkerpath));
@@ -1424,7 +1158,7 @@ pmcstat_process_elf_exec(struct pmcstat_process *pp,
 		}
 
 		if (rtldimage->pi_type == PMCSTAT_IMAGE_UNKNOWN)
-			pmcstat_image_get_elf_params(rtldimage, a);
+			pmcstat_image_get_elf_params(rtldimage);
 
 		if (rtldimage->pi_type != PMCSTAT_IMAGE_ELF32 &&
 		    rtldimage->pi_type != PMCSTAT_IMAGE_ELF64) {
@@ -1495,8 +1229,7 @@ pmcstat_process_lookup(pid_t pid, int allocate)
 
 static void
 pmcstat_process_exec(struct pmcstat_process *pp,
-    pmcstat_interned_string path, uintfptr_t entryaddr,
-    struct pmcstat_args *a)
+    pmcstat_interned_string path, uintfptr_t entryaddr)
 {
 	struct pmcstat_image *image;
 
@@ -1506,7 +1239,7 @@ pmcstat_process_exec(struct pmcstat_process *pp,
 	}
 
 	if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
-		pmcstat_image_determine_type(image, a);
+		pmcstat_image_determine_type(image);
 
 	assert(image->pi_type != PMCSTAT_IMAGE_UNKNOWN);
 
@@ -1514,12 +1247,12 @@ pmcstat_process_exec(struct pmcstat_process *pp,
 	case PMCSTAT_IMAGE_ELF32:
 	case PMCSTAT_IMAGE_ELF64:
 		pmcstat_stats.ps_exec_elf++;
-		pmcstat_process_elf_exec(pp, image, entryaddr, a);
+		pmcstat_process_elf_exec(pp, image, entryaddr);
 		break;
 
 	case PMCSTAT_IMAGE_AOUT:
 		pmcstat_stats.ps_exec_aout++;
-		pmcstat_process_aout_exec(pp, image, entryaddr, a);
+		pmcstat_process_aout_exec(pp, image, entryaddr);
 		break;
 
 	case PMCSTAT_IMAGE_INDETERMINABLE:
@@ -1537,7 +1270,7 @@ pmcstat_process_exec(struct pmcstat_process *pp,
  * Find the map entry associated with process 'p' at PC value 'pc'.
  */
 
-static struct pmcstat_pcmap *
+struct pmcstat_pcmap *
 pmcstat_process_find_map(struct pmcstat_process *p, uintfptr_t pc)
 {
 	struct pmcstat_pcmap *ppm;
@@ -1552,444 +1285,36 @@ pmcstat_process_find_map(struct pmcstat_process *p, uintfptr_t pc)
 	return (NULL);
 }
 
-static struct pmcstat_cgnode *
-pmcstat_cgnode_allocate(struct pmcstat_image *image, uintfptr_t pc)
-{
-	struct pmcstat_cgnode *cg;
-
-	if ((cg = malloc(sizeof(*cg))) == NULL)
-		err(EX_OSERR, "ERROR: Cannot allocate callgraph node");
-
-	cg->pcg_image = image;
-	cg->pcg_func = pc;
-
-	cg->pcg_count = 0;
-	cg->pcg_nchildren = 0;
-	LIST_INIT(&cg->pcg_children);
-
-	return (cg);
-}
-
-/*
- * Free a node and its children.
- */
-static void
-pmcstat_cgnode_free(struct pmcstat_cgnode *cg)
-{
-	struct pmcstat_cgnode *cgc, *cgtmp;
-
-	LIST_FOREACH_SAFE(cgc, &cg->pcg_children, pcg_sibling, cgtmp)
-		pmcstat_cgnode_free(cgc);
-	free(cg);
-}
-
-/*
- * Look for a callgraph node associated with pmc `pmcid' in the global
- * hash table that corresponds to the given `pc' value in the process
- * `pp'.
- */
-static struct pmcstat_cgnode *
-pmcstat_cgnode_hash_lookup_pc(struct pmcstat_process *pp, uint32_t pmcid,
-    uintfptr_t pc, int usermode)
-{
-	struct pmcstat_pcmap *ppm;
-	struct pmcstat_symbol *sym;
-	struct pmcstat_image *image;
-	struct pmcstat_cgnode *cg;
-	struct pmcstat_cgnode_hash *h;
-	uintfptr_t loadaddress;
-	unsigned int i, hash;
-
-	ppm = pmcstat_process_find_map(usermode ? pp : pmcstat_kernproc, pc);
-	if (ppm == NULL)
-		return (NULL);
-
-	image = ppm->ppm_image;
-
-	loadaddress = ppm->ppm_lowpc + image->pi_vaddr - image->pi_start;
-	pc -= loadaddress;	/* Convert to an offset in the image. */
-
-	/*
-	 * Try determine the function at this offset.  If we can't
-	 * find a function round leave the `pc' value alone.
-	 */
-	if ((sym = pmcstat_symbol_search(image, pc)) != NULL)
-		pc = sym->ps_start;
-
-	for (hash = i = 0; i < sizeof(uintfptr_t); i++)
-		hash += (pc >> i) & 0xFF;
-
-	hash &= PMCSTAT_HASH_MASK;
-
-	cg = NULL;
-	LIST_FOREACH(h, &pmcstat_cgnode_hash[hash], pch_next)
-	{
-		if (h->pch_pmcid != pmcid)
-			continue;
-
-		cg = h->pch_cgnode;
-
-		assert(cg != NULL);
-
-		if (cg->pcg_image == image && cg->pcg_func == pc)
-			return (cg);
-	}
-
-	/*
-	 * We haven't seen this (pmcid, pc) tuple yet, so allocate a
-	 * new callgraph node and a new hash table entry for it.
-	 */
-	cg = pmcstat_cgnode_allocate(image, pc);
-	if ((h = malloc(sizeof(*h))) == NULL)
-		err(EX_OSERR, "ERROR: Could not allocate callgraph node");
-
-	h->pch_pmcid = pmcid;
-	h->pch_cgnode = cg;
-	LIST_INSERT_HEAD(&pmcstat_cgnode_hash[hash], h, pch_next);
-
-	pmcstat_cgnode_hash_count++;
-
-	return (cg);
-}
-
-/*
- * Compare two callgraph nodes for sorting.
- */
-static int
-pmcstat_cgnode_compare(const void *a, const void *b)
-{
-	const struct pmcstat_cgnode *const *pcg1, *const *pcg2, *cg1, *cg2;
-
-	pcg1 = (const struct pmcstat_cgnode *const *) a;
-	cg1 = *pcg1;
-	pcg2 = (const struct pmcstat_cgnode *const *) b;
-	cg2 = *pcg2;
-
-	/* Sort in reverse order */
-	if (cg1->pcg_count < cg2->pcg_count)
-		return (1);
-	if (cg1->pcg_count > cg2->pcg_count)
-		return (-1);
-	return (0);
-}
-
-/*
- * Find (allocating if a needed) a callgraph node in the given
- * parent with the same (image, pcoffset) pair.
- */
-
-static struct pmcstat_cgnode *
-pmcstat_cgnode_find(struct pmcstat_cgnode *parent, struct pmcstat_image *image,
-    uintfptr_t pcoffset)
-{
-	struct pmcstat_cgnode *child;
-
-	LIST_FOREACH(child, &parent->pcg_children, pcg_sibling) {
-		if (child->pcg_image == image &&
-		    child->pcg_func == pcoffset)
-			return (child);
-	}
-
-	/*
-	 * Allocate a new structure.
-	 */
-
-	child = pmcstat_cgnode_allocate(image, pcoffset);
-
-	/*
-	 * Link it into the parent.
-	 */
-	LIST_INSERT_HEAD(&parent->pcg_children, child, pcg_sibling);
-	parent->pcg_nchildren++;
-
-	return (child);
-}
-
-/*
- * Print one callgraph node.  The output format is:
- *
- * indentation %(parent's samples) #nsamples function@object
- */
-static void
-pmcstat_cgnode_print(struct pmcstat_args *a, struct pmcstat_cgnode *cg,
-    int depth, uint32_t total)
-{
-	uint32_t n;
-	const char *space;
-	struct pmcstat_symbol *sym;
-	struct pmcstat_cgnode **sortbuffer, **cgn, *pcg;
-
-	space = " ";
-
-	if (depth > 0)
-		(void) fprintf(a->pa_graphfile, "%*s", depth, space);
-
-	if (cg->pcg_count == total)
-		(void) fprintf(a->pa_graphfile, "100.0%% ");
-	else
-		(void) fprintf(a->pa_graphfile, "%05.2f%% ",
-		    100.0 * cg->pcg_count / total);
-
-	n = fprintf(a->pa_graphfile, " [%u] ", cg->pcg_count);
-
-	/* #samples is a 12 character wide field. */
-	if (n < 12)
-		(void) fprintf(a->pa_graphfile, "%*s", 12 - n, space);
-
-	if (depth > 0)
-		(void) fprintf(a->pa_graphfile, "%*s", depth, space);
-
-	sym = pmcstat_symbol_search(cg->pcg_image, cg->pcg_func);
-	if (sym)
-		(void) fprintf(a->pa_graphfile, "%s",
-		    pmcstat_string_unintern(sym->ps_name));
-	else
-		(void) fprintf(a->pa_graphfile, "%p",
-		    (void *) (cg->pcg_image->pi_vaddr + cg->pcg_func));
-
-	if (pmcstat_previous_filename_printed !=
-	    cg->pcg_image->pi_fullpath) {
-		pmcstat_previous_filename_printed = cg->pcg_image->pi_fullpath;
-		(void) fprintf(a->pa_graphfile, " @ %s\n",
-		    pmcstat_string_unintern(
-		    pmcstat_previous_filename_printed));
-	} else
-		(void) fprintf(a->pa_graphfile, "\n");
-
-	if (cg->pcg_nchildren == 0)
-		return;
-
-	if ((sortbuffer = (struct pmcstat_cgnode **)
-		malloc(sizeof(struct pmcstat_cgnode *) *
-		    cg->pcg_nchildren)) == NULL)
-		err(EX_OSERR, "ERROR: Cannot print callgraph");
-	cgn = sortbuffer;
-
-	LIST_FOREACH(pcg, &cg->pcg_children, pcg_sibling)
-	    *cgn++ = pcg;
-
-	assert(cgn - sortbuffer == (int) cg->pcg_nchildren);
-
-	qsort(sortbuffer, cg->pcg_nchildren, sizeof(struct pmcstat_cgnode *),
-	    pmcstat_cgnode_compare);
-
-	for (cgn = sortbuffer, n = 0; n < cg->pcg_nchildren; n++, cgn++)
-		pmcstat_cgnode_print(a, *cgn, depth+1, cg->pcg_count);
-
-	free(sortbuffer);
-}
-
-/*
- * Record a callchain.
- */
-
-static void
-pmcstat_record_callchain(struct pmcstat_process *pp, uint32_t pmcid,
-    uint32_t nsamples, uintfptr_t *cc, int usermode, struct pmcstat_args *a)
-{
-	uintfptr_t pc, loadaddress;
-	uint32_t n;
-	struct pmcstat_image *image;
-	struct pmcstat_pcmap *ppm;
-	struct pmcstat_symbol *sym;
-	struct pmcstat_cgnode *parent, *child;
-
-	/*
-	 * Find the callgraph node recorded in the global hash table
-	 * for this (pmcid, pc).
-	 */
-
-	pc = cc[0];
-	parent = pmcstat_cgnode_hash_lookup_pc(pp, pmcid, pc, usermode);
-	if (parent == NULL) {
-		pmcstat_stats.ps_callchain_dubious_frames++;
-		return;
-	}
-
-	parent->pcg_count++;
-
-	/*
-	 * For each return address in the call chain record, subject
-	 * to the maximum depth desired.
-	 * - Find the image associated with the sample.  Stop if there
-	 *   there is no valid image at that address.
-	 * - Find the function that overlaps the return address.
-	 * - If found: use the start address of the function.
-	 *   If not found (say an object's symbol table is not present or
-	 *   is incomplete), round down to th gprof bucket granularity.
-	 * - Convert return virtual address to an offset in the image.
-	 * - Look for a child with the same {offset,image} tuple,
-	 *   inserting one if needed.
-	 * - Increment the count of occurrences of the child.
-	 */
-
-	for (n = 1; n < (uint32_t) a->pa_graphdepth && n < nsamples; n++,
-	    parent = child) {
-		pc = cc[n];
-
-		ppm = pmcstat_process_find_map(usermode ? pp :
-		    pmcstat_kernproc, pc);
-		if (ppm == NULL)
-			return;
-
-		image = ppm->ppm_image;
-		loadaddress = ppm->ppm_lowpc + image->pi_vaddr -
-		    image->pi_start;
-		pc -= loadaddress;
-
-		if ((sym = pmcstat_symbol_search(image, pc)) != NULL)
-			pc = sym->ps_start;
-
-		child = pmcstat_cgnode_find(parent, image, pc);
-		child->pcg_count++;
-	}
-}
-
-/*
- * Printing a callgraph for a PMC.
- */
-static void
-pmcstat_callgraph_print_for_pmcid(struct pmcstat_args *a,
-    struct pmcstat_pmcrecord *pmcr)
-{
-	int n, nentries;
-	uint32_t nsamples, pmcid;
-	struct pmcstat_cgnode **sortbuffer, **cgn;
-	struct pmcstat_cgnode_hash *pch;
-
-	/*
-	 * We pull out all callgraph nodes in the top-level hash table
-	 * with a matching PMC id.  We then sort these based on the
-	 * frequency of occurrence.  Each callgraph node is then
-	 * printed.
-	 */
-
-	nsamples = 0;
-	pmcid = pmcr->pr_pmcid;
-	if ((sortbuffer = (struct pmcstat_cgnode **)
-	    malloc(sizeof(struct pmcstat_cgnode *) *
-	    pmcstat_cgnode_hash_count)) == NULL)
-		err(EX_OSERR, "ERROR: Cannot sort callgraph");
-	cgn = sortbuffer;
-
-	memset(sortbuffer, 0xFF, pmcstat_cgnode_hash_count *
-	    sizeof(struct pmcstat_cgnode **));
-
-	for (n = 0; n < PMCSTAT_NHASH; n++)
-		LIST_FOREACH(pch, &pmcstat_cgnode_hash[n], pch_next)
-		    if (pch->pch_pmcid == pmcid) {
-			    nsamples += pch->pch_cgnode->pcg_count;
-			    *cgn++ = pch->pch_cgnode;
-		    }
-
-	nentries = cgn - sortbuffer;
-	assert(nentries <= pmcstat_cgnode_hash_count);
-
-	if (nentries == 0)
-		return;
-
-	qsort(sortbuffer, nentries, sizeof(struct pmcstat_cgnode *),
-	    pmcstat_cgnode_compare);
-
-	(void) fprintf(a->pa_graphfile,
-	    "@ %s [%u samples]\n\n",
-	    pmcstat_string_unintern(pmcr->pr_pmcname),
-	    nsamples);
-
-	for (cgn = sortbuffer, n = 0; n < nentries; n++, cgn++) {
-		pmcstat_previous_filename_printed = NULL;
-		pmcstat_cgnode_print(a, *cgn, 0, nsamples);
-		(void) fprintf(a->pa_graphfile, "\n");
-	}
-
-	free(sortbuffer);
-}
-
-/*
- * Print out callgraphs.
- */
-
-static void
-pmcstat_callgraph_print(struct pmcstat_args *a)
-{
-	struct pmcstat_pmcrecord *pmcr;
-
-	LIST_FOREACH(pmcr, &pmcstat_pmcs, pr_next)
-	    pmcstat_callgraph_print_for_pmcid(a, pmcr);
-}
-
-static void
-pmcstat_cgnode_do_gmon_arcs(struct pmcstat_cgnode *cg, pmc_id_t pmcid)
-{
-	struct pmcstat_cgnode *cgc;
-
-	/*
-	 * Look for child nodes that belong to the same image.
-	 */
-
-	LIST_FOREACH(cgc, &cg->pcg_children, pcg_sibling) {
-		if (cgc->pcg_image == cg->pcg_image)
-			pmcstat_gmon_append_arc(cg->pcg_image, pmcid,
-			    cgc->pcg_func, cg->pcg_func, cgc->pcg_count);
-		if (cgc->pcg_nchildren > 0)
-			pmcstat_cgnode_do_gmon_arcs(cgc, pmcid);
-	}
-}
-
-static void
-pmcstat_callgraph_do_gmon_arcs_for_pmcid(pmc_id_t pmcid)
-{
-	int n;
-	struct pmcstat_cgnode_hash *pch;
-
-	for (n = 0; n < PMCSTAT_NHASH; n++)
-		LIST_FOREACH(pch, &pmcstat_cgnode_hash[n], pch_next)
-			if (pch->pch_pmcid == pmcid &&
-			    pch->pch_cgnode->pcg_nchildren > 1)
-				pmcstat_cgnode_do_gmon_arcs(pch->pch_cgnode,
-				    pmcid);
-}
-
-
-static void
-pmcstat_callgraph_do_gmon_arcs(void)
-{
-	struct pmcstat_pmcrecord *pmcr;
-
-	LIST_FOREACH(pmcr, &pmcstat_pmcs, pr_next)
-		pmcstat_callgraph_do_gmon_arcs_for_pmcid(pmcr->pr_pmcid);
-}
-
 /*
  * Convert a hwpmc(4) log to profile information.  A system-wide
  * callgraph is generated if FLAG_DO_CALLGRAPHS is set.  gmon.out
  * files usable by gprof(1) are created if FLAG_DO_GPROF is set.
  */
 static int
-pmcstat_analyze_log(struct pmcstat_args *a)
+pmcstat_analyze_log(void)
 {
 	uint32_t cpu, cpuflags;
-	uintfptr_t pc, newpc;
+	uintfptr_t pc;
 	pid_t pid;
 	struct pmcstat_image *image;
-	struct pmcstat_symbol *sym;
 	struct pmcstat_process *pp, *ppnew;
 	struct pmcstat_pcmap *ppm, *ppmtmp;
 	struct pmclog_ev ev;
+	struct pmcstat_pmcrecord *pmcr;
 	pmcstat_interned_string image_path;
 
-	assert(a->pa_flags & FLAG_DO_ANALYSIS);
+	assert(args.pa_flags & FLAG_DO_ANALYSIS);
 
 	if (elf_version(EV_CURRENT) == EV_NONE)
 		err(EX_UNAVAILABLE, "Elf library intialization failed");
 
-	while (pmclog_read(a->pa_logparser, &ev) == 0) {
+	while (pmclog_read(args.pa_logparser, &ev) == 0) {
 		assert(ev.pl_state == PMCLOG_OK);
 
 		switch (ev.pl_type) {
 		case PMCLOG_TYPE_INITIALIZE:
 			if ((ev.pl_u.pl_i.pl_version & 0xFF000000) !=
-			    PMC_VERSION_MAJOR << 24 && a->pa_verbosity > 0)
+			    PMC_VERSION_MAJOR << 24 && args.pa_verbosity > 0)
 				warnx("WARNING: Log version 0x%x does not "
 				    "match compiled version 0x%x.",
 				    ev.pl_u.pl_i.pl_version,
@@ -2019,7 +1344,7 @@ pmcstat_analyze_log(struct pmcstat_args *a)
 			    pl_pathname);
 			image = pmcstat_image_from_path(image_path, pid == -1);
 			if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN)
-				pmcstat_image_determine_type(image, a);
+				pmcstat_image_determine_type(image);
 			if (image->pi_type != PMCSTAT_IMAGE_INDETERMINABLE)
 				pmcstat_image_link(pp, image,
 				    ev.pl_u.pl_mi.pl_start);
@@ -2059,16 +1384,23 @@ pmcstat_analyze_log(struct pmcstat_args *a)
 			pc = ev.pl_u.pl_s.pl_pc;
 			pp = pmcstat_process_lookup(ev.pl_u.pl_s.pl_pid,
 			    PMCSTAT_ALLOCATE);
-			if ((ppm = pmcstat_process_find_map(pp, pc)) == NULL &&
-			    (ppm = pmcstat_process_find_map(pmcstat_kernproc,
-				pc)) == NULL) {	/* unknown process,offset pair */
-				pmcstat_stats.ps_samples_unknown_offset++;
-				break;
-			}
 
-			pmcstat_image_increment_bucket(ppm, pc,
-			    ev.pl_u.pl_s.pl_pmcid, a);
+			/* Get PMC record. */
+			pmcr = pmcstat_lookup_pmcid(ev.pl_u.pl_s.pl_pmcid);
+			assert(pmcr != NULL);
 
+			/*
+			 * Call the plugins processing
+			 * TODO: move pmcstat_process_find_map inside plugins
+			 */
+
+			if (plugins[args.pa_pplugin].pl_process != NULL)
+				plugins[args.pa_pplugin].pl_process(
+				    pp, pmcr, 1, &pc,
+				    pmcstat_process_find_map(pp, pc) != NULL, 0);
+			plugins[args.pa_plugin].pl_process(
+			    pp, pmcr, 1, &pc,
+			    pmcstat_process_find_map(pp, pc) != NULL, 0);
 			break;
 
 		case PMCLOG_TYPE_CALLCHAIN:
@@ -2078,7 +1410,7 @@ pmcstat_analyze_log(struct pmcstat_args *a)
 			cpu = PMC_CALLCHAIN_CPUFLAGS_TO_CPU(cpuflags);
 
 			/* Filter on the CPU id. */
-			if ((a->pa_cpumask & (1 << cpu)) == 0) {
+			if ((args.pa_cpumask & (1 << cpu)) == 0) {
 				pmcstat_stats.ps_samples_skipped++;
 				break;
 			}
@@ -2086,45 +1418,27 @@ pmcstat_analyze_log(struct pmcstat_args *a)
 			pp = pmcstat_process_lookup(ev.pl_u.pl_cc.pl_pid,
 			    PMCSTAT_ALLOCATE);
 
-			if ((a->pa_flags & FLAG_WANTS_MAPPINGS) == 0)
-				pmcstat_record_callchain(pp,
-				    ev.pl_u.pl_cc.pl_pmcid,
-				    ev.pl_u.pl_cc.pl_npc, ev.pl_u.pl_cc.pl_pc,
-			PMC_CALLCHAIN_CPUFLAGS_TO_USERMODE(cpuflags), a);
-
-			if ((a->pa_flags &
-			    (FLAG_DO_GPROF | FLAG_WANTS_MAPPINGS)) == 0)
-				break;
-
-			pc = ev.pl_u.pl_cc.pl_pc[0];
-			if (PMC_CALLCHAIN_CPUFLAGS_TO_USERMODE(cpuflags) == 0)
-				pp = pmcstat_kernproc;
-			ppm = pmcstat_process_find_map(pp, pc);
-			if (ppm == NULL) {
-
-				/* Unknown offset. */
-				pmcstat_stats.ps_samples_unknown_offset++;
-				break;
-			}
-			if (a->pa_flags & FLAG_WANTS_MAPPINGS) {
-				image = ppm->ppm_image;
-				newpc = pc - (ppm->ppm_lowpc +
-				    (image->pi_vaddr - image->pi_start));
-				sym = pmcstat_symbol_search(image, newpc);
-				if (sym == NULL)
-					break;
-				fprintf(a->pa_graphfile, "%p %s 0x%jx 0x%jx\n",
-				    (void *)pc,
-				    pmcstat_string_unintern(sym->ps_name),
-				    (uintmax_t)(sym->ps_start +
-				    image->pi_vaddr), (uintmax_t)(sym->ps_end +
-				    image->pi_vaddr));
-				break;
-			}
+			/* Get PMC record. */
+			pmcr = pmcstat_lookup_pmcid(ev.pl_u.pl_cc.pl_pmcid);
+			assert(pmcr != NULL);
 
-			pmcstat_image_increment_bucket(ppm, pc,
-			    ev.pl_u.pl_cc.pl_pmcid, a);
+			/*
+			 * Call the plugins processing
+			 */
 
+			if (plugins[args.pa_pplugin].pl_process != NULL)
+				plugins[args.pa_pplugin].pl_process(
+				    pp, pmcr,
+				    ev.pl_u.pl_cc.pl_npc,
+				    ev.pl_u.pl_cc.pl_pc,
+				    PMC_CALLCHAIN_CPUFLAGS_TO_USERMODE(cpuflags),
+				    cpu);
+			plugins[args.pa_plugin].pl_process(
+			    pp, pmcr,
+			    ev.pl_u.pl_cc.pl_npc,
+			    ev.pl_u.pl_cc.pl_pc,
+			    PMC_CALLCHAIN_CPUFLAGS_TO_USERMODE(cpuflags),
+			    cpu);
 			break;
 
 		case PMCLOG_TYPE_PMCALLOCATE:
@@ -2133,7 +1447,7 @@ pmcstat_analyze_log(struct pmcstat_args *a)
 			 * PMC and its name.
 			 */
 			pmcstat_pmcid_add(ev.pl_u.pl_a.pl_pmcid,
-			    pmcstat_string_intern(ev.pl_u.pl_a.pl_evname), a);
+			    pmcstat_string_intern(ev.pl_u.pl_a.pl_evname));
 			break;
 
 		case PMCLOG_TYPE_PROCEXEC:
@@ -2156,7 +1470,7 @@ pmcstat_analyze_log(struct pmcstat_args *a)
 				ev.pl_u.pl_x.pl_pathname);
 			assert(image_path != NULL);
 			pmcstat_process_exec(pp, image_path,
-			    ev.pl_u.pl_x.pl_entryaddr, a);
+			    ev.pl_u.pl_x.pl_entryaddr);
 			break;
 
 		case PMCLOG_TYPE_PROCEXIT:
@@ -2224,16 +1538,16 @@ pmcstat_analyze_log(struct pmcstat_args *a)
  */
 
 static int
-pmcstat_print_log(struct pmcstat_args *a)
+pmcstat_print_log(void)
 {
 	struct pmclog_ev ev;
 	uint32_t npc;
 
-	while (pmclog_read(a->pa_logparser, &ev) == 0) {
+	while (pmclog_read(args.pa_logparser, &ev) == 0) {
 		assert(ev.pl_state == PMCLOG_OK);
 		switch (ev.pl_type) {
 		case PMCLOG_TYPE_CALLCHAIN:
-			PMCSTAT_PRINT_ENTRY(a, "callchain",
+			PMCSTAT_PRINT_ENTRY("callchain",
 			    "%d 0x%x %d %d %c", ev.pl_u.pl_cc.pl_pid,
 			    ev.pl_u.pl_cc.pl_pmcid,
 			    PMC_CALLCHAIN_CPUFLAGS_TO_CPU(ev.pl_u.pl_cc. \
@@ -2241,95 +1555,95 @@ pmcstat_print_log(struct pmcstat_args *a)
 			    PMC_CALLCHAIN_CPUFLAGS_TO_USERMODE(ev.pl_u.pl_cc.\
 			        pl_cpuflags) ? 'u' : 's');
 			for (npc = 0; npc < ev.pl_u.pl_cc.pl_npc; npc++)
-				PMCSTAT_PRINT_ENTRY(a, "...", "%p",
+				PMCSTAT_PRINT_ENTRY("...", "%p",
 				    (void *) ev.pl_u.pl_cc.pl_pc[npc]);
 			break;
 		case PMCLOG_TYPE_CLOSELOG:
-			PMCSTAT_PRINT_ENTRY(a,"closelog",);
+			PMCSTAT_PRINT_ENTRY("closelog",);
 			break;
 		case PMCLOG_TYPE_DROPNOTIFY:
-			PMCSTAT_PRINT_ENTRY(a,"drop",);
+			PMCSTAT_PRINT_ENTRY("drop",);
 			break;
 		case PMCLOG_TYPE_INITIALIZE:
-			PMCSTAT_PRINT_ENTRY(a,"initlog","0x%x \"%s\"",
+			PMCSTAT_PRINT_ENTRY("initlog","0x%x \"%s\"",
 			    ev.pl_u.pl_i.pl_version,
 			    pmc_name_of_cputype(ev.pl_u.pl_i.pl_arch));
 			if ((ev.pl_u.pl_i.pl_version & 0xFF000000) !=
-			    PMC_VERSION_MAJOR << 24 && a->pa_verbosity > 0)
+			    PMC_VERSION_MAJOR << 24 && args.pa_verbosity > 0)
 				warnx("WARNING: Log version 0x%x != expected "
 				    "version 0x%x.", ev.pl_u.pl_i.pl_version,
 				    PMC_VERSION);
 			break;
 		case PMCLOG_TYPE_MAP_IN:
-			PMCSTAT_PRINT_ENTRY(a,"map-in","%d %p \"%s\"",
+			PMCSTAT_PRINT_ENTRY("map-in","%d %p \"%s\"",
 			    ev.pl_u.pl_mi.pl_pid,
 			    (void *) ev.pl_u.pl_mi.pl_start,
 			    ev.pl_u.pl_mi.pl_pathname);
 			break;
 		case PMCLOG_TYPE_MAP_OUT:
-			PMCSTAT_PRINT_ENTRY(a,"map-out","%d %p %p",
+			PMCSTAT_PRINT_ENTRY("map-out","%d %p %p",
 			    ev.pl_u.pl_mo.pl_pid,
 			    (void *) ev.pl_u.pl_mo.pl_start,
 			    (void *) ev.pl_u.pl_mo.pl_end);
 			break;
 		case PMCLOG_TYPE_PCSAMPLE:
-			PMCSTAT_PRINT_ENTRY(a,"sample","0x%x %d %p %c",
+			PMCSTAT_PRINT_ENTRY("sample","0x%x %d %p %c",
 			    ev.pl_u.pl_s.pl_pmcid,
 			    ev.pl_u.pl_s.pl_pid,
 			    (void *) ev.pl_u.pl_s.pl_pc,
 			    ev.pl_u.pl_s.pl_usermode ? 'u' : 's');
 			break;
 		case PMCLOG_TYPE_PMCALLOCATE:
-			PMCSTAT_PRINT_ENTRY(a,"allocate","0x%x \"%s\" 0x%x",
+			PMCSTAT_PRINT_ENTRY("allocate","0x%x \"%s\" 0x%x",
 			    ev.pl_u.pl_a.pl_pmcid,
 			    ev.pl_u.pl_a.pl_evname,
 			    ev.pl_u.pl_a.pl_flags);
 			break;
 		case PMCLOG_TYPE_PMCATTACH:
-			PMCSTAT_PRINT_ENTRY(a,"attach","0x%x %d \"%s\"",
+			PMCSTAT_PRINT_ENTRY("attach","0x%x %d \"%s\"",
 			    ev.pl_u.pl_t.pl_pmcid,
 			    ev.pl_u.pl_t.pl_pid,
 			    ev.pl_u.pl_t.pl_pathname);
 			break;
 		case PMCLOG_TYPE_PMCDETACH:
-			PMCSTAT_PRINT_ENTRY(a,"detach","0x%x %d",
+			PMCSTAT_PRINT_ENTRY("detach","0x%x %d",
 			    ev.pl_u.pl_d.pl_pmcid,
 			    ev.pl_u.pl_d.pl_pid);
 			break;
 		case PMCLOG_TYPE_PROCCSW:
-			PMCSTAT_PRINT_ENTRY(a,"cswval","0x%x %d %jd",
+			PMCSTAT_PRINT_ENTRY("cswval","0x%x %d %jd",
 			    ev.pl_u.pl_c.pl_pmcid,
 			    ev.pl_u.pl_c.pl_pid,
 			    ev.pl_u.pl_c.pl_value);
 			break;
 		case PMCLOG_TYPE_PROCEXEC:
-			PMCSTAT_PRINT_ENTRY(a,"exec","0x%x %d %p \"%s\"",
+			PMCSTAT_PRINT_ENTRY("exec","0x%x %d %p \"%s\"",
 			    ev.pl_u.pl_x.pl_pmcid,
 			    ev.pl_u.pl_x.pl_pid,
 			    (void *) ev.pl_u.pl_x.pl_entryaddr,
 			    ev.pl_u.pl_x.pl_pathname);
 			break;
 		case PMCLOG_TYPE_PROCEXIT:
-			PMCSTAT_PRINT_ENTRY(a,"exitval","0x%x %d %jd",
+			PMCSTAT_PRINT_ENTRY("exitval","0x%x %d %jd",
 			    ev.pl_u.pl_e.pl_pmcid,
 			    ev.pl_u.pl_e.pl_pid,
 			    ev.pl_u.pl_e.pl_value);
 			break;
 		case PMCLOG_TYPE_PROCFORK:
-			PMCSTAT_PRINT_ENTRY(a,"fork","%d %d",
+			PMCSTAT_PRINT_ENTRY("fork","%d %d",
 			    ev.pl_u.pl_f.pl_oldpid,
 			    ev.pl_u.pl_f.pl_newpid);
 			break;
 		case PMCLOG_TYPE_USERDATA:
-			PMCSTAT_PRINT_ENTRY(a,"userdata","0x%x",
+			PMCSTAT_PRINT_ENTRY("userdata","0x%x",
 			    ev.pl_u.pl_u.pl_userdata);
 			break;
 		case PMCLOG_TYPE_SYSEXIT:
-			PMCSTAT_PRINT_ENTRY(a,"exit","%d",
+			PMCSTAT_PRINT_ENTRY("exit","%d",
 			    ev.pl_u.pl_se.pl_pid);
 			break;
 		default:
-			fprintf(a->pa_printfile, "unknown event (type %d).\n",
+			fprintf(args.pa_printfile, "unknown event (type %d).\n",
 			    ev.pl_type);
 		}
 	}
@@ -2354,13 +1668,13 @@ pmcstat_print_log(struct pmcstat_args *a)
  */
 
 int
-pmcstat_close_log(struct pmcstat_args *a)
+pmcstat_close_log(void)
 {
 	if (pmc_flush_logfile() < 0 ||
 	    pmc_configure_logfile(-1) < 0)
 		err(EX_OSERR, "ERROR: logging failed");
-	a->pa_flags &= ~(FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE);
-	return (a->pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING :
+	args.pa_flags &= ~(FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE);
+	return (args.pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING :
 	    PMCSTAT_FINISHED);
 }
 
@@ -2456,17 +1770,208 @@ pmcstat_open_log(const char *path, int mode)
  */
 
 int
-pmcstat_process_log(struct pmcstat_args *a)
+pmcstat_process_log(void)
 {
 
 	/*
 	 * If analysis has not been asked for, just print the log to
 	 * the current output file.
 	 */
-	if (a->pa_flags & FLAG_DO_PRINT)
-		return (pmcstat_print_log(a));
+	if (args.pa_flags & FLAG_DO_PRINT)
+		return (pmcstat_print_log());
 	else
-		return (pmcstat_analyze_log(a));
+		return (pmcstat_analyze_log());
+}
+
+/*
+ * Refresh top display.
+ */
+
+static void
+pmcstat_refresh_top(void)
+{
+	char pmcname[40];
+
+	/* If in pause mode do not refresh display. */
+	if (pmcstat_pause)
+		return;
+
+	/* Format PMC name. */
+	if (pmcstat_mergepmc)
+		snprintf(pmcname, sizeof(pmcname), "[%s]",
+		    pmcstat_pmcindex_to_name(pmcstat_pmcinfilter));
+	else
+		snprintf(pmcname, sizeof(pmcname), "%s.%d",
+		    pmcstat_pmcindex_to_name(pmcstat_pmcinfilter),
+		    pmcstat_pmcinfilter);
+
+	PMCSTAT_PRINTBEGIN();
+	PMCSTAT_PRINTW("PMC: %s Samples: %u processed, %u invalid\n\n",
+	    pmcname,
+	    pmcstat_stats.ps_samples_total,
+	    pmcstat_stats.ps_samples_unknown_offset +
+	    pmcstat_stats.ps_samples_indeterminable +
+	    pmcstat_stats.ps_callchain_dubious_frames);
+	if (plugins[args.pa_plugin].pl_topdisplay != NULL)
+		plugins[args.pa_plugin].pl_topdisplay();
+	PMCSTAT_PRINTEND();
+}
+
+/*
+ * Find the next pmc index to display.
+ */
+
+static void
+pmcstat_changefilter(void)
+{
+	int pmcin;
+	struct pmcstat_pmcrecord *pmcr;
+
+	/*
+	 * Find the next merge target.
+	 */
+	if (pmcstat_mergepmc) {
+		pmcin = pmcstat_pmcinfilter;
+
+		do {
+			pmcr = pmcstat_pmcindex_to_pmcr(pmcstat_pmcinfilter);
+			if (pmcr == pmcr->pr_merge)
+				break;
+
+			pmcstat_pmcinfilter++;
+			if (pmcstat_pmcinfilter >= pmcstat_npmcs)
+				pmcstat_pmcinfilter = 0;
+
+		} while (pmcstat_pmcinfilter != pmcin);
+	}
+}
+
+/*
+ * Top mode keypress.
+ */
+
+int
+pmcstat_keypress_log(void)
+{
+	int c, ret = 0;
+	WINDOW *w;
+
+	w = newwin(1, 0, 1, 0);
+	c = wgetch(w);
+	wprintw(w, "Key: %c => ", c);
+	switch (c) {
+	case 'c':
+		wprintw(w, "enter mode 'd' or 'a' => ");
+		c = wgetch(w);
+		if (c == 'd') {
+			args.pa_topmode = PMCSTAT_TOP_DELTA;
+			wprintw(w, "switching to delta mode");
+		} else {
+			args.pa_topmode = PMCSTAT_TOP_ACCUM;
+			wprintw(w, "switching to accumulation mode");
+		}
+		break;
+	case 'm':
+		pmcstat_mergepmc = !pmcstat_mergepmc;
+		/*
+		 * Changing merge state require data reset.
+		 */
+		if (plugins[args.pa_plugin].pl_shutdown != NULL)
+			plugins[args.pa_plugin].pl_shutdown(NULL);
+		bzero(&pmcstat_stats, sizeof(struct pmcstat_stats));
+		if (plugins[args.pa_plugin].pl_init != NULL)
+			plugins[args.pa_plugin].pl_init();
+
+		/* Update filter to be on a merge target. */
+		pmcstat_changefilter();
+		wprintw(w, "merge PMC %s", pmcstat_mergepmc ? "on" : "off");
+		break;
+	case 'n':
+		/* Close current plugin. */
+		if (plugins[args.pa_plugin].pl_shutdown != NULL)
+			plugins[args.pa_plugin].pl_shutdown(NULL);
+
+		/* Find next top display available. */
+		do {
+			args.pa_plugin++;
+			if (plugins[args.pa_plugin].pl_name == NULL)
+				args.pa_plugin = 0;
+		} while (plugins[args.pa_plugin].pl_topdisplay == NULL);
+
+		/* Open new plugin. */
+		bzero(&pmcstat_stats, sizeof(struct pmcstat_stats));
+		if (plugins[args.pa_plugin].pl_init != NULL)
+			plugins[args.pa_plugin].pl_init();
+		wprintw(w, "switching to plugin %s",
+		    plugins[args.pa_plugin].pl_name);
+		break;
+	case 'p':
+		pmcstat_pmcinfilter++;
+		if (pmcstat_pmcinfilter >= pmcstat_npmcs)
+			pmcstat_pmcinfilter = 0;
+		pmcstat_changefilter();
+		wprintw(w, "switching to PMC %s.%d",
+		    pmcstat_pmcindex_to_name(pmcstat_pmcinfilter),
+		    pmcstat_pmcinfilter);
+		break;
+	case ' ':
+		pmcstat_pause = !pmcstat_pause;
+		if (pmcstat_pause)
+			wprintw(w, "pause => press space again to continue");
+		break;
+	case 'q':
+		wprintw(w, "exiting...");
+		ret = 1;
+	default:
+		if (plugins[args.pa_plugin].pl_topkeypress != NULL)
+			if (plugins[args.pa_plugin].pl_topkeypress(c, w))
+				ret = 1;
+	}
+
+	wrefresh(w);
+	delwin(w);
+	return ret;
+}
+
+
+/*
+ * Top mode display.
+ */
+
+void
+pmcstat_display_log(void)
+{
+
+	pmcstat_refresh_top();
+
+	/* Reset everythings if delta mode. */
+	if (args.pa_topmode == PMCSTAT_TOP_DELTA) {
+		if (plugins[args.pa_plugin].pl_shutdown != NULL)
+			plugins[args.pa_plugin].pl_shutdown(NULL);
+		bzero(&pmcstat_stats, sizeof(struct pmcstat_stats));
+		if (plugins[args.pa_plugin].pl_init != NULL)
+			plugins[args.pa_plugin].pl_init();
+	}
+
+}
+
+/*
+ * Configure a plugins.
+ */
+
+void
+pmcstat_pluginconfigure_log(char *opt)
+{
+
+	if (strncmp(opt, "threshold=", 10) == 0) {
+		pmcstat_threshold = atof(opt+10);
+	} else {
+		if (plugins[args.pa_plugin].pl_configure != NULL) {
+			if (!plugins[args.pa_plugin].pl_configure(opt))
+				err(EX_USAGE,
+				    "ERROR: unknown option <%s>.", opt);
+		}
+	}
 }
 
 /*
@@ -2474,12 +1979,10 @@ pmcstat_process_log(struct pmcstat_args *a)
  */
 
 void
-pmcstat_initialize_logging(struct pmcstat_args *a)
+pmcstat_initialize_logging(void)
 {
 	int i;
 
-	(void) a;
-
 	/* use a convenient format for 'ldd' output */
 	if (setenv("LD_TRACE_LOADED_OBJECTS_FMT1","%o \"%p\" %x\n",1) != 0)
 		err(EX_OSERR, "ERROR: Cannot setenv");
@@ -2499,6 +2002,21 @@ pmcstat_initialize_logging(struct pmcstat_args *a)
 	if ((pmcstat_kernproc = pmcstat_process_lookup((pid_t) -1,
 		 PMCSTAT_ALLOCATE)) == NULL)
 		err(EX_OSERR, "ERROR: Cannot initialize logging");
+
+	/* PMC count. */
+	pmcstat_npmcs = 0;
+
+	/* Merge PMC with same name. */
+	pmcstat_mergepmc = args.pa_mergepmc;
+
+	/*
+	 * Initialize plugins
+	 */
+
+	if (plugins[args.pa_pplugin].pl_init != NULL)
+		plugins[args.pa_pplugin].pl_init();
+	if (plugins[args.pa_plugin].pl_init != NULL)
+		plugins[args.pa_plugin].pl_init();
 }
 
 /*
@@ -2506,99 +2024,57 @@ pmcstat_initialize_logging(struct pmcstat_args *a)
  */
 
 void
-pmcstat_shutdown_logging(struct pmcstat_args *a)
+pmcstat_shutdown_logging(void)
 {
 	int i;
 	FILE *mf;
-	struct pmcstat_gmonfile *pgf, *pgftmp;
 	struct pmcstat_image *pi, *pitmp;
 	struct pmcstat_process *pp, *pptmp;
-	struct pmcstat_cgnode_hash *pch, *pchtmp;
+	struct pmcstat_pcmap *ppm, *ppmtmp;
 
 	/* determine where to send the map file */
 	mf = NULL;
-	if (a->pa_mapfilename != NULL)
-		mf = (strcmp(a->pa_mapfilename, "-") == 0) ?
-		    a->pa_printfile : fopen(a->pa_mapfilename, "w");
+	if (args.pa_mapfilename != NULL)
+		mf = (strcmp(args.pa_mapfilename, "-") == 0) ?
+		    args.pa_printfile : fopen(args.pa_mapfilename, "w");
 
-	if (mf == NULL && a->pa_flags & FLAG_DO_GPROF &&
-	    a->pa_verbosity >= 2)
-		mf = a->pa_printfile;
+	if (mf == NULL && args.pa_flags & FLAG_DO_GPROF &&
+	    args.pa_verbosity >= 2)
+		mf = args.pa_printfile;
 
 	if (mf)
 		(void) fprintf(mf, "MAP:\n");
 
-
-	if (a->pa_flags & FLAG_DO_CALLGRAPHS)
-		pmcstat_callgraph_print(a);
-
-	/*
-	 * Sync back all gprof flat profile data.
-	 */
-	for (i = 0; i < PMCSTAT_NHASH; i++) {
-		LIST_FOREACH(pi, &pmcstat_image_hash[i], pi_next) {
-			if (mf)
-				(void) fprintf(mf, " \"%s\" => \"%s\"",
-				    pmcstat_string_unintern(pi->pi_execpath),
-				    pmcstat_string_unintern(
-				    pi->pi_samplename));
-
-			/* flush gmon.out data to disk */
-			LIST_FOREACH(pgf, &pi->pi_gmlist, pgf_next) {
-				pmcstat_gmon_unmap_file(pgf);
-				if (mf)
-					(void) fprintf(mf, " %s/%d",
-					    pmcstat_pmcid_to_name(
-					    pgf->pgf_pmcid),
-					    pgf->pgf_nsamples);
-				if (pgf->pgf_overflow && a->pa_verbosity >= 1)
-					warnx("WARNING: profile \"%s\" "
-					    "overflowed.",
-					    pmcstat_string_unintern(
-					        pgf->pgf_name));
-			}
-
-			if (mf)
-				(void) fprintf(mf, "\n");
-		}
-	}
-
 	/*
-	 * Compute arcs and add these to the gprof files.
+	 * Shutdown the plugins
 	 */
-	if (a->pa_flags & FLAG_DO_GPROF && a->pa_graphdepth > 1)
-		pmcstat_callgraph_do_gmon_arcs();
 
-	/*
-	 * Free memory.
-	 */
-	for (i = 0; i < PMCSTAT_NHASH; i++) {
-		LIST_FOREACH_SAFE(pch, &pmcstat_cgnode_hash[i], pch_next,
-		    pchtmp) {
-			pmcstat_cgnode_free(pch->pch_cgnode);
-			free(pch);
-		}
-	}
+	if (plugins[args.pa_plugin].pl_shutdown != NULL)
+		plugins[args.pa_plugin].pl_shutdown(mf);
+	if (plugins[args.pa_pplugin].pl_shutdown != NULL)
+		plugins[args.pa_pplugin].pl_shutdown(mf);
 
 	for (i = 0; i < PMCSTAT_NHASH; i++) {
-		LIST_FOREACH_SAFE(pi, &pmcstat_image_hash[i], pi_next, pitmp)
-		{
-			LIST_FOREACH_SAFE(pgf, &pi->pi_gmlist, pgf_next,
-			    pgftmp) {
-				if (pgf->pgf_file)
-					(void) fclose(pgf->pgf_file);
-				LIST_REMOVE(pgf, pgf_next);
-				free(pgf);
-			}
-			if (pi->pi_symbols)
-				free(pi->pi_symbols);
-
+		LIST_FOREACH_SAFE(pi, &pmcstat_image_hash[i], pi_next,
+		    pitmp) {
+			if (plugins[args.pa_plugin].pl_shutdownimage != NULL)
+				plugins[args.pa_plugin].pl_shutdownimage(pi);
+			if (plugins[args.pa_pplugin].pl_shutdownimage != NULL)
+				plugins[args.pa_pplugin].pl_shutdownimage(pi);
+
+			free(pi->pi_symbols);
+			if (pi->pi_addr2line != NULL)
+				pclose(pi->pi_addr2line);
 			LIST_REMOVE(pi, pi_next);
 			free(pi);
 		}
 
 		LIST_FOREACH_SAFE(pp, &pmcstat_process_hash[i], pp_next,
 		    pptmp) {
+			TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next, ppmtmp) {
+				TAILQ_REMOVE(&pp->pp_map, ppm, ppm_next);
+				free(ppm);
+			}
 			LIST_REMOVE(pp, pp_next);
 			free(pp);
 		}
@@ -2610,23 +2086,23 @@ pmcstat_shutdown_logging(struct pmcstat_args *a)
 	 * Print errors unless -q was specified.  Print all statistics
 	 * if verbosity > 1.
 	 */
-#define	PRINT(N,V,A) do {						\
-		if (pmcstat_stats.ps_##V || (A)->pa_verbosity >= 2)	\
-			(void) fprintf((A)->pa_printfile, " %-40s %d\n",\
+#define	PRINT(N,V) do {							\
+		if (pmcstat_stats.ps_##V || args.pa_verbosity >= 2)	\
+			(void) fprintf(args.pa_printfile, " %-40s %d\n",\
 			    N, pmcstat_stats.ps_##V);			\
 	} while (0)
 
-	if (a->pa_verbosity >= 1 && a->pa_flags & FLAG_DO_GPROF) {
-		(void) fprintf(a->pa_printfile, "CONVERSION STATISTICS:\n");
-		PRINT("#exec/a.out", exec_aout, a);
-		PRINT("#exec/elf", exec_elf, a);
-		PRINT("#exec/unknown", exec_indeterminable, a);
-		PRINT("#exec handling errors", exec_errors, a);
-		PRINT("#samples/total", samples_total, a);
-		PRINT("#samples/unclaimed", samples_unknown_offset, a);
-		PRINT("#samples/unknown-object", samples_indeterminable, a);
-		PRINT("#callchain/dubious-frames", callchain_dubious_frames,
-		    a);
+	if (args.pa_verbosity >= 1 && (args.pa_flags & FLAG_DO_ANALYSIS) &&
+	    (args.pa_flags & FLAG_DO_TOP) == 0) {
+		(void) fprintf(args.pa_printfile, "CONVERSION STATISTICS:\n");
+		PRINT("#exec/a.out", exec_aout);
+		PRINT("#exec/elf", exec_elf);
+		PRINT("#exec/unknown", exec_indeterminable);
+		PRINT("#exec handling errors", exec_errors);
+		PRINT("#samples/total", samples_total);
+		PRINT("#samples/unclaimed", samples_unknown_offset);
+		PRINT("#samples/unknown-object", samples_indeterminable);
+		PRINT("#callchain/dubious-frames", callchain_dubious_frames);
 	}
 
 	if (mf)
diff --git a/usr.sbin/pmcstat/pmcstat_log.h b/usr.sbin/pmcstat/pmcstat_log.h
new file mode 100644
index 0000000..de92649
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcstat_log.h
@@ -0,0 +1,196 @@
+/*-
+ * Copyright (c) 2005-2007, Joseph Koshy
+ * Copyright (c) 2007 The FreeBSD Foundation
+ * Copyright (c) 2009, Fabien Thomas
+ * All rights reserved.
+ *
+ * Portions of this software were developed by A. Joseph Koshy under
+ * sponsorship from the FreeBSD Foundation and Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_LOG_H_
+#define	_PMCSTAT_LOG_H_
+
+typedef const void *pmcstat_interned_string;
+
+/*
+ * A 'pmcstat_process' structure models processes.  Each process is
+ * associated with a set of pmcstat_pcmap structures that map
+ * addresses inside it to executable objects.  This set is implemented
+ * as a list, kept sorted in ascending order of mapped addresses.
+ *
+ * 'pp_pid' holds the pid of the process.  When a process exits, the
+ * 'pp_isactive' field is set to zero, but the process structure is
+ * not immediately reclaimed because there may still be samples in the
+ * log for this process.
+ */
+
+struct pmcstat_process {
+	LIST_ENTRY(pmcstat_process) pp_next;	/* hash-next */
+	pid_t			pp_pid;		/* associated pid */
+	int			pp_isactive;	/* whether active */
+	uintfptr_t		pp_entryaddr;	/* entry address */
+	TAILQ_HEAD(,pmcstat_pcmap) pp_map;	/* address range map */
+};
+extern LIST_HEAD(pmcstat_process_hash_list, pmcstat_process) pmcstat_process_hash[PMCSTAT_NHASH];
+
+/*
+ * A 'pmcstat_image' structure describes an executable program on
+ * disk.  'pi_execpath' is a cookie representing the pathname of
+ * the executable.  'pi_start' and 'pi_end' are the least and greatest
+ * virtual addresses for the text segments in the executable.
+ * 'pi_gmonlist' contains a linked list of gmon.out files associated
+ * with this image.
+ */
+
+enum pmcstat_image_type {
+	PMCSTAT_IMAGE_UNKNOWN = 0,	/* never looked at the image */
+	PMCSTAT_IMAGE_INDETERMINABLE,	/* can't tell what the image is */
+	PMCSTAT_IMAGE_ELF32,		/* ELF 32 bit object */
+	PMCSTAT_IMAGE_ELF64,		/* ELF 64 bit object */
+	PMCSTAT_IMAGE_AOUT		/* AOUT object */
+};
+
+struct pmcstat_image {
+	LIST_ENTRY(pmcstat_image) pi_next;	/* hash link */
+	TAILQ_ENTRY(pmcstat_image) pi_lru;	/* LRU list */
+	pmcstat_interned_string	pi_execpath;    /* cookie */
+	pmcstat_interned_string pi_samplename;  /* sample path name */
+	pmcstat_interned_string pi_fullpath;    /* path to FS object */
+	pmcstat_interned_string pi_name;	/* display name */
+
+	enum pmcstat_image_type pi_type;	/* executable type */
+
+	/*
+	 * Executables have pi_start and pi_end; these are zero
+	 * for shared libraries.
+	 */
+	uintfptr_t	pi_start;	/* start address (inclusive) */
+	uintfptr_t	pi_end;		/* end address (exclusive) */
+	uintfptr_t	pi_entry;	/* entry address */
+	uintfptr_t	pi_vaddr;	/* virtual address where loaded */
+	int		pi_isdynamic;	/* whether a dynamic object */
+	int		pi_iskernelmodule;
+	pmcstat_interned_string pi_dynlinkerpath; /* path in .interp */
+
+	/* All symbols associated with this object. */
+	struct pmcstat_symbol *pi_symbols;
+	size_t		pi_symcount;
+
+	/* Handle to addr2line for this image. */
+	FILE *pi_addr2line;
+
+	/*
+	 * Plugins private data
+	 */
+
+	/* gprof:
+	 * An image can be associated with one or more gmon.out files;
+	 * one per PMC.
+	 */
+	LIST_HEAD(,pmcstat_gmonfile) pi_gmlist;
+};
+extern LIST_HEAD(pmcstat_image_hash_list, pmcstat_image) pmcstat_image_hash[PMCSTAT_NHASH];
+
+/*
+ * A 'pmcstat_pcmap' structure maps a virtual address range to an
+ * underlying 'pmcstat_image' descriptor.
+ */
+struct pmcstat_pcmap {
+	TAILQ_ENTRY(pmcstat_pcmap) ppm_next;
+	uintfptr_t	ppm_lowpc;
+	uintfptr_t	ppm_highpc;
+	struct pmcstat_image *ppm_image;
+};
+
+/*
+ * Each function symbol tracked by pmcstat(8).
+ */
+
+struct pmcstat_symbol {
+	pmcstat_interned_string ps_name;
+	uint64_t	ps_start;
+	uint64_t	ps_end;
+};
+
+/*
+ * 'pmcstat_pmcrecord' is a mapping from PMC ids to human-readable
+ * names.
+ */
+
+struct pmcstat_pmcrecord {
+	LIST_ENTRY(pmcstat_pmcrecord)	pr_next;
+	pmc_id_t			pr_pmcid;
+	int				pr_pmcin;
+	pmcstat_interned_string		pr_pmcname;
+	struct pmcstat_pmcrecord	*pr_merge;
+};
+extern LIST_HEAD(pmcstat_pmcs, pmcstat_pmcrecord) pmcstat_pmcs; /* PMC list */
+
+/*
+ * Misc. statistics
+ */
+struct pmcstat_stats {
+	int ps_exec_aout;	/* # a.out executables seen */
+	int ps_exec_elf;	/* # elf executables seen */
+	int ps_exec_errors;	/* # errors processing executables */
+	int ps_exec_indeterminable; /* # unknown executables seen */
+	int ps_samples_total;	/* total number of samples processed */
+	int ps_samples_skipped; /* #samples filtered out for any reason */
+	int ps_samples_unknown_offset;	/* #samples of rank 0 not in a map */
+	int ps_samples_indeterminable;	/* #samples in indeterminable images */
+	int ps_callchain_dubious_frames;/* #dubious frame pointers seen */
+};
+extern struct pmcstat_stats pmcstat_stats; /* statistics */
+
+extern struct pmcstat_process *pmcstat_kernproc; /* kernel 'process' */
+
+extern int pmcstat_npmcs; /* PMC count. */
+
+/*
+ * Top mode global options.
+ */
+float pmcstat_threshold; /* Threshold to filter node. */
+int pmcstat_pmcinfilter; /* PMC index displayed. */
+
+/* Function prototypes */
+const char *pmcstat_pmcid_to_name(pmc_id_t _pmcid);
+const char *pmcstat_pmcindex_to_name(int pmcin);
+struct pmcstat_pmcrecord *pmcstat_pmcindex_to_pmcr(int pmcin);
+struct pmcstat_pcmap *pmcstat_process_find_map(struct pmcstat_process *_p,
+	uintfptr_t _pc);
+struct pmcstat_symbol *pmcstat_symbol_search(struct pmcstat_image *image,
+	uintfptr_t addr);
+const char *pmcstat_string_unintern(pmcstat_interned_string _is);
+pmcstat_interned_string pmcstat_string_intern(const char *_s);
+void pmcstat_image_determine_type(struct pmcstat_image *_image);
+pmcstat_interned_string pmcstat_string_lookup(const char *_s);
+int pmcstat_image_addr2line(struct pmcstat_image *image, uintfptr_t addr,
+    char *sourcefile, size_t sourcefile_len, unsigned *sourceline,
+    char *funcname, size_t funcname_len);
+
+#endif	/* _PMCSTAT_LOG_H_ */
+
diff --git a/usr.sbin/pmcstat/pmcstat_top.h b/usr.sbin/pmcstat/pmcstat_top.h
new file mode 100644
index 0000000..281410b
--- /dev/null
+++ b/usr.sbin/pmcstat/pmcstat_top.h
@@ -0,0 +1,75 @@
+/*-
+ * Copyright (c) 2009, Fabien Thomas
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_PMCSTAT_TOP_H_
+#define	_PMCSTAT_TOP_H_
+
+/* Return the ncurses attributes for the given value. */
+#define PMCSTAT_ATTRPERCENT(b) 					\
+    ((b) > 10.0 ? (args.pa_topcolor ? COLOR_PAIR(1) : A_BOLD) :	\
+    ((b) >  5.0 ? (args.pa_topcolor ? COLOR_PAIR(2) : 0) : 	\
+    ((b) >  2.5 ? (args.pa_topcolor ? COLOR_PAIR(3) : 0) : 0)))
+
+/* Print to the default ncurse windows if on a terminal or to the file. */
+#define PMCSTAT_PRINTW(...) do {			\
+	if (args.pa_toptty)				\
+		printw(__VA_ARGS__);			\
+	else						\
+		fprintf(args.pa_printfile, __VA_ARGS__);\
+} while (0)
+
+/* If ncurses mode active set attributes. */
+#define PMCSTAT_ATTRON(b) do {				\
+	if (args.pa_toptty)				\
+		attron(b);				\
+} while (0)
+
+/* If ncurses mode active unset attributes. */
+#define PMCSTAT_ATTROFF(b) do {				\
+	if (args.pa_toptty)				\
+		attroff(b);				\
+} while (0)
+
+/* Erase screen and set cursor to top left. */
+#define PMCSTAT_PRINTBEGIN() do {			\
+	if (args.pa_toptty)				\
+		clear();				\
+} while (0)
+
+/* Flush buffer to backend. */
+#define PMCSTAT_PRINTEND() do {				\
+	if (!args.pa_toptty) {				\
+		PMCSTAT_PRINTW("---\n");		\
+		fflush(args.pa_printfile);		\
+	} else						\
+		refresh();				\
+} while (0)
+
+/* Function prototypes */
+
+#endif	/* _PMCSTAT_TOP_H_ */