A first cut at teaching libkvm how to deal with dynamic per-CPU storage

(DPCPU):

A new API, kvm_dpcpu_setcpu(3), selects the active CPU for the purposes
of DPCPU.  Calls to kvm_nlist(3) will automatically translate DPCPU
symbols and return a pointer to the current CPU's version of the data.
Consumers needing to read the same symbol on several CPUs will invoke a
series of setcpu/nlist calls, one per CPU of interest.

This addition makes it possible for tools like netstat(1) to query the
values of DPCPU variables during crashdump analysis, and is based on
similar code handling virtualized global variables.

MFC after:	1 week
Sponsored by:	Juniper Networks, Inc.

6 files changed, 210 insertions, 7 deletions

diff --git a/lib/libkvm/Makefile b/lib/libkvm/Makefile
index 0cb3ad7..e62d7ca 100644
--- a/lib/libkvm/Makefile
+++ b/lib/libkvm/Makefile
@@ -23,6 +23,7 @@ MAN=	kvm.3 kvm_getcptime.3 kvm_geterr.3 kvm_getfiles.3 kvm_getloadavg.3 \
 	kvm_read.3
 
 MLINKS+=kvm_getpcpu.3 kvm_getmaxcpu.3
+MLINKS+=kvm_getpcpu.3 kvm_dpcpu_setcpu.3
 MLINKS+=kvm_getprocs.3 kvm_getargv.3 kvm_getprocs.3 kvm_getenvv.3
 MLINKS+=kvm_open.3 kvm_close.3 kvm_open.3 kvm_openfiles.3
 MLINKS+=kvm_read.3 kvm_write.3
diff --git a/lib/libkvm/kvm.c b/lib/libkvm/kvm.c
index 50f752a..37f6a72 100644
--- a/lib/libkvm/kvm.c
+++ b/lib/libkvm/kvm.c
@@ -416,6 +416,8 @@ _kvm_nlist(kvm_t *kd, struct nlist *nl, int initialize)
 	struct kld_sym_lookup lookup;
 	int error;
 	char *prefix = "", symname[1024]; /* XXX-BZ symbol name length limit? */
+	int tried_vnet, tried_dpcpu;
+
 	/*
 	 * If we can't use the kld symbol lookup, revert to the
 	 * slow library call.
@@ -429,6 +431,10 @@ _kvm_nlist(kvm_t *kd, struct nlist *nl, int initialize)
 			error = kvm_fdnlist_prefix(kd, nl, error,
 			    VNET_SYMPREFIX, _kvm_vnet_validaddr);
 
+		if (error > 0 && _kvm_dpcpu_initialized(kd, initialize))
+			error = kvm_fdnlist_prefix(kd, nl, error,
+			    "pcpu_entry_", _kvm_dpcpu_validaddr);
+
 		return (error);
 	}
 
@@ -437,6 +443,8 @@ _kvm_nlist(kvm_t *kd, struct nlist *nl, int initialize)
 	 * and look it up with a kldsym(2) syscall.
 	 */
 	nvalid = 0;
+	tried_vnet = 0;
+	tried_dpcpu = 0;
 again:
 	for (p = nl; p->n_name && p->n_name[0]; ++p) {
 		if (p->n_type != N_UNDF)
@@ -464,6 +472,10 @@ again:
 			    !strcmp(prefix, VNET_SYMPREFIX))
 				p->n_value =
 				    _kvm_vnet_validaddr(kd, lookup.symvalue);
+			else if (_kvm_dpcpu_initialized(kd, initialize) &&
+			    !strcmp(prefix, "pcpu_entry_"))
+				p->n_value =
+				    _kvm_dpcpu_validaddr(kd, lookup.symvalue);
 			else
 				p->n_value = lookup.symvalue;
 			++nvalid;
@@ -473,14 +485,19 @@ again:
 
 	/*
 	 * Check the number of entries that weren't found. If they exist,
-	 * try again with a prefix for virtualized symbol names.
+	 * try again with a prefix for virtualized or DPCPU symbol names.
 	 */
 	error = ((p - nl) - nvalid);
-	if (error && _kvm_vnet_initialized(kd, initialize) &&
-	    strcmp(prefix, VNET_SYMPREFIX)) {
+	if (error && _kvm_vnet_initialized(kd, initialize) && !tried_vnet) {
+		tried_vnet = 1;
 		prefix = VNET_SYMPREFIX;
 		goto again;
 	}
+	if (error && _kvm_dpcpu_initialized(kd, initialize) && !tried_dpcpu) {
+		tried_dpcpu = 1;
+		prefix = "pcpu_entry_";
+		goto again;
+	}
 
 	/*
 	 * Return the number of entries that weren't found. If they exist,
diff --git a/lib/libkvm/kvm.h b/lib/libkvm/kvm.h
index 0427bd1..6b5160b 100644
--- a/lib/libkvm/kvm.h
+++ b/lib/libkvm/kvm.h
@@ -69,6 +69,7 @@ struct kvm_swap {
 
 __BEGIN_DECLS
 int	  kvm_close(kvm_t *);
+int	  kvm_dpcpu_setcpu(kvm_t *, u_int);
 char	**kvm_getargv(kvm_t *, const struct kinfo_proc *, int);
 int	  kvm_getcptime(kvm_t *, long *);
 char	**kvm_getenvv(kvm_t *, const struct kinfo_proc *, int);
diff --git a/lib/libkvm/kvm_getpcpu.3 b/lib/libkvm/kvm_getpcpu.3
index 40f16ac..f2deda9 100644
--- a/lib/libkvm/kvm_getpcpu.3
+++ b/lib/libkvm/kvm_getpcpu.3
@@ -28,10 +28,11 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 19, 2008
+.Dd February 28, 2010
 .Dt KVM_GETPCPU 3
 .Os
 .Sh NAME
+.Nm kvm_dpcpu_setcpu
 .Nm kvm_getmaxcpu ,
 .Nm kvm_getpcpu
 .Nd access per-CPU data
@@ -43,20 +44,30 @@
 .In sys/sysctl.h
 .In kvm.h
 .Ft int
+.Fn kvm_dpcpu_setcpu "kvm_t *kd" "u_int cpu"
+.Ft int
 .Fn kvm_getmaxcpu "kvm_t *kd"
 .Ft void *
 .Fn kvm_getpcpu "kvm_t *kd" "int cpu"
 .Sh DESCRIPTION
 The
-.Fn kvm_getmaxcpu
+.Fn kvm_dpcpu_setcpu ,
+.Fn kvm_getmaxcpu ,
 and
 .Fn kvm_getpcpu
 functions are used to access the per-CPU data of active processors in the
 kernel indicated by
 .Fa kd .
+Per-CPU storage comes in two flavours: data stored directly in a
+.Vt "struct pcpu"
+associated with each CPU, and dynamic per-CPU storage (DPCPU), in which a
+single kernel symbol refers to different data depending on what CPU it is
+accessed from.
+.Pp
 The
 .Fn kvm_getmaxcpu
 function returns the maximum number of CPUs supported by the kernel.
+.Pp
 The
 .Fn kvm_getpcpu
 function returns a buffer holding the per-CPU data for a single CPU.
@@ -71,8 +82,22 @@ If
 is not active, then
 .Dv NULL
 is returned instead.
+.Pp
+Symbols for dynamic per-CPU data are accessed via
+.Xr kvm_nlist 3
+as with other symbols.
+.Nm libkvm
+maintains a notion of the "current CPU", set by
+.Xr kvm_dpcpu_setcpu ,
+which defaults to 0.
+Once another CPU is selected,
+.Xr kvm_nlist 3
+will return pointers to that data on the appropriate CPU.
 .Sh CACHING
-These functions cache the nlist values for various kernel variables which are
+.Fn kvm_getmaxcpu
+and
+.Vn kvm_getpcpu
+cache the nlist values for various kernel variables which are
 reused in successive calls.
 You may call either function with
 .Fa kd
@@ -93,7 +118,11 @@ function returns a pointer to an allocated buffer or
 If an error occurs,
 it returns -1 instead.
 .Pp
-If either function encounters an error,
+On success, the
+.Fn kvm_dpcpu_setcpu
+call returns 0; if an error occurs, it returns -1 instead.
+.Pp
+If any function encounters an error,
 then an error message may be retrieved via
 .Xr kvm_geterr 3.
 .Sh SEE ALSO
diff --git a/lib/libkvm/kvm_pcpu.c b/lib/libkvm/kvm_pcpu.c
index e4f8909..484d2ea 100644
--- a/lib/libkvm/kvm_pcpu.c
+++ b/lib/libkvm/kvm_pcpu.c
@@ -1,8 +1,15 @@
 /*-
+ * Copyright (c) 2010 Juniper Networks, Inc.
+ * Copyright (c) 2009 Robert N. M. Watson
+ * Copyright (c) 2009 Bjoern A. Zeeb <bz@FreeBSD.org>
  * Copyright (c) 2008 Yahoo!, Inc.
  * All rights reserved.
+ *
  * Written by: John Baldwin <jhb@FreeBSD.org>
  *
+ * This software was developed by Robert N. M. Watson under contract
+ * to Juniper Networks, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -49,6 +56,10 @@ static struct nlist kvm_pcpu_nl[] = {
 /*
  * Kernel per-CPU data state.  We cache this stuff on the first
  * access.	
+ *
+ * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the
+ * consumer has multiple handles in flight to differently configured
+ * kernels/crashdumps.
  */
 static void **pcpu_data;
 static int maxcpu;
@@ -150,3 +161,132 @@ kvm_getmaxcpu(kvm_t *kd)
 			return (-1);
 	return (maxcpu);
 }
+
+static int
+_kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error)
+{
+
+	if (!kd->dpcpu_initialized) {
+		if (report_error)
+			_kvm_err(kd, kd->program, "%s: not initialized",
+			    __func__);
+		return (-1);
+	}
+	if (cpu >= kd->dpcpu_maxcpus) {
+		if (report_error)
+			_kvm_err(kd, kd->program, "%s: CPU %u too big",
+			    __func__, cpu);
+		return (-1);
+	}
+	if (kd->dpcpu_off[cpu] == 0) {
+		if (report_error)
+			_kvm_err(kd, kd->program, "%s: CPU %u not found",
+			    __func__, cpu);
+		return (-1);
+	}
+	kd->dpcpu_curcpu = cpu;
+	kd->dpcpu_curoff = kd->dpcpu_off[cpu];
+	return (0);
+}
+
+/*
+ * Set up libkvm to handle dynamic per-CPU memory.
+ */
+static int
+_kvm_dpcpu_init(kvm_t *kd)
+{
+	struct nlist nl[] = {
+#define	NLIST_START_SET_PCPU	0
+		{ "___start_set_pcpu" },
+#define	NLIST_STOP_SET_PCPU	1
+		{ "___stop_set_pcpu" },
+#define	NLIST_DPCPU_OFF		2
+		{ "_dpcpu_off" },
+#define	NLIST_MP_MAXCPUS	3
+		{ "_mp_maxcpus" },
+		{ NULL },
+	};
+	uintptr_t *dpcpu_off_buf;
+	size_t len;
+	u_int dpcpu_maxcpus;
+
+	/*
+	 * Locate and cache locations of important symbols using the internal
+	 * version of _kvm_nlist, turning off initialization to avoid
+	 * recursion in case of unresolveable symbols.
+	 */
+	if (_kvm_nlist(kd, nl, 0) != 0)
+		return (-1);
+	if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus,
+	    sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus))
+		return (-1);
+	len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf);
+	dpcpu_off_buf = malloc(len);
+	if (dpcpu_off_buf == NULL)
+		return (-1);
+	if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) !=
+	    len) {
+		free(dpcpu_off_buf);
+		return (-1);
+	}
+	kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value;
+	kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value;
+	kd->dpcpu_maxcpus = dpcpu_maxcpus;
+	kd->dpcpu_off = dpcpu_off_buf;
+	kd->dpcpu_initialized = 1;
+	(void)_kvm_dpcpu_setcpu(kd, 0, 0);
+	return (0);
+}
+
+/*
+ * Check whether the dpcpu module has been initialized sucessfully or not,
+ * initialize it if permitted.
+ */
+int
+_kvm_dpcpu_initialized(kvm_t *kd, int intialize)
+{
+
+	if (kd->dpcpu_initialized || !intialize)
+		return (kd->dpcpu_initialized);
+
+	(void)_kvm_dpcpu_init(kd);
+
+	return (kd->dpcpu_initialized);
+}
+
+/*
+ * Check whether the value is within the dpcpu symbol range and only if so
+ * adjust the offset relative to the current offset.
+ */
+uintptr_t
+_kvm_dpcpu_validaddr(kvm_t *kd, uintptr_t value)
+{
+
+	if (value == 0)
+		return (value);
+
+	if (!kd->dpcpu_initialized)
+		return (value);
+
+	if (value < kd->dpcpu_start || value >= kd->dpcpu_stop)
+		return (value);
+
+	return (kd->dpcpu_curoff + value);
+}
+
+int
+kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu)
+{
+	int ret;
+
+	if (!kd->dpcpu_initialized) {
+		ret = _kvm_dpcpu_init(kd);
+		if (ret != 0) {
+			_kvm_err(kd, kd->program, "%s: init failed",
+			    __func__);
+			return (ret);
+		}
+	}
+
+	return (_kvm_dpcpu_setcpu(kd, cpu, 1));
+}
diff --git a/lib/libkvm/kvm_private.h b/lib/libkvm/kvm_private.h
index cc073db..69b1658 100644
--- a/lib/libkvm/kvm_private.h
+++ b/lib/libkvm/kvm_private.h
@@ -68,6 +68,19 @@ struct __kvm {
 	uintptr_t	vnet_stop;	/* stop of kernel's vnet region */
 	uintptr_t	vnet_current;	/* vnet we're working with */
 	uintptr_t	vnet_base;	/* vnet base of current vnet */
+
+	/*
+	 * Dynamic per-CPU kernel memory.  We translate symbols, on-demand,
+	 * to the data associated with dpcpu_curcpu, set with
+	 * kvm_dpcpu_setcpu().
+	 */
+	int		dpcpu_initialized;	/* dpcpu fields set up */
+	uintptr_t	dpcpu_start;	/* start of kernel's dpcpu region */
+	uintptr_t	dpcpu_stop;	/* stop of kernel's dpcpu region */
+	u_int		dpcpu_maxcpus;	/* size of base array */
+	uintptr_t	*dpcpu_off;	/* base array, indexed by CPU ID */
+	u_int		dpcpu_curcpu;	/* CPU we're currently working with */
+	uintptr_t	dpcpu_curoff;	/* dpcpu base of current CPU */
 };
 
 /*
@@ -88,6 +101,8 @@ int	 _kvm_uvatop(kvm_t *, const struct proc *, u_long, u_long *);
 int	 _kvm_vnet_selectpid(kvm_t *, pid_t);
 int	 _kvm_vnet_initialized(kvm_t *, int);
 uintptr_t _kvm_vnet_validaddr(kvm_t *, uintptr_t);
+int	 _kvm_dpcpu_initialized(kvm_t *, int);
+uintptr_t _kvm_dpcpu_validaddr(kvm_t *, uintptr_t);
 
 #if defined(__amd64__) || defined(__i386__) || defined(__arm__)
 void	 _kvm_minidump_freevtop(kvm_t *);