Teach netstat -Q to work with -N and -M by adding libkvm versions of data

query routines.  This code is necessarily more fragile in the presence of
kernel changes than querying the kernel via sysctl (the default), but
useful when investigating crashes or live kernel state via firewire.

MFC after:	1 week
Sponsored by:	Juniper Networks

3 files changed, 278 insertions, 26 deletions

diff --git a/usr.bin/netstat/main.c b/usr.bin/netstat/main.c
index 6cdd7e4..f54db4e 100644
--- a/usr.bin/netstat/main.c
+++ b/usr.bin/netstat/main.c
@@ -530,9 +530,11 @@ main(int argc, char *argv[])
 		exit(0);
 	}
 	if (Qflag) {
-		if (!live)
-			usage();
-		netisr_stats();
+		if (!live) {
+			if (kread(0, NULL, 0) == 0)
+				netisr_stats(kvmd);
+		} else
+			netisr_stats(NULL);
 		exit(0);
 	}
 #if 0
diff --git a/usr.bin/netstat/netisr.c b/usr.bin/netstat/netisr.c
index c2503f4..d427080 100644
--- a/usr.bin/netstat/netisr.c
+++ b/usr.bin/netstat/netisr.c
@@ -31,15 +31,22 @@
 
 __FBSDID("$FreeBSD$");
 
-#include <sys/types.h>
+#include <sys/param.h>
 #include <sys/sysctl.h>
 
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
+
+#define	_WANT_NETISR_INTERNAL
 #include <net/netisr.h>
+#include <net/netisr_internal.h>
 
 #include <err.h>
+#include <kvm.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "netstat.h"
 
@@ -65,6 +72,90 @@ static u_int				 workstream_array_len;
 static struct sysctl_netisr_work	*work_array;
 static u_int				 work_array_len;
 
+static u_int				*nws_array;
+
+static u_int				 maxprot;
+
+static void
+netisr_load_kvm_uint(kvm_t *kd, char *name, u_int *p)
+{
+	struct nlist nl[] = {
+		{ .n_name = name },
+		{ .n_name = NULL },
+	};
+	int ret;
+
+	ret = kvm_nlist(kd, nl);
+	if (ret < 0)
+		errx(-1, "%s: kvm_nlist(%s): %s", __func__, name,
+		    kvm_geterr(kd));
+	if (ret != 0)
+		errx(-1, "%s: kvm_nlist(%s): unresolved symbol", __func__,
+		    name);
+	if (kvm_read(kd, nl[0].n_value, p, sizeof(*p)) != sizeof(*p))
+		errx(-1, "%s: kvm_read(%s): %s", __func__, name,
+		    kvm_geterr(kd));
+}
+
+/*
+ * Load a nul-terminated string from KVM up to 'limit', guarantee that the
+ * string in local memory is nul-terminated.
+ */
+static void
+netisr_load_kvm_string(kvm_t *kd, uintptr_t addr, char *dest, u_int limit)
+{
+	u_int i;
+
+	for (i = 0; i < limit; i++) {
+		if (kvm_read(kd, addr + i, &dest[i], sizeof(dest[i])) !=
+		    sizeof(dest[i]))
+			err(-1, "%s: kvm_read: %s", __func__,
+			    kvm_geterr(kd));
+		if (dest[i] == '\0')
+			break;
+	}
+	dest[limit - 1] = '\0';
+}
+
+static const char *
+netisr_proto2name(u_int proto)
+{
+	u_int i;
+
+	for (i = 0; i < proto_array_len; i++) {
+		if (proto_array[i].snp_proto == proto)
+			return (proto_array[i].snp_name);
+	}
+	return ("unknown");
+}
+
+static int
+netisr_protoispresent(u_int proto)
+{
+	u_int i;
+
+	for (i = 0; i < proto_array_len; i++) {
+		if (proto_array[i].snp_proto == proto)
+			return (1);
+	}
+	return (0);
+}
+
+static void
+netisr_load_kvm_config(kvm_t *kd)
+{
+
+	netisr_load_kvm_uint(kd, "_netisr_bindthreads", &bindthreads);
+	netisr_load_kvm_uint(kd, "_netisr_maxthreads", &maxthreads);
+	netisr_load_kvm_uint(kd, "_nws_count", &numthreads);
+
+	netisr_load_kvm_uint(kd, "_netisr_defaultqlimit", &defaultqlimit);
+	netisr_load_kvm_uint(kd, "_netisr_maxqlimit", &maxqlimit);
+
+	netisr_load_kvm_uint(kd, "_netisr_direct", &direct);
+	netisr_load_kvm_uint(kd, "_netisr_direct_force", &direct_force);
+}
+
 static void
 netisr_load_sysctl_uint(const char *name, u_int *p)
 {
@@ -78,7 +169,7 @@ netisr_load_sysctl_uint(const char *name, u_int *p)
 }
 
 static void
-netisr_load_config(void)
+netisr_load_sysctl_config(void)
 {
 
 	netisr_load_sysctl_uint("net.isr.bindthreads", &bindthreads);
@@ -93,7 +184,80 @@ netisr_load_config(void)
 }
 
 static void
-netisr_load_proto(void)
+netisr_load_kvm_proto(kvm_t *kd)
+{
+	struct nlist nl[] = {
+#define	NLIST_NETISR_PROTO	0
+		{ .n_name = "_netisr_proto" },
+		{ .n_name = NULL },
+	};
+	struct netisr_proto *np_array, *npp;
+	u_int i, protocount;
+	struct sysctl_netisr_proto *snpp;
+	size_t len;
+	int ret;
+
+	/*
+	 * Kernel compile-time and user compile-time definitions of
+	 * NETISR_MAXPROT must match, as we use that to size work arrays.
+	 */
+	netisr_load_kvm_uint(kd, "_netisr_maxprot", &maxprot);
+	if (maxprot != NETISR_MAXPROT)
+		errx(-1, "%s: NETISR_MAXPROT mismatch", __func__);
+	len = maxprot * sizeof(*np_array);
+	np_array = malloc(len);
+	if (np_array == NULL)
+		err(-1, "%s: malloc", __func__);
+	ret = kvm_nlist(kd, nl);
+	if (ret < 0)
+		errx(-1, "%s: kvm_nlist(_netisr_proto): %s", __func__,
+		    kvm_geterr(kd));
+	if (ret != 0)
+		errx(-1, "%s: kvm_nlist(_netisr_proto): unresolved symbol",
+		    __func__);
+	if (kvm_read(kd, nl[NLIST_NETISR_PROTO].n_value, np_array, len) !=
+	    (ssize_t)len)
+		errx(-1, "%s: kvm_read(_netisr_proto): %s", __func__,
+		    kvm_geterr(kd));
+
+	/*
+	 * Size and allocate memory to hold only live protocols.
+	 */
+	protocount = 0;
+	for (i = 0; i < maxprot; i++) {
+		if (np_array[i].np_name == NULL)
+			continue;
+		protocount++;
+	}
+	proto_array = calloc(protocount, sizeof(*proto_array));
+	if (proto_array == NULL)
+		err(-1, "malloc");
+	protocount = 0;
+	for (i = 0; i < maxprot; i++) {
+		npp = &np_array[i];
+		if (npp->np_name == NULL)
+			continue;
+		snpp = &proto_array[protocount];
+		snpp->snp_version = sizeof(*snpp);
+		netisr_load_kvm_string(kd, (uintptr_t)npp->np_name,
+		    snpp->snp_name, sizeof(snpp->snp_name));
+		snpp->snp_proto = i;
+		snpp->snp_qlimit = npp->np_qlimit;
+		snpp->snp_policy = npp->np_policy;
+		if (npp->np_m2flow != NULL)
+			snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW;
+		if (npp->np_m2cpuid != NULL)
+			snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID;
+		if (npp->np_drainedcpu != NULL)
+			snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU;
+		protocount++;
+	}
+	proto_array_len = protocount;
+	free(np_array);
+}
+
+static void
+netisr_load_sysctl_proto(void)
 {
 	size_t len;
 
@@ -116,7 +280,96 @@ netisr_load_proto(void)
 }
 
 static void
-netisr_load_workstream(void)
+netisr_load_kvm_workstream(kvm_t *kd)
+{
+	struct nlist nl[] = {
+#define	NLIST_NWS_ARRAY		0
+		{ .n_name = "_nws_array" },
+		{ .n_name = NULL },
+	};
+	struct netisr_workstream nws;
+	struct sysctl_netisr_workstream *snwsp;
+	struct sysctl_netisr_work *snwp;
+	struct netisr_work *nwp;
+	struct nlist nl_nws[2];
+	u_int counter, cpuid, proto, wsid;
+	size_t len;
+	int ret;
+
+	len = numthreads * sizeof(*nws_array);
+	nws_array = malloc(len);
+	if (nws_array == NULL)
+		err(-1, "malloc");
+	ret = kvm_nlist(kd, nl);
+	if (ret < 0)
+		errx(-1, "%s: kvm_nlist: %s", __func__, kvm_geterr(kd));
+	if (ret != 0)
+		errx(-1, "%s: kvm_nlist: unresolved symbol", __func__);
+	if (kvm_read(kd, nl[NLIST_NWS_ARRAY].n_value, nws_array, len) !=
+	    (ssize_t)len)
+		errx(-1, "%s: kvm_read(_nws_array): %s", __func__,
+		    kvm_geterr(kd));
+	workstream_array = calloc(numthreads, sizeof(*workstream_array));
+	if (workstream_array == NULL)
+		err(-1, "calloc");
+	workstream_array_len = numthreads;
+	work_array = calloc(numthreads * proto_array_len, sizeof(*work_array));
+	if (work_array == NULL)
+		err(-1, "calloc");
+	counter = 0;
+	for (wsid = 0; wsid < numthreads; wsid++) {
+		cpuid = nws_array[wsid];
+		if (kvm_dpcpu_setcpu(kd, cpuid) < 0)
+			errx(-1, "%s: kvm_dpcpu_setcpu(%u): %s", __func__,
+			    cpuid, kvm_geterr(kd));
+		bzero(nl_nws, sizeof(nl_nws));
+		nl_nws[0].n_name = "_nws";
+		ret = kvm_nlist(kd, nl_nws);
+		if (ret < 0)
+			errx(-1, "%s: kvm_nlist looking up nws on CPU %u: %s",
+			    __func__, cpuid, kvm_geterr(kd));
+		if (ret != 0)
+			errx(-1, "%s: kvm_nlist(nws): unresolved symbol on "
+			    "CPU %u", __func__, cpuid);
+		if (kvm_read(kd, nl_nws[0].n_value, &nws, sizeof(nws)) !=
+		    sizeof(nws))
+			errx(-1, "%s: kvm_read(nw): %s", __func__,
+			    kvm_geterr(kd));
+		snwsp = &workstream_array[wsid];
+		snwsp->snws_version = sizeof(*snwsp);
+		snwsp->snws_wsid = cpuid;
+		snwsp->snws_cpu = cpuid;
+		if (nws.nws_intr_event != NULL)
+			snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR;
+
+		/*
+		 * Extract the CPU's per-protocol work information.
+		 */
+		printf("counting to maxprot: %u\n", maxprot);
+		for (proto = 0; proto < maxprot; proto++) {
+			if (!netisr_protoispresent(proto))
+				continue;
+			nwp = &nws.nws_work[proto];
+			snwp = &work_array[counter];
+			snwp->snw_version = sizeof(*snwp);
+			snwp->snw_wsid = cpuid;
+			snwp->snw_proto = proto;
+			snwp->snw_len = nwp->nw_len;
+			snwp->snw_watermark = nwp->nw_watermark;
+			snwp->snw_dispatched = nwp->nw_dispatched;
+			snwp->snw_hybrid_dispatched =
+			    nwp->nw_hybrid_dispatched;
+			snwp->snw_qdrops = nwp->nw_qdrops;
+			snwp->snw_queued = nwp->nw_queued;
+			snwp->snw_handled = nwp->nw_handled;
+			counter++;
+		}
+	}
+	work_array_len = counter;
+}
+
+static void
+netisr_load_sysctl_workstream(void)
 {
 	size_t len;
 
@@ -140,7 +393,7 @@ netisr_load_workstream(void)
 }
 
 static void
-netisr_load_work(void)
+netisr_load_sysctl_work(void)
 {
 	size_t len;
 
@@ -179,18 +432,6 @@ netisr_print_proto(struct sysctl_netisr_proto *snpp)
 	    (snpp->snp_flags & NETISR_SNP_FLAGS_M2FLOW) ? "F" : "-");
 }
 
-static const char *
-netisr_proto2name(u_int proto)
-{
-	u_int i;
-
-	for (i = 0; i < proto_array_len; i++) {
-		if (proto_array[i].snp_proto == proto)
-			return (proto_array[i].snp_name);
-	}
-	return ("unknown");
-}
-
 static void
 netisr_print_workstream(struct sysctl_netisr_workstream *snwsp)
 {
@@ -223,16 +464,25 @@ netisr_print_workstream(struct sysctl_netisr_workstream *snwsp)
 }
 
 void
-netisr_stats(void)
+netisr_stats(void *kvmd)
 {
 	struct sysctl_netisr_workstream *snwsp;
 	struct sysctl_netisr_proto *snpp;
+	kvm_t *kd = kvmd;
 	u_int i;
 
-	netisr_load_config();
-	netisr_load_proto();
-	netisr_load_workstream();
-	netisr_load_work();
+	if (live) {
+		netisr_load_sysctl_config();
+		netisr_load_sysctl_proto();
+		netisr_load_sysctl_workstream();
+		netisr_load_sysctl_work();
+	} else {
+		if (kd == NULL)
+			errx(-1, "netisr_stats: !live but !kd");
+		netisr_load_kvm_config(kd);
+		netisr_load_kvm_proto(kd);
+		netisr_load_kvm_workstream(kd);		/* Also does work. */
+	}
 
 	printf("Configuration:\n");
 	printf("%-25s %12s %12s\n", "Setting", "Value", "Maximum");
diff --git a/usr.bin/netstat/netstat.h b/usr.bin/netstat/netstat.h
index 009a0c8..da3f8f3 100644
--- a/usr.bin/netstat/netstat.h
+++ b/usr.bin/netstat/netstat.h
@@ -115,7 +115,7 @@ void	pfkey_stats(u_long, const char *, int, int);
 
 void	mbpr(void *, u_long);
 
-void	netisr_stats(void);
+void	netisr_stats(void *);
 
 void	hostpr(u_long, u_long);
 void	impstats(u_long, u_long);