- Remove rt_metrics_lite and simply put its members into rtentry.

- Use counter(9) for rt_pksent (former rt_rmx.rmx_pksent). This
  removes another cache trashing ++ from packet forwarding path.
- Create zini/fini methods for the rtentry UMA zone. Via initialize
  mutex and counter in them.
- Fix reporting of rmx_pksent to routing socket.
- Fix netstat(1) to report "Use" both in kvm(3) and sysctl(3) mode.

The change is mostly targeted for stable/10 merge. For head,
rt_pksent is expected to just disappear.

Discussed with:		melifaro
Sponsored by:		Netflix
Sponsored by:		Nginx, Inc.

3 files changed, 37 insertions, 31 deletions

diff --git a/usr.bin/netstat/main.c b/usr.bin/netstat/main.c
index 88ea55a..42e7080 100644
--- a/usr.bin/netstat/main.c
+++ b/usr.bin/netstat/main.c
@@ -763,6 +763,19 @@ kread(u_long addr, void *buf, size_t size)
 }
 
 /*
+ * Read single counter(9).
+ */
+uint64_t
+kread_counter(u_long addr)
+{
+
+	if (kvmd_init() < 0)
+		return (-1);
+
+	return (kvm_counter_u64_fetch(kvmd, addr));
+}
+
+/*
  * Read an array of N counters in kernel memory into array of N uint64_t's.
  */
 int
diff --git a/usr.bin/netstat/netstat.h b/usr.bin/netstat/netstat.h
index fd4ec96..414b4ce 100644
--- a/usr.bin/netstat/netstat.h
+++ b/usr.bin/netstat/netstat.h
@@ -60,6 +60,7 @@ extern int	live;	/* true if we are examining a live system */
 
 struct nlist;
 int	kread(u_long addr, void *buf, size_t size);
+uint64_t kread_counter(u_long addr);
 int	kread_counters(u_long addr, void *buf, size_t size);
 int	kresolve_list(struct nlist *);
 const char *plural(uintmax_t);
diff --git a/usr.bin/netstat/route.c b/usr.bin/netstat/route.c
index 75c255f..56a0f06 100644
--- a/usr.bin/netstat/route.c
+++ b/usr.bin/netstat/route.c
@@ -181,7 +181,7 @@ routepr(int fibnum, int af)
 	/*
 	 * Since kernel & userland use different timebase
 	 * (time_uptime vs time_second) and we are reading kernel memory
-	 * directly we should do rt_rmx.rmx_expire --> expire_time conversion.
+	 * directly we should do rt_expire --> expire_time conversion.
 	 */
 	if (clock_gettime(CLOCK_UPTIME, &uptime) < 0)
 		err(EX_OSERR, "clock_gettime() failed");
@@ -256,8 +256,7 @@ pr_family(int af1)
 static int wid_dst;
 static int wid_gw;
 static int wid_flags;
-static int wid_refs;
-static int wid_use;
+static int wid_pksent;
 static int wid_mtu;
 static int wid_if;
 static int wid_expire;
@@ -268,8 +267,7 @@ size_cols(int ef, struct radix_node *rn)
 	wid_dst = WID_DST_DEFAULT(ef);
 	wid_gw = WID_GW_DEFAULT(ef);
 	wid_flags = 6;
-	wid_refs = 6;
-	wid_use = 8;
+	wid_pksent = 8;
 	wid_mtu = 6;
 	wid_if = WID_IF_DEFAULT(ef);
 	wid_expire = 6;
@@ -329,16 +327,10 @@ size_cols_rtentry(struct rtentry *rt)
 	len = strlen(bp);
 	wid_flags = MAX(len, wid_flags);
 
-	if (addr.u_sa.sa_family == AF_INET || Wflag) {
-		len = snprintf(buffer, sizeof(buffer), "%d", rt->rt_refcnt);
-		wid_refs = MAX(len, wid_refs);
-		len = snprintf(buffer, sizeof(buffer), "%lu", rt->rt_use);
-		wid_use = MAX(len, wid_use);
-		if (Wflag && rt->rt_rmx.rmx_mtu != 0) {
-			len = snprintf(buffer, sizeof(buffer),
-				       "%lu", rt->rt_rmx.rmx_mtu);
-			wid_mtu = MAX(len, wid_mtu);
-		}
+	if (Wflag) {
+		len = snprintf(buffer, sizeof(buffer), "%lu",
+		    kread_counter((u_long )rt->rt_pksent));
+		wid_pksent = MAX(len, wid_pksent);
 	}
 	if (rt->rt_ifp) {
 		if (rt->rt_ifp != lastif) {
@@ -349,11 +341,11 @@ size_cols_rtentry(struct rtentry *rt)
 			lastif = rt->rt_ifp;
 			wid_if = MAX(len, wid_if);
 		}
-		if (rt->rt_rmx.rmx_expire) {
+		if (rt->rt_expire) {
 			time_t expire_time;
 
 			if ((expire_time =
-			    rt->rt_rmx.rmx_expire - uptime.tv_sec) > 0) {
+			    rt->rt_expire - uptime.tv_sec) > 0) {
 				len = snprintf(buffer, sizeof(buffer), "%d",
 					       (int)expire_time);
 				wid_expire = MAX(len, wid_expire);
@@ -373,10 +365,11 @@ pr_rthdr(int af1)
 	if (Aflag)
 		printf("%-8.8s ","Address");
 	if (Wflag) {
-		printf("%-*.*s %-*.*s %-*.*s %*.*s %*.*s %*s\n",
+		printf("%-*.*s %-*.*s %-*.*s %*.*s %*.*s %*.*s %*s\n",
 			wid_dst,	wid_dst,	"Destination",
 			wid_gw,		wid_gw,		"Gateway",
 			wid_flags,	wid_flags,	"Flags",
+			wid_pksent,	wid_pksent,	"Use",
 			wid_mtu,	wid_mtu,	"Mtu",
 			wid_if,		wid_if,		"Netif",
 			wid_expire,			"Expire");
@@ -661,6 +654,8 @@ p_rtentry_sysctl(struct rt_msghdr *rtm)
 	snprintf(buffer, sizeof(buffer), "%%-%d.%ds ", wid_flags, wid_flags);
 	p_flags(rtm->rtm_flags, buffer);
 	if (Wflag) {
+		printf("%*lu ", wid_pksent, rtm->rtm_rmx.rmx_pksent);
+
 		if (rtm->rtm_rmx.rmx_mtu != 0)
 			printf("%*lu ", wid_mtu, rtm->rtm_rmx.rmx_mtu);
 		else
@@ -870,17 +865,14 @@ p_rtentry_kvm(struct rtentry *rt)
 	p_sockaddr(kgetsa(rt->rt_gateway), NULL, RTF_HOST, wid_gw);
 	snprintf(buffer, sizeof(buffer), "%%-%d.%ds ", wid_flags, wid_flags);
 	p_flags(rt->rt_flags, buffer);
-	if (addr.u_sa.sa_family == AF_INET || Wflag) {
-#if 0
-		printf("%*d %*lu ", wid_refs, rt->rt_refcnt,
-				     wid_use, rt->rt_use);
-#endif
-		if (Wflag) {
-			if (rt->rt_rmx.rmx_mtu != 0)
-				printf("%*lu ", wid_mtu, rt->rt_rmx.rmx_mtu);
-			else
-				printf("%*s ", wid_mtu, "");
-		}
+	if (Wflag) {
+		printf("%*lu ", wid_pksent,
+		    kread_counter((u_long )rt->rt_pksent));
+
+		if (rt->rt_mtu != 0)
+			printf("%*lu ", wid_mtu, rt->rt_mtu);
+		else
+			printf("%*s ", wid_mtu, "");
 	}
 	if (rt->rt_ifp) {
 		if (rt->rt_ifp != lastif) {
@@ -892,11 +884,11 @@ p_rtentry_kvm(struct rtentry *rt)
 			lastif = rt->rt_ifp;
 		}
 		printf("%*.*s", wid_if, wid_if, prettyname);
-		if (rt->rt_rmx.rmx_expire) {
+		if (rt->rt_expire) {
 			time_t expire_time;
 
 			if ((expire_time =
-			    rt->rt_rmx.rmx_expire - uptime.tv_sec) > 0)
+			    rt->rt_expire - uptime.tv_sec) > 0)
 				printf(" %*d", wid_expire, (int)expire_time);
 		}
 		if (rt->rt_nodes[0].rn_dupedkey)