Introduce new structure sfstat for collecting sendfile's statistics

and remove corresponding fields from struct mbstat. Use PCPU counters
and SFSTAT_INC() macro for update these statistics.

Discussed with:	glebius

12 files changed, 63 insertions, 27 deletions

diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c
index 29a213f..3bb76e2 100644
--- a/sys/arm/arm/vm_machdep.c
+++ b/sys/arm/arm/vm_machdep.c
@@ -251,7 +251,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			goto done;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_buf_freelist, &sf_buf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index c462b4c..92e0f52 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -833,7 +833,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			goto done;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_buf_freelist, &sf_buf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 8243a81..0b5b5f8 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -401,8 +401,6 @@ mbuf_init(void *dummy)
 	mbstat.m_numtypes = MT_NTYPES;
 
 	mbstat.m_mcfail = mbstat.m_mpfail = 0;
-	mbstat.sf_iocnt = 0;
-	mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
 }
 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
 
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index a477820..46ceef2 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -115,6 +115,7 @@ static int getsockname1(struct thread *td, struct getsockname_args *uap,
 static int getpeername1(struct thread *td, struct getpeername_args *uap,
 			int compat);
 
+counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
 /*
  * NSFBUFS-related variables and associated sysctls
  */
@@ -129,6 +130,27 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufspeak, CTLFLAG_RD, &nsfbufspeak, 0,
 SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufsused, CTLFLAG_RD, &nsfbufsused, 0,
     "Number of sendfile(2) sf_bufs in use");
 
+static void
+sfstat_init(const void *unused)
+{
+
+	COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t),
+	    M_WAITOK);
+}
+SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL);
+
+static int
+sfstat_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct sfstat s;
+
+	COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t));
+	if (req->newptr)
+		COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t));
+	return (SYSCTL_OUT(req, &s, sizeof(s)));
+}
+SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW,
+    NULL, 0, sfstat_sysctl, "I", "sendfile statistics");
 /*
  * Convert a user file descriptor to a kernel file entry and check if required
  * capability rights are present.
@@ -2241,7 +2263,7 @@ retry_space:
 				vm_page_io_finish(pg);
 				if (!error)
 					VM_OBJECT_WUNLOCK(obj);
-				mbstat.sf_iocnt++;
+				SFSTAT_INC(sf_iocnt);
 			}
 			if (error) {
 				vm_page_lock(pg);
@@ -2273,7 +2295,7 @@ retry_space:
 			sf = sf_buf_alloc(pg, (mnw || m != NULL) ? SFB_NOWAIT :
 			    SFB_CATCH);
 			if (sf == NULL) {
-				mbstat.sf_allocfail++;
+				SFSTAT_INC(sf_allocfail);
 				vm_page_lock(pg);
 				vm_page_unwire(pg, 0);
 				KASSERT(pg->object != NULL,
diff --git a/sys/mips/mips/vm_machdep.c b/sys/mips/mips/vm_machdep.c
index 4c2460a..0323bb3 100644
--- a/sys/mips/mips/vm_machdep.c
+++ b/sys/mips/mips/vm_machdep.c
@@ -540,7 +540,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			break;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_freelist, &sf_freelist.sf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c
index 8deb821..785f22a 100644
--- a/sys/powerpc/aim/vm_machdep.c
+++ b/sys/powerpc/aim/vm_machdep.c
@@ -299,7 +299,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
                         goto done;
 
                 sf_buf_alloc_want++;
-                mbstat.sf_allocwait++;
+                SFSTAT_INC(sf_allocwait);
                 error = msleep(&sf_buf_freelist, &sf_buf_lock,
                     (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
                 sf_buf_alloc_want--;
diff --git a/sys/powerpc/booke/vm_machdep.c b/sys/powerpc/booke/vm_machdep.c
index 54a0b01..3303794 100644
--- a/sys/powerpc/booke/vm_machdep.c
+++ b/sys/powerpc/booke/vm_machdep.c
@@ -300,7 +300,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 			goto done;
 
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_buf_freelist, &sf_buf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/sparc64/sparc64/vm_machdep.c b/sys/sparc64/sparc64/vm_machdep.c
index 0196ec2..8cec001 100644
--- a/sys/sparc64/sparc64/vm_machdep.c
+++ b/sys/sparc64/sparc64/vm_machdep.c
@@ -445,7 +445,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			break;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_freelist, &sf_freelist.sf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 82f48d3..4c8ade0 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -338,10 +338,7 @@ struct mbstat {
 	/* Number of mbtypes (gives # elems in mbtypes[] array) */
 	short	m_numtypes;
 
-	/* XXX: Sendfile stats should eventually move to their own struct */
-	u_long	sf_iocnt;	/* times sendfile had to do disk I/O */
-	u_long	sf_allocfail;	/* times sfbuf allocation failed */
-	u_long	sf_allocwait;	/* times sfbuf allocation had to wait */
+	u_long	spare[3];
 };
 
 /*
diff --git a/sys/sys/sf_buf.h b/sys/sys/sf_buf.h
index af42065..1b62fd3 100644
--- a/sys/sys/sf_buf.h
+++ b/sys/sys/sf_buf.h
@@ -29,8 +29,6 @@
 #ifndef _SYS_SF_BUF_H_
 #define _SYS_SF_BUF_H_
 
-#include <machine/sf_buf.h>
-
 /*
  * Options to sf_buf_alloc() are specified through its flags argument.  This
  * argument's value should be the result of a bitwise or'ing of one or more
@@ -48,6 +46,23 @@ extern  int nsfbufs;                    /* Number of sendfile(2) bufs alloced */
 extern  int nsfbufspeak;                /* Peak of nsfbufsused */
 extern  int nsfbufsused;                /* Number of sendfile(2) bufs in use */
 
+struct sfstat {				/* sendfile statistics */
+	uint64_t	sf_iocnt;	/* times sendfile had to do disk I/O */
+	uint64_t	sf_allocfail;	/* times sfbuf allocation failed */
+	uint64_t	sf_allocwait;	/* times sfbuf allocation had to wait */
+};
+
+#ifdef _KERNEL
+#include <machine/sf_buf.h>
+#include <sys/counter.h>
+
+extern counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
+#define	SFSTAT_ADD(name, val)	\
+    counter_u64_add(sfstat[offsetof(struct sfstat, name) / sizeof(uint64_t)],\
+	(val))
+#define	SFSTAT_INC(name)	SFSTAT_ADD(name, 1)
+#endif /* _KERNEL */
+
 struct sf_buf *
 	sf_buf_alloc(struct vm_page *m, int flags);
 void	sf_buf_free(struct sf_buf *sf);
diff --git a/usr.bin/netstat/main.c b/usr.bin/netstat/main.c
index 249650a..569c394 100644
--- a/usr.bin/netstat/main.c
+++ b/usr.bin/netstat/main.c
@@ -184,6 +184,8 @@ static struct nlist nl[] = {
 	{ .n_name = "_arpstat" },
 #define	N_UNP_SPHEAD	56
 	{ .n_name = "unp_sphead" },
+#define	N_SFSTAT	57
+	{ .n_name = "_sfstat"},
 	{ .n_name = NULL },
 };
 
@@ -543,7 +545,7 @@ main(int argc, char *argv[])
 	if (mflag) {
 		if (!live) {
 			if (kread(0, NULL, 0) == 0)
-				mbpr(kvmd, nl[N_MBSTAT].n_value);
+				mbpr(kvmd, nl[N_SFSTAT].n_value);
 		} else
 			mbpr(NULL, 0);
 		exit(0);
diff --git a/usr.bin/netstat/mbuf.c b/usr.bin/netstat/mbuf.c
index 401a03c..d32304d 100644
--- a/usr.bin/netstat/mbuf.c
+++ b/usr.bin/netstat/mbuf.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/mbuf.h>
 #include <sys/protosw.h>
+#include <sys/sf_buf.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
@@ -81,7 +82,7 @@ mbpr(void *kvmd, u_long mbaddr)
 	uintmax_t jumbo16_failures, jumbo16_sleeps, jumbo16_size;
 	uintmax_t bytes_inuse, bytes_incache, bytes_total;
 	int nsfbufs, nsfbufspeak, nsfbufsused;
-	struct mbstat mbstat;
+	struct sfstat sfstat;
 	size_t mlen;
 	int error;
 
@@ -308,20 +309,21 @@ mbpr(void *kvmd, u_long mbaddr)
 		    &mlen, NULL, 0))
 			printf("%d/%d/%d sfbufs in use (current/peak/max)\n",
 			    nsfbufsused, nsfbufspeak, nsfbufs);
-		mlen = sizeof(mbstat);
-		if (sysctlbyname("kern.ipc.mbstat", &mbstat, &mlen, NULL, 0)) {
-			warn("kern.ipc.mbstat");
+		mlen = sizeof(sfstat);
+		if (sysctlbyname("kern.ipc.sfstat", &sfstat, &mlen, NULL, 0)) {
+			warn("kern.ipc.sfstat");
 			goto out;
 		}
 	} else {
-		if (kread(mbaddr, (char *)&mbstat, sizeof mbstat) != 0)
+		if (kread_counters(mbaddr, (char *)&sfstat, sizeof sfstat) != 0)
 			goto out;
 	}
-	printf("%lu requests for sfbufs denied\n", mbstat.sf_allocfail);
-	printf("%lu requests for sfbufs delayed\n", mbstat.sf_allocwait);
-	printf("%lu requests for I/O initiated by sendfile\n",
-	    mbstat.sf_iocnt);
-	printf("%lu calls to protocol drain routines\n", mbstat.m_drain);
+	printf("%ju requests for sfbufs denied\n",
+	    (uintmax_t)sfstat.sf_allocfail);
+	printf("%ju requests for sfbufs delayed\n",
+	    (uintmax_t)sfstat.sf_allocwait);
+	printf("%ju requests for I/O initiated by sendfile\n",
+	    (uintmax_t)sfstat.sf_iocnt);
 out:
 	memstat_mtl_free(mtlp);
 }