Introduce new structure sfstat for collecting sendfile's statistics

and remove corresponding fields from struct mbstat. Use PCPU counters
and SFSTAT_INC() macro for update these statistics.

Discussed with:	glebius

10 files changed, 48 insertions, 16 deletions

diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c
index 29a213f..3bb76e2 100644
--- a/sys/arm/arm/vm_machdep.c
+++ b/sys/arm/arm/vm_machdep.c
@@ -251,7 +251,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			goto done;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_buf_freelist, &sf_buf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index c462b4c..92e0f52 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -833,7 +833,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			goto done;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_buf_freelist, &sf_buf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 8243a81..0b5b5f8 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -401,8 +401,6 @@ mbuf_init(void *dummy)
 	mbstat.m_numtypes = MT_NTYPES;
 
 	mbstat.m_mcfail = mbstat.m_mpfail = 0;
-	mbstat.sf_iocnt = 0;
-	mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
 }
 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
 
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index a477820..46ceef2 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -115,6 +115,7 @@ static int getsockname1(struct thread *td, struct getsockname_args *uap,
 static int getpeername1(struct thread *td, struct getpeername_args *uap,
 			int compat);
 
+counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
 /*
  * NSFBUFS-related variables and associated sysctls
  */
@@ -129,6 +130,27 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufspeak, CTLFLAG_RD, &nsfbufspeak, 0,
 SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufsused, CTLFLAG_RD, &nsfbufsused, 0,
     "Number of sendfile(2) sf_bufs in use");
 
+static void
+sfstat_init(const void *unused)
+{
+
+	COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t),
+	    M_WAITOK);
+}
+SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL);
+
+static int
+sfstat_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct sfstat s;
+
+	COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t));
+	if (req->newptr)
+		COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t));
+	return (SYSCTL_OUT(req, &s, sizeof(s)));
+}
+SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW,
+    NULL, 0, sfstat_sysctl, "I", "sendfile statistics");
 /*
  * Convert a user file descriptor to a kernel file entry and check if required
  * capability rights are present.
@@ -2241,7 +2263,7 @@ retry_space:
 				vm_page_io_finish(pg);
 				if (!error)
 					VM_OBJECT_WUNLOCK(obj);
-				mbstat.sf_iocnt++;
+				SFSTAT_INC(sf_iocnt);
 			}
 			if (error) {
 				vm_page_lock(pg);
@@ -2273,7 +2295,7 @@ retry_space:
 			sf = sf_buf_alloc(pg, (mnw || m != NULL) ? SFB_NOWAIT :
 			    SFB_CATCH);
 			if (sf == NULL) {
-				mbstat.sf_allocfail++;
+				SFSTAT_INC(sf_allocfail);
 				vm_page_lock(pg);
 				vm_page_unwire(pg, 0);
 				KASSERT(pg->object != NULL,
diff --git a/sys/mips/mips/vm_machdep.c b/sys/mips/mips/vm_machdep.c
index 4c2460a..0323bb3 100644
--- a/sys/mips/mips/vm_machdep.c
+++ b/sys/mips/mips/vm_machdep.c
@@ -540,7 +540,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			break;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_freelist, &sf_freelist.sf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c
index 8deb821..785f22a 100644
--- a/sys/powerpc/aim/vm_machdep.c
+++ b/sys/powerpc/aim/vm_machdep.c
@@ -299,7 +299,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
                         goto done;
 
                 sf_buf_alloc_want++;
-                mbstat.sf_allocwait++;
+                SFSTAT_INC(sf_allocwait);
                 error = msleep(&sf_buf_freelist, &sf_buf_lock,
                     (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
                 sf_buf_alloc_want--;
diff --git a/sys/powerpc/booke/vm_machdep.c b/sys/powerpc/booke/vm_machdep.c
index 54a0b01..3303794 100644
--- a/sys/powerpc/booke/vm_machdep.c
+++ b/sys/powerpc/booke/vm_machdep.c
@@ -300,7 +300,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 			goto done;
 
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_buf_freelist, &sf_buf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/sparc64/sparc64/vm_machdep.c b/sys/sparc64/sparc64/vm_machdep.c
index 0196ec2..8cec001 100644
--- a/sys/sparc64/sparc64/vm_machdep.c
+++ b/sys/sparc64/sparc64/vm_machdep.c
@@ -445,7 +445,7 @@ sf_buf_alloc(struct vm_page *m, int flags)
 		if (flags & SFB_NOWAIT)
 			break;
 		sf_buf_alloc_want++;
-		mbstat.sf_allocwait++;
+		SFSTAT_INC(sf_allocwait);
 		error = msleep(&sf_freelist, &sf_freelist.sf_lock,
 		    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
 		sf_buf_alloc_want--;
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 82f48d3..4c8ade0 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -338,10 +338,7 @@ struct mbstat {
 	/* Number of mbtypes (gives # elems in mbtypes[] array) */
 	short	m_numtypes;
 
-	/* XXX: Sendfile stats should eventually move to their own struct */
-	u_long	sf_iocnt;	/* times sendfile had to do disk I/O */
-	u_long	sf_allocfail;	/* times sfbuf allocation failed */
-	u_long	sf_allocwait;	/* times sfbuf allocation had to wait */
+	u_long	spare[3];
 };
 
 /*
diff --git a/sys/sys/sf_buf.h b/sys/sys/sf_buf.h
index af42065..1b62fd3 100644
--- a/sys/sys/sf_buf.h
+++ b/sys/sys/sf_buf.h
@@ -29,8 +29,6 @@
 #ifndef _SYS_SF_BUF_H_
 #define _SYS_SF_BUF_H_
 
-#include <machine/sf_buf.h>
-
 /*
  * Options to sf_buf_alloc() are specified through its flags argument.  This
  * argument's value should be the result of a bitwise or'ing of one or more
@@ -48,6 +46,23 @@ extern  int nsfbufs;                    /* Number of sendfile(2) bufs alloced */
 extern  int nsfbufspeak;                /* Peak of nsfbufsused */
 extern  int nsfbufsused;                /* Number of sendfile(2) bufs in use */
 
+struct sfstat {				/* sendfile statistics */
+	uint64_t	sf_iocnt;	/* times sendfile had to do disk I/O */
+	uint64_t	sf_allocfail;	/* times sfbuf allocation failed */
+	uint64_t	sf_allocwait;	/* times sfbuf allocation had to wait */
+};
+
+#ifdef _KERNEL
+#include <machine/sf_buf.h>
+#include <sys/counter.h>
+
+extern counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)];
+#define	SFSTAT_ADD(name, val)	\
+    counter_u64_add(sfstat[offsetof(struct sfstat, name) / sizeof(uint64_t)],\
+	(val))
+#define	SFSTAT_INC(name)	SFSTAT_ADD(name, 1)
+#endif /* _KERNEL */
+
 struct sf_buf *
 	sf_buf_alloc(struct vm_page *m, int flags);
 void	sf_buf_free(struct sf_buf *sf);