Device Polling code for -current.

Non-SMP, i386-only, no polling in the idle loop at the moment.

To use this code you must compile a kernel with

        options DEVICE_POLLING

and at runtime enable polling with

        sysctl kern.polling.enable=1

The percentage of CPU reserved to userland can be set with

        sysctl kern.polling.user_frac=NN (default is 50)

while the remainder is used by polling device drivers and netisr's.
These are the only two variables that you should need to touch. There
are a few more parameters in kern.polling but the default values
are adequate for all purposes. See the code in kern_poll.c for
more details on them.

Polling in the idle loop will be implemented shortly by introducing
a kernel thread which does the job. Until then, the amount of CPU
dedicated to polling will never exceed (100-user_frac).
The equivalent (actually, better) code for -stable is at

	http://info.iet.unipi.it/~luigi/polling/

and also supports polling in the idle loop.

NOTE to Alpha developers:
There is really nothing in this code that is i386-specific.
If you move the 2 lines supporting the new option from
sys/conf/{files,options}.i386 to sys/conf/{files,options} I am
pretty sure that this should work on the Alpha as well, just that
I do not have a suitable test box to try it. If someone feels like
trying it, I would appreciate it.

NOTE to other developers:
sure some things could be done better, and as always I am open to
constructive criticism, which a few of you have already given and
I greatly appreciated.
However, before proposing radical architectural changes, please
take some time to possibly try out this code, or at the very least
read the comments in kern_poll.c, especially re. the reason why I
am using a soft netisr and cannot (I believe) replace it with a
simple timeout.

Quick description of files touched by this commit:

sys/conf/files.i386
        new file kern/kern_poll.c
sys/conf/options.i386
        new option
sys/i386/i386/trap.c
        poll in trap (disabled by default)
sys/kern/kern_clock.c
        initialization and hardclock hooks.
sys/kern/kern_intr.c
        minor swi_net changes
sys/kern/kern_poll.c
        the bulk of the code.
sys/net/if.h
        new flag
sys/net/if_var.h
        declaration for functions used in device drivers.
sys/net/netisr.h
        NETISR_POLL
sys/dev/fxp/if_fxp.c
sys/dev/fxp/if_fxpvar.h
sys/pci/if_dc.c
sys/pci/if_dcreg.h
sys/pci/if_sis.c
sys/pci/if_sisreg.h
        device driver modifications

3 files changed, 448 insertions, 0 deletions

diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 4c2ccc0..a338dca 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -69,6 +69,12 @@
 #include <sys/gmon.h>
 #endif
 
+#ifdef DEVICE_POLLING
+#include <net/netisr.h>		/* for NETISR_POLL */
+ 
+extern void ether_poll1(void);
+extern void hardclock_device_poll(void);
+#endif /* DEVICE_POLLING */
 
 static void initclocks __P((void *dummy));
 SYSINIT(clocks, SI_SUB_CLOCKS, SI_ORDER_FIRST, initclocks, NULL)
@@ -140,6 +146,9 @@ initclocks(dummy)
 	psdiv = pscnt = 1;
 	cpu_initclocks();
 
+#ifdef DEVICE_POLLING
+	register_netisr(NETISR_POLL, ether_poll1);
+#endif
 	/*
 	 * Compute profhz/stathz, and fix profhz if needed.
 	 */
@@ -212,6 +221,9 @@ hardclock(frame)
 		statclock(frame);
 
 	tc_windup();
+#ifdef DEVICE_POLLING
+	hardclock_device_poll();
+#endif /* DEVICE_POLLING */
 
 	/*
 	 * Process callouts at a very low cpu priority, so we don't keep the
diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c
index e873c6e..88868dc 100644
--- a/sys/kern/kern_intr.c
+++ b/sys/kern/kern_intr.c
@@ -623,7 +623,16 @@ swi_net(void *dummy)
 	u_int bits;
 	int i;
 
+#ifdef DEVICE_POLLING
+    for (;;) {
+	int pollmore;
+#endif
 	bits = atomic_readandclear_int(&netisr);
+#ifdef DEVICE_POLLING
+	if (bits == 0)
+		return;
+	pollmore = bits & (1 << NETISR_POLL);
+#endif
 	while ((i = ffs(bits)) != 0) {
 		i--;
 		if (netisrs[i] != NULL)
@@ -632,6 +641,11 @@ swi_net(void *dummy)
 			printf("swi_net: unregistered isr number: %d.\n", i);
 		bits &= ~(1 << i);
 	}
+#ifdef DEVICE_POLLING
+	if (pollmore)
+		ether_pollmore();
+    }
+#endif
 }
 
 /* 
diff --git a/sys/kern/kern_poll.c b/sys/kern/kern_poll.c
new file mode 100644
index 0000000..763e5cb
--- /dev/null
+++ b/sys/kern/kern_poll.c
@@ -0,0 +1,422 @@
+/*-
+ * Copyright (c) 2001 Luigi Rizzo
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/socket.h>			/* needed by net/if.h		*/
+#include <sys/sysctl.h>
+
+#include <net/if.h>			/* for IFF_* flags		*/
+#include <net/netisr.h>			/* for NETISR_POLL		*/
+
+#ifdef SMP
+#error DEVICE_POLLING is not compatible with SMP
+#endif
+
+void ether_poll1(void);
+void ether_poll(int);			/* polling while in trap	*/
+void ether_pollmore(void);
+void hardclock_device_poll(void);
+
+/*
+ * Polling support for [network] device drivers.
+ *
+ * Drivers which support this feature try to register with the
+ * polling code.
+ *
+ * If registration is successful, the driver must disable interrupts,
+ * and further I/O is performed through the handler, which is invoked
+ * (at least once per clock tick) with 3 arguments: the "arg" passed at
+ * register time (a struct ifnet pointer), a command, and a "count" limit.
+ *
+ * The command can be one of the following:
+ *  POLL_ONLY: quick move of "count" packets from input/output queues.
+ *  POLL_AND_CHECK_STATUS: as above, plus check status registers or do
+ *	other more expensive operations. This command is issued periodically
+ *	but less frequently than POLL_ONLY.
+ *  POLL_DEREGISTER: deregister and return to interrupt mode.
+ *
+ * The first two commands are only issued if the interface is marked as
+ * 'IFF_UP and IFF_RUNNING', the last one only if IFF_RUNNING is set.
+ *
+ * The count limit specifies how much work the handler can do during the
+ * call -- typically this is the number of packets to be received, or
+ * transmitted, etc. (drivers are free to interpret this number, as long
+ * as the max time spent in the function grows roughly linearly with the
+ * count).
+ *
+ * Deregistration can be requested by the driver itself (typically in the
+ * *_stop() routine), or by the polling code, by invoking the handler.
+ *
+ * Polling can be globally enabled or disabled with the sysctl variable
+ * kern.polling.enable (default is 0, disabled)
+ *
+ * A second variable controls the sharing of CPU between polling/kernel
+ * network processing, and other activities (typically userlevel tasks):
+ * kern.polling.user_frac (between 0 and 100, default 50) sets the share
+ * of CPU allocated to user tasks. CPU is allocated proportionally to the
+ * shares, by dynamically adjusting the "count" (poll_burst).
+ *
+ * Other parameters can should be left to their default values.
+ * The following constraints hold
+ *
+ *	1 <= poll_each_burst <= poll_burst <= poll_burst_max
+ *	0 <= poll_in_trap <= poll_each_burst
+ *	MIN_POLL_BURST_MAX <= poll_burst_max <= MAX_POLL_BURST_MAX
+ */
+
+#define MIN_POLL_BURST_MAX	10
+#define MAX_POLL_BURST_MAX	1000
+
+SYSCTL_NODE(_kern, OID_AUTO, polling, CTLFLAG_RW, 0,
+	"Device polling parameters");
+
+static u_int32_t poll_burst = 5;
+SYSCTL_ULONG(_kern_polling, OID_AUTO, burst, CTLFLAG_RW,
+	&poll_burst, 0, "Current polling burst size");
+
+static u_int32_t poll_each_burst = 5;
+SYSCTL_ULONG(_kern_polling, OID_AUTO, each_burst, CTLFLAG_RW,
+	&poll_each_burst, 0, "Max size of each burst");
+
+static u_int32_t poll_burst_max = 150;	/* good for 100Mbit net and HZ=1000 */
+SYSCTL_ULONG(_kern_polling, OID_AUTO, burst_max, CTLFLAG_RW,
+	&poll_burst_max, 0, "Max Polling burst size");
+
+u_int32_t poll_in_trap;			/* used in trap.c */
+SYSCTL_ULONG(_kern_polling, OID_AUTO, poll_in_trap, CTLFLAG_RW,
+	&poll_in_trap, 0, "Poll burst size during a trap");
+
+static u_int32_t user_frac = 50;
+SYSCTL_ULONG(_kern_polling, OID_AUTO, user_frac, CTLFLAG_RW,
+	&user_frac, 0, "Desired user fraction of cpu time");
+
+static u_int32_t reg_frac = 20 ;
+SYSCTL_ULONG(_kern_polling, OID_AUTO, reg_frac, CTLFLAG_RW,
+	&reg_frac, 0, "Every this many cycles poll register");
+
+static u_int32_t short_ticks;
+SYSCTL_ULONG(_kern_polling, OID_AUTO, short_ticks, CTLFLAG_RW,
+	&short_ticks, 0, "Hardclock ticks shorter than they should be");
+
+static u_int32_t lost_polls;
+SYSCTL_ULONG(_kern_polling, OID_AUTO, lost_polls, CTLFLAG_RW,
+	&lost_polls, 0, "How many times we would have lost a poll tick");
+
+static u_int32_t poll_handlers; /* next free entry in pr[]. */
+SYSCTL_ULONG(_kern_polling, OID_AUTO, handlers, CTLFLAG_RD,
+	&poll_handlers, 0, "Number of registered poll handlers");
+
+static int polling = 0;		/* global polling enable */
+SYSCTL_ULONG(_kern_polling, OID_AUTO, enable, CTLFLAG_RW,
+	&polling, 0, "Polling enabled");
+
+
+static u_int32_t poll1_active;
+static u_int32_t need_poll_again;
+
+#define POLL_LIST_LEN  128
+struct pollrec {
+	poll_handler_t	*handler;
+	struct ifnet	*ifp;
+};
+
+static struct pollrec pr[POLL_LIST_LEN];
+
+/*
+ * Hook from hardclock. Tries to schedule a netisr, but keeps track
+ * of lost ticks due to the previous handler taking too long.
+ * The first part of the code is just for debugging purposes, and tries
+ * to count how often hardclock ticks are shorter than they should,
+ * meaning either stray interrupts or delayed events.
+ */
+void
+hardclock_device_poll(void)
+{
+	static struct timeval prev_t, t;
+	int delta;
+
+	microuptime(&t);
+	delta = (t.tv_usec - prev_t.tv_usec) +
+		(t.tv_sec - prev_t.tv_sec)*1000000;
+	if (delta * hz < 500000)
+		short_ticks++;
+	else
+		prev_t = t;
+
+	if (poll_handlers > 0) {
+		if (poll1_active) {
+			lost_polls++;
+			need_poll_again++;
+		} else {
+			poll1_active = 1;
+			schednetisr(NETISR_POLL);
+		}
+	}
+}
+
+/*
+ * ether_poll is called from the idle loop or from the trap handler.
+ */
+void
+ether_poll(int count)
+{
+	int i;
+	int s = splimp();
+
+	mtx_lock(&Giant);
+
+	if (count > poll_each_burst)
+		count = poll_each_burst;
+	for (i = 0 ; i < poll_handlers ; i++)
+		if (pr[i].handler && (IFF_UP|IFF_RUNNING) ==
+		    (pr[i].ifp->if_flags & (IFF_UP|IFF_RUNNING)) )
+			pr[i].handler(pr[i].ifp, 0, count); /* quick check */
+	mtx_unlock(&Giant);
+	splx(s);
+}
+
+/*
+ * ether_pollmore is called after other netisr's, possibly scheduling
+ * another NETISR_POLL call, or adapting the burst size for the next cycle.
+ *
+ * It is very bad to fetch large bursts of packets from a single card at once,
+ * because the burst could take a long time to be completely processed, or
+ * could saturate the intermediate queue (ipintrq or similar) leading to
+ * losses or unfairness. To reduce the problem, and also to account better for
+ * time spent in network-related processnig, we split the burst in smaller
+ * chunks of fixed size, giving control to the other netisr's between chunks.
+ * This helps in improving the fairness, reducing livelock (because we
+ * emulate more closely the "process to completion" that we have with
+ * fastforwarding) and accounting for the work performed in low level
+ * handling and forwarding.
+ */
+
+static int residual_burst = 0;
+
+static struct timeval poll_start_t;
+
+void
+ether_pollmore()
+{
+	struct timeval t;
+	int kern_load;
+	int s = splhigh();
+
+	if (residual_burst > 0) {
+		schednetisr(NETISR_POLL);
+		/* will run immediately on return, followed by netisrs */
+		splx(s);
+		return ;
+	}
+	/* here we can account time spent in netisr's in this tick */
+	microuptime(&t);
+	kern_load = (t.tv_usec - poll_start_t.tv_usec) +
+		(t.tv_sec - poll_start_t.tv_sec)*1000000;	/* us */
+	kern_load = (kern_load * hz) / 10000;			/* 0..100 */
+	if (kern_load > (100 - user_frac)) { /* try decrease ticks */
+		if (poll_burst > 1)
+			poll_burst--;
+	} else {
+		if (poll_burst < poll_burst_max)
+			poll_burst++;
+	}
+
+	if (need_poll_again) {
+		/*
+		 * Last cycle was long and caused us to miss one or more
+		 * hardclock ticks. Restart processnig again, but slightly
+		 * reduce the burst size to prevent that this happens again.
+		 */
+		need_poll_again--;
+		poll_burst -= (poll_burst / 8);
+		if (poll_burst < 1)
+			poll_burst = 1;
+		schednetisr(NETISR_POLL);
+	} else
+		poll1_active = 0;
+	splx(s);
+}
+
+/*
+ * ether_poll1 is called by schednetisr when appropriate, typically once
+ * per tick. It is called at splnet() so first thing to do is to upgrade to
+ * splimp(), and call all registered handlers.
+ */
+void
+ether_poll1(void)
+{
+	static int reg_frac_count;
+	int i, cycles;
+	enum poll_cmd arg = POLL_ONLY;
+	int s=splimp();
+	mtx_lock(&Giant);
+
+	if (residual_burst == 0) { /* first call in this tick */
+		microuptime(&poll_start_t);
+		/*
+		 * Check that paremeters are consistent with runtime
+		 * variables. Some of these tests could be done at sysctl
+		 * time, but the savings would be very limited because we
+		 * still have to check against reg_frac_count and
+		 * poll_each_burst. So, instead of writing separate sysctl
+		 * handlers, we do all here.
+		 */
+
+		if (reg_frac > hz)
+			reg_frac = hz;
+		else if (reg_frac < 1)
+			reg_frac = 1;
+		if (reg_frac_count > reg_frac)
+			reg_frac_count = reg_frac - 1;
+		if (reg_frac_count-- == 0) {
+			arg = POLL_AND_CHECK_STATUS;
+			reg_frac_count = reg_frac - 1;
+		}
+		if (poll_burst_max < MIN_POLL_BURST_MAX)
+			poll_burst_max = MIN_POLL_BURST_MAX;
+		else if (poll_burst_max > MAX_POLL_BURST_MAX)
+			poll_burst_max = MAX_POLL_BURST_MAX;
+
+		if (poll_each_burst < 1)
+			poll_each_burst = 1;
+		else if (poll_each_burst > poll_burst_max)
+			poll_each_burst = poll_burst_max;
+
+		residual_burst = poll_burst;
+	}
+	cycles = (residual_burst < poll_each_burst) ?
+		residual_burst : poll_each_burst;
+	residual_burst -= cycles;
+
+	if (polling) {
+		for (i = 0 ; i < poll_handlers ; i++)
+			if (pr[i].handler && (IFF_UP|IFF_RUNNING) ==
+			    (pr[i].ifp->if_flags & (IFF_UP|IFF_RUNNING)) )
+				pr[i].handler(pr[i].ifp, arg, cycles);
+	} else {	/* unregister */
+		for (i = 0 ; i < poll_handlers ; i++) {
+			if (pr[i].handler &&
+			    pr[i].ifp->if_flags & IFF_RUNNING) {
+				pr[i].ifp->if_ipending &= ~IFF_POLLING;
+				pr[i].handler(pr[i].ifp, POLL_DEREGISTER, 1);
+			}
+			pr[i].handler=NULL;
+		}
+		residual_burst = 0;
+		poll_handlers = 0;
+	}
+	/* on -stable, schednetisr(NETISR_POLLMORE); */
+	mtx_unlock(&Giant);
+	splx(s);
+}
+
+/*
+ * Try to register routine for polling. Returns 1 if successful
+ * (and polling should be enabled), 0 otherwise.
+ * A device is not supposed to register itself multiple times.
+ *
+ * This is called from within the *_intr() function, so we should
+ * probably not need further locking. XXX
+ */
+int
+ether_poll_register(poll_handler_t *h, struct ifnet *ifp)
+{
+	int s;
+
+	if (polling == 0) /* polling disabled, cannot register */
+		return 0;
+	if (h == NULL || ifp == NULL)		/* bad arguments	*/
+		return 0;
+	if ( !(ifp->if_flags & IFF_UP) )	/* must be up		*/
+		return 0;
+	if (ifp->if_ipending & IFF_POLLING)	/* already polling	*/
+		return 0;
+
+	s = splhigh();
+	if (poll_handlers >= POLL_LIST_LEN) {
+		/*
+		 * List full, cannot register more entries.
+		 * This should never happen; if it does, it is probably a
+		 * broken driver trying to register multiple times. Checking
+		 * this at runtime is expensive, and won't solve the problem
+		 * anyways, so just report a few times and then give up.
+		 */
+		static int verbose = 10 ;
+		splx(s);
+		if (verbose >0) {
+			printf("poll handlers list full, "
+				"maybe a broken driver ?\n");
+			verbose--;
+		}
+		return 0; /* no polling for you */
+	}
+
+	pr[poll_handlers].handler = h;
+	pr[poll_handlers].ifp = ifp;
+	poll_handlers++;
+	ifp->if_ipending |= IFF_POLLING;
+	splx(s);
+	return 1; /* polling enabled in next call */
+}
+
+/*
+ * Remove the interface from the list of polling ones.
+ * Normally run by *_stop().
+ * We allow it being called with IFF_POLLING clear, the
+ * call is sufficiently rare so it is preferable to save the
+ * space for the extra test in each device in exchange of one
+ * additional function call.
+ */
+int
+ether_poll_deregister(struct ifnet *ifp)
+{
+	int i;
+
+	mtx_lock(&Giant);
+	if ( !ifp || !(ifp->if_ipending & IFF_POLLING) ) {
+		mtx_unlock(&Giant);
+		return 0;
+	}
+	for (i = 0 ; i < poll_handlers ; i++)
+		if (pr[i].ifp == ifp) /* found it */
+			break;
+	ifp->if_ipending &= ~IFF_POLLING; /* found or not... */
+	if (i == poll_handlers) {
+		mtx_unlock(&Giant);
+		printf("ether_poll_deregister: ifp not found!!!\n");
+		return 0;
+	}
+	poll_handlers--;
+	if (i < poll_handlers) { /* Last entry replaces this one. */
+		pr[i].handler = pr[poll_handlers].handler;
+		pr[i].ifp = pr[poll_handlers].ifp;
+	}
+	mtx_unlock(&Giant);
+	return 1;
+}