Update the Tigon 1 and 2 driver to use the sf_buf API for implementing

zero-copy receive of jumbo frames.  This eliminates the need for the
jumbo frame allocator implemented in kern/uipc_jumbo.c and sys/jumbo.h.
Remove it.

Note: Zero-copy receive of jumbo frames did not work without these changes;
I believe there was insufficient locking on the jumbo vm object.

Tested by: ken@
Discussed with: gallatin@

6 files changed, 50 insertions, 350 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 44a25fa..a2d8bb1 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1167,7 +1167,6 @@ kern/tty_tty.c		standard
 kern/uipc_accf.c	optional inet
 kern/uipc_cow.c		optional zero_copy_sockets
 kern/uipc_domain.c	standard
-kern/uipc_jumbo.c	standard
 kern/uipc_mbuf.c	standard
 kern/uipc_mbuf2.c	standard
 kern/uipc_proto.c	standard
diff --git a/sys/dev/ti/if_ti.c b/sys/dev/ti/if_ti.c
index f9ee1e4..461ca32 100644
--- a/sys/dev/ti/if_ti.c
+++ b/sys/dev/ti/if_ti.c
@@ -120,6 +120,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sockio.h>
 #include <sys/uio.h>
 #include <sys/lock.h>
+#include <sys/sf_buf.h>
 #include <vm/vm_extern.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
@@ -131,7 +132,6 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_object.h>
 #include <vm/vm_kern.h>
 #include <sys/proc.h>
-#include <sys/jumbo.h>
 #endif /* !TI_PRIVATE_JUMBOS */
 
 #include <dev/pci/pcireg.h>
@@ -1251,8 +1251,9 @@ ti_newbuf_jumbo(sc, idx, m_old)
 	struct mbuf		*m[3] = {NULL, NULL, NULL};
 	struct ti_rx_desc_ext	*r;
 	vm_page_t		frame;
+	static int		color;
 				/* 1 extra buf to make nobufs easy*/
-	caddr_t			buf[3] = {NULL, NULL, NULL};
+	struct sf_buf		*sf[3] = {NULL, NULL, NULL};
 	int			i;
 
 	if (m_old != NULL) {
@@ -1291,20 +1292,33 @@ ti_newbuf_jumbo(sc, idx, m_old)
 				       "-- packet dropped!\n", sc->ti_unit);
 				goto nobufs;
 			}
-			if (!(frame = jumbo_pg_alloc())){
+			frame = vm_page_alloc(NULL, color++,
+			    VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
+			    VM_ALLOC_WIRED);
+			if (frame == NULL) {
+				printf("ti%d: buffer allocation failed "
+				       "-- packet dropped!\n", sc->ti_unit);
+				printf("      index %d page %d\n", idx, i);
+				goto nobufs;
+			}
+			sf[i] = sf_buf_alloc(frame, SFB_NOWAIT);
+			if (sf[i] == NULL) {
+				vm_page_lock_queues();
+				vm_page_unwire(frame, 0);
+				vm_page_free(frame);
+				vm_page_unlock_queues();
 				printf("ti%d: buffer allocation failed "
 				       "-- packet dropped!\n", sc->ti_unit);
 				printf("      index %d page %d\n", idx, i);
 				goto nobufs;
 			}
-			buf[i] = jumbo_phys_to_kva(VM_PAGE_TO_PHYS(frame));
 		}
 		for (i = 0; i < NPAYLOAD; i++){
 		/* Attach the buffer to the mbuf. */
-			m[i]->m_data = (void *)buf[i];
+			m[i]->m_data = (void *)sf_buf_kva(sf[i]);
 			m[i]->m_len = PAGE_SIZE;
-			MEXTADD(m[i], (void *)buf[i], PAGE_SIZE,
-				jumbo_freem, NULL, 0, EXT_DISPOSABLE);
+			MEXTADD(m[i], sf_buf_kva(sf[i]), PAGE_SIZE,
+			    sf_buf_mext, sf[i], 0, EXT_DISPOSABLE);
 			m[i]->m_next = m[i+1];
 		}
 		/* link the buffers to the header */
@@ -1360,8 +1374,8 @@ nobufs:
 	for (i = 0; i < 3; i++) {
 		if (m[i])
 			m_freem(m[i]);
-		if (buf[i])
-			jumbo_pg_free((vm_offset_t)buf[i]);
+		if (sf[i])
+			sf_buf_mext((void *)sf_buf_kva(sf[i]), sf[i]);
 	}
 	return (ENOBUFS);
 }
@@ -2116,12 +2130,6 @@ ti_attach(dev)
 		error = ENXIO;
 		goto fail;
 	}
-#else
-	if (!jumbo_vm_init()) {
-		printf("ti%d: VM initialization failed!\n", sc->ti_unit);
-		error = ENOMEM;
-		goto fail;
-	}
 #endif
 
 	/*
diff --git a/sys/kern/kern_subr.c b/sys/kern/kern_subr.c
index 6357c06..48c3af4 100644
--- a/sys/kern/kern_subr.c
+++ b/sys/kern/kern_subr.c
@@ -83,9 +83,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
 	 * First lookup the kernel page.
 	 */
 	kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
-	/*
-	 * XXX The vm object containing kern_pg needs locking.
-	 */
+
 	if ((vm_map_lookup(&map, uaddr,
 			   VM_PROT_WRITE, &entry, &uobject,
 			   &upindex, &prot, &wired)) != KERN_SUCCESS) {
@@ -111,7 +109,8 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
 		else
 			panic("vm_pgmoveco: renaming busy page");
 	}
-	vm_page_rename(kern_pg, uobject, upindex);
+	vm_page_insert(kern_pg, uobject, upindex);
+	vm_page_dirty(kern_pg);
 	kern_pg->valid = VM_PAGE_BITS_ALL;
 	vm_page_unlock_queues();
 	VM_OBJECT_UNLOCK(uobject);
@@ -221,12 +220,11 @@ userspaceco(void *cp, u_int cnt, struct uio *uio, struct vm_object *obj,
 	iov = uio->uio_iov;
 	if (uio->uio_rw == UIO_READ) {
 		if ((so_zero_copy_receive != 0)
-		 && (obj != NULL)
+		 && (obj == NULL)
 		 && ((cnt & PAGE_MASK) == 0)
 		 && ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
 		 && ((uio->uio_offset & PAGE_MASK) == 0)
 		 && ((((intptr_t) cp) & PAGE_MASK) == 0)
-		 && (obj->type == OBJT_DEFAULT)
 		 && (disposable != 0)) {
 			/* SOCKET: use page-trading */
 			/*
diff --git a/sys/kern/uipc_jumbo.c b/sys/kern/uipc_jumbo.c
deleted file mode 100644
index f0aba95..0000000
--- a/sys/kern/uipc_jumbo.c
+++ /dev/null
@@ -1,254 +0,0 @@
-/*-
- * Copyright (c) 1997, Duke University
- * All rights reserved.
- *
- * Author:
- *         Andrew Gallatin <gallatin@cs.duke.edu>  
- *            
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of Duke University may not be used to endorse or promote 
- *    products derived from this software without specific prior written 
- *    permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY DUKE UNIVERSITY ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DUKE UNIVERSITY BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITSOR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.  
- */
-
-/*
- * This is a set of routines for allocating large-sized mbuf payload
- * areas, and is primarily intended for use in receive side mbuf
- * allocation.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/types.h>
-#include <sys/sockio.h>
-#include <sys/uio.h>
-#include <sys/lock.h>
-#include <sys/kernel.h>
-#include <sys/mutex.h>
-#include <sys/malloc.h>
-#include <vm/vm.h>
-#include <vm/pmap.h>
-#include <vm/vm_extern.h>
-#include <vm/vm_map.h>
-#include <vm/vm_param.h>
-#include <vm/vm_pageout.h>
-#include <sys/vmmeter.h>
-#include <vm/vm_page.h>
-#include <vm/vm_object.h>
-#include <vm/vm_kern.h>
-#include <sys/proc.h>
-#include <sys/jumbo.h>
-
-/*
- * XXX this may be too high or too low.
- */
-#define JUMBO_MAX_PAGES 3072
-
-struct jumbo_kmap {
-	vm_offset_t kva;
-	SLIST_ENTRY(jumbo_kmap) entries;     /* Singly-linked List. */
-};
-
-static SLIST_HEAD(jumbo_kmap_head, jumbo_kmap) jumbo_kmap_free,
-					       jumbo_kmap_inuse;
-
-static struct mtx jumbo_mutex;
-MTX_SYSINIT(jumbo_lock, &jumbo_mutex, "jumbo mutex", MTX_DEF);
-
-static struct vm_object *jumbo_vm_object;
-static unsigned long jumbo_vmuiomove_pgs_freed = 0;
-#if 0
-static int jumbo_vm_wakeup_wanted = 0;
-#endif
-vm_offset_t jumbo_basekva;
-
-int
-jumbo_vm_init(void)
-{
-	int i;
-	struct jumbo_kmap *entry;
-
-	mtx_lock(&jumbo_mutex);
-
-	if (jumbo_vm_object != NULL) {
-		mtx_unlock(&jumbo_mutex);
-		return (1);
-	}
-
-	/* allocate our object */
-	jumbo_vm_object = vm_object_allocate_wait(OBJT_DEFAULT, JUMBO_MAX_PAGES,
-						  M_NOWAIT);
-
-	if (jumbo_vm_object == NULL) {
-		mtx_unlock(&jumbo_mutex);
-		return (0);
-	}
-
-	SLIST_INIT(&jumbo_kmap_free);
-	SLIST_INIT(&jumbo_kmap_inuse);
-
-	/* grab some kernel virtual address space */
-	jumbo_basekva = kmem_alloc_nofault(kernel_map,
-		PAGE_SIZE * JUMBO_MAX_PAGES);
-	if (jumbo_basekva == 0) {
-		vm_object_deallocate(jumbo_vm_object);
-		jumbo_vm_object = NULL;
-		mtx_unlock(&jumbo_mutex);
-		return 0;
-	}
-	for (i = 0; i < JUMBO_MAX_PAGES; i++) {
-		entry = malloc(sizeof(struct jumbo_kmap), M_TEMP, M_NOWAIT);
-		if (!entry && !i)  {
-			mtx_unlock(&jumbo_mutex);
-			panic("jumbo_vm_init: unable to allocated kvas");
-		} else if (!entry) {
-			printf("warning: jumbo_vm_init allocated only %d kva\n",
-			       i);
-			mtx_unlock(&jumbo_mutex);
-			return 1;
-		}
-		entry->kva = jumbo_basekva + (vm_offset_t)i * PAGE_SIZE;
-		SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries);
-	}
-	mtx_unlock(&jumbo_mutex);
-	return 1;
-}
-
-void
-jumbo_freem(void *addr, void *args)
-{
-	vm_page_t frame;
-
-	frame = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)addr));
-
-	/*
-	 * Need giant for looking at the hold count below.  Convert this
-	 * to the vm mutex once the VM code has been moved out from under
-	 * giant.
-	 */
-	GIANT_REQUIRED;
-
-	if (frame->hold_count == 0)
-		jumbo_pg_free((vm_offset_t)addr);
-	else
-		printf("jumbo_freem: hold count for %p is %d!!??\n",
-		       frame, frame->hold_count);
-}
-
-void 
-jumbo_pg_steal(vm_page_t pg)
-{
-	vm_offset_t addr;
-	struct jumbo_kmap *entry;
-
-	addr = ptoa(pg->pindex) + jumbo_basekva;
-
-	if (pg->object != jumbo_vm_object)
-		panic("stealing a non jumbo_vm_object page");
-	vm_page_remove(pg);
-
-	mtx_lock(&jumbo_mutex);
-
-	pmap_qremove(addr,1);
-	entry = SLIST_FIRST(&jumbo_kmap_inuse);
-	entry->kva = addr;
-	SLIST_REMOVE_HEAD(&jumbo_kmap_inuse, entries);
-	SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries);
-
-	mtx_unlock(&jumbo_mutex);
-
-#if 0
-	if (jumbo_vm_wakeup_wanted)
-		wakeup(jumbo_vm_object);
-#endif
-}
-
-
-vm_page_t 
-jumbo_pg_alloc(void)
-{
-	vm_page_t pg;
-	vm_pindex_t pindex;
-	struct jumbo_kmap *entry;
-
- 	pg = NULL;
-	mtx_lock(&jumbo_mutex);
-
-	entry = SLIST_FIRST(&jumbo_kmap_free);
-	if (entry != NULL){
-		pindex = atop(entry->kva - jumbo_basekva);
-		VM_OBJECT_LOCK(jumbo_vm_object);
-		pg = vm_page_alloc(jumbo_vm_object, pindex, VM_ALLOC_INTERRUPT);
-		VM_OBJECT_UNLOCK(jumbo_vm_object);
-		if (pg != NULL) {
-			SLIST_REMOVE_HEAD(&jumbo_kmap_free, entries);
-			SLIST_INSERT_HEAD(&jumbo_kmap_inuse, entry, entries);
-			pmap_qenter(entry->kva, &pg, 1);
-		}
-	}
-	mtx_unlock(&jumbo_mutex);
-	return(pg);
-}
-
-void 
-jumbo_pg_free(vm_offset_t addr)
-{
-	struct jumbo_kmap *entry;
-	vm_paddr_t paddr;
-	vm_page_t pg;
-
-	paddr = pmap_kextract((vm_offset_t)addr);
-	pg = PHYS_TO_VM_PAGE(paddr);
-
-	VM_OBJECT_LOCK(jumbo_vm_object);
-	if (pg->object != jumbo_vm_object) {
-		jumbo_vmuiomove_pgs_freed++;
-/*		if(vm_page_lookup(jumbo_vm_object, atop(addr - jumbo_basekva)))
-			panic("vm_page_rename didn't");
-		printf("freeing uiomoved pg:\t pindex = %d, padd = 0x%lx\n",
-		       atop(addr - jumbo_basekva), paddr);
-*/
-	} else {
-		vm_page_lock_queues();
-		vm_page_free(pg);
-		vm_page_unlock_queues();
-	}
-	VM_OBJECT_UNLOCK(jumbo_vm_object);
-
-	mtx_lock(&jumbo_mutex);
-
-	pmap_qremove(addr,1);
-	entry = SLIST_FIRST(&jumbo_kmap_inuse);
-	entry->kva = addr;
-	SLIST_REMOVE_HEAD(&jumbo_kmap_inuse, entries);
-	SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries);
-
-	mtx_unlock(&jumbo_mutex);
-
-#if 0
-	if (jumbo_vm_wakeup_wanted)
-		wakeup(jumbo_vm_object);
-#endif
-}
diff --git a/sys/pci/if_ti.c b/sys/pci/if_ti.c
index f9ee1e4..461ca32 100644
--- a/sys/pci/if_ti.c
+++ b/sys/pci/if_ti.c
@@ -120,6 +120,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sockio.h>
 #include <sys/uio.h>
 #include <sys/lock.h>
+#include <sys/sf_buf.h>
 #include <vm/vm_extern.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
@@ -131,7 +132,6 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_object.h>
 #include <vm/vm_kern.h>
 #include <sys/proc.h>
-#include <sys/jumbo.h>
 #endif /* !TI_PRIVATE_JUMBOS */
 
 #include <dev/pci/pcireg.h>
@@ -1251,8 +1251,9 @@ ti_newbuf_jumbo(sc, idx, m_old)
 	struct mbuf		*m[3] = {NULL, NULL, NULL};
 	struct ti_rx_desc_ext	*r;
 	vm_page_t		frame;
+	static int		color;
 				/* 1 extra buf to make nobufs easy*/
-	caddr_t			buf[3] = {NULL, NULL, NULL};
+	struct sf_buf		*sf[3] = {NULL, NULL, NULL};
 	int			i;
 
 	if (m_old != NULL) {
@@ -1291,20 +1292,33 @@ ti_newbuf_jumbo(sc, idx, m_old)
 				       "-- packet dropped!\n", sc->ti_unit);
 				goto nobufs;
 			}
-			if (!(frame = jumbo_pg_alloc())){
+			frame = vm_page_alloc(NULL, color++,
+			    VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
+			    VM_ALLOC_WIRED);
+			if (frame == NULL) {
+				printf("ti%d: buffer allocation failed "
+				       "-- packet dropped!\n", sc->ti_unit);
+				printf("      index %d page %d\n", idx, i);
+				goto nobufs;
+			}
+			sf[i] = sf_buf_alloc(frame, SFB_NOWAIT);
+			if (sf[i] == NULL) {
+				vm_page_lock_queues();
+				vm_page_unwire(frame, 0);
+				vm_page_free(frame);
+				vm_page_unlock_queues();
 				printf("ti%d: buffer allocation failed "
 				       "-- packet dropped!\n", sc->ti_unit);
 				printf("      index %d page %d\n", idx, i);
 				goto nobufs;
 			}
-			buf[i] = jumbo_phys_to_kva(VM_PAGE_TO_PHYS(frame));
 		}
 		for (i = 0; i < NPAYLOAD; i++){
 		/* Attach the buffer to the mbuf. */
-			m[i]->m_data = (void *)buf[i];
+			m[i]->m_data = (void *)sf_buf_kva(sf[i]);
 			m[i]->m_len = PAGE_SIZE;
-			MEXTADD(m[i], (void *)buf[i], PAGE_SIZE,
-				jumbo_freem, NULL, 0, EXT_DISPOSABLE);
+			MEXTADD(m[i], sf_buf_kva(sf[i]), PAGE_SIZE,
+			    sf_buf_mext, sf[i], 0, EXT_DISPOSABLE);
 			m[i]->m_next = m[i+1];
 		}
 		/* link the buffers to the header */
@@ -1360,8 +1374,8 @@ nobufs:
 	for (i = 0; i < 3; i++) {
 		if (m[i])
 			m_freem(m[i]);
-		if (buf[i])
-			jumbo_pg_free((vm_offset_t)buf[i]);
+		if (sf[i])
+			sf_buf_mext((void *)sf_buf_kva(sf[i]), sf[i]);
 	}
 	return (ENOBUFS);
 }
@@ -2116,12 +2130,6 @@ ti_attach(dev)
 		error = ENXIO;
 		goto fail;
 	}
-#else
-	if (!jumbo_vm_init()) {
-		printf("ti%d: VM initialization failed!\n", sc->ti_unit);
-		error = ENOMEM;
-		goto fail;
-	}
 #endif
 
 	/*
diff --git a/sys/sys/jumbo.h b/sys/sys/jumbo.h
deleted file mode 100644
index 3d7cc28..0000000
--- a/sys/sys/jumbo.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * Copyright (c) 1997, Duke University
- * All rights reserved.
- *
- * Author:
- *         Andrew Gallatin <gallatin@cs.duke.edu>  
- *            
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of Duke University may not be used to endorse or promote 
- *    products derived from this software without specific prior written 
- *    permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY DUKE UNIVERSITY ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DUKE UNIVERSITY BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITSOR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.  
- *
- * $FreeBSD$
- */
-
-#ifndef _SYS_JUMBO_H_
-#define _SYS_JUMBO_H_
-
-#ifdef _KERNEL
-extern vm_offset_t jumbo_basekva;
-
-static __inline caddr_t	jumbo_phys_to_kva(vm_paddr_t pa);
-static __inline caddr_t
-jumbo_phys_to_kva(vm_paddr_t pa)
-{
-	vm_page_t pg;
-
-	pg = PHYS_TO_VM_PAGE(pa);
-	pg->flags &= ~PG_BUSY;
-	return (caddr_t)(ptoa((vm_offset_t)pg->pindex) + jumbo_basekva);
-}
-
-int		jumbo_vm_init(void);
-void		jumbo_freem(void *addr, void *args);
-vm_page_t	jumbo_pg_alloc(void);
-void		jumbo_pg_free(vm_offset_t addr);
-void 		jumbo_pg_steal(vm_page_t pg);
-#endif /* _KERNEL */
-
-#endif /* !_SYS_JUMBO_H_ */