IFC @ r227804

Pull in the virtio drivers from head.

1431 files changed, 101016 insertions, 13764 deletions

diff --git a/Makefile b/Makefile
index 61e678b..6fd82311 100644
--- a/Makefile
+++ b/Makefile
@@ -182,10 +182,12 @@ buildworld: upgrade_checks
 #
 # In the following, the first 'rm' in a series will usually remove all
 # files and directories.  If it does not, then there are probably some
-# files with chflags set, so this unsets them and tries the 'rm' a
+# files with file flags set, so this unsets them and tries the 'rm' a
 # second time.  There are situations where this target will be cleaning
 # some directories via more than one method, but that duplication is
-# needed to correctly handle all the possible situations.
+# needed to correctly handle all the possible situations.  Removing all
+# files without file flags set in the first 'rm' instance saves time,
+# because 'chflags' will need to operate on fewer files afterwards.
 #
 BW_CANONICALOBJDIR:=${MAKEOBJDIRPREFIX}${.CURDIR}
 cleanworld:
diff --git a/Makefile.inc1 b/Makefile.inc1
index 07e4301..122df0a 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -291,7 +291,7 @@ LIB32CPUFLAGS=	-mcpu=powerpc
 LIB32CPUFLAGS=	-mcpu=${TARGET_CPUTYPE}
 .endif
 LIB32WMAKEENV=	MACHINE=powerpc MACHINE_ARCH=powerpc \
-		LD="${LD} -m elf32ppc"
+		LD="${LD} -m elf32ppc_fbsd"
 .endif
 
 
@@ -313,7 +313,8 @@ LIB32WMAKEENV+=	MAKEOBJDIRPREFIX=${OBJTREE}/lib32 \
 
 LIB32WMAKE=	${LIB32WMAKEENV} ${MAKE} -DNO_CPU_CFLAGS -DCOMPAT_32BIT \
 		-DWITHOUT_BIND -DWITHOUT_MAN -DWITHOUT_INFO \
-		-DWITHOUT_HTML -DNO_CTF -DNO_LINT DESTDIR=${LIB32TMP}
+		-DWITHOUT_HTML -DNO_CTF -DNO_LINT -ECC -ECXX -EAS -ELD \
+		DESTDIR=${LIB32TMP}
 LIB32IMAKE=	${LIB32WMAKE:NINSTALL=*:NDESTDIR=*} -DNO_INCS
 .endif
 
@@ -1229,12 +1230,13 @@ _prebuild_libs=	${_kerberos5_lib_libasn1} ${_kerberos5_lib_libhdb} \
 		${_lib_libgssapi} ${_lib_libipx} \
 		lib/libkiconv lib/libkvm lib/liblzma lib/libmd \
 		lib/ncurses/ncurses lib/ncurses/ncursesw \
-		lib/libopie lib/libpam ${_lib_libthr} \
+		lib/libopie ${_lib_libthr} \
 		lib/libradius lib/libsbuf lib/libtacplus \
 		${_cddl_lib_libumem} \
 		lib/libutil ${_lib_libypclnt} lib/libz lib/msun \
 		${_secure_lib_libcrypto} ${_secure_lib_libssh} \
-		${_secure_lib_libssl}
+		${_secure_lib_libssl} \
+		lib/libpam
 
 .if ${MK_LIBTHR} != "no"
 _lib_libthr=	lib/libthr
@@ -1322,7 +1324,7 @@ ${_lib}__PL: .PHONY
 .endif
 .endfor
 
-.for _lib in ${_startup_libs} ${_prebuild_libs:Nlib/libpam} ${_generic_libs}
+.for _lib in ${_startup_libs} ${_prebuild_libs} ${_generic_libs}
 ${_lib}__L: .PHONY
 .if exists(${.CURDIR}/${_lib})
 	${_+_}@${ECHODIR} "===> ${_lib} (obj,depend,all,install)"; \
@@ -1334,17 +1336,6 @@ ${_lib}__L: .PHONY
 .endif
 .endfor
 
-# libpam is special: we need to build static PAM modules before
-# static PAM library, and dynamic PAM library before dynamic PAM
-# modules.
-lib/libpam__L: .PHONY
-	${_+_}@${ECHODIR} "===> lib/libpam (obj,depend,all,install)"; \
-		cd ${.CURDIR}/lib/libpam; \
-		${MAKE} DIRPRFX=lib/libpam/ obj; \
-		${MAKE} DIRPRFX=lib/libpam/ depend; \
-		${MAKE} DIRPRFX=lib/libpam/ -D_NO_LIBPAM_SO_YET all; \
-		${MAKE} DIRPRFX=lib/libpam/ -D_NO_LIBPAM_SO_YET install
-
 _prereq_libs: ${_prereq_libs:S/$/__PL/}
 _startup_libs: ${_startup_libs:S/$/__L/}
 _prebuild_libs: ${_prebuild_libs:S/$/__L/}
diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index 6ca9d84..8744e2f 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -44,10 +44,12 @@ OLD_FILES+=usr/share/man/man8/sysinstall.8.gz
 OLD_FILES+=usr/lib/libftpio.a
 OLD_FILES+=usr/lib/libftpio.so
 OLD_LIBS+=usr/lib/libftpio.so.8
+OLD_FILES+=usr/lib/libftpio_p.a
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_FILES+=usr/lib32/libftpio.a
 OLD_FILES+=usr/lib32/libftpio.so
 OLD_LIBS+=usr/lib32/libftpio.so.8
+OLD_FILES+=usr/lib32/libftpio_p.a
 .endif
 OLD_FILES+=usr/include/ftpio.h
 OLD_FILES+=usr/share/man/man3/ftpio.3.gz
@@ -77,11 +79,22 @@ OLD_LIBS+=usr/lib32/libopie.so.6
 OLD_LIBS+=usr/lib32/librtld_db.so.1
 OLD_LIBS+=usr/lib32/libtacplus.so.4
 .endif
+# 20110817: no more acd.4, ad.4, afd.4 and ast.4
+OLD_FILES+=usr/share/man/man4/acd.4.gz
+OLD_FILES+=usr/share/man/man4/ad.4.gz
+OLD_FILES+=usr/share/man/man4/afd.4.gz
+OLD_FILES+=usr/share/man/man4/ast.4.gz
 # 20110718: no longer useful in the age of rc.d
 OLD_FILES+=usr/sbin/named.reconfig
 OLD_FILES+=usr/sbin/named.reload
+OLD_FILES+=usr/share/man/man8/named.reconfig.8.gz
+OLD_FILES+=usr/share/man/man8/named.reload.8.gz
 # 20110716: bind 9.8.0 import
 OLD_LIBS+=usr/lib/liblwres.so.50
+OLD_FILES+=usr/share/doc/bind9/KNOWN-DEFECTS
+OLD_FILES+=usr/share/doc/bind9/NSEC3-NOTES
+OLD_FILES+=usr/share/doc/bind9/README.idnkit
+OLD_FILES+=usr/share/doc/bind9/README.pkcs11
 # 20110709: vm_map_clean.9 -> vm_map_sync.9
 OLD_FILES+=usr/share/man/man9/vm_map_clean.9.gz
 # 20110709: Catch up with removal of these functions.
diff --git a/UPDATING b/UPDATING
index f4c029c..dc17383 100644
--- a/UPDATING
+++ b/UPDATING
@@ -22,6 +22,16 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 10.x IS SLOW:
 	machines to maximize performance.  (To disable malloc debugging, run
 	ln -s aj /etc/malloc.conf.)
 
+20111108:
+	The option VFS_ALLOW_NONMPSAFE option has been added in order to
+	explicitely support non-MPSAFE filesystems.
+	It is on by default for all supported platform at this present
+	time.
+
+20111101:
+	The broken amd(4) driver has been replaced with esp(4) in the amd64,
+	i386 and pc98 GENERIC kernel configuration files.
+
 20110930:
 	sysinstall has been removed
 
diff --git a/bin/cat/cat.c b/bin/cat/cat.c
index d338742..437c013 100644
--- a/bin/cat/cat.c
+++ b/bin/cat/cat.c
@@ -64,9 +64,9 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include <stddef.h>
 
-int bflag, eflag, nflag, sflag, tflag, vflag;
-int rval;
-const char *filename;
+static int bflag, eflag, nflag, sflag, tflag, vflag;
+static int rval;
+static const char *filename;
 
 static void usage(void);
 static void scanfiles(char *argv[], int cooked);
diff --git a/bin/ln/ln.c b/bin/ln/ln.c
index e5fdeee..4169e62 100644
--- a/bin/ln/ln.c
+++ b/bin/ln/ln.c
@@ -54,16 +54,16 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-int	fflag;				/* Unlink existing files. */
-int	Fflag;				/* Remove empty directories also. */
-int	hflag;				/* Check new name for symlink first. */
-int	iflag;				/* Interactive mode. */
-int	Pflag;				/* Create hard links to symlinks. */
-int	sflag;				/* Symbolic, not hard, link. */
-int	vflag;				/* Verbose output. */
-int	wflag;				/* Warn if symlink target does not
+static int	fflag;			/* Unlink existing files. */
+static int	Fflag;			/* Remove empty directories also. */
+static int	hflag;			/* Check new name for symlink first. */
+static int	iflag;			/* Interactive mode. */
+static int	Pflag;			/* Create hard links to symlinks. */
+static int	sflag;			/* Symbolic, not hard, link. */
+static int	vflag;			/* Verbose output. */
+static int	wflag;			/* Warn if symlink target does not
 					 * exist, and -f is not enabled. */
-char	linkch;
+static char	linkch;
 
 int	linkit(const char *, const char *, int);
 void	usage(void);
diff --git a/bin/mkdir/mkdir.c b/bin/mkdir/mkdir.c
index 5fa002f..fda9c24 100644
--- a/bin/mkdir/mkdir.c
+++ b/bin/mkdir/mkdir.c
@@ -56,7 +56,7 @@ __FBSDID("$FreeBSD$");
 static int	build(char *, mode_t);
 static void	usage(void);
 
-int vflag;
+static int	vflag;
 
 int
 main(int argc, char *argv[])
diff --git a/bin/mv/mv.c b/bin/mv/mv.c
index 67108f7..e4fe007 100644
--- a/bin/mv/mv.c
+++ b/bin/mv/mv.c
@@ -68,7 +68,7 @@ __FBSDID("$FreeBSD$");
 /* Exit code for a failed exec. */
 #define EXEC_FAILED 127
 
-int fflg, iflg, nflg, vflg;
+static int	fflg, iflg, nflg, vflg;
 
 static int	copy(const char *, const char *);
 static int	do_move(const char *, const char *);
diff --git a/bin/ps/print.c b/bin/ps/print.c
index 45c7823..021415f 100644
--- a/bin/ps/print.c
+++ b/bin/ps/print.c
@@ -362,7 +362,7 @@ tdev(KINFO *k, VARENT *ve)
 	v = ve->var;
 	dev = k->ki_p->ki_tdev;
 	if (dev == NODEV)
-		str = strdup("??");
+		str = strdup("-");
 	else
 		asprintf(&str, "%#jx", (uintmax_t)dev);
 
@@ -379,7 +379,7 @@ tname(KINFO *k, VARENT *ve)
 	v = ve->var;
 	dev = k->ki_p->ki_tdev;
 	if (dev == NODEV || (ttname = devname(dev, S_IFCHR)) == NULL)
-		str = strdup("?? ");
+		str = strdup("- ");
 	else {
 		if (strncmp(ttname, "tty", 3) == 0 ||
 		    strncmp(ttname, "cua", 3) == 0)
@@ -403,7 +403,7 @@ longtname(KINFO *k, VARENT *ve)
 	v = ve->var;
 	dev = k->ki_p->ki_tdev;
 	if (dev == NODEV || (ttname = devname(dev, S_IFCHR)) == NULL)
-		ttname = "??";
+		ttname = "-";
 
 	return (strdup(ttname));
 }
diff --git a/bin/ps/ps.1 b/bin/ps/ps.1
index 2c04c21..1787b400 100644
--- a/bin/ps/ps.1
+++ b/bin/ps/ps.1
@@ -29,7 +29,7 @@
 .\"     @(#)ps.1	8.3 (Berkeley) 4/18/94
 .\" $FreeBSD$
 .\"
-.Dd October 1, 2011
+.Dd October 30, 2011
 .Dt PS 1
 .Os
 .Sh NAME
@@ -437,6 +437,10 @@ This is followed by a
 .Ql -
 if the process can no longer reach that
 controlling terminal (i.e., it has been revoked).
+A
+.Ql -
+without a preceding two letter abbreviation or pseudo-terminal device number
+indicates a process which never had a controlling terminal.
 The full pathname of the controlling terminal is available via the
 .Cm tty
 keyword.
diff --git a/bin/rm/rm.c b/bin/rm/rm.c
index af362a9..3c2ba61 100644
--- a/bin/rm/rm.c
+++ b/bin/rm/rm.c
@@ -57,10 +57,10 @@ __FBSDID("$FreeBSD$");
 #include <sysexits.h>
 #include <unistd.h>
 
-int dflag, eval, fflag, iflag, Pflag, vflag, Wflag, stdin_ok;
-int rflag, Iflag;
-uid_t uid;
-volatile sig_atomic_t info;
+static int dflag, eval, fflag, iflag, Pflag, vflag, Wflag, stdin_ok;
+static int rflag, Iflag;
+static uid_t uid;
+static volatile sig_atomic_t info;
 
 int	check(char *, char *, struct stat *);
 int	check2(char **);
diff --git a/bin/sh/arith_yacc.c b/bin/sh/arith_yacc.c
index 041e638..48c63ea 100644
--- a/bin/sh/arith_yacc.c
+++ b/bin/sh/arith_yacc.c
@@ -131,11 +131,11 @@ static arith_t do_binop(int op, arith_t a, arith_t b)
 			yyerror("divide error");
 		return op == ARITH_REM ? a % b : a / b;
 	case ARITH_MUL:
-		return a * b;
+		return (uintmax_t)a * (uintmax_t)b;
 	case ARITH_ADD:
-		return a + b;
+		return (uintmax_t)a + (uintmax_t)b;
 	case ARITH_SUB:
-		return a - b;
+		return (uintmax_t)a - (uintmax_t)b;
 	case ARITH_LSHIFT:
 		return a << b;
 	case ARITH_RSHIFT:
diff --git a/bin/sh/options.c b/bin/sh/options.c
index 9104e93..4c2c8ab 100644
--- a/bin/sh/options.c
+++ b/bin/sh/options.c
@@ -401,9 +401,10 @@ setcmd(int argc, char **argv)
 void
 getoptsreset(const char *value)
 {
-	if (number(value) == 1) {
+	while (*value == '0')
+		value++;
+	if (strcmp(value, "1") == 0)
 		shellparam.reset = 1;
-	}
 }
 
 /*
diff --git a/bin/sh/sh.1 b/bin/sh/sh.1
index bbb0f49..9376842 100644
--- a/bin/sh/sh.1
+++ b/bin/sh/sh.1
@@ -32,7 +32,7 @@
 .\"	from: @(#)sh.1	8.6 (Berkeley) 5/4/95
 .\" $FreeBSD$
 .\"
-.Dd July 10, 2011
+.Dd November 5, 2011
 .Dt SH 1
 .Os
 .Sh NAME
@@ -1160,6 +1160,11 @@ of alphabetics, numerics, and underscores.
 The first letter of a variable name must not be numeric.
 A parameter can also be denoted by a number
 or a special character as explained below.
+.Pp
+Assignments are expanded differently from other words:
+tilde expansion is also performed after the equals sign and after any colon
+and usernames are also terminated by colons,
+and field splitting and pathname expansion are not performed.
 .Ss Positional Parameters
 A positional parameter is a parameter denoted by a number greater than zero.
 The shell sets these initially to the values of its command line
@@ -1273,11 +1278,15 @@ used in tilde expansion and as a default directory for the
 built-in.
 .It Va IFS
 Input Field Separators.
-This is normally set to
+The default value is
 .Aq space ,
 .Aq tab ,
 and
-.Aq newline .
+.Aq newline
+in that order.
+This default also applies if
+.Va IFS
+is unset, but not if it is set to the empty string.
 See the
 .Sx White Space Splitting
 section for more details.
@@ -1423,12 +1432,12 @@ part of the name.
 If a parameter expansion occurs inside double-quotes:
 .Bl -enum
 .It
-Pathname expansion is not performed on the results of the
-expansion.
-.It
 Field splitting is not performed on the results of the
 expansion, with the exception of the special parameter
 .Va @ .
+.It
+Pathname expansion is not performed on the results of the
+expansion.
 .El
 .Pp
 In addition, a parameter expansion can be modified by using one of the
@@ -1641,16 +1650,51 @@ and contain integer constants.
 .Pp
 The result of the expression is substituted in decimal.
 .Ss White Space Splitting (Field Splitting)
-After parameter expansion, command substitution, and
+In certain contexts,
+after parameter expansion, command substitution, and
 arithmetic expansion the shell scans the results of
 expansions and substitutions that did not occur in double-quotes for
 field splitting and multiple fields can result.
 .Pp
-The shell treats each character of the
+Characters in
+.Va IFS
+that are whitespace
+.Po
+.Aq space ,
+.Aq tab ,
+and
+.Aq newline
+.Pc
+are treated differently from other characters in
+.Va IFS .
+.Pp
+Whitespace in
+.Va IFS
+at the beginning or end of a word is discarded.
+.Pp
+Subsequently, a field is delimited by either
+.Bl -enum
+.It
+a non-whitespace character in
+.Va IFS
+with any whitespace in
+.Va IFS
+surrounding it, or
+.It
+one or more whitespace characters in
+.Va IFS .
+.El
+.Pp
+If a word ends with a non-whitespace character in
+.Va IFS ,
+there is no empty field after this character.
+.Pp
+If no field is delimited, the word is discarded.
+In particular, if a word consists solely of an unquoted substitution
+and the result of the substitution is null,
+it is removed by field splitting even if
 .Va IFS
-variable as a delimiter and uses
-the delimiters to split the results of parameter expansion and command
-substitution into fields.
+is null.
 .Ss Pathname Expansion (File Name Generation)
 Unless the
 .Fl f
diff --git a/bin/test/test.c b/bin/test/test.c
index 62c0309..394988e 100644
--- a/bin/test/test.c
+++ b/bin/test/test.c
@@ -118,7 +118,7 @@ enum token_types {
 	PAREN
 };
 
-struct t_op {
+static struct t_op {
 	const char *op_text;
 	short op_num, op_type;
 } const ops [] = {
@@ -165,10 +165,10 @@ struct t_op {
 	{0,	0,	0}
 };
 
-struct t_op const *t_wp_op;
-int nargc;
-char **t_wp;
-int parenlevel;
+static struct t_op const *t_wp_op;
+static int nargc;
+static char **t_wp;
+static int parenlevel;
 
 static int	aexpr(enum token);
 static int	binop(void);
diff --git a/cddl/contrib/opensolaris/cmd/zdb/zdb.8 b/cddl/contrib/opensolaris/cmd/zdb/zdb.8
index f601825..1ed1803 100644
--- a/cddl/contrib/opensolaris/cmd/zdb/zdb.8
+++ b/cddl/contrib/opensolaris/cmd/zdb/zdb.8
@@ -3,7 +3,8 @@
 .\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License").  You may not use this file except in compliance with the License.
 .\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.  See the License for the specific language governing permissions and limitations under the License.
 .\" When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE.  If applicable, add the following below this CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
-.TH zdb 1M "31 Oct 2005" "SunOS 5.11" "System Administration Commands"
+.\" Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
+.TH ZDB 8 "Oct 31, 2005"
 .SH NAME
 zdb \- ZFS debugger
 .SH SYNOPSIS
@@ -15,70 +16,50 @@ zdb \- ZFS debugger
 .SH DESCRIPTION
 .sp
 .LP
-The \fBzdb\fR command is used by support engineers to diagnose failures and gather statistics. Since the \fBZFS\fR file system is always consistent on disk and is self-repairing, \fBzdb\fR should only be run under the direction by a support engineer.
+The \fBzdb\fR command is used by support engineers to diagnose failures and
+gather statistics. Since the \fBZFS\fR file system is always consistent on disk
+and is self-repairing, \fBzdb\fR should only be run under the direction by a
+support engineer.
 .sp
 .LP
-If no arguments are specified, \fBzdb\fR, performs basic consistency checks on the pool and associated datasets, and report any problems detected.
+If no arguments are specified, \fBzdb\fR, performs basic consistency checks on
+the pool and associated datasets, and report any problems detected.
 .sp
 .LP
-Any options supported by this command are internal to Sun and subject to change at any time.
+Any options supported by this command are internal to Sun and subject to change
+at any time.
 .SH EXIT STATUS
 .sp
 .LP
 The following exit values are returned:
 .sp
 .ne 2
-.mk
 .na
 \fB\fB0\fR\fR
 .ad
 .RS 5n
-.rt  
 The pool is consistent.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB1\fR\fR
 .ad
 .RS 5n
-.rt  
 An error was detected.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB2\fR\fR
 .ad
 .RS 5n
-.rt  
 Invalid command line options were specified.
 .RE
 
-.SH ATTRIBUTES
-.sp
-.LP
-See \fBattributes\fR(5) for descriptions of the following attributes:
-.sp
-
-.sp
-.TS
-tab() box;
-cw(2.75i) |cw(2.75i) 
-lw(2.75i) |lw(2.75i) 
-.
-ATTRIBUTE TYPEATTRIBUTE VALUE
-_
-AvailabilitySUNWzfsu
-_
-Interface StabilityUnstable
-.TE
-
 .SH SEE ALSO
 .sp
 .LP
-\fBzfs\fR(1M), \fBzpool\fR(1M), \fBattributes\fR(5)
+zfs(8), zpool(8)
diff --git a/cddl/contrib/opensolaris/cmd/zfs/zfs.8 b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
index 03deef2..c32bfa2 100644
--- a/cddl/contrib/opensolaris/cmd/zfs/zfs.8
+++ b/cddl/contrib/opensolaris/cmd/zfs/zfs.8
@@ -8,7 +8,9 @@
 .\" the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
 .\" Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
 .\" Copyright 2011 by Delphix.  All rights reserved.
-.TH zfs 1M "24 Sep 2009" "SunOS 5.11" "System Administration Commands"
+.\" Portions Copyright 2011 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+.\" Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
+.TH ZFS 8 "September 24, 2009" FreeBSD
 .SH NAME
 zfs \- configures ZFS file systems
 .SH SYNOPSIS
@@ -39,7 +41,7 @@ zfs \- configures ZFS file systems
 
 .LP
 .nf
-\fBzfs\fR \fBsnapshot\fR [\fB-r\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR]... 
+\fBzfs\fR \fBsnapshot\fR [\fB-r\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR]...
       \fIfilesystem@snapname\fR|\fIvolume@snapname\fR
 .fi
 
@@ -60,7 +62,7 @@ zfs \- configures ZFS file systems
 
 .LP
 .nf
-\fBzfs\fR \fBrename\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR 
+\fBzfs\fR \fBrename\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR
      \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR
 .fi
 
@@ -125,7 +127,7 @@ zfs \- configures ZFS file systems
 
 .LP
 .nf
-\fBzfs\fR \fBmount\fR 
+\fBzfs\fR \fBmount\fR
 .fi
 
 .LP
@@ -170,7 +172,7 @@ zfs \- configures ZFS file systems
 
 .LP
 .nf
-\fBzfs\fR \fBallow\fR [\fB-ldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...] \fIperm\fR|\fI@setname\fR[,...] 
+\fBzfs\fR \fBallow\fR [\fB-ldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...] \fIperm\fR|\fI@setname\fR[,...]
      \fIfilesystem\fR|\fIvolume\fR
 .fi
 
@@ -191,7 +193,7 @@ zfs \- configures ZFS file systems
 
 .LP
 .nf
-\fBzfs\fR \fBunallow\fR [\fB-rldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...] [\fIperm\fR|@\fIsetname\fR[,... ]] 
+\fBzfs\fR \fBunallow\fR [\fB-rldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...] [\fIperm\fR|@\fIsetname\fR[,... ]]
      \fIfilesystem\fR|\fIvolume\fR
 .fi
 
@@ -225,17 +227,22 @@ zfs \- configures ZFS file systems
 \fBzfs\fR \fBrelease\fR [\fB-r\fR] \fItag\fR \fIsnapshot\fR...
 .fi
 
-\fBzfs\fR \fBjail\fR \fBjailid\fR \fB\fIfilesystem\fR\fR
+.LP
+.nf
+\fBzfs\fR \fBjail\fR \fIjailid\fR \fIfilesystem\fR
 .fi
+
 .LP
 .nf
-\fBzfs\fR \fBunjail\fR \fBjailid\fR \fB\fIfilesystem\fR\fR
+\fBzfs\fR \fBunjail\fR \fIjailid\fR \fIfilesystem\fR
 .fi
 
 .SH DESCRIPTION
 .sp
 .LP
-The \fBzfs\fR command configures \fBZFS\fR datasets within a \fBZFS\fR storage pool, as described in \fBzpool\fR(1M). A dataset is identified by a unique path within the \fBZFS\fR namespace. For example:
+The \fBzfs\fR command configures \fBZFS\fR datasets within a \fBZFS\fR storage
+pool, as described in \fBzpool\fR(8). A dataset is identified by a unique path
+within the \fBZFS\fR namespace. For example:
 .sp
 .in +2
 .nf
@@ -252,115 +259,164 @@ where the maximum length of a dataset name is \fBMAXNAMELEN\fR (256 bytes).
 A dataset can be one of the following:
 .sp
 .ne 2
-.mk
 .na
 \fB\fIfile system\fR\fR
 .ad
 .sp .6
 .RS 4n
-A \fBZFS\fR dataset of type \fBfilesystem\fR can be mounted within the standard system namespace and behaves like other file systems. While \fBZFS\fR file systems are designed to be \fBPOSIX\fR compliant, known issues exist that prevent compliance in some cases. Applications that depend on standards conformance might fail due to nonstandard behavior when checking file system free space.
+A \fBZFS\fR dataset of type \fBfilesystem\fR can be mounted within the standard
+system namespace and behaves like other file systems. While \fBZFS\fR file
+systems are designed to be \fBPOSIX\fR compliant, known issues exist that
+prevent compliance in some cases. Applications that depend on standards
+conformance might fail due to nonstandard behavior when checking file system
+free space.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-A logical volume exported as a raw or block device. This type of dataset should only be used under special circumstances. File systems are typically used in most environments.
+A logical volume exported as a raw or block device. This type of dataset should
+only be used under special circumstances. File systems are typically used in
+most environments.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-A read-only version of a file system or volume at a given point in time. It is specified as \fIfilesystem@name\fR or \fIvolume@name\fR.
+A read-only version of a file system or volume at a given point in time. It is
+specified as \fIfilesystem@name\fR or \fIvolume@name\fR.
 .RE
 
 .SS "ZFS File System Hierarchy"
 .sp
 .LP
-A \fBZFS\fR storage pool is a logical collection of devices that provide space for datasets. A storage pool is also the root of the \fBZFS\fR file system hierarchy.
+A \fBZFS\fR storage pool is a logical collection of devices that provide space
+for datasets. A storage pool is also the root of the \fBZFS\fR file system
+hierarchy.
 .sp
 .LP
-The root of the pool can be accessed as a file system, such as mounting and unmounting, taking snapshots, and setting properties. The physical storage characteristics, however, are managed by the \fBzpool\fR(1M) command.
+The root of the pool can be accessed as a file system, such as mounting and
+unmounting, taking snapshots, and setting properties. The physical storage
+characteristics, however, are managed by the \fBzpool\fR(8) command.
 .sp
 .LP
-See \fBzpool\fR(1M) for more information on creating and administering pools.
+See \fBzpool\fR(8) for more information on creating and administering pools.
 .SS "Snapshots"
 .sp
 .LP
-A snapshot is a read-only copy of a file system or volume. Snapshots can be created extremely quickly, and initially consume no additional space within the pool. As data within the active dataset changes, the snapshot consumes more data than would otherwise be shared with the active dataset.
+A snapshot is a read-only copy of a file system or volume. Snapshots can be
+created extremely quickly, and initially consume no additional space within the
+pool. As data within the active dataset changes, the snapshot consumes more
+data than would otherwise be shared with the active dataset.
 .sp
 .LP
-Snapshots can have arbitrary names. Snapshots of volumes can be cloned or rolled back, but cannot be accessed independently.
+Snapshots can have arbitrary names. Snapshots of volumes can be cloned or
+rolled back, but cannot be accessed independently.
 .sp
 .LP
-File system snapshots can be accessed under the \fB\&.zfs/snapshot\fR directory in the root of the file system. Snapshots are automatically mounted on demand and may be unmounted at regular intervals. The visibility of the \fB\&.zfs\fR directory can be controlled by the \fBsnapdir\fR property.
+File system snapshots can be accessed under the \fB\&.zfs/snapshot\fR directory
+in the root of the file system. Snapshots are automatically mounted on demand
+and may be unmounted at regular intervals. The visibility of the \fB\&.zfs\fR
+directory can be controlled by the \fBsnapdir\fR property.
 .SS "Clones"
 .sp
 .LP
-A clone is a writable volume or file system whose initial contents are the same as another dataset. As with snapshots, creating a clone is nearly instantaneous, and initially consumes no additional space.
+A clone is a writable volume or file system whose initial contents are the same
+as another dataset. As with snapshots, creating a clone is nearly
+instantaneous, and initially consumes no additional space.
 .sp
 .LP
-Clones can only be created from a snapshot. When a snapshot is cloned, it creates an implicit dependency between the parent and child. Even though the clone is created somewhere else in the dataset hierarchy, the original snapshot cannot be destroyed as long as a clone exists. The \fBorigin\fR property exposes this dependency, and the \fBdestroy\fR command lists any such dependencies, if they exist.
+Clones can only be created from a snapshot. When a snapshot is cloned, it
+creates an implicit dependency between the parent and child. Even though the
+clone is created somewhere else in the dataset hierarchy, the original snapshot
+cannot be destroyed as long as a clone exists. The \fBorigin\fR property
+exposes this dependency, and the \fBdestroy\fR command lists any such
+dependencies, if they exist.
 .sp
 .LP
-The clone parent-child dependency relationship can be reversed by using the \fBpromote\fR subcommand. This causes the "origin" file system to become a clone of the specified file system, which makes it possible to destroy the file system that the clone was created from.
+The clone parent-child dependency relationship can be reversed by using the
+\fBpromote\fR subcommand. This causes the "origin" file system to become a
+clone of the specified file system, which makes it possible to destroy the file
+system that the clone was created from.
 .SS "Mount Points"
 .sp
 .LP
-Creating a \fBZFS\fR file system is a simple operation, so the number of file systems per system is likely to be numerous. To cope with this, \fBZFS\fR automatically manages mounting and unmounting file systems without the need to edit the \fB/etc/vfstab\fR file. All automatically managed file systems are mounted by \fBZFS\fR at boot time.
-.sp
-.LP
-By default, file systems are mounted under \fB/\fIpath\fR\fR, where \fIpath\fR is the name of the file system in the \fBZFS\fR namespace. Directories are created and destroyed as needed.
-.sp
-.LP
-A file system can also have a mount point set in the \fBmountpoint\fR property. This directory is created as needed, and \fBZFS\fR automatically mounts the file system when the \fBzfs mount -a\fR command is invoked (without editing \fB/etc/vfstab\fR). The \fBmountpoint\fR property can be inherited, so if \fBpool/home\fR has a mount point of \fB/export/stuff\fR, then \fBpool/home/user\fR automatically inherits a mount point of \fB/export/stuff/user\fR.
-.sp
-.LP
-A file system \fBmountpoint\fR property of \fBnone\fR prevents the file system from being mounted.
-.sp
-.LP
-If needed, \fBZFS\fR file systems can also be managed with traditional tools (\fBmount\fR, \fBumount\fR, \fB/etc/vfstab\fR). If a file system's mount point is set to \fBlegacy\fR, \fBZFS\fR makes no attempt to manage the file system, and the administrator is responsible for mounting and unmounting the file system.
-.SS "Zones"
+Creating a \fBZFS\fR file system is a simple operation, so the number of file
+systems per system is likely to be numerous. To cope with this, \fBZFS\fR
+automatically manages mounting and unmounting file systems without the need to
+edit the \fB/etc/fstab\fR file. All automatically managed file systems are
+mounted by \fBZFS\fR at boot time.
 .sp
 .LP
-A \fBZFS\fR file system can be added to a non-global zone by using the \fBzonecfg\fR \fBadd fs\fR subcommand. A \fBZFS\fR file system that is added to a non-global zone must have its \fBmountpoint\fR property set to \fBlegacy\fR.
+By default, file systems are mounted under \fB/\fIpath\fR\fR, where \fIpath\fR
+is the name of the file system in the \fBZFS\fR namespace. Directories are
+created and destroyed as needed.
 .sp
 .LP
-The physical properties of an added file system are controlled by the global administrator. However, the zone administrator can create, modify, or destroy files within the added file system, depending on how the file system is mounted.
+A file system can also have a mount point set in the \fBmountpoint\fR property.
+This directory is created as needed, and \fBZFS\fR automatically mounts the
+file system when the \fBzfs mount -a\fR command is invoked (without editing
+\fB/etc/fstab\fR). The \fBmountpoint\fR property can be inherited, so if
+\fBpool/home\fR has a mount point of \fB/export/stuff\fR, then
+\fBpool/home/user\fR automatically inherits a mount point of
+\fB/export/stuff/user\fR.
 .sp
 .LP
-A dataset can also be delegated to a non-global zone by using the \fBzonecfg\fR \fBadd dataset\fR subcommand. You cannot delegate a dataset to one zone and the children of the same dataset to another zone. The zone administrator can change properties of the dataset or any of its children. However, the \fBquota\fR property is controlled by the global administrator.
+A file system \fBmountpoint\fR property of \fBnone\fR prevents the file system
+from being mounted.
 .sp
 .LP
-A \fBZFS\fR volume can be added as a device to a non-global zone by using the \fBzonecfg\fR \fBadd device\fR subcommand. However, its physical properties can be modified only by the global administrator.
+If needed, \fBZFS\fR file systems can also be managed with traditional tools
+(\fBmount\fR, \fBumount\fR, \fB/etc/fstab\fR). If a file system's mount point
+is set to \fBlegacy\fR, \fBZFS\fR makes no attempt to manage the file system,
+and the administrator is responsible for mounting and unmounting the file
+system.
+.SS "Jails"
 .sp
 .LP
-For more information about \fBzonecfg\fR syntax, see \fBzonecfg\fR(1M).
+A \fBZFS\fR dataset can be attached to a jail by using the \fBzfs\fR
+\fBjail\fR subcommand. You cannot attach a dataset to one jail and the
+children of the same dataset to another jails. To allow managment of the
+dataset from within a jail, the "\fBjailed\fR" property has to be set.
+The \fBquota\fR property cannot be changed from within a jail.
 .sp
 .LP
-After a dataset is delegated to a non-global zone, the \fBzoned\fR property is automatically set. A zoned file system cannot be mounted in the global zone, since the zone administrator might have to set the mount point to an unacceptable value.
+A \fBZFS\fR dataset can be detached from a jail using the \fBzfs\fR
+\fBunjail\fR subcommand.
 .sp
 .LP
-The global administrator can forcibly clear the \fBzoned\fR property, though this should be done with extreme care. The global administrator should verify that all the mount points are acceptable before clearing the property.
+After a dataset is attached to a jail and the \fBjailed\fR property is set,
+a jailed file system cannot be mounted outside the jail, since the jail
+administrator might have set the mount point to an unacceptable value.
 .SS "Native Properties"
 .sp
 .LP
-Properties are divided into two types, native properties and user-defined (or "user") properties. Native properties either export internal statistics or control \fBZFS\fR behavior. In addition, native properties are either editable or read-only. User properties have no effect on \fBZFS\fR behavior, but you can use them to annotate datasets in a way that is meaningful in your environment. For more information about user properties, see the "User Properties" section, below.
+Properties are divided into two types, native properties and user-defined (or
+"user") properties. Native properties either export internal statistics or
+control \fBZFS\fR behavior. In addition, native properties are either editable
+or read-only. User properties have no effect on \fBZFS\fR behavior, but you can
+use them to annotate datasets in a way that is meaningful in your environment.
+For more information about user properties, see the "User Properties" section,
+below.
 .sp
 .LP
-Every dataset has a set of properties that export statistics about the dataset as well as control various behaviors. Properties are inherited from the parent unless overridden by the child. Some properties apply only to certain types of datasets (file systems, volumes, or snapshots).
+Every dataset has a set of properties that export statistics about the dataset
+as well as control various behaviors. Properties are inherited from the parent
+unless overridden by the child. Some properties apply only to certain types of
+datasets (file systems, volumes, or snapshots).
 .sp
 .LP
-The values of numeric properties can be specified using human-readable suffixes (for example, \fBk\fR, \fBKB\fR, \fBM\fR, \fBGb\fR, and so forth, up to \fBZ\fR for zettabyte). The following are all valid (and equal) specifications: 
+The values of numeric properties can be specified using human-readable suffixes
+(for example, \fBk\fR, \fBKB\fR, \fBM\fR, \fBGb\fR, and so forth, up to \fBZ\fR
+for zettabyte). The following are all valid (and equal) specifications:
 .sp
 .in +2
 .nf
@@ -371,37 +427,47 @@ The values of numeric properties can be specified using human-readable suffixes
 
 .sp
 .LP
-The values of non-numeric properties are case sensitive and must be lowercase, except for \fBmountpoint\fR, \fBsharenfs\fR, and \fBsharesmb\fR.
+The values of non-numeric properties are case sensitive and must be lowercase,
+except for \fBmountpoint\fR, \fBsharenfs\fR, and \fBsharesmb\fR.
 .sp
 .LP
-The following native properties consist of read-only statistics about the dataset. These properties can be neither set, nor inherited. Native properties apply to all dataset types unless otherwise noted.
+The following native properties consist of read-only statistics about the
+dataset. These properties can be neither set, nor inherited. Native properties
+apply to all dataset types unless otherwise noted.
 .sp
 .ne 2
-.mk
 .na
 \fB\fBavailable\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space available to the dataset and all its children, assuming that there is no other activity in the pool. Because space is shared within a pool, availability can be limited by any number of factors, including physical pool size, quotas, reservations, or other datasets within the pool.
+The amount of space available to the dataset and all its children, assuming
+that there is no other activity in the pool. Because space is shared within a
+pool, availability can be limited by any number of factors, including physical
+pool size, quotas, reservations, or other datasets within the pool.
 .sp
-This property can also be referred to by its shortened column name, \fBavail\fR.
+This property can also be referred to by its shortened column name,
+\fBavail\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcompressratio\fR\fR
 .ad
 .sp .6
 .RS 4n
-For non-snapshots, the compression ratio achieved for the \fBused\fR space of this dataset, expressed as a multiplier.  The \fBused\fR property includes descendant datasets, and, for clones, does not include the space shared with the origin snapshot.  For snapshots, the \fBcompressratio\fR is the same as the \fBrefcompressratio\fR property. Compression can be turned on by running: \fBzfs set compression=on \fIdataset\fR\fR. The default value is \fBoff\fR.
+For non-snapshots, the compression ratio achieved for the \fBused\fR
+space of this dataset, expressed as a multiplier.  The \fBused\fR
+property includes descendant datasets, and, for clones, does not include
+the space shared with the origin snapshot.  For snapshots, the
+\fBcompressratio\fR is the same as the \fBrefcompressratio\fR property.
+Compression can be turned on by running: \fBzfs set compression=on
+\fIdataset\fR\fR. The default value is \fBoff\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcreation\fR\fR
 .ad
@@ -412,64 +478,69 @@ The time this dataset was created.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBdefer_destroy\fR\fR
 .ad
 .sp .6
 .RS 4n
-This property is \fBon\fR if the snapshot has been marked for deferred destroy by using the \fBzfs destroy\fR \fB-d\fR command. Otherwise, the property is \fBoff\fR.
+This property is \fBon\fR if the snapshot has been marked for deferred destroy
+by using the \fBzfs destroy\fR \fB-d\fR command. Otherwise, the property is
+\fBoff\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBmounted\fR\fR
 .ad
 .sp .6
 .RS 4n
-For file systems, indicates whether the file system is currently mounted. This property can be either \fByes\fR or \fBno\fR.
+For file systems, indicates whether the file system is currently mounted. This
+property can be either \fByes\fR or \fBno\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBorigin\fR\fR
 .ad
 .sp .6
 .RS 4n
-For cloned file systems or volumes, the snapshot from which the clone was created. The origin cannot be destroyed (even with the \fB-r\fR or \fB-f\fR options) so long as a clone exists.
+For cloned file systems or volumes, the snapshot from which the clone was
+created. The origin cannot be destroyed (even with the \fB-r\fR or \fB-f\fR
+options) so long as a clone exists.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBreferenced\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of data that is accessible by this dataset, which may or may not be shared with other datasets in the pool. When a snapshot or clone is created, it initially references the same amount of space as the file system or snapshot it was created from, since its contents are identical.
+The amount of data that is accessible by this dataset, which may or may not be
+shared with other datasets in the pool. When a snapshot or clone is created, it
+initially references the same amount of space as the file system or snapshot it
+was created from, since its contents are identical.
 .sp
-This property can also be referred to by its shortened column name, \fBrefer\fR.
+This property can also be referred to by its shortened column name,
+\fBrefer\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBrefcompressratio\fR\fR
 .ad
 .sp .6
 .RS 4n
-The compression ratio achieved for the \fBreferenced\fR space of this dataset, expressed as a multiplier.  See also the \fBcompressratio\fR property.
+The compression ratio achieved for the \fBreferenced\fR space of this
+dataset, expressed as a multiplier.  See also the \fBcompressratio\fR
+property.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBtype\fR\fR
 .ad
@@ -480,87 +551,113 @@ The type of dataset: \fBfilesystem\fR, \fBvolume\fR, or \fBsnapshot\fR.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBused\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space consumed by this dataset and all its descendents. This is the value that is checked against this dataset's quota and reservation. The space used does not include this dataset's reservation, but does take into account the reservations of any descendent datasets. The amount of space that a dataset consumes from its parent, as well as the amount of space that are freed if this dataset is recursively destroyed, is the greater of its space used and its reservation.
+The amount of space consumed by this dataset and all its descendents. This is
+the value that is checked against this dataset's quota and reservation. The
+space used does not include this dataset's reservation, but does take into
+account the reservations of any descendent datasets. The amount of space that a
+dataset consumes from its parent, as well as the amount of space that are freed
+if this dataset is recursively destroyed, is the greater of its space used and
+its reservation.
 .sp
-When snapshots (see the "Snapshots" section) are created, their space is initially shared between the snapshot and the file system, and possibly with previous snapshots. As the file system changes, space that was previously shared becomes unique to the snapshot, and counted in the snapshot's space used. Additionally, deleting snapshots can increase the amount of space unique to (and used by) other snapshots.
+When snapshots (see the "Snapshots" section) are created, their space is
+initially shared between the snapshot and the file system, and possibly with
+previous snapshots. As the file system changes, space that was previously
+shared becomes unique to the snapshot, and counted in the snapshot's space
+used. Additionally, deleting snapshots can increase the amount of space unique
+to (and used by) other snapshots.
 .sp
-The amount of space used, available, or referenced does not take into account pending changes. Pending changes are generally accounted for within a few seconds. Committing a change to a disk using \fBfsync\fR(3c) or \fBO_SYNC\fR does not necessarily guarantee that the space usage information is updated immediately.
+The amount of space used, available, or referenced does not take into account
+pending changes. Pending changes are generally accounted for within a few
+seconds. Committing a change to a disk using \fBfsync\fR(2) or \fBO_SYNC\fR
+does not necessarily guarantee that the space usage information is updated
+immediately.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBusedby*\fR\fR
 .ad
 .sp .6
 .RS 4n
-The \fBusedby*\fR properties decompose the \fBused\fR properties into the various reasons that space is used. Specifically, \fBused\fR = \fBusedbychildren\fR + \fBusedbydataset\fR + \fBusedbyrefreservation\fR +, \fBusedbysnapshots\fR. These properties are only available for datasets created on \fBzpool\fR "version 13" pools.
+The \fBusedby*\fR properties decompose the \fBused\fR properties into the
+various reasons that space is used. Specifically, \fBused\fR =
+\fBusedbychildren\fR + \fBusedbydataset\fR + \fBusedbyrefreservation\fR +,
+\fBusedbysnapshots\fR. These properties are only available for datasets created
+on \fBzpool\fR "version 13" pools.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBusedbychildren\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space used by children of this dataset, which would be freed if all the dataset's children were destroyed.
+The amount of space used by children of this dataset, which would be freed if
+all the dataset's children were destroyed.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBusedbydataset\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space used by this dataset itself, which would be freed if the dataset were destroyed (after first removing any \fBrefreservation\fR and destroying any necessary snapshots or descendents).
+The amount of space used by this dataset itself, which would be freed if the
+dataset were destroyed (after first removing any \fBrefreservation\fR and
+destroying any necessary snapshots or descendents).
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBusedbyrefreservation\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space used by a \fBrefreservation\fR set on this dataset, which would be freed if the \fBrefreservation\fR was removed.
+The amount of space used by a \fBrefreservation\fR set on this dataset, which
+would be freed if the \fBrefreservation\fR was removed.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBusedbysnapshots\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space consumed by snapshots of this dataset. In particular, it is the amount of space that would be freed if all of this dataset's snapshots were destroyed. Note that this is not simply the sum of the snapshots' \fBused\fR properties because space can be shared by multiple snapshots.
+The amount of space consumed by snapshots of this dataset. In particular, it is
+the amount of space that would be freed if all of this dataset's snapshots were
+destroyed. Note that this is not simply the sum of the snapshots' \fBused\fR
+properties because space can be shared by multiple snapshots.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBuserused@\fR\fIuser\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space consumed by the specified user in this dataset. Space is charged to the owner of each file, as displayed by \fBls\fR \fB-l\fR. The amount of space charged is displayed by \fBdu\fR and \fBls\fR \fB-s\fR. See the \fBzfs userspace\fR subcommand for more information.
+The amount of space consumed by the specified user in this dataset. Space is
+charged to the owner of each file, as displayed by \fBls\fR \fB-l\fR. The
+amount of space charged is displayed by \fBdu\fR and \fBls\fR \fB-s\fR. See the
+\fBzfs userspace\fR subcommand for more information.
 .sp
-Unprivileged users can access only their own space usage. The root user, or a user who has been granted the \fBuserused\fR privilege with \fBzfs allow\fR, can access everyone's usage.
+Unprivileged users can access only their own space usage. The root user, or a
+user who has been granted the \fBuserused\fR privilege with \fBzfs allow\fR,
+can access everyone's usage.
 .sp
-The \fBuserused@\fR... properties are not displayed by \fBzfs get all\fR. The user's name must be appended after the \fB@\fR symbol, using one of the following forms:
+The \fBuserused@\fR... properties are not displayed by \fBzfs get all\fR. The
+user's name must be appended after the \fB@\fR symbol, using one of the
+following forms:
 .RS +4
 .TP
 .ie t \(bu
@@ -589,218 +686,310 @@ The \fBuserused@\fR... properties are not displayed by \fBzfs get all\fR. The us
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBuserrefs\fR\fR
 .ad
 .sp .6
 .RS 4n
-This property is set to the number of user holds on this snapshot. User holds are set by using the \fBzfs hold\fR command.
+This property is set to the number of user holds on this snapshot. User holds
+are set by using the \fBzfs hold\fR command.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBgroupused@\fR\fIgroup\fR\fR
 .ad
 .sp .6
 .RS 4n
-The amount of space consumed by the specified group in this dataset. Space is charged to the group of each file, as displayed by \fBls\fR \fB-l\fR. See the \fBuserused@\fR\fIuser\fR property for more information.
+The amount of space consumed by the specified group in this dataset. Space is
+charged to the group of each file, as displayed by \fBls\fR \fB-l\fR. See the
+\fBuserused@\fR\fIuser\fR property for more information.
 .sp
-Unprivileged users can only access their own groups' space usage. The root user, or a user who has been granted the \fBgroupused\fR privilege with \fBzfs allow\fR, can access all groups' usage.
+Unprivileged users can only access their own groups' space usage. The root
+user, or a user who has been granted the \fBgroupused\fR privilege with \fBzfs
+allow\fR, can access all groups' usage.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBvolblocksize\fR=\fIblocksize\fR\fR
 .ad
 .sp .6
 .RS 4n
-For volumes, specifies the block size of the volume. The \fBblocksize\fR cannot be changed once the volume has been written, so it should be set at volume creation time. The default \fBblocksize\fR for volumes is 8 Kbytes. Any power of 2 from 512 bytes to 128 Kbytes is valid.
+For volumes, specifies the block size of the volume. The \fBblocksize\fR cannot
+be changed once the volume has been written, so it should be set at volume
+creation time. The default \fBblocksize\fR for volumes is 8 Kbytes. Any power
+of 2 from 512 bytes to 128 Kbytes is valid.
 .sp
-This property can also be referred to by its shortened column name, \fBvolblock\fR.
+This property can also be referred to by its shortened column name,
+\fBvolblock\fR.
 .RE
 
 .sp
 .LP
-The following native properties can be used to change the behavior of a \fBZFS\fR dataset.
+The following native properties can be used to change the behavior of a
+\fBZFS\fR dataset.
 .sp
 .ne 2
-.mk
 .na
-\fB\fBaclinherit\fR=\fBdiscard\fR | \fBnoallow\fR | \fBrestricted\fR | \fBpassthrough\fR | \fBpassthrough-x\fR\fR
+\fB\fBaclinherit\fR=\fBdiscard\fR | \fBnoallow\fR | \fBrestricted\fR |
+\fBpassthrough\fR | \fBpassthrough-x\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls how \fBACL\fR entries are inherited when files and directories are created. A file system with an \fBaclinherit\fR property of \fBdiscard\fR does not inherit any \fBACL\fR entries. A file system with an \fBaclinherit\fR property value of \fBnoallow\fR only inherits inheritable \fBACL\fR entries that specify "deny" permissions. The property value \fBrestricted\fR (the default) removes the \fBwrite_acl\fR and \fBwrite_owner\fR permissions when the \fBACL\fR entry is inherited. A file system with an \fBaclinherit\fR property value of \fBpassthrough\fR inherits all inheritable \fBACL\fR entries without any modifications made to the \fBACL\fR entries when they are inherited. A file system with an \fBaclinherit\fR property value of \fBpassthrough-x\fR has the same meaning as \fBpassthrough\fR, except that the \fBowner@\fR, \fBgroup@\fR, and \fBeveryone@\fR \fBACE\fRs inherit the execute permission only if the file creation mode also requests the execute bit.
+Controls how \fBACL\fR entries are inherited when files and directories are
+created. A file system with an \fBaclinherit\fR property of \fBdiscard\fR does
+not inherit any \fBACL\fR entries. A file system with an \fBaclinherit\fR
+property value of \fBnoallow\fR only inherits inheritable \fBACL\fR entries
+that specify "deny" permissions. The property value \fBrestricted\fR (the
+default) removes the \fBwrite_acl\fR and \fBwrite_owner\fR permissions when the
+\fBACL\fR entry is inherited. A file system with an \fBaclinherit\fR property
+value of \fBpassthrough\fR inherits all inheritable \fBACL\fR entries without
+any modifications made to the \fBACL\fR entries when they are inherited. A file
+system with an \fBaclinherit\fR property value of \fBpassthrough-x\fR has the
+same meaning as \fBpassthrough\fR, except that the \fBowner@\fR, \fBgroup@\fR,
+and \fBeveryone@\fR \fBACE\fRs inherit the execute permission only if the file
+creation mode also requests the execute bit.
 .sp
-When the property value is set to \fBpassthrough\fR, files are created with a mode determined by the inheritable \fBACE\fRs. If no inheritable \fBACE\fRs exist that affect the mode, then the mode is set in accordance to the requested mode from the application.
+When the property value is set to \fBpassthrough\fR, files are created with a
+mode determined by the inheritable \fBACE\fRs. If no inheritable \fBACE\fRs
+exist that affect the mode, then the mode is set in accordance to the requested
+mode from the application.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBaclmode\fR=\fBdiscard\fR | \fBgroupmask\fR | \fBpassthrough\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls how an \fBACL\fR is modified during \fBchmod\fR(2). A file system with an \fBaclmode\fR property of \fBdiscard\fR (the default) deletes all \fBACL\fR entries that do not represent the mode of the file. An \fBaclmode\fR property of \fBgroupmask\fR reduces permissions granted in all \fBALLOW\fR entries found in the \fBACL\fR such that they are no greater than the group permissions specified by \fBchmod\fR.  A file system with an \fBaclmode\fR property of \fBpassthrough\fR indicates that no changes are made to the \fBACL\fR other than creating or updating the necessary \fBACL\fR entries to represent the new mode of the file or directory.
+Controls how an \fBACL\fR is modified during \fBchmod\fR(2). A file system with
+an \fBaclmode\fR property of \fBdiscard\fR (the default) deletes all \fBACL\fR
+entries that do not represent the mode of the file. An \fBaclmode\fR property
+of \fBgroupmask\fR reduces permissions granted in all \fBALLOW\fR entries found
+in the \fBACL\fR such that they are no greater than the group permissions
+specified by \fBchmod\fR.  A file system with an \fBaclmode\fR property of
+\fBpassthrough\fR indicates that no changes are made to the \fBACL\fR other
+than creating or updating the necessary \fBACL\fR entries to
+represent the new mode of the file or directory.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBatime\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the access time for files is updated when they are read. Turning this property off avoids producing write traffic when reading files and can result in significant performance gains, though it might confuse mailers and other similar utilities. The default value is \fBon\fR.
+Controls whether the access time for files is updated when they are read.
+Turning this property off avoids producing write traffic when reading files and
+can result in significant performance gains, though it might confuse mailers
+and other similar utilities. The default value is \fBon\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcanmount\fR=\fBon\fR | \fBoff\fR | \fBnoauto\fR\fR
 .ad
 .sp .6
 .RS 4n
-If this property is set to \fBoff\fR, the file system cannot be mounted, and is ignored by \fBzfs mount -a\fR. Setting this property to \fBoff\fR is similar to setting the \fBmountpoint\fR property to \fBnone\fR, except that the dataset still has a normal \fBmountpoint\fR property, which can be inherited. Setting this property to \fBoff\fR allows datasets to be used solely as a mechanism to inherit properties. One example of setting \fBcanmount=\fR\fBoff\fR is to have two datasets with the same \fBmountpoint\fR, so that the children of both datasets appear in the same directory, but might have different inherited characteristics.
+If this property is set to \fBoff\fR, the file system cannot be mounted, and is
+ignored by \fBzfs mount -a\fR. Setting this property to \fBoff\fR is similar to
+setting the \fBmountpoint\fR property to \fBnone\fR, except that the dataset
+still has a normal \fBmountpoint\fR property, which can be inherited. Setting
+this property to \fBoff\fR allows datasets to be used solely as a mechanism to
+inherit properties. One example of setting \fBcanmount=\fR\fBoff\fR is to have
+two datasets with the same \fBmountpoint\fR, so that the children of both
+datasets appear in the same directory, but might have different inherited
+characteristics.
 .sp
-When the \fBnoauto\fR option is set, a dataset can only be mounted and unmounted explicitly. The dataset is not mounted automatically when the dataset is created or imported, nor is it mounted by the \fBzfs mount -a\fR command or unmounted by the \fBzfs unmount -a\fR command.
+When the \fBnoauto\fR option is set, a dataset can only be mounted and
+unmounted explicitly. The dataset is not mounted automatically when the dataset
+is created or imported, nor is it mounted by the \fBzfs mount -a\fR command or
+unmounted by the \fBzfs unmount -a\fR command.
 .sp
 This property is not inherited.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBchecksum\fR=\fBon\fR | \fBoff\fR | \fBfletcher2,\fR| \fBfletcher4\fR | \fBsha256\fR\fR
+\fB\fBchecksum\fR=\fBon\fR | \fBoff\fR | \fBfletcher2,\fR| \fBfletcher4\fR |
+\fBsha256\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls the checksum used to verify data integrity. The default value is \fBon\fR, which automatically selects an appropriate algorithm (currently, \fBfletcher4\fR, but this may change in future releases). The value \fBoff\fR disables integrity checking on user data. Disabling checksums is \fBNOT\fR a recommended practice.
+Controls the checksum used to verify data integrity. The default value is
+\fBon\fR, which automatically selects an appropriate algorithm (currently,
+\fBfletcher4\fR, but this may change in future releases). The value \fBoff\fR
+disables integrity checking on user data. Disabling checksums is \fBNOT\fR a
+recommended practice.
 .sp
 Changing this property affects only newly-written data.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBcompression\fR=\fBon\fR | \fBoff\fR | \fBlzjb\fR | \fBgzip\fR | \fBgzip-\fR\fIN\fR\fR
+\fB\fBcompression\fR=\fBon\fR | \fBoff\fR | \fBlzjb\fR | \fBgzip\fR |
+\fBgzip-\fR\fIN\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls the compression algorithm used for this dataset. The \fBlzjb\fR compression algorithm is optimized for performance while providing decent data compression. Setting compression to \fBon\fR uses the \fBlzjb\fR compression algorithm. The \fBgzip\fR compression algorithm uses the same compression as the \fBgzip\fR(1) command. You can specify the \fBgzip\fR level by using the value \fBgzip-\fR\fIN\fR where \fIN\fR is an integer from 1 (fastest) to 9 (best compression ratio). Currently, \fBgzip\fR is equivalent to \fBgzip-6\fR (which is also the default for \fBgzip\fR(1)).
+Controls the compression algorithm used for this dataset. The \fBlzjb\fR
+compression algorithm is optimized for performance while providing decent data
+compression. Setting compression to \fBon\fR uses the \fBlzjb\fR compression
+algorithm. The \fBgzip\fR compression algorithm uses the same compression as
+the \fBgzip\fR(1) command. You can specify the \fBgzip\fR level by using the
+value \fBgzip-\fR\fIN\fR where \fIN\fR is an integer from 1 (fastest) to 9
+(best compression ratio). Currently, \fBgzip\fR is equivalent to \fBgzip-6\fR
+(which is also the default for \fBgzip\fR(1)).
 .sp
-This property can also be referred to by its shortened column name \fBcompress\fR. Changing this property affects only newly-written data.
+This property can also be referred to by its shortened column name
+\fBcompress\fR. Changing this property affects only newly-written data.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcopies\fR=\fB1\fR | \fB2\fR | \fB3\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls the number of copies of data stored for this dataset. These copies are in addition to any redundancy provided by the pool, for example, mirroring or RAID-Z. The copies are stored on different disks, if possible. The space used by multiple copies is charged to the associated file and dataset, changing the \fBused\fR property and counting against quotas and reservations.
+Controls the number of copies of data stored for this dataset. These copies are
+in addition to any redundancy provided by the pool, for example, mirroring or
+RAID-Z. The copies are stored on different disks, if possible. The space used
+by multiple copies is charged to the associated file and dataset, changing the
+\fBused\fR property and counting against quotas and reservations.
 .sp
-Changing this property only affects newly-written data. Therefore, set this property at file system creation time by using the \fB-o\fR \fBcopies=\fR\fIN\fR option.
+Changing this property only affects newly-written data. Therefore, set this
+property at file system creation time by using the \fB-o\fR
+\fBcopies=\fR\fIN\fR option.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBdevices\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether device nodes can be opened on this file system. The default value is \fBon\fR.
+Controls whether device nodes can be opened on this file system. The default
+value is \fBon\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBexec\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether processes can be executed from within this file system. The default value is \fBon\fR.
+Controls whether processes can be executed from within this file system. The
+default value is \fBon\fR.
+.RE
+
+.sp
+.ne 2
+.na
+\fB\fBjailed\fR=\fBon\fR | \fBoff\fR\fR
+.ad
+.sp .6
+.RS 4n
+Controls whether the dataset is managed from within a jail. See the "Jails"
+section for more information. The default value is \fBoff\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBmountpoint\fR=\fIpath\fR | \fBnone\fR | \fBlegacy\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls the mount point used for this file system. See the "Mount Points" section for more information on how this property is used. 
+Controls the mount point used for this file system. See the "Mount Points"
+section for more information on how this property is used.
 .sp
-When the \fBmountpoint\fR property is changed for a file system, the file system and any children that inherit the mount point are unmounted. If the new value is \fBlegacy\fR, then they remain unmounted. Otherwise, they are automatically remounted in the new location if the property was previously \fBlegacy\fR or \fBnone\fR, or if they were mounted before the property was changed. In addition, any shared file systems are unshared and shared in the new location.
+When the \fBmountpoint\fR property is changed for a file system, the file
+system and any children that inherit the mount point are unmounted. If the new
+value is \fBlegacy\fR, then they remain unmounted. Otherwise, they are
+automatically remounted in the new location if the property was previously
+\fBlegacy\fR or \fBnone\fR, or if they were mounted before the property was
+changed. In addition, any shared file systems are unshared and shared in the
+new location.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBnbmand\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the file system should be mounted with \fBnbmand\fR (Non Blocking mandatory locks). This is used for \fBCIFS\fR clients. Changes to this property only take effect when the file system is umounted and remounted. See \fBmount\fR(1M) for more information on \fBnbmand\fR mounts.
+Controls whether the file system should be mounted with \fBnbmand\fR (Non
+Blocking mandatory locks). This is used for \fBCIFS\fR clients. Changes to this
+property only take effect when the file system is umounted and remounted. See
+\fBmount\fR(8) for more information on \fBnbmand\fR mounts.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBprimarycache\fR=\fBall\fR | \fBnone\fR | \fBmetadata\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls what is cached in the primary cache (ARC). If this property is set to \fBall\fR, then both user data and metadata is cached. If this property is set to \fBnone\fR, then neither user data nor metadata is cached. If this property is set to \fBmetadata\fR, then only metadata is cached. The default value is \fBall\fR.
+Controls what is cached in the primary cache (ARC). If this property is set to
+\fBall\fR, then both user data and metadata is cached. If this property is set
+to \fBnone\fR, then neither user data nor metadata is cached. If this property
+is set to \fBmetadata\fR, then only metadata is cached. The default value is
+\fBall\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBquota\fR=\fIsize\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-Limits the amount of space a dataset and its descendents can consume. This property enforces a hard limit on the amount of space used. This includes all space consumed by descendents, including file systems and snapshots. Setting a quota on a descendent of a dataset that already has a quota does not override the ancestor's quota, but rather imposes an additional limit.
+Limits the amount of space a dataset and its descendents can consume. This
+property enforces a hard limit on the amount of space used. This includes all
+space consumed by descendents, including file systems and snapshots. Setting a
+quota on a descendent of a dataset that already has a quota does not override
+the ancestor's quota, but rather imposes an additional limit.
 .sp
-Quotas cannot be set on volumes, as the \fBvolsize\fR property acts as an implicit quota.
+Quotas cannot be set on volumes, as the \fBvolsize\fR property acts as an
+implicit quota.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBuserquota@\fR\fIuser\fR=\fIsize\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-Limits the amount of space consumed by the specified user. User space consumption is identified by the \fBuserspace@\fR\fIuser\fR property.
+Limits the amount of space consumed by the specified user. User space
+consumption is identified by the \fBuserspace@\fR\fIuser\fR property.
 .sp
-Enforcement of user quotas may be delayed by several seconds. This delay means that a user might exceed their quota before the system notices that they are over quota and begins to refuse additional writes with the \fBEDQUOT\fR error message . See the \fBzfs userspace\fR subcommand for more information.
+Enforcement of user quotas may be delayed by several seconds. This delay means
+that a user might exceed their quota before the system notices that they are
+over quota and begins to refuse additional writes with the \fBEDQUOT\fR error
+message . See the \fBzfs userspace\fR subcommand for more information.
 .sp
-Unprivileged users can only access their own groups' space usage. The root user, or a user who has been granted the \fBuserquota\fR privilege with \fBzfs allow\fR, can get and set everyone's quota.
+Unprivileged users can only access their own groups' space usage. The root
+user, or a user who has been granted the \fBuserquota\fR privilege with \fBzfs
+allow\fR, can get and set everyone's quota.
 .sp
-This property is not available on volumes, on file systems before version 4, or on pools before version 15. The \fBuserquota@\fR... properties are not displayed by \fBzfs get all\fR. The user's name must be appended after the \fB@\fR symbol, using one of the following forms:
+This property is not available on volumes, on file systems before version 4, or
+on pools before version 15. The \fBuserquota@\fR... properties are not
+displayed by \fBzfs get all\fR. The user's name must be appended after the
+\fB@\fR symbol, using one of the following forms:
 .RS +4
 .TP
 .ie t \(bu
@@ -829,20 +1018,21 @@ This property is not available on volumes, on file systems before version 4, or
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBgroupquota@\fR\fIgroup\fR=\fIsize\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-Limits the amount of space consumed by the specified group. Group space consumption is identified by the \fBuserquota@\fR\fIuser\fR property.
+Limits the amount of space consumed by the specified group. Group space
+consumption is identified by the \fBuserquota@\fR\fIuser\fR property.
 .sp
-Unprivileged users can access only their own groups' space usage. The root user, or a user who has been granted the \fBgroupquota\fR privilege with \fBzfs allow\fR, can get and set all groups' quotas.
+Unprivileged users can access only their own groups' space usage. The root
+user, or a user who has been granted the \fBgroupquota\fR privilege with \fBzfs
+allow\fR, can get and set all groups' quotas.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBreadonly\fR=\fBon\fR | \fBoff\fR\fR
 .ad
@@ -850,269 +1040,300 @@ Unprivileged users can access only their own groups' space usage. The root user,
 .RS 4n
 Controls whether this dataset can be modified. The default value is \fBoff\fR.
 .sp
-This property can also be referred to by its shortened column name, \fBrdonly\fR.
+This property can also be referred to by its shortened column name,
+\fBrdonly\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBrecordsize\fR=\fIsize\fR\fR
 .ad
 .sp .6
 .RS 4n
-Specifies a suggested block size for files in the file system. This property is designed solely for use with database workloads that access files in fixed-size records. \fBZFS\fR automatically tunes block sizes according to internal algorithms optimized for typical access patterns. 
+Specifies a suggested block size for files in the file system. This property is
+designed solely for use with database workloads that access files in fixed-size
+records. \fBZFS\fR automatically tunes block sizes according to internal
+algorithms optimized for typical access patterns.
 .sp
-For databases that create very large files but access them in small random chunks, these algorithms may be suboptimal. Specifying a \fBrecordsize\fR greater than or equal to the record size of the database can result in significant performance gains. Use of this property for general purpose file systems is strongly discouraged, and may adversely affect performance.
+For databases that create very large files but access them in small random
+chunks, these algorithms may be suboptimal. Specifying a \fBrecordsize\fR
+greater than or equal to the record size of the database can result in
+significant performance gains. Use of this property for general purpose file
+systems is strongly discouraged, and may adversely affect performance.
 .sp
-The size specified must be a power of two greater than or equal to 512 and less than or equal to 128 Kbytes.
+The size specified must be a power of two greater than or equal to 512 and less
+than or equal to 128 Kbytes.
 .sp
-Changing the file system's \fBrecordsize\fR affects only files created afterward; existing files are unaffected.
+Changing the file system's \fBrecordsize\fR affects only files created
+afterward; existing files are unaffected.
 .sp
-This property can also be referred to by its shortened column name, \fBrecsize\fR.
+This property can also be referred to by its shortened column name,
+\fBrecsize\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBrefquota\fR=\fIsize\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-Limits the amount of space a dataset can consume. This property enforces a hard limit on the amount of space used. This hard limit does not include space used by descendents, including file systems and snapshots.
+Limits the amount of space a dataset can consume. This property enforces a hard
+limit on the amount of space used. This hard limit does not include space used
+by descendents, including file systems and snapshots.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBrefreservation\fR=\fIsize\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-The minimum amount of space guaranteed to a dataset, not including its descendents. When the amount of space used is below this value, the dataset is treated as if it were taking up the amount of space specified by \fBrefreservation\fR. The \fBrefreservation\fR reservation is accounted for in the parent datasets' space used, and counts against the parent datasets' quotas and reservations.
+The minimum amount of space guaranteed to a dataset, not including its
+descendents. When the amount of space used is below this value, the dataset is
+treated as if it were taking up the amount of space specified by
+\fBrefreservation\fR. The \fBrefreservation\fR reservation is accounted for in
+the parent datasets' space used, and counts against the parent datasets' quotas
+and reservations.
 .sp
-If \fBrefreservation\fR is set, a snapshot is only allowed if there is enough free pool space outside of this reservation to accommodate the current number of "referenced" bytes in the dataset.
+If \fBrefreservation\fR is set, a snapshot is only allowed if there is enough
+free pool space outside of this reservation to accommodate the current number
+of "referenced" bytes in the dataset.
 .sp
-This property can also be referred to by its shortened column name, \fBrefreserv\fR.
+This property can also be referred to by its shortened column name,
+\fBrefreserv\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBreservation\fR=\fIsize\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-The minimum amount of space guaranteed to a dataset and its descendents. When the amount of space used is below this value, the dataset is treated as if it were taking up the amount of space specified by its reservation. Reservations are accounted for in the parent datasets' space used, and count against the parent datasets' quotas and reservations.
+The minimum amount of space guaranteed to a dataset and its descendents. When
+the amount of space used is below this value, the dataset is treated as if it
+were taking up the amount of space specified by its reservation. Reservations
+are accounted for in the parent datasets' space used, and count against the
+parent datasets' quotas and reservations.
 .sp
-This property can also be referred to by its shortened column name, \fBreserv\fR.
+This property can also be referred to by its shortened column name,
+\fBreserv\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBsecondarycache\fR=\fBall\fR | \fBnone\fR | \fBmetadata\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls what is cached in the secondary cache (L2ARC). If this property is set to \fBall\fR, then both user data and metadata is cached. If this property is set to \fBnone\fR, then neither user data nor metadata is cached. If this property is set to \fBmetadata\fR, then only metadata is cached. The default value is \fBall\fR.
+Controls what is cached in the secondary cache (L2ARC). If this property is set
+to \fBall\fR, then both user data and metadata is cached. If this property is
+set to \fBnone\fR, then neither user data nor metadata is cached. If this
+property is set to \fBmetadata\fR, then only metadata is cached. The default
+value is \fBall\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBsetuid\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the set-\fBUID\fR bit is respected for the file system. The default value is \fBon\fR.
-.RE
-
-.sp
-.ne 2
-.mk
-.na
-\fB\fBshareiscsi\fR=\fBon\fR | \fBoff\fR\fR
-.ad
-.sp .6
-.RS 4n
-Like the \fBsharenfs\fR property, \fBshareiscsi\fR indicates whether a \fBZFS\fR volume is exported as an \fBiSCSI\fR target. The acceptable values for this property are \fBon\fR, \fBoff\fR, and \fBtype=disk\fR. The default value is \fBoff\fR. In the future, other target types might be supported. For example, \fBtape\fR.
-.sp
-You might want to set \fBshareiscsi=on\fR for a file system so that all \fBZFS\fR volumes within the file system are shared by default. However, setting this property on a file system has no direct effect.
+Controls whether the set-\fBUID\fR bit is respected for the file system. The
+default value is \fBon\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBsharesmb\fR=\fBon\fR | \fBoff\fR | \fIopts\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the file system is shared by using the Solaris \fBCIFS\fR service, and what options are to be used. A file system with the \fBsharesmb\fR property set to \fBoff\fR is managed through traditional tools such as \fBsharemgr\fR(1M). Otherwise, the file system is automatically shared and unshared with the \fBzfs share\fR and \fBzfs unshare\fR commands. If the property is set to \fBon\fR, the \fBsharemgr\fR(1M) command is invoked with no options. Otherwise, the \fBsharemgr\fR(1M) command is invoked with options equivalent to the contents of this property.
-.sp
-Because \fBSMB\fR shares requires a resource name, a unique resource name is constructed from the dataset name. The constructed name is a copy of the dataset name except that the characters in the dataset name, which would be illegal in the resource name, are replaced with underscore (\fB_\fR) characters. A pseudo property "name" is also supported that allows you to replace the data set name with a specified name. The specified name is then used to replace the prefix dataset in the case of inheritance. For example, if the dataset \fBdata/home/john\fR is set to \fBname=john\fR, then \fBdata/home/john\fR has a resource name of \fBjohn\fR. If a child dataset of \fBdata/home/john/backups\fR, it has a resource name of \fBjohn_backups\fR.
-.sp
-When SMB shares are created, the SMB share name appears as an entry in the \fB\&.zfs/shares\fR directory. You can use the \fBls\fR or \fBchmod\fR command to display the share-level ACLs on the entries in this directory.
-.sp
-When the \fBsharesmb\fR property is changed for a dataset, the dataset and any children inheriting the property are re-shared with the new options, only if the property was previously set to \fBoff\fR, or if they were shared before the property was changed. If the new property is set to \fBoff\fR, the file systems are unshared.
+The \fB\fBsharesmb\fR property has currently no effect on FreeBSD.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBsharenfs\fR=\fBon\fR | \fBoff\fR | \fIopts\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the file system is shared via \fBNFS\fR, and what options are used. A file system with a \fBsharenfs\fR property of \fBoff\fR is managed through traditional tools such as \fBshare\fR(1M), \fBunshare\fR(1M), and \fBdfstab\fR(4). Otherwise, the file system is automatically shared and unshared with the \fBzfs share\fR and \fBzfs unshare\fR commands. If the property is set to \fBon\fR, the \fBshare\fR(1M) command is invoked with no options. Otherwise, the \fBshare\fR(1M) command is invoked with options equivalent to the contents of this property.
+Controls whether the file system is shared via \fBNFS\fR, and what options are
+used. A file system with a \fBsharenfs\fR property of \fBoff\fR is managed
+the traditional way via \fBexports\fR(5). Otherwise, the file system is
+automatically shared and unshared with the \fBzfs share\fR and
+\fBzfs unshare\fR commands. If the property is set to \fBon\fR, no NFS export
+options are used. Otherwise, NFS export options are equivalent to the contents
+of this property.
 .sp
-When the \fBsharenfs\fR property is changed for a dataset, the dataset and any children inheriting the property are re-shared with the new options, only if the property was previously \fBoff\fR, or if they were shared before the property was changed. If the new property is \fBoff\fR, the file systems are unshared.
+When the \fBsharenfs\fR property is changed for a dataset, the \fBmountd\fR(8)
+daemon is reloaded.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBlogbias\fR = \fBlatency\fR | \fBthroughput\fR\fR
 .ad
 .sp .6
 .RS 4n
-Provide a hint to ZFS about handling of synchronous requests in this dataset. If \fBlogbias\fR is set to \fBlatency\fR (the default), ZFS will use pool log devices (if configured) to handle the requests at low latency. If \fBlogbias\fR is set to \fBthroughput\fR, ZFS will not use configured pool log devices. ZFS will instead optimize synchronous operations for global pool throughput and efficient use of resources.
+Provide a hint to ZFS about handling of synchronous requests in this dataset.
+If \fBlogbias\fR is set to \fBlatency\fR (the default), ZFS will use pool log
+devices (if configured) to handle the requests at low latency. If \fBlogbias\fR
+is set to \fBthroughput\fR, ZFS will not use configured pool log devices. ZFS
+will instead optimize synchronous operations for global pool throughput and
+efficient use of resources.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBsnapdir\fR=\fBhidden\fR | \fBvisible\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the \fB\&.zfs\fR directory is hidden or visible in the root of the file system as discussed in the "Snapshots" section. The default value is \fBhidden\fR.
+Controls whether the \fB\&.zfs\fR directory is hidden or visible in the root of
+the file system as discussed in the "Snapshots" section. The default value is
+\fBhidden\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBversion\fR=\fB1\fR | \fB2\fR | \fBcurrent\fR\fR
 .ad
 .sp .6
 .RS 4n
-The on-disk version of this file system, which is independent of the pool version. This property can only be set to later supported versions. See the \fBzfs upgrade\fR command.
+The on-disk version of this file system, which is independent of the pool
+version. This property can only be set to later supported versions. See the
+\fBzfs upgrade\fR command.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBvolsize\fR=\fIsize\fR\fR
 .ad
 .sp .6
 .RS 4n
-For volumes, specifies the logical size of the volume. By default, creating a volume establishes a reservation of equal size. For storage pools with a version number of 9 or higher, a \fBrefreservation\fR is set instead. Any changes to \fBvolsize\fR are reflected in an equivalent change to the reservation (or \fBrefreservation\fR). The \fBvolsize\fR can only be set to a multiple of \fBvolblocksize\fR, and cannot be zero.
+For volumes, specifies the logical size of the volume. By default, creating a
+volume establishes a reservation of equal size. For storage pools with a
+version number of 9 or higher, a \fBrefreservation\fR is set instead. Any
+changes to \fBvolsize\fR are reflected in an equivalent change to the
+reservation (or \fBrefreservation\fR). The \fBvolsize\fR can only be set to a
+multiple of \fBvolblocksize\fR, and cannot be zero.
 .sp
-The reservation is kept equal to the volume's logical size to prevent unexpected behavior for consumers. Without the reservation, the volume could run out of space, resulting in undefined behavior or data corruption, depending on how the volume is used. These effects can also occur when the volume size is changed while it is in use (particularly when shrinking the size). Extreme care should be used when adjusting the volume size.
+The reservation is kept equal to the volume's logical size to prevent
+unexpected behavior for consumers. Without the reservation, the volume could
+run out of space, resulting in undefined behavior or data corruption, depending
+on how the volume is used. These effects can also occur when the volume size is
+changed while it is in use (particularly when shrinking the size). Extreme care
+should be used when adjusting the volume size.
 .sp
-Though not recommended, a "sparse volume" (also known as "thin provisioning") can be created by specifying the \fB-s\fR option to the \fBzfs create -V\fR command, or by changing the reservation after the volume has been created. A "sparse volume" is a volume where the reservation is less then the volume size. Consequently, writes to a sparse volume can fail with \fBENOSPC\fR when the pool is low on space. For a sparse volume, changes to \fBvolsize\fR are not reflected in the reservation.
+Though not recommended, a "sparse volume" (also known as "thin provisioning")
+can be created by specifying the \fB-s\fR option to the \fBzfs create -V\fR
+command, or by changing the reservation after the volume has been created. A
+"sparse volume" is a volume where the reservation is less then the volume size.
+Consequently, writes to a sparse volume can fail with \fBENOSPC\fR when the
+pool is low on space. For a sparse volume, changes to \fBvolsize\fR are not
+reflected in the reservation.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBvscan\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether regular files should be scanned for viruses when a file is opened and closed. In addition to enabling this property, the virus scan service must also be enabled for virus scanning to occur. The default value is \fBoff\fR.
+Controls whether regular files should be scanned for viruses when a file is
+opened and closed. In addition to enabling this property, the virus scan
+service must also be enabled for virus scanning to occur. The default value is
+\fBoff\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBxattr\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether extended attributes are enabled for this file system. The default value is \fBon\fR.
-.RE
-
-.sp
-.ne 2
-.mk
-.na
-\fB\fBzoned\fR=\fBon\fR | \fBoff\fR\fR
-.ad
-.sp .6
-.RS 4n
-Controls whether the dataset is managed from a non-global zone. See the "Zones" section for more information. The default value is \fBoff\fR.
+Controls whether extended attributes are enabled for this file system. The
+default value is \fBon\fR.
 .RE
 
 .sp
 .LP
-The following three properties cannot be changed after the file system is created, and therefore, should be set when the file system is created. If the properties are not set with the \fBzfs create\fR or \fBzpool create\fR commands, these properties are inherited from the parent dataset. If the parent dataset lacks these properties due to having been created prior to these features being supported, the new file system will have the default values for these properties.
+The following three properties cannot be changed after the file system is
+created, and therefore, should be set when the file system is created. If the
+properties are not set with the \fBzfs create\fR or \fBzpool create\fR
+commands, these properties are inherited from the parent dataset. If the parent
+dataset lacks these properties due to having been created prior to these
+features being supported, the new file system will have the default values for
+these properties.
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcasesensitivity\fR=\fBsensitive\fR | \fBinsensitive\fR | \fBmixed\fR\fR
 .ad
 .sp .6
 .RS 4n
-Indicates whether the file name matching algorithm used by the file system should be case-sensitive, case-insensitive, or allow a combination of both styles of matching. The default value for the \fBcasesensitivity\fR property is \fBsensitive\fR. Traditionally, UNIX and POSIX file systems have case-sensitive file names.
-.sp
-The \fBmixed\fR value for the \fBcasesensitivity\fR property indicates that the file system can support requests for both case-sensitive and case-insensitive matching behavior. Currently, case-insensitive matching behavior on a file system that supports mixed behavior is limited to the Solaris CIFS server product. For more information about the \fBmixed\fR value behavior, see the \fISolaris ZFS Administration Guide\fR.
-.RE
-
-.sp
-.ne 2
-.mk
-.na
-\fB\fBnormalization\fR = \fBnone\fR | \fBformC\fR | \fBformD\fR | \fBformKC\fR | \fBformKD\fR\fR
-.ad
-.sp .6
-.RS 4n
-Indicates whether the file system should perform a \fBunicode\fR normalization of file names whenever two file names are compared, and which normalization algorithm should be used. File names are always stored unmodified, names are normalized as part of any comparison process. If this property is set to a legal value other than \fBnone\fR, and the \fButf8only\fR property was left unspecified, the \fButf8only\fR property is automatically set to \fBon\fR. The default value of the \fBnormalization\fR property is \fBnone\fR. This property cannot be changed after the file system is created.
+The \fB\fBcasesensitivity\fR property is currently not supported on FreeBSD.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fBjailed =\fIon\fR | \fIoff\fR\fR
+\fB\fBnormalization\fR = \fBnone\fR | \fBformC\fR | \fBformD\fR | \fBformKC\fR
+| \fBformKD\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether the dataset is managed from within a jail. The default value is "off".
+Indicates whether the file system should perform a \fBunicode\fR normalization
+of file names whenever two file names are compared, and which normalization
+algorithm should be used. File names are always stored unmodified, names are
+normalized as part of any comparison process. If this property is set to a
+legal value other than \fBnone\fR, and the \fButf8only\fR property was left
+unspecified, the \fButf8only\fR property is automatically set to \fBon\fR. The
+default value of the \fBnormalization\fR property is \fBnone\fR. This property
+cannot be changed after the file system is created.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fButf8only\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Indicates whether the file system should reject file names that include characters that are not present in the \fBUTF-8\fR character code set. If this property is explicitly set to \fBoff\fR, the normalization property must either not be explicitly set or be set to \fBnone\fR. The default value for the \fButf8only\fR property is \fBoff\fR. This property cannot be changed after the file system is created.
+Indicates whether the file system should reject file names that include
+characters that are not present in the \fBUTF-8\fR character code set. If this
+property is explicitly set to \fBoff\fR, the normalization property must either
+not be explicitly set or be set to \fBnone\fR. The default value for the
+\fButf8only\fR property is \fBoff\fR. This property cannot be changed after the
+file system is created.
 .RE
 
 .sp
 .LP
-The \fBcasesensitivity\fR, \fBnormalization\fR, and \fButf8only\fR properties are also new permissions that can be assigned to non-privileged users by using the \fBZFS\fR delegated administration feature.
+The \fBcasesensitivity\fR, \fBnormalization\fR, and \fButf8only\fR properties
+are also new permissions that can be assigned to non-privileged users by using
+the \fBZFS\fR delegated administration feature.
 .SS "Temporary Mount Point Properties"
 .sp
 .LP
-When a file system is mounted, either through \fBmount\fR(1M) for legacy mounts or the \fBzfs mount\fR command for normal file systems, its mount options are set according to its properties. The correlation between properties and mount options is as follows:
+When a file system is mounted, either through \fBmount\fR(8) for legacy mounts
+or the \fBzfs mount\fR command for normal file systems, its mount options are
+set according to its properties. The correlation between properties and mount
+options is as follows:
 .sp
 .in +2
 .nf
@@ -1128,37 +1349,60 @@ When a file system is mounted, either through \fBmount\fR(1M) for legacy mounts
 
 .sp
 .LP
-In addition, these options can be set on a per-mount basis using the \fB-o\fR option, without affecting the property that is stored on disk. The values specified on the command line override the values stored in the dataset. The \fB-nosuid\fR option is an alias for \fBnodevices,nosetuid\fR. These properties are reported as "temporary" by the \fBzfs get\fR command. If the properties are changed while the dataset is mounted, the new setting overrides any temporary settings.
+In addition, these options can be set on a per-mount basis using the \fB-o\fR
+option, without affecting the property that is stored on disk. The values
+specified on the command line override the values stored in the dataset. The
+\fB-nosuid\fR option is an alias for \fBnodevices,nosetuid\fR. These properties
+are reported as "temporary" by the \fBzfs get\fR command. If the properties are
+changed while the dataset is mounted, the new setting overrides any temporary
+settings.
 .SS "User Properties"
 .sp
 .LP
-In addition to the standard native properties, \fBZFS\fR supports arbitrary user properties. User properties have no effect on \fBZFS\fR behavior, but applications or administrators can use them to annotate datasets (file systems, volumes, and snapshots).
+In addition to the standard native properties, \fBZFS\fR supports arbitrary
+user properties. User properties have no effect on \fBZFS\fR behavior, but
+applications or administrators can use them to annotate datasets (file systems,
+volumes, and snapshots).
 .sp
 .LP
-User property names must contain a colon (\fB:\fR) character to distinguish them from native properties. They may contain lowercase letters, numbers, and the following punctuation characters: colon (\fB:\fR), dash (\fB-\fR), period (\fB\&.\fR), and underscore (\fB_\fR). The expected convention is that the property name is divided into two portions such as \fImodule\fR\fB:\fR\fIproperty\fR, but this namespace is not enforced by \fBZFS\fR. User property names can be at most 256 characters, and cannot begin with a dash (\fB-\fR).
+User property names must contain a colon (\fB:\fR) character to distinguish
+them from native properties. They may contain lowercase letters, numbers, and
+the following punctuation characters: colon (\fB:\fR), dash (\fB-\fR), period
+(\fB\&.\fR), and underscore (\fB_\fR). The expected convention is that the
+property name is divided into two portions such as
+\fImodule\fR\fB:\fR\fIproperty\fR, but this namespace is not enforced by
+\fBZFS\fR. User property names can be at most 256 characters, and cannot begin
+with a dash (\fB-\fR).
 .sp
 .LP
-When making programmatic use of user properties, it is strongly suggested to use a reversed \fBDNS\fR domain name for the \fImodule\fR component of property names to reduce the chance that two independently-developed packages use the same property name for different purposes. Property names beginning with \fBcom.sun\fR. are reserved for use by Sun Microsystems.
+When making programmatic use of user properties, it is strongly suggested to
+use a reversed \fBDNS\fR domain name for the \fImodule\fR component of property
+names to reduce the chance that two independently-developed packages use the
+same property name for different purposes. Property names beginning with
+\fBcom.sun\fR. are reserved for use by Sun Microsystems.
 .sp
 .LP
-The values of user properties are arbitrary strings, are always inherited, and are never validated. All of the commands that operate on properties (\fBzfs list\fR, \fBzfs get\fR, \fBzfs set\fR, and so forth) can be used to manipulate both native properties and user properties. Use the \fBzfs inherit\fR command to clear a user property . If the property is not defined in any parent dataset, it is removed entirely. Property values are limited to 1024 characters.
-.SS "ZFS Volumes as Swap or Dump Devices"
-.sp
-.LP
-During an initial installation or a live upgrade from a \fBUFS\fR file system, a swap device and dump device are created on \fBZFS\fR volumes in the \fBZFS\fR root pool. By default, the swap area size is based on 1/2 the size of physical memory up to 2 Gbytes. The size of the dump device depends on the kernel's requirements at installation time. Separate \fBZFS\fR volumes must be used for the swap area and dump devices. Do not swap to a file on a \fBZFS\fR file system. A \fBZFS\fR swap file configuration is not supported. 
-.sp
-.LP
-If you need to change your swap area or dump device after the system is installed or upgraded, use the \fBswap\fR(1M) and \fBdumpadm\fR(1M) commands. If you need to change the size of your swap area or dump device, see the \fISolaris ZFS Administration Guide\fR.
+The values of user properties are arbitrary strings, are always inherited, and
+are never validated. All of the commands that operate on properties (\fBzfs
+list\fR, \fBzfs get\fR, \fBzfs set\fR, and so forth) can be used to manipulate
+both native properties and user properties. Use the \fBzfs inherit\fR command
+to clear a user property . If the property is not defined in any parent
+dataset, it is removed entirely. Property values are limited to 1024
+characters.
 .SH SUBCOMMANDS
 .sp
 .LP
-All subcommands that modify state are logged persistently to the pool in their original form.
+All subcommands that modify state are logged persistently to the pool in their
+original form.
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs ?\fR\fR
 .ad
+.br
+.na
+\fB\fBzfs rename\fR \fB-u\fR [\fB-p\fR] \fIfilesystem\fR \fIfilesystem\fR\fR
+.ad
 .sp .6
 .RS 4n
 Displays a help message.
@@ -1166,106 +1410,136 @@ Displays a help message.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs create\fR [\fB-p\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ... \fIfilesystem\fR\fR
+\fB\fBzfs create\fR [\fB-p\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ...
+\fIfilesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates a new \fBZFS\fR file system. The file system is automatically mounted according to the \fBmountpoint\fR property inherited from the parent.
+Creates a new \fBZFS\fR file system. The file system is automatically mounted
+according to the \fBmountpoint\fR property inherited from the parent.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-p\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates all the non-existing parent datasets. Datasets created in this manner are automatically mounted according to the \fBmountpoint\fR property inherited from their parent. Any property specified on the command line using the \fB-o\fR option is ignored. If the target filesystem already exists, the operation completes successfully.
+Creates all the non-existing parent datasets. Datasets created in this manner
+are automatically mounted according to the \fBmountpoint\fR property inherited
+from their parent. Any property specified on the command line using the
+\fB-o\fR option is ignored. If the target filesystem already exists, the
+operation completes successfully.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty\fR=\fIvalue\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets the specified property as if the command \fBzfs set\fR \fIproperty\fR=\fIvalue\fR was invoked at the same time the dataset was created. Any editable \fBZFS\fR property can also be set at creation time. Multiple \fB-o\fR options can be specified. An error results if the same property is specified in multiple \fB-o\fR options.
+Sets the specified property as if the command \fBzfs set\fR
+\fIproperty\fR=\fIvalue\fR was invoked at the same time the dataset was
+created. Any editable \fBZFS\fR property can also be set at creation time.
+Multiple \fB-o\fR options can be specified. An error results if the same
+property is specified in multiple \fB-o\fR options.
+.RE
+
+.sp
+.ne 2
+.mk
+.na
+\fB\fB-u\fR\fR
+.ad
+.sp .6
+.RS 4n
+Do not remount file systems during rename. If a file system's \fBmountpoint\fR property is set to \fBlegacy\fR or \fBnone\fR, file system is not unmounted even if this option is not given.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs create\fR [\fB-ps\fR] [\fB-b\fR \fIblocksize\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ... \fB-V\fR \fIsize\fR \fIvolume\fR\fR
+\fB\fBzfs create\fR [\fB-ps\fR] [\fB-b\fR \fIblocksize\fR] [\fB-o\fR
+\fIproperty\fR=\fIvalue\fR] ... \fB-V\fR \fIsize\fR \fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates a volume of the given size. The volume is exported as a block device in \fB/dev/zvol/{dsk,rdsk}/\fR\fIpath\fR, where \fIpath\fR is the name of the volume in the \fBZFS\fR namespace. The size represents the logical size as exported by the device. By default, a reservation of equal size is created.
+Creates a volume of the given size. The volume is exported as a block device in
+\fB/dev/zvol/{dsk,rdsk}/\fR\fIpath\fR, where \fIpath\fR is the name of the
+volume in the \fBZFS\fR namespace. The size represents the logical size as
+exported by the device. By default, a reservation of equal size is created.
 .sp
-\fIsize\fR is automatically rounded up to the nearest 128 Kbytes to ensure that the volume has an integral number of blocks regardless of \fIblocksize\fR.
+\fIsize\fR is automatically rounded up to the nearest 128 Kbytes to ensure that
+the volume has an integral number of blocks regardless of \fIblocksize\fR.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-p\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates all the non-existing parent datasets. Datasets created in this manner are automatically mounted according to the \fBmountpoint\fR property inherited from their parent. Any property specified on the command line using the \fB-o\fR option is ignored. If the target filesystem already exists, the operation completes successfully.
+Creates all the non-existing parent datasets. Datasets created in this manner
+are automatically mounted according to the \fBmountpoint\fR property inherited
+from their parent. Any property specified on the command line using the
+\fB-o\fR option is ignored. If the target filesystem already exists, the
+operation completes successfully.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-s\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates a sparse volume with no reservation. See \fBvolsize\fR in the Native Properties section for more information about sparse volumes.
+Creates a sparse volume with no reservation. See \fBvolsize\fR in the Native
+Properties section for more information about sparse volumes.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty\fR=\fIvalue\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets the specified property as if the \fBzfs set\fR \fIproperty\fR=\fIvalue\fR command was invoked at the same time the dataset was created. Any editable \fBZFS\fR property can also be set at creation time. Multiple \fB-o\fR options can be specified. An error results if the same property is specified in multiple \fB-o\fR options.
+Sets the specified property as if the \fBzfs set\fR \fIproperty\fR=\fIvalue\fR
+command was invoked at the same time the dataset was created. Any editable
+\fBZFS\fR property can also be set at creation time. Multiple \fB-o\fR options
+can be specified. An error results if the same property is specified in
+multiple \fB-o\fR options.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-b\fR \fIblocksize\fR\fR
 .ad
 .sp .6
 .RS 4n
-Equivalent to \fB-o\fR \fBvolblocksize\fR=\fIblocksize\fR. If this option is specified in conjunction with \fB-o\fR \fBvolblocksize\fR, the resulting behavior is undefined.
+Equivalent to \fB-o\fR \fBvolblocksize\fR=\fIblocksize\fR. If this option is
+specified in conjunction with \fB-o\fR \fBvolblocksize\fR, the resulting
+behavior is undefined.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs destroy\fR [\fB-rRf\fR] \fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Destroys the given dataset. By default, the command unshares any file systems that are currently shared, unmounts any file systems that are currently mounted, and refuses to destroy a dataset that has active dependents (children or clones).
+Destroys the given dataset. By default, the command unshares any file systems
+that are currently shared, unmounts any file systems that are currently
+mounted, and refuses to destroy a dataset that has active dependents (children
+or clones).
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
@@ -1276,43 +1550,48 @@ Recursively destroy all children.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively destroy all dependents, including cloned file systems outside the target hierarchy.
+Recursively destroy all dependents, including cloned file systems outside the
+target hierarchy.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .sp .6
 .RS 4n
-Force an unmount of any file systems using the \fBunmount -f\fR command. This option has no effect on non-file systems or unmounted file systems.
+Force an unmount of any file systems using the \fBunmount -f\fR command. This
+option has no effect on non-file systems or unmounted file systems.
 .RE
 
-Extreme care should be taken when applying either the \fB-r\fR or the \fB-R\fR options, as they can destroy large portions of a pool and cause unexpected behavior for mounted file systems in use. 
+Extreme care should be taken when applying either the \fB-r\fR or the \fB-R\fR
+options, as they can destroy large portions of a pool and cause unexpected
+behavior for mounted file systems in use.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs destroy\fR [\fB-rRd\fR] \fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-The given snapshot is destroyed immediately if and only if the \fBzfs destroy\fR command without the \fB-d\fR option would have destroyed it. Such immediate destruction would occur, for example, if the snapshot had no clones and the user-initiated reference count were zero.
+The given snapshot is destroyed immediately if and only if the \fBzfs
+destroy\fR command without the \fB-d\fR option would have destroyed it. Such
+immediate destruction would occur, for example, if the snapshot had no clones
+and the user-initiated reference count were zero.
 .sp
-If the snapshot does not qualify for immediate destruction, it is marked for deferred deletion. In this state, it exists as a usable, visible snapshot until both of the preconditions listed above are met, at which point it is destroyed.
+If the snapshot does not qualify for immediate destruction, it is marked for
+deferred deletion. In this state, it exists as a usable, visible snapshot until
+both of the preconditions listed above are met, at which point it is destroyed.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR\fR
 .ad
@@ -1323,18 +1602,17 @@ Defer snapshot deletion.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Destroy (or mark for deferred deletion) all snapshots with this name in descendent file systems.
+Destroy (or mark for deferred deletion) all snapshots with this name in
+descendent file systems.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR\fR
 .ad
@@ -1347,27 +1625,29 @@ Recursively destroy all dependents.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs snapshot\fR [\fB-r\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ... \fIfilesystem@snapname\fR|\fIvolume@snapname\fR\fR
+\fB\fBzfs snapshot\fR [\fB-r\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ...
+\fIfilesystem@snapname\fR|\fIvolume@snapname\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates a snapshot with the given name. All previous modifications by successful system calls to the file system are part of the snapshot. See the "Snapshots" section for details.
+Creates a snapshot with the given name. All previous modifications by
+successful system calls to the file system are part of the snapshot. See the
+"Snapshots" section for details.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively create snapshots of all descendent datasets. Snapshots are taken atomically, so that all recursive snapshots correspond to the same moment in time.
+Recursively create snapshots of all descendent datasets. Snapshots are taken
+atomically, so that all recursive snapshots correspond to the same moment in
+time.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty\fR=\fIvalue\fR\fR
 .ad
@@ -1380,18 +1660,24 @@ Sets the specified property; see \fBzfs create\fR for details.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs rollback\fR [\fB-rRf\fR] \fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Roll back the given dataset to a previous snapshot. When a dataset is rolled back, all data that has changed since the snapshot is discarded, and the dataset reverts to the state at the time of the snapshot. By default, the command refuses to roll back to a snapshot other than the most recent one. In order to do so, all intermediate snapshots must be destroyed by specifying the \fB-r\fR option.
+Roll back the given dataset to a previous snapshot. When a dataset is rolled
+back, all data that has changed since the snapshot is discarded, and the
+dataset reverts to the state at the time of the snapshot. By default, the
+command refuses to roll back to a snapshot other than the most recent one. In
+order to do so, all intermediate snapshots must be destroyed by specifying the
+\fB-r\fR option.
 .sp
-The \fB-rR\fR options do not recursively destroy the child snapshots of a recursive snapshot. Only the top-level recursive snapshot is destroyed by either of these options. To completely roll back a recursive snapshot, you must rollback the individual child snapshots.
+The \fB-rR\fR options do not recursively destroy the child snapshots of a
+recursive snapshot. Only the top-level recursive snapshot is destroyed by
+either of these options. To completely roll back a recursive snapshot, you must
+rollback the individual child snapshots.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
@@ -1402,51 +1688,54 @@ Recursively destroy any snapshots more recent than the one specified.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively destroy any more recent snapshots, as well as any clones of those snapshots.
+Recursively destroy any more recent snapshots, as well as any clones of those
+snapshots.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .sp .6
 .RS 4n
-Used with the \fB-R\fR option to force an unmount of any clone file systems that are to be destroyed.
+Used with the \fB-R\fR option to force an unmount of any clone file systems
+that are to be destroyed.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs clone\fR [\fB-p\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ... \fIsnapshot\fR \fIfilesystem\fR|\fIvolume\fR\fR
+\fB\fBzfs clone\fR [\fB-p\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR] ...
+\fIsnapshot\fR \fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates a clone of the given snapshot. See the "Clones" section for details. The target dataset can be located anywhere in the \fBZFS\fR hierarchy, and is created as the same type as the original.
+Creates a clone of the given snapshot. See the "Clones" section for details.
+The target dataset can be located anywhere in the \fBZFS\fR hierarchy, and is
+created as the same type as the original.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-p\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates all the non-existing parent datasets. Datasets created in this manner are automatically mounted according to the \fBmountpoint\fR property inherited from their parent. If the target filesystem or volume already exists, the operation completes successfully.
+Creates all the non-existing parent datasets. Datasets created in this manner
+are automatically mounted according to the \fBmountpoint\fR property inherited
+from their parent. If the target filesystem or volume already exists, the
+operation completes successfully.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty\fR=\fIvalue\fR\fR
 .ad
@@ -1459,20 +1748,27 @@ Sets the specified property; see \fBzfs create\fR for details.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs promote\fR \fIclone-filesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Promotes a clone file system to no longer be dependent on its "origin" snapshot. This makes it possible to destroy the file system that the clone was created from. The clone parent-child dependency relationship is reversed, so that the origin file system becomes a clone of the specified file system. 
+Promotes a clone file system to no longer be dependent on its "origin"
+snapshot. This makes it possible to destroy the file system that the clone was
+created from. The clone parent-child dependency relationship is reversed, so
+that the origin file system becomes a clone of the specified file system.
 .sp
-The snapshot that was cloned, and any snapshots previous to this snapshot, are now owned by the promoted clone. The space they use moves from the origin file system to the promoted clone, so enough space must be available to accommodate these snapshots. No new space is consumed by this operation, but the space accounting is adjusted. The promoted clone must not have any conflicting snapshot names of its own. The \fBrename\fR subcommand can be used to rename any conflicting snapshots.
+The snapshot that was cloned, and any snapshots previous to this snapshot, are
+now owned by the promoted clone. The space they use moves from the origin file
+system to the promoted clone, so enough space must be available to accommodate
+these snapshots. No new space is consumed by this operation, but the space
+accounting is adjusted. The promoted clone must not have any conflicting
+snapshot names of its own. The \fBrename\fR subcommand can be used to rename
+any conflicting snapshots.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs rename\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR\fR
 .ad
@@ -1482,95 +1778,93 @@ The snapshot that was cloned, and any snapshots previous to this snapshot, are n
 .ad
 .br
 .na
-\fB\fBzfs rename\fR [\fB-p\fR] \fIfilesystem\fR|\fIvolume\fR \fIfilesystem\fR|\fIvolume\fR\fR
-.ad
-.br
-.na
-\fB\fBzfs rename\fR \fB-u\fR [\fB-p\fR] \fIfilesystem\fR \fIfilesystem\fR\fR
+\fB\fBzfs rename\fR [\fB-p\fR] \fIfilesystem\fR|\fIvolume\fR
+\fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Renames the given dataset. The new target can be located anywhere in the \fBZFS\fR hierarchy, with the exception of snapshots. Snapshots can only be renamed within the parent file system or volume. When renaming a snapshot, the parent file system of the snapshot does not need to be specified as part of the second argument. Renamed file systems can inherit new mount points, in which case they are unmounted and remounted at the new mount point.
+Renames the given dataset. The new target can be located anywhere in the
+\fBZFS\fR hierarchy, with the exception of snapshots. Snapshots can only be
+renamed within the parent file system or volume. When renaming a snapshot, the
+parent file system of the snapshot does not need to be specified as part of the
+second argument. Renamed file systems can inherit new mount points, in which
+case they are unmounted and remounted at the new mount point.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-p\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates all the nonexistent parent datasets. Datasets created in this manner are automatically mounted according to the \fBmountpoint\fR property inherited from their parent.
-.RE
-
-.sp
-.ne 2
-.mk
-.na
-\fB\fB-u\fR\fR
-.ad
-.sp .6
-.RS 4n
-Do not remount file systems during rename. If a file system's \fBmountpoint\fR property is set to \fBlegacy\fR or \fBnone\fR, file system is not unmounted even if this option is not given.
+Creates all the nonexistent parent datasets. Datasets created in this manner
+are automatically mounted according to the \fBmountpoint\fR property inherited
+from their parent.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs rename\fR \fB-r\fR \fIsnapshot\fR \fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively rename the snapshots of all descendent datasets. Snapshots are the only dataset that can be renamed recursively.
+Recursively rename the snapshots of all descendent datasets. Snapshots are the
+only dataset that can be renamed recursively.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs\fR \fBlist\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR] [\fB-H\fR] [\fB-o\fR \fIproperty\fR[,\fI\&...\fR]] [ \fB-t\fR \fItype\fR[,\fI\&...\fR]] [ \fB-s\fR \fIproperty\fR ] ... [ \fB-S\fR \fIproperty\fR ] ... [\fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR] ...\fR
+\fB\fBzfs\fR \fBlist\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR] [\fB-H\fR] [\fB-o\fR
+\fIproperty\fR[,\fI\&...\fR]] [ \fB-t\fR \fItype\fR[,\fI\&...\fR]] [ \fB-s\fR
+\fIproperty\fR ] ... [ \fB-S\fR \fIproperty\fR ] ...
+[\fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR] ...\fR
 .ad
 .sp .6
 .RS 4n
-Lists the property information for the given datasets in tabular form. If specified, you can list property information by the absolute pathname or the relative pathname. By default, all file systems and volumes are displayed. Snapshots are displayed if the \fBlistsnaps\fR property is \fBon\fR (the default is \fBoff\fR) . The following fields are displayed, \fBname,used,available,referenced,mountpoint\fR.
+Lists the property information for the given datasets in tabular form. If
+specified, you can list property information by the absolute pathname or the
+relative pathname. By default, all file systems and volumes are displayed.
+Snapshots are displayed if the \fBlistsnaps\fR property is \fBon\fR (the
+default is \fBoff\fR) . The following fields are displayed,
+\fBname,used,available,referenced,mountpoint\fR.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-H\fR\fR
 .ad
 .sp .6
 .RS 4n
-Used for scripting mode. Do not print headers and separate fields by a single tab instead of arbitrary white space.
+Used for scripting mode. Do not print headers and separate fields by a single
+tab instead of arbitrary white space.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively display any children of the dataset on the command line. 
+Recursively display any children of the dataset on the command line.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR \fIdepth\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively display any children of the dataset, limiting the recursion to \fIdepth\fR. A depth of \fB1\fR will display only the dataset and its direct children.
+Recursively display any children of the dataset, limiting the recursion to
+\fIdepth\fR. A depth of \fB1\fR will display only the dataset and its direct
+children.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty\fR\fR
 .ad
@@ -1599,19 +1893,26 @@ The value \fBname\fR to display the dataset name
 .TP
 .ie t \(bu
 .el o
-The value \fBspace\fR to display space usage properties on file systems and volumes. This is a shortcut for specifying \fB-o name,avail,used,usedsnap,usedds,usedrefreserv,usedchild\fR \fB-t filesystem,volume\fR syntax.
+The value \fBspace\fR to display space usage properties on file systems and
+volumes. This is a shortcut for specifying \fB-o
+name,avail,used,usedsnap,usedds,usedrefreserv,usedchild\fR \fB-t
+filesystem,volume\fR syntax.
 .RE
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-s\fR \fIproperty\fR\fR
 .ad
 .sp .6
 .RS 4n
-A property for sorting the output by column in ascending order based on the value of the property. The property must be one of the properties described in the "Properties" section, or the special value \fBname\fR to sort by the dataset name. Multiple properties can be specified at one time using multiple \fB-s\fR property options. Multiple \fB-s\fR options are evaluated from left to right in decreasing order of importance.
+A property for sorting the output by column in ascending order based on the
+value of the property. The property must be one of the properties described in
+the "Properties" section, or the special value \fBname\fR to sort by the
+dataset name. Multiple properties can be specified at one time using multiple
+\fB-s\fR property options. Multiple \fB-s\fR options are evaluated from left to
+right in decreasing order of importance.
 .sp
 The following is a list of sorting criteria:
 .RS +4
@@ -1630,60 +1931,72 @@ String types sort in alphabetical order.
 .TP
 .ie t \(bu
 .el o
-Types inappropriate for a row sort that row to the literal bottom, regardless of the specified ordering.
+Types inappropriate for a row sort that row to the literal bottom, regardless
+of the specified ordering.
 .RE
 .RS +4
 .TP
 .ie t \(bu
 .el o
-If no sorting options are specified the existing behavior of \fBzfs list\fR is preserved.
+If no sorting options are specified the existing behavior of \fBzfs list\fR is
+preserved.
 .RE
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-S\fR \fIproperty\fR\fR
 .ad
 .sp .6
 .RS 4n
-Same as the \fB-s\fR option, but sorts by property in descending order. 
+Same as the \fB-s\fR option, but sorts by property in descending order.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-t\fR \fItype\fR\fR
 .ad
 .sp .6
 .RS 4n
-A comma-separated list of types to display, where \fItype\fR is one of \fBfilesystem\fR, \fBsnapshot\fR , \fBvolume\fR, or \fBall\fR. For example, specifying \fB-t snapshot\fR displays only snapshots.
+A comma-separated list of types to display, where \fItype\fR is one of
+\fBfilesystem\fR, \fBsnapshot\fR , \fBvolume\fR, or \fBall\fR. For example,
+specifying \fB-t snapshot\fR displays only snapshots.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs set\fR \fIproperty\fR=\fIvalue\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR ...\fR
+\fB\fBzfs set\fR \fIproperty\fR=\fIvalue\fR
+\fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Sets the property to the given value for each dataset. Only some properties can be edited. See the "Properties" section for more information on what properties can be set and acceptable values. Numeric values can be specified as exact values, or in a human-readable form with a suffix of \fBB\fR, \fBK\fR, \fBM\fR, \fBG\fR, \fBT\fR, \fBP\fR, \fBE\fR, \fBZ\fR (for bytes, kilobytes, megabytes, gigabytes, terabytes, petabytes, exabytes, or zettabytes, respectively). User properties can be set on snapshots. For more information, see the "User Properties" section.
+Sets the property to the given value for each dataset. Only some properties can
+be edited. See the "Properties" section for more information on what properties
+can be set and acceptable values. Numeric values can be specified as exact
+values, or in a human-readable form with a suffix of \fBB\fR, \fBK\fR, \fBM\fR,
+\fBG\fR, \fBT\fR, \fBP\fR, \fBE\fR, \fBZ\fR (for bytes, kilobytes, megabytes,
+gigabytes, terabytes, petabytes, exabytes, or zettabytes, respectively). User
+properties can be set on snapshots. For more information, see the "User
+Properties" section.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs get\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR] [\fB-Hp\fR] [\fB-o\fR \fIfield\fR[,...] [\fB-s\fR \fIsource\fR[,...] "\fIall\fR" | \fIproperty\fR[,...] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR ...\fR
+\fB\fBzfs get\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR] [\fB-Hp\fR] [\fB-o\fR
+\fIfield\fR[,...] [\fB-s\fR \fIsource\fR[,...] "\fIall\fR" |
+\fIproperty\fR[,...] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Displays properties for the given datasets. If no datasets are specified, then the command displays properties for all datasets on the system. For each property, the following columns are displayed:
+Displays properties for the given datasets. If no datasets are specified, then
+the command displays properties for all datasets on the system. For each
+property, the following columns are displayed:
 .sp
 .in +2
 .nf
@@ -1696,12 +2009,14 @@ Displays properties for the given datasets. If no datasets are specified, then t
 .in -2
 .sp
 
-All columns are displayed by default, though this can be controlled by using the \fB-o\fR option. This command takes a comma-separated list of properties as described in the "Native Properties" and "User Properties" sections.
+All columns are displayed by default, though this can be controlled by using
+the \fB-o\fR option. This command takes a comma-separated list of properties as
+described in the "Native Properties" and "User Properties" sections.
 .sp
-The special value \fBall\fR can be used to display all properties that apply to the given dataset's type (filesystem, volume, or snapshot).
+The special value \fBall\fR can be used to display all properties that apply to
+the given dataset's type (filesystem, volume, or snapshot).
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
@@ -1712,51 +2027,54 @@ Recursively display properties for any children.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR \fIdepth\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively display any children of the dataset, limiting the recursion to \fIdepth\fR. A depth of \fB1\fR will display only the dataset and its direct children.
+Recursively display any children of the dataset, limiting the recursion to
+\fIdepth\fR. A depth of \fB1\fR will display only the dataset and its direct
+children.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-H\fR\fR
 .ad
 .sp .6
 .RS 4n
-Display output in a form more easily parsed by scripts. Any headers are omitted, and fields are explicitly separated by a single tab instead of an arbitrary amount of space.
+Display output in a form more easily parsed by scripts. Any headers are
+omitted, and fields are explicitly separated by a single tab instead of an
+arbitrary amount of space.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIfield\fR\fR
 .ad
 .sp .6
 .RS 4n
-A comma-separated list of columns to display. \fBname,property,value,source\fR is the default value. 
+A comma-separated list of columns to display. \fBname,property,value,source\fR
+is the default value.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-s\fR \fIsource\fR\fR
 .ad
 .sp .6
 .RS 4n
-A comma-separated list of sources to display. Those properties coming from a source other than those in this list are ignored. Each source must be one of the following: \fBlocal,default,inherited,temporary,none\fR. The default value is all sources.
+A comma-separated list of sources to display. Those properties coming from a
+source other than those in this list are ignored. Each source must be one of
+the following: \fBlocal,default,inherited,temporary,none\fR. The default value
+is all sources.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-p\fR\fR
 .ad
@@ -1769,16 +2087,18 @@ Display numbers in parseable (exact) values.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs inherit\fR [\fB-r\fR] \fIproperty\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR ...\fR
+\fB\fBzfs inherit\fR [\fB-r\fR] \fIproperty\fR
+\fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Clears the specified property, causing it to be inherited from an ancestor. If no ancestor has the property set, then the default value is used. See the "Properties" section for a listing of default values, and details on which properties can be inherited.
+Clears the specified property, causing it to be inherited from an ancestor. If
+no ancestor has the property set, then the default value is used. See the
+"Properties" section for a listing of default values, and details on which
+properties can be inherited.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
@@ -1791,7 +2111,6 @@ Recursively inherit the given property for all children.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs upgrade\fR [\fB-v\fR]\fR
 .ad
@@ -1802,20 +2121,25 @@ Displays a list of file systems that are not the most recent version.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs upgrade\fR [\fB-r\fR] [\fB-V\fR \fIversion\fR] [\fB-a\fR | \fIfilesystem\fR]\fR
+\fB\fBzfs upgrade\fR [\fB-r\fR] [\fB-V\fR \fIversion\fR] [\fB-a\fR |
+\fIfilesystem\fR]\fR
 .ad
 .sp .6
 .RS 4n
-Upgrades file systems to a new on-disk version. Once this is done, the file systems will no longer be accessible on systems running older versions of the software. \fBzfs send\fR streams generated from new snapshots of these file systems cannot be accessed on systems running older versions of the software.
+Upgrades file systems to a new on-disk version. Once this is done, the file
+systems will no longer be accessible on systems running older versions of the
+software. \fBzfs send\fR streams generated from new snapshots of these file
+systems cannot be accessed on systems running older versions of the software.
 .sp
-In general, the file system version is independent of the pool version. See \fBzpool\fR(1M) for information on the \fBzpool upgrade\fR command. 
+In general, the file system version is independent of the pool version. See
+\fBzpool\fR(8) for information on the \fBzpool upgrade\fR command.
 .sp
-In some cases, the file system version and the pool version are interrelated and the pool version must be upgraded before the file system version can be upgraded.
+In some cases, the file system version and the pool version are interrelated
+and the pool version must be upgraded before the file system version can be
+upgraded.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
@@ -1826,51 +2150,53 @@ Upgrade all file systems on all imported pools.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIfilesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Upgrade the specified file system. 
+Upgrade the specified file system.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Upgrade the specified file system and all descendent file systems 
+Upgrade the specified file system and all descendent file systems
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-V\fR \fIversion\fR\fR
 .ad
 .sp .6
 .RS 4n
-Upgrade to the specified \fIversion\fR. If the \fB-V\fR flag is not specified, this command upgrades to the most recent version. This option can only be used to increase the version number, and only up to the most recent version supported by this software.
+Upgrade to the specified \fIversion\fR. If the \fB-V\fR flag is not specified,
+this command upgrades to the most recent version. This option can only be used
+to increase the version number, and only up to the most recent version
+supported by this software.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs userspace\fR [\fB-niHp\fR] [\fB-o\fR \fIfield\fR[,...]] [\fB-sS\fR \fIfield\fR]... [\fB-t\fR \fItype\fR [,...]] \fIfilesystem\fR | \fIsnapshot\fR\fR
+\fB\fBzfs userspace\fR [\fB-niHp\fR] [\fB-o\fR \fIfield\fR[,...]] [\fB-sS\fR
+\fIfield\fR]... [\fB-t\fR \fItype\fR [,...]] \fIfilesystem\fR |
+\fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Displays space consumed by, and quotas on, each user in the specified filesystem or snapshot. This corresponds to the \fBuserused@\fR\fIuser\fR and \fBuserquota@\fR\fIuser\fR properties.
+Displays space consumed by, and quotas on, each user in the specified
+filesystem or snapshot. This corresponds to the \fBuserused@\fR\fIuser\fR and
+\fBuserquota@\fR\fIuser\fR properties.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-n\fR\fR
 .ad
@@ -1881,7 +2207,6 @@ Print numeric ID instead of user/group name.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-H\fR\fR
 .ad
@@ -1892,7 +2217,6 @@ Do not print headers, use tab-delimited output.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-p\fR\fR
 .ad
@@ -1903,29 +2227,29 @@ Use exact (parseable) numeric output.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIfield\fR[,...]\fR
 .ad
 .sp .6
 .RS 4n
-Display only the specified fields from the following set, \fBtype,name,used,quota\fR.The default is to display all fields.
+Display only the specified fields from the following set,
+\fBtype,name,used,quota\fR.The default is to display all fields.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-s\fR \fIfield\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sort output by this field. The \fIs\fR and \fIS\fR flags may be specified multiple times to sort first by one field, then by another. The default is \fB-s type\fR \fB-s name\fR.
+Sort output by this field. The \fIs\fR and \fIS\fR flags may be specified
+multiple times to sort first by one field, then by another. The default is
+\fB-s type\fR \fB-s name\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-S\fR \fIfield\fR\fR
 .ad
@@ -1936,13 +2260,13 @@ Sort by this field in reverse order. See \fB-s\fR.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-t\fR \fItype\fR[,...]\fR
 .ad
 .sp .6
 .RS 4n
-Print only the specified types from the following set, \fBall,posixuser,smbuser,posixgroup,smbgroup\fR.
+Print only the specified types from the following set,
+\fBall,posixuser,smbuser,posixgroup,smbgroup\fR.
 .sp
 The default is \fB-t posixuser,smbuser\fR
 .sp
@@ -1951,26 +2275,35 @@ The default can be changed to include group types.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-i\fR\fR
 .ad
 .sp .6
 .RS 4n
-Translate SID to POSIX ID. The POSIX ID may be ephemeral if no mapping exists. Normal POSIX interfaces (for example, \fBstat\fR(2), \fBls\fR \fB-l\fR) perform this translation, so the \fB-i\fR option allows the output from \fBzfs userspace\fR to be compared directly with those utilities. However, \fB-i\fR may lead to confusion if some files were created by an SMB user before a SMB-to-POSIX name mapping was established. In such a case, some files are owned by the SMB entity and some by the POSIX entity. However, the \fB-i\fR option will report that the POSIX entity has the total usage and quota for both.
+Translate SID to POSIX ID. The POSIX ID may be ephemeral if no mapping exists.
+Normal POSIX interfaces (for example, \fBstat\fR(2), \fBls\fR \fB-l\fR) perform
+this translation, so the \fB-i\fR option allows the output from \fBzfs
+userspace\fR to be compared directly with those utilities. However, \fB-i\fR
+may lead to confusion if some files were created by an SMB user before a
+SMB-to-POSIX name mapping was established. In such a case, some files are owned
+by the SMB entity and some by the POSIX entity. However, the \fB-i\fR option
+will report that the POSIX entity has the total usage and quota for both.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs groupspace\fR [\fB-niHp\fR] [\fB-o\fR \fIfield\fR[,...]] [\fB-sS\fR \fIfield\fR]... [\fB-t\fR \fItype\fR [,...]] \fIfilesystem\fR | \fIsnapshot\fR\fR
+\fB\fBzfs groupspace\fR [\fB-niHp\fR] [\fB-o\fR \fIfield\fR[,...]] [\fB-sS\fR
+\fIfield\fR]... [\fB-t\fR \fItype\fR [,...]] \fIfilesystem\fR |
+\fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Displays space consumed by, and quotas on, each group in the specified filesystem or snapshot. This subcommand is identical to \fBzfs userspace\fR, except that the default types to display are \fB-t posixgroup,smbgroup\fR.
+Displays space consumed by, and quotas on, each group in the specified
+filesystem or snapshot. This subcommand is identical to \fBzfs userspace\fR,
+except that the default types to display are \fB-t posixgroup,smbgroup\fR.
 .sp
 .in +2
 .nf
@@ -1983,7 +2316,6 @@ Displays space consumed by, and quotas on, each group in the specified filesyste
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs mount\fR\fR
 .ad
@@ -1994,38 +2326,38 @@ Displays all \fBZFS\fR file systems currently mounted.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs mount\fR [\fB-vO\fR] [\fB-o\fR \fIoptions\fR] \fB-a\fR | \fIfilesystem\fR\fR
+\fB\fBzfs mount\fR [\fB-vO\fR] [\fB-o\fR \fIoptions\fR] \fB-a\fR |
+\fIfilesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Mounts \fBZFS\fR file systems. Invoked automatically as part of the boot process.
+Mounts \fBZFS\fR file systems. Invoked automatically as part of the boot
+process.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIoptions\fR\fR
 .ad
 .sp .6
 .RS 4n
-An optional, comma-separated list of mount options to use temporarily for the duration of the mount. See the "Temporary Mount Point Properties" section for details.
+An optional, comma-separated list of mount options to use temporarily for the
+duration of the mount. See the "Temporary Mount Point Properties" section for
+details.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-O\fR\fR
 .ad
 .sp .6
 .RS 4n
-Perform an overlay mount. See \fBmount\fR(1M) for more information.
+Perform an overlay mount. See \fBmount\fR(8) for more information.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-v\fR\fR
 .ad
@@ -2036,18 +2368,17 @@ Report mount progress.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
 .sp .6
 .RS 4n
-Mount all available \fBZFS\fR file systems. Invoked automatically as part of the boot process. 
+Mount all available \fBZFS\fR file systems. Invoked automatically as part of
+the boot process.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIfilesystem\fR\fR
 .ad
@@ -2060,16 +2391,15 @@ Mount the specified filesystem.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs unmount\fR [\fB-f\fR] \fB-a\fR | \fIfilesystem\fR|\fImountpoint\fR\fR
 .ad
 .sp .6
 .RS 4n
-Unmounts currently mounted \fBZFS\fR file systems. Invoked automatically as part of the shutdown process.
+Unmounts currently mounted \fBZFS\fR file systems. Invoked automatically as
+part of the shutdown process.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
@@ -2080,143 +2410,158 @@ Forcefully unmount the file system, even if it is currently in use.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
 .sp .6
 .RS 4n
-Unmount all available \fBZFS\fR file systems. Invoked automatically as part of the boot process. 
+Unmount all available \fBZFS\fR file systems. Invoked automatically as part of
+the boot process.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIfilesystem\fR|\fImountpoint\fR\fR
 .ad
 .sp .6
 .RS 4n
-Unmount the specified filesystem. The command can also be given a path to a \fBZFS\fR file system mount point on the system.
+Unmount the specified filesystem. The command can also be given a path to a
+\fBZFS\fR file system mount point on the system.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs share\fR \fB-a\fR | \fIfilesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Shares available \fBZFS\fR file systems. 
+Shares available \fBZFS\fR file systems.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
 .sp .6
 .RS 4n
-Share all available \fBZFS\fR file systems. Invoked automatically as part of the boot process. 
+Share all available \fBZFS\fR file systems. Invoked automatically as part of
+the boot process.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIfilesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Share the specified filesystem according to the \fBsharenfs\fR and \fBsharesmb\fR properties. File systems are shared when the \fBsharenfs\fR or \fBsharesmb\fR property is set.
+Share the specified filesystem according to the \fBsharenfs\fR and
+\fBsharesmb\fR properties. File systems are shared when the \fBsharenfs\fR or
+\fBsharesmb\fR property is set.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs unshare\fR \fB-a\fR | \fIfilesystem\fR|\fImountpoint\fR\fR
 .ad
 .sp .6
 .RS 4n
-Unshares currently shared \fBZFS\fR file systems. This is invoked automatically as part of the shutdown process.
+Unshares currently shared \fBZFS\fR file systems. This is invoked automatically
+as part of the shutdown process.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
 .sp .6
 .RS 4n
-Unshare all available \fBZFS\fR file systems. Invoked automatically as part of the boot process. 
+Unshare all available \fBZFS\fR file systems. Invoked automatically as part of
+the boot process.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fIfilesystem\fR|\fImountpoint\fR\fR
 .ad
 .sp .6
 .RS 4n
-Unshare the specified filesystem. The command can also be given a path to a \fBZFS\fR file system shared on the system.
+Unshare the specified filesystem. The command can also be given a path to a
+\fBZFS\fR file system shared on the system.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs send\fR [\fB-vR\fR] [\fB-\fR[\fBiI\fR] \fIsnapshot\fR] \fIsnapshot\fR\fR
+\fB\fBzfs send\fR [\fB-vR\fR] [\fB-\fR[\fBiI\fR] \fIsnapshot\fR]
+\fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Creates a stream representation of the second \fIsnapshot\fR, which is written to standard output. The output can be redirected to a file or to a different system (for example, using \fBssh\fR(1). By default, a full stream is generated.
+Creates a stream representation of the second \fIsnapshot\fR, which is written
+to standard output. The output can be redirected to a file or to a different
+system (for example, using \fBssh\fR(1). By default, a full stream is
+generated.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-i\fR \fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Generate an incremental stream from the first \fIsnapshot\fR to the second \fIsnapshot\fR. The incremental source (the first \fIsnapshot\fR) can be specified as the last component of the snapshot name (for example, the part after the \fB@\fR), and it is assumed to be from the same file system as the second \fIsnapshot\fR.
+Generate an incremental stream from the first \fIsnapshot\fR to the second
+\fIsnapshot\fR. The incremental source (the first \fIsnapshot\fR) can be
+specified as the last component of the snapshot name (for example, the part
+after the \fB@\fR), and it is assumed to be from the same file system as the
+second \fIsnapshot\fR.
 .sp
-If the destination is a clone, the source may be the origin snapshot, which must be fully specified (for example, \fBpool/fs@origin\fR, not just \fB@origin\fR).
+If the destination is a clone, the source may be the origin snapshot, which
+must be fully specified (for example, \fBpool/fs@origin\fR, not just
+\fB@origin\fR).
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-I\fR \fIsnapshot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Generate a stream package that sends all intermediary snapshots from the first snapshot to the second snapshot. For example, \fB-I @a fs@d\fR is similar to \fB-i @a fs@b; -i @b fs@c; -i @c fs@d\fR. The incremental source snapshot may be specified as with the \fB-i\fR option.
+Generate a stream package that sends all intermediary snapshots from the first
+snapshot to the second snapshot. For example, \fB-I @a fs@d\fR is similar to
+\fB-i @a fs@b; -i @b fs@c; -i @c fs@d\fR. The incremental source snapshot may
+be specified as with the \fB-i\fR option.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR\fR
 .ad
 .sp .6
 .RS 4n
-Generate a replication stream package, which will replicate the specified filesystem, and all descendent file systems, up to the named snapshot. When received, all properties, snapshots, descendent file systems, and clones are preserved.
+Generate a replication stream package, which will replicate the specified
+filesystem, and all descendent file systems, up to the named snapshot. When
+received, all properties, snapshots, descendent file systems, and clones are
+preserved.
 .sp
-If the \fB-i\fR or \fB-I\fR flags are used in conjunction with the \fB-R\fR flag, an incremental replication stream is generated. The current values of properties, and current snapshot and file system names are set when the stream is received. If the \fB-F\fR flag is specified when this stream is received, snapshots and file systems that do not exist on the sending side are destroyed. 
+If the \fB-i\fR or \fB-I\fR flags are used in conjunction with the \fB-R\fR
+flag, an incremental replication stream is generated. The current values of
+properties, and current snapshot and file system names are set when the stream
+is received. If the \fB-F\fR flag is specified when this stream is received,
+snapshots and file systems that do not exist on the sending side are destroyed.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-v\fR\fR
 .ad
@@ -2225,14 +2570,15 @@ If the \fB-i\fR or \fB-I\fR flags are used in conjunction with the \fB-R\fR flag
 Print verbose information about the stream package generated.
 .RE
 
-The format of the stream is committed. You will be able to receive your streams on future versions of \fBZFS\fR.
+The format of the stream is committed. You will be able to receive your streams
+on future versions of \fBZFS\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs receive\fR [\fB-vnFu\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR\fR
+\fB\fBzfs receive\fR [\fB-vnFu\fR]
+\fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR\fR
 .ad
 .br
 .na
@@ -2240,29 +2586,46 @@ The format of the stream is committed. You will be able to receive your streams
 .ad
 .sp .6
 .RS 4n
-Creates a snapshot whose contents are as specified in the stream provided on standard input. If a full stream is received, then a new file system is created as well. Streams are created using the \fBzfs send\fR subcommand, which by default creates a full stream. \fBzfs recv\fR can be used as an alias for \fBzfs receive\fR.
+Creates a snapshot whose contents are as specified in the stream provided on
+standard input. If a full stream is received, then a new file system is created
+as well. Streams are created using the \fBzfs send\fR subcommand, which by
+default creates a full stream. \fBzfs recv\fR can be used as an alias for
+\fBzfs receive\fR.
 .sp
-If an incremental stream is received, then the destination file system must already exist, and its most recent snapshot must match the incremental stream's source. For \fBzvols\fR, the destination device link is destroyed and recreated, which means the \fBzvol\fR cannot be accessed during the \fBreceive\fR operation.
+If an incremental stream is received, then the destination file system must
+already exist, and its most recent snapshot must match the incremental stream's
+source. For \fBzvols\fR, the destination device link is destroyed and
+recreated, which means the \fBzvol\fR cannot be accessed during the
+\fBreceive\fR operation.
 .sp
-When a snapshot replication package stream that is generated by using the \fBzfs send\fR \fB-R\fR command is  received, any snapshots that do not exist on the sending location are destroyed by using the \fBzfs destroy\fR \fB-d\fR command.
+When a snapshot replication package stream that is generated by using the
+\fBzfs send\fR \fB-R\fR command is  received, any snapshots that do not exist
+on the sending location are destroyed by using the \fBzfs destroy\fR \fB-d\fR
+command.
 .sp
-The name of the snapshot (and file system, if a full stream is received) that this subcommand creates depends on the argument type and the \fB-d\fR option.
+The name of the snapshot (and file system, if a full stream is received) that
+this subcommand creates depends on the argument type and the \fB-d\fR option.
 .sp
-If the argument is a snapshot name, the specified \fIsnapshot\fR is created. If the argument is a file system or volume name, a snapshot with the same name as the sent snapshot is created within the specified \fIfilesystem\fR or \fIvolume\fR. If the \fB-d\fR option is specified, the snapshot name is determined by appending the sent snapshot's name to the specified \fIfilesystem\fR. If the \fB-d\fR option is specified, any required file systems within the specified one are created.
+If the argument is a snapshot name, the specified \fIsnapshot\fR is created. If
+the argument is a file system or volume name, a snapshot with the same name as
+the sent snapshot is created within the specified \fIfilesystem\fR or
+\fIvolume\fR. If the \fB-d\fR option is specified, the snapshot name is
+determined by appending the sent snapshot's name to the specified
+\fIfilesystem\fR. If the \fB-d\fR option is specified, any required file
+systems within the specified one are created.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR\fR
 .ad
 .sp .6
 .RS 4n
-Use the name of the sent snapshot to determine the name of the new snapshot as described in the paragraph above.
+Use the name of the sent snapshot to determine the name of the new snapshot as
+described in the paragraph above.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-u\fR\fR
 .ad
@@ -2273,101 +2636,117 @@ File system that is associated with the received stream is not mounted.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-v\fR\fR
 .ad
 .sp .6
 .RS 4n
-Print verbose information about the stream and the time required to perform the receive operation.
+Print verbose information about the stream and the time required to perform the
+receive operation.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-n\fR\fR
 .ad
 .sp .6
 .RS 4n
-Do not actually receive the stream. This can be useful in conjunction with the \fB-v\fR option to verify the name the receive operation would use.
+Do not actually receive the stream. This can be useful in conjunction with the
+\fB-v\fR option to verify the name the receive operation would use.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-F\fR\fR
 .ad
 .sp .6
 .RS 4n
-Force a rollback of the file system to the most recent snapshot before performing the receive operation. If receiving an incremental replication stream (for example, one generated by \fBzfs send -R -[iI]\fR), destroy snapshots and file systems that do not exist on the sending side.
+Force a rollback of the file system to the most recent snapshot before
+performing the receive operation. If receiving an incremental replication
+stream (for example, one generated by \fBzfs send -R -[iI]\fR), destroy
+snapshots and file systems that do not exist on the sending side.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs allow\fR \fIfilesystem\fR | \fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Displays permissions that have been delegated on the specified filesystem or volume. See the other forms of \fBzfs allow\fR for more information.
+Displays permissions that have been delegated on the specified filesystem or
+volume. See the other forms of \fBzfs allow\fR for more information.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs allow\fR [\fB-ldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...] \fIperm\fR|@\fIsetname\fR[,...] \fIfilesystem\fR| \fIvolume\fR\fR
+\fB\fBzfs allow\fR [\fB-ldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...]
+\fIperm\fR|@\fIsetname\fR[,...] \fIfilesystem\fR| \fIvolume\fR\fR
 .ad
 .br
 .na
-\fB\fBzfs allow\fR [\fB-ld\fR] \fB-e\fR \fIperm\fR|@\fIsetname\fR[,...] \fIfilesystem\fR | \fIvolume\fR\fR
+\fB\fBzfs allow\fR [\fB-ld\fR] \fB-e\fR \fIperm\fR|@\fIsetname\fR[,...]
+\fIfilesystem\fR | \fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Delegates \fBZFS\fR administration permission for the file systems to non-privileged users.
+Delegates \fBZFS\fR administration permission for the file systems to
+non-privileged users.
 .sp
 .ne 2
-.mk
 .na
 \fB[\fB-ug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...]\fR
 .ad
 .sp .6
 .RS 4n
-Specifies to whom the permissions are delegated. Multiple entities can be specified as a comma-separated list. If neither of the \fB-ug\fR options are specified, then the argument is interpreted preferentially as the keyword "everyone", then as a user name, and lastly as a group name. To specify a user or group named "everyone", use the \fB-u\fR or \fB-g\fR options. To specify a group with the same name as a user, use the \fB-g\fR options.
+Specifies to whom the permissions are delegated. Multiple entities can be
+specified as a comma-separated list. If neither of the \fB-ug\fR options are
+specified, then the argument is interpreted preferentially as the keyword
+"everyone", then as a user name, and lastly as a group name. To specify a user
+or group named "everyone", use the \fB-u\fR or \fB-g\fR options. To specify a
+group with the same name as a user, use the \fB-g\fR options.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB[\fB-e\fR] \fIperm\fR|@\fIsetname\fR[,...]\fR
 .ad
 .sp .6
 .RS 4n
-Specifies that the permissions be delegated to "everyone." Multiple permissions may be specified as a comma-separated list. Permission names are the same as \fBZFS\fR subcommand and property names. See the property list below. Property set names, which begin with an at sign (\fB@\fR) , may be specified. See the \fB-s\fR form below for details.
+Specifies that the permissions be delegated to "everyone." Multiple permissions
+may be specified as a comma-separated list. Permission names are the same as
+\fBZFS\fR subcommand and property names. See the property list below. Property
+set names, which begin with an at sign (\fB@\fR) , may be specified. See the
+\fB-s\fR form below for details.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB[\fB-ld\fR] \fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Specifies where the permissions are delegated. If neither of the \fB-ld\fR options are specified, or both are, then the permissions are allowed for the file system or volume, and all of its descendents. If only the \fB-l\fR option is used, then is allowed "locally" only for the specified file system. If only the \fB-d\fR option is used, then is allowed only for the descendent file systems.
+Specifies where the permissions are delegated. If neither of the \fB-ld\fR
+options are specified, or both are, then the permissions are allowed for the
+file system or volume, and all of its descendents. If only the \fB-l\fR option
+is used, then is allowed "locally" only for the specified file system. If only
+the \fB-d\fR option is used, then is allowed only for the descendent file
+systems.
 .RE
 
 .RE
 
 .sp
 .LP
-Permissions are generally the ability to use a \fBZFS\fR subcommand or change a \fBZFS\fR property. The following permissions are available:
+Permissions are generally the ability to use a \fBZFS\fR subcommand or change a
+\fBZFS\fR property. The following permissions are available:
 .sp
 .in +2
 .nf
@@ -2385,7 +2764,7 @@ receive          subcommand     Must also have the 'mount' and 'create' ability
 rename           subcommand     Must also have the 'mount' and 'create'
                                 ability in the new parent
 rollback         subcommand     Must also have the 'mount' ability
-send             subcommand     
+send             subcommand
 share            subcommand     Allows sharing file systems over NFS or SMB
                                 protocols
 snapshot         subcommand     Must also have the 'mount' ability
@@ -2395,74 +2774,81 @@ userprop         other          Allows changing any user property
 userquota        other          Allows accessing any userquota@... property
 userused         other          Allows reading any userused@... property
 
-aclinherit       property       
-aclmode          property       
-atime            property       
-canmount         property       
-casesensitivity  property       
-checksum         property       
-compression      property       
-copies           property       
-devices          property       
-exec             property       
-mountpoint       property       
-nbmand           property       
-normalization    property       
-primarycache     property       
-quota            property       
-readonly         property       
-recordsize       property       
-refquota         property       
-refreservation   property       
-reservation      property       
-secondarycache   property       
-setuid           property       
-shareiscsi       property       
-sharenfs         property       
-sharesmb         property       
-snapdir          property       
-utf8only         property       
-version          property       
-volblocksize     property       
-volsize          property       
-vscan            property       
-xattr            property       
-zoned            property       
+aclinherit       property
+aclmode          property
+atime            property
+canmount         property
+casesensitivity  property
+checksum         property
+compression      property
+copies           property
+devices          property
+exec             property
+jailed           property
+mountpoint       property
+nbmand           property
+normalization    property
+primarycache     property
+quota            property
+readonly         property
+recordsize       property
+refquota         property
+refreservation   property
+reservation      property
+secondarycache   property
+setuid           property
+sharenfs         property
+sharesmb         property
+snapdir          property
+utf8only         property
+version          property
+volblocksize     property
+volsize          property
+vscan            property
+xattr            property
 .fi
 .in -2
 .sp
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs allow\fR \fB-c\fR \fIperm\fR|@\fIsetname\fR[,...] \fIfilesystem\fR|\fIvolume\fR\fR
+\fB\fBzfs allow\fR \fB-c\fR \fIperm\fR|@\fIsetname\fR[,...]
+\fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets "create time" permissions. These permissions are granted (locally) to the creator of any newly-created descendent file system.
+Sets "create time" permissions. These permissions are granted (locally) to the
+creator of any newly-created descendent file system.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs allow\fR \fB-s\fR @\fIsetname\fR \fIperm\fR|@\fIsetname\fR[,...] \fIfilesystem\fR|\fIvolume\fR\fR
+\fB\fBzfs allow\fR \fB-s\fR @\fIsetname\fR \fIperm\fR|@\fIsetname\fR[,...]
+\fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .sp .6
 .RS 4n
-Defines or adds permissions to a permission set. The set can be used by other \fBzfs allow\fR commands for the specified file system and its descendents. Sets are evaluated dynamically, so changes to a set are immediately reflected. Permission sets follow the same naming restrictions as ZFS file systems, but the name must begin with an "at sign" (\fB@\fR), and can be no more than 64 characters long.
+Defines or adds permissions to a permission set. The set can be used by other
+\fBzfs allow\fR commands for the specified file system and its descendents.
+Sets are evaluated dynamically, so changes to a set are immediately reflected.
+Permission sets follow the same naming restrictions as ZFS file systems, but
+the name must begin with an "at sign" (\fB@\fR), and can be no more than 64
+characters long.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs unallow\fR [\fB-rldug\fR] "\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...] [\fIperm\fR|@\fIsetname\fR[, ...]] \fIfilesystem\fR|\fIvolume\fR\fR
+\fB\fBzfs unallow\fR [\fB-rldug\fR]
+"\fIeveryone\fR"|\fIuser\fR|\fIgroup\fR[,...]
+[\fIperm\fR|@\fIsetname\fR[, ...]] \fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .br
 .na
-\fB\fBzfs unallow\fR [\fB-rld\fR] \fB-e\fR [\fIperm\fR|@\fIsetname\fR [,...]] \fIfilesystem\fR|\fIvolume\fR\fR
+\fB\fBzfs unallow\fR [\fB-rld\fR] \fB-e\fR [\fIperm\fR|@\fIsetname\fR [,...]]
+\fIfilesystem\fR|\fIvolume\fR\fR
 .ad
 .br
 .na
@@ -2474,10 +2860,16 @@ Defines or adds permissions to a permission set. The set can be used by other \f
 .ad
 .sp .6
 .RS 4n
-Removes permissions that were granted with the \fBzfs allow\fR command. No permissions are explicitly denied, so other permissions granted are still in effect. For example, if the permission is granted by an ancestor. If no permissions are specified, then all permissions for the specified \fIuser\fR, \fIgroup\fR, or \fIeveryone\fR are removed. Specifying "everyone" (or using the \fB-e\fR option) only removes the permissions that were granted to "everyone", not all permissions for every user and group. See the \fBzfs allow\fR command for a description of the \fB-ldugec\fR options.
+Removes permissions that were granted with the \fBzfs allow\fR command. No
+permissions are explicitly denied, so other permissions granted are still in
+effect. For example, if the permission is granted by an ancestor. If no
+permissions are specified, then all permissions for the specified \fIuser\fR,
+\fIgroup\fR, or \fIeveryone\fR are removed. Specifying "everyone" (or using the
+\fB-e\fR option) only removes the permissions that were granted to "everyone",
+not all permissions for every user and group. See the \fBzfs allow\fR command
+for a description of the \fB-ldugec\fR options.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
@@ -2490,9 +2882,9 @@ Recursively remove the permissions from this file system and all descendents.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzfs unallow\fR [\fB-r\fR] \fB-s\fR @\fIsetname\fR [\fIperm\fR|@\fIsetname\fR[,...]]\fR
+\fB\fBzfs unallow\fR [\fB-r\fR] \fB-s\fR @\fIsetname\fR
+[\fIperm\fR|@\fIsetname\fR[,...]]\fR
 .ad
 .br
 .na
@@ -2500,36 +2892,38 @@ Recursively remove the permissions from this file system and all descendents.
 .ad
 .sp .6
 .RS 4n
-Removes permissions from a permission set. If no permissions are specified, then all permissions are removed, thus removing the set entirely.
+Removes permissions from a permission set. If no permissions are specified,
+then all permissions are removed, thus removing the set entirely.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs hold\fR [\fB-r\fR] \fItag\fR \fIsnapshot\fR...\fR
 .ad
 .sp .6
 .RS 4n
-Adds a single reference, named with the \fItag\fR argument, to the specified snapshot or snapshots. Each snapshot has its own tag namespace, and tags must be unique within that space.
+Adds a single reference, named with the \fItag\fR argument, to the specified
+snapshot or snapshots. Each snapshot has its own tag namespace, and tags must
+be unique within that space.
 .sp
-If a hold exists on a snapshot, attempts to destroy that snapshot by using the \fBzfs destroy\fR command return \fBEBUSY\fR.
+If a hold exists on a snapshot, attempts to destroy that snapshot by using the
+\fBzfs destroy\fR command return \fBEBUSY\fR.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Specifies that a hold with the given tag is applied recursively to the snapshots of all descendent file systems.
+Specifies that a hold with the given tag is applied recursively to the
+snapshots of all descendent file systems.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs holds\fR [\fB-r\fR] \fIsnapshot\fR...\fR
 .ad
@@ -2538,47 +2932,54 @@ Specifies that a hold with the given tag is applied recursively to the snapshots
 Lists all existing user references for the given snapshot or snapshots.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Lists the holds that are set on the named descendent snapshots, in addition to listing the holds on the named snapshot.
+Lists the holds that are set on the named descendent snapshots, in addition to
+listing the holds on the named snapshot.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzfs release\fR [\fB-r\fR] \fItag\fR \fIsnapshot\fR...\fR
 .ad
 .sp .6
 .RS 4n
-Removes a single reference, named with the \fItag\fR argument, from the specified snapshot or snapshots. The tag must already exist for each snapshot.
+Removes a single reference, named with the \fItag\fR argument, from the
+specified snapshot or snapshots. The tag must already exist for each snapshot.
 .sp
-If a hold exists on a snapshot, attempts to destroy that snapshot by using the \fBzfs destroy\fR command return \fBEBUSY\fR.
+If a hold exists on a snapshot, attempts to destroy that snapshot by using the
+\fBzfs destroy\fR command return \fBEBUSY\fR.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-r\fR\fR
 .ad
 .sp .6
 .RS 4n
-Recursively releases a hold with the given tag on the snapshots of all descendent file systems.
+Recursively releases a hold with the given tag on the snapshots of all
+descendent file systems.
 .RE
 
 .RE
 
+.sp
+.ne 2
+.na
 \fB\fBzfs jail\fR \fIjailid\fR \fIfilesystem\fR\fR
 .ad
 .sp .6
 .RS 4n
-Attaches the given file system to the given jail. From now on this file system tree can be managed from within a jail if the "\fBjailed\fR" property has been set.
-To use this functionality, sysctl \fBsecurity.jail.enforce_statfs\fR should be set to 0 and sysctl \fBsecurity.jail.mount_allowed\fR should be set to 1.
+Attaches the given file system to the given jail. From now on this file system
+tree can be managed from within a jail if the "\fBjailed\fR" property has been
+set. To use this functionality, sysctl \fBsecurity.jail.enforce_statfs\fR
+should be set to 0 and sysctl \fBsecurity.jail.mount_allowed\fR should be set
+to 1.
 .RE
 
 .sp
@@ -2592,12 +2993,17 @@ To use this functionality, sysctl \fBsecurity.jail.enforce_statfs\fR should be s
 Detaches the given file system from the given jail.
 .RE
 
+.RE
+
 .SH EXAMPLES
 .LP
 \fBExample 1 \fRCreating a ZFS File System Hierarchy
 .sp
 .LP
-The following commands create a file system named \fBpool/home\fR and a file system named \fBpool/home/bob\fR. The mount point \fB/export/home\fR is set for the parent file system, and is automatically inherited by the child file system.
+The following commands create a file system named \fBpool/home\fR and a file
+system named \fBpool/home/bob\fR. The mount point \fB/export/home\fR is set for
+the parent file system, and is automatically inherited by the child file
+system.
 
 .sp
 .in +2
@@ -2613,7 +3019,9 @@ The following commands create a file system named \fBpool/home\fR and a file sys
 \fBExample 2 \fRCreating a ZFS Snapshot
 .sp
 .LP
-The following command creates a snapshot named \fByesterday\fR. This snapshot is mounted on demand in the \fB\&.zfs/snapshot\fR directory at the root of the \fBpool/home/bob\fR file system.
+The following command creates a snapshot named \fByesterday\fR. This snapshot
+is mounted on demand in the \fB\&.zfs/snapshot\fR directory at the root of the
+\fBpool/home/bob\fR file system.
 
 .sp
 .in +2
@@ -2627,7 +3035,10 @@ The following command creates a snapshot named \fByesterday\fR. This snapshot is
 \fBExample 3 \fRCreating and Destroying Multiple Snapshots
 .sp
 .LP
-The following command creates snapshots named \fByesterday\fR of \fBpool/home\fR and all of its descendent file systems. Each snapshot is mounted on demand in the \fB\&.zfs/snapshot\fR directory at the root of its file system. The second command destroys the newly created snapshots.
+The following command creates snapshots named \fByesterday\fR of
+\fBpool/home\fR and all of its descendent file systems. Each snapshot is
+mounted on demand in the \fB\&.zfs/snapshot\fR directory at the root of its
+file system. The second command destroys the newly created snapshots.
 
 .sp
 .in +2
@@ -2642,7 +3053,9 @@ The following command creates snapshots named \fByesterday\fR of \fBpool/home\fR
 \fBExample 4 \fRDisabling and Enabling File System Compression
 .sp
 .LP
-The following command disables the \fBcompression\fR property for all file systems under \fBpool/home\fR. The next command explicitly enables \fBcompression\fR for \fBpool/home/anne\fR.
+The following command disables the \fBcompression\fR property for all file
+systems under \fBpool/home\fR. The next command explicitly enables
+\fBcompression\fR for \fBpool/home/anne\fR.
 
 .sp
 .in +2
@@ -2657,7 +3070,10 @@ The following command disables the \fBcompression\fR property for all file syste
 \fBExample 5 \fRListing ZFS Datasets
 .sp
 .LP
-The following command lists all active file systems and volumes in the system. Snapshots are displayed if the \fBlistsnaps\fR property is \fBon\fR. The default is \fBoff\fR. See \fBzpool\fR(1M) for more information on pool properties.
+The following command lists all active file systems and volumes in the system.
+Snapshots are displayed if the \fBlistsnaps\fR property is \fBon\fR. The
+default is \fBoff\fR. See \fBzpool\fR(8) for more information on pool
+properties.
 
 .sp
 .in +2
@@ -2716,12 +3132,11 @@ pool/home/bob  devices               on                     default
 pool/home/bob  exec                  on                     default
 pool/home/bob  setuid                on                     default
 pool/home/bob  readonly              off                    default
-pool/home/bob  zoned                 off                    default
+pool/home/bob  jailed                off                    default
 pool/home/bob  snapdir               hidden                 default
 pool/home/bob  aclmode               discard                default
 pool/home/bob  aclinherit            restricted             default
 pool/home/bob  canmount              on                     default
-pool/home/bob  shareiscsi            off                    default
 pool/home/bob  xattr                 on                     default
 pool/home/bob  copies                1                      default
 pool/home/bob  version               4                      -
@@ -2758,7 +3173,8 @@ on
 
 .sp
 .LP
-The following command lists all properties with local settings for \fBpool/home/bob\fR.
+The following command lists all properties with local settings for
+\fBpool/home/bob\fR.
 
 .sp
 .in +2
@@ -2775,7 +3191,8 @@ pool/home/bob  compression           on
 \fBExample 8 \fRRolling Back a ZFS File System
 .sp
 .LP
-The following command reverts the contents of \fBpool/home/anne\fR to the snapshot named \fByesterday\fR, deleting all intermediate snapshots.
+The following command reverts the contents of \fBpool/home/anne\fR to the
+snapshot named \fByesterday\fR, deleting all intermediate snapshots.
 
 .sp
 .in +2
@@ -2789,7 +3206,8 @@ The following command reverts the contents of \fBpool/home/anne\fR to the snapsh
 \fBExample 9 \fRCreating a ZFS Clone
 .sp
 .LP
-The following command creates a writable file system whose initial contents are the same as \fBpool/home/bob@yesterday\fR.
+The following command creates a writable file system whose initial contents are
+the same as \fBpool/home/bob@yesterday\fR.
 
 .sp
 .in +2
@@ -2803,7 +3221,9 @@ The following command creates a writable file system whose initial contents are
 \fBExample 10 \fRPromoting a ZFS Clone
 .sp
 .LP
-The following commands illustrate how to test out changes to a file system, and then replace the original file system with the changed one, using clones, clone promotion, and renaming:
+The following commands illustrate how to test out changes to a file system, and
+then replace the original file system with the changed one, using clones, clone
+promotion, and renaming:
 
 .sp
 .in +2
@@ -2826,7 +3246,8 @@ once the legacy version is no longer needed, it can be destroyed
 \fBExample 11 \fRInheriting ZFS Properties
 .sp
 .LP
-The following command causes \fBpool/home/bob\fR and \fBpool/home/anne\fR to inherit the \fBchecksum\fR property from their parent.
+The following command causes \fBpool/home/bob\fR and \fBpool/home/anne\fR to
+inherit the \fBchecksum\fR property from their parent.
 
 .sp
 .in +2
@@ -2840,7 +3261,11 @@ The following command causes \fBpool/home/bob\fR and \fBpool/home/anne\fR to inh
 \fBExample 12 \fRRemotely Replicating ZFS Data
 .sp
 .LP
-The following commands send a full stream and then an incremental stream to a remote machine, restoring them into \fBpoolB/received/fs@a\fRand \fBpoolB/received/fs@b\fR, respectively. \fBpoolB\fR must contain the file system \fBpoolB/received\fR, and must not initially contain \fBpoolB/received/fs\fR.
+The following commands send a full stream and then an incremental stream to a
+remote machine, restoring them into \fBpoolB/received/fs@a\fRand
+\fBpoolB/received/fs@b\fR, respectively. \fBpoolB\fR must contain the file
+system \fBpoolB/received\fR, and must not initially contain
+\fBpoolB/received/fs\fR.
 
 .sp
 .in +2
@@ -2857,7 +3282,12 @@ The following commands send a full stream and then an incremental stream to a re
 \fBExample 13 \fRUsing the \fBzfs receive\fR \fB-d\fR Option
 .sp
 .LP
-The following command sends a full stream of \fBpoolA/fsA/fsB@snap\fR to a remote machine, receiving it into \fBpoolB/received/fsA/fsB@snap\fR. The \fBfsA/fsB@snap\fR portion of the received snapshot's name is determined from the name of the sent snapshot. \fBpoolB\fR must contain the file system \fBpoolB/received\fR. If \fBpoolB/received/fsA\fR does not exist, it is created as an empty file system.
+The following command sends a full stream of \fBpoolA/fsA/fsB@snap\fR to a
+remote machine, receiving it into \fBpoolB/received/fsA/fsB@snap\fR. The
+\fBfsA/fsB@snap\fR portion of the received snapshot's name is determined from
+the name of the sent snapshot. \fBpoolB\fR must contain the file system
+\fBpoolB/received\fR. If \fBpoolB/received/fsA\fR does not exist, it is created
+as an empty file system.
 
 .sp
 .in +2
@@ -2872,7 +3302,8 @@ The following command sends a full stream of \fBpoolA/fsA/fsB@snap\fR to a remot
 \fBExample 14 \fRSetting User Properties
 .sp
 .LP
-The following example sets the user-defined \fBcom.example:department\fR property for a dataset.
+The following example sets the user-defined \fBcom.example:department\fR
+property for a dataset.
 
 .sp
 .in +2
@@ -2883,33 +3314,13 @@ The following example sets the user-defined \fBcom.example:department\fR propert
 .sp
 
 .LP
-\fBExample 15 \fRCreating a ZFS Volume as an iSCSI Target Device
-.sp
-.LP
-The following example shows how to create a \fBZFS\fR volume as an \fBiSCSI\fR target. 
-
-.sp
-.in +2
-.nf
-# \fBzfs create -V 2g pool/volumes/vol1\fR
-# \fBzfs set shareiscsi=on pool/volumes/vol1\fR
-# \fBiscsitadm list target\fR
-Target: pool/volumes/vol1
- iSCSI Name: 
- iqn.1986-03.com.sun:02:7b4b02a6-3277-eb1b-e686-a24762c52a8c
- Connections: 0
-.fi
-.in -2
-.sp
-
-.sp
-.LP
-After the \fBiSCSI\fR target is created, set up the \fBiSCSI\fR initiator. For more information about the Solaris \fBiSCSI\fR initiator, see \fBiscsitadm\fR(1M).
-.LP
 \fBExample 16 \fRPerforming a Rolling Snapshot
 .sp
 .LP
-The following example shows how to maintain a history of snapshots with a consistent naming scheme. To keep a week's worth of snapshots, the user destroys the oldest snapshot, renames the remaining snapshots, and then creates a new snapshot, as follows:
+The following example shows how to maintain a history of snapshots with a
+consistent naming scheme. To keep a week's worth of snapshots, the user
+destroys the oldest snapshot, renames the remaining snapshots, and then creates
+a new snapshot, as follows:
 
 .sp
 .in +2
@@ -2931,7 +3342,9 @@ The following example shows how to maintain a history of snapshots with a consis
 \fBExample 17 \fRSetting \fBsharenfs\fR Property Options on a ZFS File System
 .sp
 .LP
-The following commands show how to set \fBsharenfs\fR property options to enable \fBrw\fR access for a set of \fBIP\fR addresses and to enable root access for system \fBneo\fR on the \fBtank/home\fR file system.
+The following commands show how to set \fBsharenfs\fR property options to
+enable \fBrw\fR access for a set of \fBIP\fR addresses and to enable root
+access for system \fBneo\fR on the \fBtank/home\fR file system.
 
 .sp
 .in +2
@@ -2943,13 +3356,16 @@ The following commands show how to set \fBsharenfs\fR property options to enable
 
 .sp
 .LP
-If you are using \fBDNS\fR for host name resolution, specify the fully qualified hostname.
+If you are using \fBDNS\fR for host name resolution, specify the fully
+qualified hostname.
 
 .LP
 \fBExample 18 \fRDelegating ZFS Administration Permissions on a ZFS Dataset
 .sp
 .LP
-The following example shows how to set permissions so that user \fBcindys\fR can create, destroy, mount, and take snapshots on \fBtank/cindys\fR. The permissions on \fBtank/cindys\fR are also displayed.
+The following example shows how to set permissions so that user \fBcindys\fR
+can create, destroy, mount, and take snapshots on \fBtank/cindys\fR. The
+permissions on \fBtank/cindys\fR are also displayed.
 
 .sp
 .in +2
@@ -2966,7 +3382,9 @@ Local+Descendent permissions on (tank/cindys)
 
 .sp
 .LP
-Because the \fBtank/cindys\fR mount point permission is set to 755 by default, user \fBcindys\fR will be unable to mount file systems under \fBtank/cindys\fR. Set an \fBACL\fR similar to the following syntax to provide mount point access:
+Because the \fBtank/cindys\fR mount point permission is set to 755 by default,
+user \fBcindys\fR will be unable to mount file systems under \fBtank/cindys\fR.
+Set an \fBACL\fR similar to the following syntax to provide mount point access:
 .sp
 .in +2
 .nf
@@ -2979,7 +3397,10 @@ Because the \fBtank/cindys\fR mount point permission is set to 755 by default, u
 \fBExample 19 \fRDelegating Create Time Permissions on a ZFS Dataset
 .sp
 .LP
-The following example shows how to grant anyone in the group \fBstaff\fR to create file systems in \fBtank/users\fR. This syntax also allows staff members to destroy their own file systems, but not destroy anyone else's file system. The permissions on \fBtank/users\fR are also displayed.
+The following example shows how to grant anyone in the group \fBstaff\fR to
+create file systems in \fBtank/users\fR. This syntax also allows staff members
+to destroy their own file systems, but not destroy anyone else's file system.
+The permissions on \fBtank/users\fR are also displayed.
 
 .sp
 .in +2
@@ -2992,7 +3413,7 @@ Create time permissions on (tank/users)
           create,destroy
 Local+Descendent permissions on (tank/users)
           group staff create,mount
-------------------------------------------------------------- 
+-------------------------------------------------------------
 .fi
 .in -2
 .sp
@@ -3001,7 +3422,9 @@ Local+Descendent permissions on (tank/users)
 \fBExample 20 \fRDefining and Granting a Permission Set on a ZFS Dataset
 .sp
 .LP
-The following example shows how to define and grant a permission set on the \fBtank/users\fR file system. The permissions on \fBtank/users\fR are also displayed.
+The following example shows how to define and grant a permission set on the
+\fBtank/users\fR file system. The permissions on \fBtank/users\fR are also
+displayed.
 
 .sp
 .in +2
@@ -3025,7 +3448,9 @@ Local+Descendent permissions on (tank/users)
 \fBExample 21 \fRDelegating Property Permissions on a ZFS Dataset
 .sp
 .LP
-The following example shows to grant the ability to set quotas and reservations on the \fBusers/home\fR file system. The permissions on \fBusers/home\fR are also displayed.
+The following example shows to grant the ability to set quotas and reservations
+on the \fBusers/home\fR file system. The permissions on \fBusers/home\fR are
+also displayed.
 
 .sp
 .in +2
@@ -3039,7 +3464,7 @@ Local+Descendent permissions on (users/home)
 cindys% \fBzfs set quota=10G users/home/marks\fR
 cindys% \fBzfs get quota users/home/marks\fR
 NAME              PROPERTY  VALUE             SOURCE
-users/home/marks  quota     10G               local 
+users/home/marks  quota     10G               local
 .fi
 .in -2
 .sp
@@ -3048,7 +3473,9 @@ users/home/marks  quota     10G               local
 \fBExample 22 \fRRemoving ZFS Delegated Permissions on a ZFS Dataset
 .sp
 .LP
-The following example shows how to remove the snapshot permission from the \fBstaff\fR group on the \fBtank/users\fR file system. The permissions on \fBtank/users\fR are also displayed.
+The following example shows how to remove the snapshot permission from the
+\fBstaff\fR group on the \fBtank/users\fR file system. The permissions on
+\fBtank/users\fR are also displayed.
 
 .sp
 .in +2
@@ -3062,7 +3489,7 @@ Create time permissions on (tank/users)
         create,destroy
 Local+Descendent permissions on (tank/users)
         group staff @pset,create,mount
-------------------------------------------------------------- 
+-------------------------------------------------------------
 .fi
 .in -2
 .sp
@@ -3073,29 +3500,26 @@ Local+Descendent permissions on (tank/users)
 The following exit values are returned:
 .sp
 .ne 2
-.mk
 .na
 \fB\fB0\fR\fR
 .ad
 .sp .6
 .RS 4n
-Successful completion. 
+Successful completion.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB1\fR\fR
 .ad
 .sp .6
 .RS 4n
 An error occurred.
-.RE
 
+.RE
 .sp
 .ne 2
-.mk
 .na
 \fB\fB2\fR\fR
 .ad
@@ -3104,32 +3528,7 @@ An error occurred.
 Invalid command line options were specified.
 .RE
 
-.SH ATTRIBUTES
-.sp
-.LP
-See \fBattributes\fR(5) for descriptions of the following attributes:
-.sp
-
-.sp
-.TS
-tab() box;
-cw(2.75i) |cw(2.75i) 
-lw(2.75i) |lw(2.75i) 
-.
-ATTRIBUTE TYPEATTRIBUTE VALUE
-_
-AvailabilitySUNWzfsu
-_
-Interface StabilityCommitted
-.TE
-
 .SH SEE ALSO
 .sp
 .LP
-\fBssh\fR(1), \fBiscsitadm\fR(1M), \fBmount\fR(1M), \fBshare\fR(1M), \fBsharemgr\fR(1M), \fBunshare\fR(1M), \fBzonecfg\fR(1M), \fBzpool\fR(1M), \fBchmod\fR(2), \fBstat\fR(2), \fBwrite\fR(2), \fBfsync\fR(3C), \fBdfstab\fR(4), \fBattributes\fR(5)
-.sp
-.LP
-See the \fBgzip\fR(1) man page, which is not part of the SunOS man page collection.
-.sp
-.LP
-For information about using the \fBZFS\fR web-based management tool and other \fBZFS\fR features, see the \fISolaris ZFS Administration Guide\fR.
+chmod(2), fsync(2), exports(5), fstab(5), jail(8), mount(8), zpool(8)
diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool.8 b/cddl/contrib/opensolaris/cmd/zpool/zpool.8
index c700b7f..8c41f42 100644
--- a/cddl/contrib/opensolaris/cmd/zpool/zpool.8
+++ b/cddl/contrib/opensolaris/cmd/zpool/zpool.8
@@ -3,7 +3,9 @@
 .\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License"). You may not use this file except in compliance with the License. You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.
 .\" See the License for the specific language governing permissions and limitations under the License. When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE. If applicable, add the following below this CDDL HEADER, with the
 .\" fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
-.TH zpool 1M "21 Sep 2009" "SunOS 5.11" "System Administration Commands"
+.\" Portions Copyright 2011 Justin T. Gibbs <gibbs@FreeBSD.org>
+.\" Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
+.TH ZPOOL 8 "September 21, 2009" FreeBSD
 .SH NAME
 zpool \- configures ZFS storage pools
 .SH SYNOPSIS
@@ -29,7 +31,7 @@ zpool \- configures ZFS storage pools
 
 .LP
 .nf
-\fBzpool create\fR [\fB-fn\fR] [\fB-o\fR \fIproperty=value\fR] ... [\fB-O\fR \fIfile-system-property=value\fR] 
+\fBzpool create\fR [\fB-fn\fR] [\fB-o\fR \fIproperty=value\fR] ... [\fB-O\fR \fIfile-system-property=value\fR]
      ... [\fB-m\fR \fImountpoint\fR] [\fB-R\fR \fIroot\fR] \fIpool\fR \fIvdev\fR ...
 .fi
 
@@ -65,7 +67,7 @@ zpool \- configures ZFS storage pools
 
 .LP
 .nf
-\fBzpool import\fR [\fB-o \fImntopts\fR\fR] [\fB-o\fR \fIproperty=value\fR] ... [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR] 
+\fBzpool import\fR [\fB-o \fImntopts\fR\fR] [\fB-o\fR \fIproperty=value\fR] ... [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR]
      [\fB-D\fR] [\fB-f\fR] [\fB-R\fR \fIroot\fR] \fB-a\fR
 .fi
 
@@ -127,7 +129,7 @@ zpool \- configures ZFS storage pools
 
 .LP
 .nf
-\fBzpool upgrade\fR 
+\fBzpool upgrade\fR
 .fi
 
 .LP
@@ -143,50 +145,58 @@ zpool \- configures ZFS storage pools
 .SH DESCRIPTION
 .sp
 .LP
-The \fBzpool\fR command configures \fBZFS\fR storage pools. A storage pool is a collection of devices that provides physical storage and data replication for \fBZFS\fR datasets.
+The \fBzpool\fR command configures \fBZFS\fR storage pools. A storage pool is a
+collection of devices that provides physical storage and data replication for
+\fBZFS\fR datasets.
 .sp
 .LP
-All datasets within a storage pool share the same space. See \fBzfs\fR(1M) for information on managing datasets.
+All datasets within a storage pool share the same space. See \fBzfs\fR(8) for
+information on managing datasets.
 .SS "Virtual Devices (\fBvdev\fRs)"
 .sp
 .LP
-A "virtual device" describes a single device or a collection of devices organized according to certain performance and fault characteristics. The following virtual devices are supported:
+A "virtual device" describes a single device or a collection of devices
+organized according to certain performance and fault characteristics. The
+following virtual devices are supported:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBdisk\fR\fR
 .ad
 .RS 10n
-.rt  
-A block device, typically located under \fB/dev/dsk\fR. \fBZFS\fR can use individual slices or partitions, though the recommended mode of operation is to use whole disks. A disk can be specified by a full path, or it can be a shorthand name (the relative portion of the path under "/dev/dsk"). A whole disk can be specified by omitting the slice or partition designation. For example, "c0t0d0" is equivalent to "/dev/dsk/c0t0d0s2". When given a whole disk, \fBZFS\fR automatically labels the disk, if necessary.
+A block device, typically located under \fB/dev\fR. \fBZFS\fR can use
+individual slices, partitions or \fBgeom(4)\fR objects, though the recommended
+mode of operation is to use whole disks. A disk can be specified by a full
+path, or it can be a shorthand name (the relative portion of the path under
+"/dev"). For example, "da0" is equivalent to "/dev/da0".
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBfile\fR\fR
 .ad
 .RS 10n
-.rt  
-A regular file. The use of files as a backing store is strongly discouraged. It is designed primarily for experimental purposes, as the fault tolerance of a file is only as good as the file system of which it is a part. A file must be specified by a full path.
+A regular file. The use of files as a backing store is strongly discouraged. It
+is designed primarily for experimental purposes, as the fault tolerance of a
+file is only as good as the file system of which it is a part. A file must be
+specified by a full path.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBmirror\fR\fR
 .ad
 .RS 10n
-.rt  
-A mirror of two or more devices. Data is replicated in an identical fashion across all components of a mirror. A mirror with \fIN\fR disks of size \fIX\fR can hold \fIX\fR bytes and can withstand (\fIN-1\fR) devices failing before data integrity is compromised.
+A mirror of two or more devices. Data is replicated in an identical fashion
+across all components of a mirror. A mirror with \fIN\fR disks of size \fIX\fR
+can hold \fIX\fR bytes and can withstand (\fIN-1\fR) devices failing before
+data integrity is compromised.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBraidz\fR\fR
 .ad
@@ -203,60 +213,80 @@ A mirror of two or more devices. Data is replicated in an identical fashion acro
 \fB\fBraidz3\fR\fR
 .ad
 .RS 10n
-.rt  
-A variation on \fBRAID-5\fR that allows for better distribution of parity and eliminates the "\fBRAID-5\fR write hole" (in which data and parity become inconsistent after a power loss). Data and parity is striped across all disks within a \fBraidz\fR group.
+A variation on \fBRAID-5\fR that allows for better distribution of parity and
+eliminates the "\fBRAID-5\fR write hole" (in which data and parity become
+inconsistent after a power loss). Data and parity is striped across all disks
+within a \fBraidz\fR group.
 .sp
-A \fBraidz\fR group can have single-, double- , or triple parity, meaning that the \fBraidz\fR group can sustain one, two, or three failures, respectively, without losing any data. The \fBraidz1\fR \fBvdev\fR type specifies a single-parity \fBraidz\fR group; the \fBraidz2\fR \fBvdev\fR type specifies a double-parity \fBraidz\fR group; and the \fBraidz3\fR \fBvdev\fR type specifies a triple-parity \fBraidz\fR group. The \fBraidz\fR \fBvdev\fR type is an alias for \fBraidz1\fR.
+A \fBraidz\fR group can have single-, double- , or triple parity, meaning that
+the \fBraidz\fR group can sustain one, two, or three failures, respectively,
+without losing any data. The \fBraidz1\fR \fBvdev\fR type specifies a
+single-parity \fBraidz\fR group; the \fBraidz2\fR \fBvdev\fR type specifies a
+double-parity \fBraidz\fR group; and the \fBraidz3\fR \fBvdev\fR type specifies
+a triple-parity \fBraidz\fR group. The \fBraidz\fR \fBvdev\fR type is an alias
+for \fBraidz1\fR.
 .sp
-A \fBraidz\fR group with \fIN\fR disks of size \fIX\fR with \fIP\fR parity disks can hold approximately (\fIN-P\fR)*\fIX\fR bytes and can withstand \fIP\fR device(s) failing before data integrity is compromised. The minimum number of devices in a \fBraidz\fR group is one more than the number of parity disks. The recommended number is between 3 and 9 to help increase performance.
+A \fBraidz\fR group with \fIN\fR disks of size \fIX\fR with \fIP\fR parity
+disks can hold approximately (\fIN-P\fR)*\fIX\fR bytes and can withstand
+\fIP\fR device(s) failing before data integrity is compromised. The minimum
+number of devices in a \fBraidz\fR group is one more than the number of parity
+disks. The recommended number is between 3 and 9 to help increase performance.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBspare\fR\fR
 .ad
 .RS 10n
-.rt  
-A special pseudo-\fBvdev\fR which keeps track of available hot spares for a pool. For more information, see the "Hot Spares" section.
+A special pseudo-\fBvdev\fR which keeps track of available hot spares for a
+pool. For more information, see the "Hot Spares" section.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBlog\fR\fR
 .ad
 .RS 10n
-.rt  
-A separate-intent log device. If more than one log device is specified, then writes are load-balanced between devices. Log devices can be mirrored. However, \fBraidz\fR \fBvdev\fR types are not supported for the intent log. For more information, see the "Intent Log" section.
+A separate-intent log device. If more than one log device is specified, then
+writes are load-balanced between devices. Log devices can be mirrored. However,
+\fBraidz\fR \fBvdev\fR types are not supported for the intent log. For more
+information, see the "Intent Log" section.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcache\fR\fR
 .ad
 .RS 10n
-.rt  
-A device used to cache storage pool data. A cache device cannot be cannot be configured as a mirror or \fBraidz\fR group. For more information, see the "Cache Devices" section.
+A device used to cache storage pool data. A cache device cannot be cannot be
+configured as a mirror or \fBraidz\fR group. For more information, see the
+"Cache Devices" section.
 .RE
 
 .sp
 .LP
-Virtual devices cannot be nested, so a mirror or \fBraidz\fR virtual device can only contain files or disks. Mirrors of mirrors (or other combinations) are not allowed.
+Virtual devices cannot be nested, so a mirror or \fBraidz\fR virtual device can
+only contain files or disks. Mirrors of mirrors (or other combinations) are not
+allowed.
 .sp
 .LP
-A pool can have any number of virtual devices at the top of the configuration (known as "root vdevs"). Data is dynamically distributed across all top-level devices to balance data among devices. As new virtual devices are added, \fBZFS\fR automatically places data on the newly available devices.
+A pool can have any number of virtual devices at the top of the configuration
+(known as "root vdevs"). Data is dynamically distributed across all top-level
+devices to balance data among devices. As new virtual devices are added,
+\fBZFS\fR automatically places data on the newly available devices.
 .sp
 .LP
-Virtual devices are specified one at a time on the command line, separated by whitespace. The keywords "mirror" and "raidz" are used to distinguish where a group ends and another begins. For example, the following creates two root vdevs, each a mirror of two disks:
+Virtual devices are specified one at a time on the command line, separated by
+whitespace. The keywords "mirror" and "raidz" are used to distinguish where a
+group ends and another begins. For example, the following creates two root
+vdevs, each a mirror of two disks:
 .sp
 .in +2
 .nf
-# \fBzpool create mypool mirror c0t0d0 c0t1d0 mirror c1t0d0 c1t1d0\fR
+# \fBzpool create mypool mirror da0 da1 mirror da2 da3\fR
 .fi
 .in -2
 .sp
@@ -264,362 +294,439 @@ Virtual devices are specified one at a time on the command line, separated by wh
 .SS "Device Failure and Recovery"
 .sp
 .LP
-\fBZFS\fR supports a rich set of mechanisms for handling device failure and data corruption. All metadata and data is checksummed, and \fBZFS\fR automatically repairs bad data from a good copy when corruption is detected.
+\fBZFS\fR supports a rich set of mechanisms for handling device failure and
+data corruption. All metadata and data is checksummed, and \fBZFS\fR
+automatically repairs bad data from a good copy when corruption is detected.
 .sp
 .LP
-In order to take advantage of these features, a pool must make use of some form of redundancy, using either mirrored or \fBraidz\fR groups. While \fBZFS\fR supports running in a non-redundant configuration, where each root vdev is simply a disk or file, this is strongly discouraged. A single case of bit corruption can render some or all of your data unavailable.
+In order to take advantage of these features, a pool must make use of some form
+of redundancy, using either mirrored or \fBraidz\fR groups. While \fBZFS\fR
+supports running in a non-redundant configuration, where each root vdev is
+simply a disk or file, this is strongly discouraged. A single case of bit
+corruption can render some or all of your data unavailable.
 .sp
 .LP
-A pool's health status is described by one of three states: online, degraded, or faulted. An online pool has all devices operating normally. A degraded pool is one in which one or more devices have failed, but the data is still available due to a redundant configuration. A faulted pool has corrupted metadata, or one or more faulted devices, and insufficient replicas to continue functioning. 
+A pool's health status is described by one of three states: online, degraded,
+or faulted. An online pool has all devices operating normally. A degraded pool
+is one in which one or more devices have failed, but the data is still
+available due to a redundant configuration. A faulted pool has corrupted
+metadata, or one or more faulted devices, and insufficient replicas to continue
+functioning.
 .sp
 .LP
-The health of the top-level vdev, such as mirror or \fBraidz\fR device, is potentially impacted by the state of its associated vdevs, or component devices. A top-level vdev or component device is in one of the following states:
+The health of the top-level vdev, such as mirror or \fBraidz\fR device, is
+potentially impacted by the state of its associated vdevs, or component
+devices. A top-level vdev or component device is in one of the following
+states:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBDEGRADED\fR\fR
 .ad
 .RS 12n
-.rt  
-One or more top-level vdevs is in the degraded state because one or more component devices are offline. Sufficient replicas exist to continue functioning.
+One or more top-level vdevs is in the degraded state because one or more
+component devices are offline. Sufficient replicas exist to continue
+functioning.
 .sp
-One or more component devices is in the degraded or faulted state, but sufficient replicas exist to continue functioning. The underlying conditions are as follows:
+One or more component devices is in the degraded or faulted state, but
+sufficient replicas exist to continue functioning. The underlying conditions
+are as follows:
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The number of checksum errors exceeds acceptable levels and the device is degraded as an indication that something may be wrong. \fBZFS\fR continues to use the device as necessary.
+The number of checksum errors exceeds acceptable levels and the device is
+degraded as an indication that something may be wrong. \fBZFS\fR continues to
+use the device as necessary.
 .RE
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The number of I/O errors exceeds acceptable levels. The device could not be marked as faulted because there are insufficient replicas to continue functioning.
+The number of I/O errors exceeds acceptable levels. The device could not be
+marked as faulted because there are insufficient replicas to continue
+functioning.
 .RE
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBFAULTED\fR\fR
 .ad
 .RS 12n
-.rt  
-One or more top-level vdevs is in the faulted state because one or more component devices are offline. Insufficient replicas exist to continue functioning. 
+One or more top-level vdevs is in the faulted state because one or more
+component devices are offline. Insufficient replicas exist to continue
+functioning.
 .sp
-One or more component devices is in the faulted state, and insufficient replicas exist to continue functioning. The underlying conditions are as follows:
+One or more component devices is in the faulted state, and insufficient
+replicas exist to continue functioning. The underlying conditions are as
+follows:
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The device could be opened, but the contents did not match expected values. 
+The device could be opened, but the contents did not match expected values.
 .RE
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The number of I/O errors exceeds acceptable levels and the device is faulted to prevent further use of the device.
+The number of I/O errors exceeds acceptable levels and the device is faulted to
+prevent further use of the device.
 .RE
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBOFFLINE\fR\fR
 .ad
 .RS 12n
-.rt  
 The device was explicitly taken offline by the "\fBzpool offline\fR" command.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBONLINE\fR\fR
 .ad
 .RS 12n
-.rt  
 The device is online and functioning.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBREMOVED\fR\fR
 .ad
 .RS 12n
-.rt  
-The device was physically removed while the system was running. Device removal detection is hardware-dependent and may not be supported on all platforms.
+The device was physically removed while the system was running. Device removal
+detection is hardware-dependent and may not be supported on all platforms.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBUNAVAIL\fR\fR
 .ad
 .RS 12n
-.rt  
-The device could not be opened. If a pool is imported when a device was unavailable, then the device will be identified by a unique identifier instead of its path since the path was never correct in the first place.
+The device could not be opened. If a pool is imported when a device was
+unavailable, then the device will be identified by a unique identifier instead
+of its path since the path was never correct in the first place.
 .RE
 
 .sp
 .LP
-If a device is removed and later re-attached to the system, \fBZFS\fR attempts to put the device online automatically. Device attach detection is hardware-dependent and might not be supported on all platforms.
+If a device is removed and later re-attached to the system, \fBZFS\fR attempts
+to put the device online automatically. Device attach detection is
+hardware-dependent and might not be supported on all platforms.
 .SS "Hot Spares"
 .sp
 .LP
-\fBZFS\fR allows devices to be associated with pools as "hot spares". These devices are not actively used in the pool, but when an active device fails, it is automatically replaced by a hot spare. To create a pool with hot spares, specify a "spare" \fBvdev\fR with any number of devices. For example, 
+\fBZFS\fR allows devices to be associated with pools as "hot spares". These
+devices are not actively used in the pool, but when an active device fails, it
+is automatically replaced by a hot spare. To create a pool with hot spares,
+specify a "spare" \fBvdev\fR with any number of devices. For example,
 .sp
 .in +2
 .nf
-# zpool create pool mirror c0d0 c1d0 spare c2d0 c3d0
+# zpool create pool mirror da0 da1 spare da2 da3
 .fi
 .in -2
 .sp
 
 .sp
 .LP
-Spares can be shared across multiple pools, and can be added with the "\fBzpool add\fR" command and removed with the "\fBzpool remove\fR" command. Once a spare replacement is initiated, a new "spare" \fBvdev\fR is created within the configuration that will remain there until the original device is replaced. At this point, the hot spare becomes available again if another device fails.
+Spares can be shared across multiple pools, and can be added with the "\fBzpool
+add\fR" command and removed with the "\fBzpool remove\fR" command. Once a spare
+replacement is initiated, a new "spare" \fBvdev\fR is created within the
+configuration that will remain there until the original device is replaced. At
+this point, the hot spare becomes available again if another device fails.
 .sp
 .LP
-If a pool has a shared spare that is currently being used, the pool can not be exported since other pools may use this shared spare, which may lead to potential data corruption.
+If a pool has a shared spare that is currently being used, the pool can not be
+exported since other pools may use this shared spare, which may lead to
+potential data corruption.
 .sp
 .LP
-An in-progress spare replacement can be cancelled by detaching the hot spare. If the original faulted device is detached, then the hot spare assumes its place in the configuration, and is removed from the spare list of all active pools.
+An in-progress spare replacement can be cancelled by detaching the hot spare.
+If the original faulted device is detached, then the hot spare assumes its
+place in the configuration, and is removed from the spare list of all active
+pools.
 .sp
 .LP
 Spares cannot replace log devices.
 .SS "Intent Log"
 .sp
 .LP
-The \fBZFS\fR Intent Log (\fBZIL\fR) satisfies \fBPOSIX\fR requirements for synchronous transactions. For instance, databases often require their transactions to be on stable storage devices when returning from a system call. \fBNFS\fR and other applications can also use \fBfsync\fR() to ensure data stability. By default, the intent log is allocated from blocks within the main pool. However, it might be possible to get better performance using separate intent log devices such as \fBNVRAM\fR or a dedicated disk. For example:
+The \fBZFS\fR Intent Log (\fBZIL\fR) satisfies \fBPOSIX\fR requirements for
+synchronous transactions. For instance, databases often require their
+transactions to be on stable storage devices when returning from a system call.
+\fBNFS\fR and other applications can also use \fBfsync\fR() to ensure data
+stability. By default, the intent log is allocated from blocks within the main
+pool. However, it might be possible to get better performance using separate
+intent log devices such as \fBNVRAM\fR or a dedicated disk. For example:
 .sp
 .in +2
 .nf
-\fB# zpool create pool c0d0 c1d0 log c2d0\fR
+\fB# zpool create pool da0 da1 log da3\fR
 .fi
 .in -2
 .sp
 
 .sp
 .LP
-Multiple log devices can also be specified, and they can be mirrored. See the EXAMPLES section for an example of mirroring multiple log devices.
+Multiple log devices can also be specified, and they can be mirrored. See the
+EXAMPLES section for an example of mirroring multiple log devices.
 .sp
 .LP
-Log devices can be added, replaced, attached, detached, and imported and exported as part of the larger pool. Mirrored log devices can be removed by specifying the top-level mirror for the log.
+Log devices can be added, replaced, attached, detached, and imported and
+exported as part of the larger pool. Mirrored log devices can be removed by
+specifying the top-level mirror for the log.
 .SS "Cache Devices"
 .sp
 .LP
-Devices can be added to a storage pool as "cache devices." These devices provide an additional layer of caching between main memory and disk. For read-heavy workloads, where the working set size is much larger than what can be cached in main memory, using cache devices allow much more of this working set to be served from low latency media. Using cache devices provides the greatest performance improvement for random read-workloads of mostly static content.
+Devices can be added to a storage pool as "cache devices." These devices
+provide an additional layer of caching between main memory and disk. For
+read-heavy workloads, where the working set size is much larger than what can
+be cached in main memory, using cache devices allow much more of this working
+set to be served from low latency media. Using cache devices provides the
+greatest performance improvement for random read-workloads of mostly static
+content.
 .sp
 .LP
-To create a pool with cache devices, specify a "cache" \fBvdev\fR with any number of devices. For example:
+To create a pool with cache devices, specify a "cache" \fBvdev\fR with any
+number of devices. For example:
 .sp
 .in +2
 .nf
-\fB# zpool create pool c0d0 c1d0 cache c2d0 c3d0\fR
+\fB# zpool create pool da0 da1 cache da2 da3\fR
 .fi
 .in -2
 .sp
 
 .sp
 .LP
-Cache devices cannot be mirrored or part of a \fBraidz\fR configuration. If a read error is encountered on a cache device, that read \fBI/O\fR is reissued to the original storage pool device, which might be part of a mirrored or \fBraidz\fR configuration.
+Cache devices cannot be mirrored or part of a \fBraidz\fR configuration. If a
+read error is encountered on a cache device, that read \fBI/O\fR is reissued to
+the original storage pool device, which might be part of a mirrored or
+\fBraidz\fR configuration.
 .sp
 .LP
-The content of the cache devices is considered volatile, as is the case with other system caches.
+The content of the cache devices is considered volatile, as is the case with
+other system caches.
 .SS "Properties"
 .sp
 .LP
-Each pool has several properties associated with it. Some properties are read-only statistics while others are configurable and change the behavior of the pool. The following are read-only properties:
+Each pool has several properties associated with it. Some properties are
+read-only statistics while others are configurable and change the behavior of
+the pool. The following are read-only properties:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBavailable\fR\fR
 .ad
 .RS 20n
-.rt  
-Amount of storage available within the pool. This property can also be referred to by its shortened column name, "avail".
+Amount of storage available within the pool. This property can also be referred
+to by its shortened column name, "avail".
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcapacity\fR\fR
 .ad
 .RS 20n
-.rt  
-Percentage of pool space used. This property can also be referred to by its shortened column name, "cap".
+Percentage of pool space used. This property can also be referred to by its
+shortened column name, "cap".
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBhealth\fR\fR
 .ad
 .RS 20n
-.rt  
-The current health of the pool. Health can be "\fBONLINE\fR", "\fBDEGRADED\fR", "\fBFAULTED\fR", " \fBOFFLINE\fR", "\fBREMOVED\fR", or "\fBUNAVAIL\fR".
+The current health of the pool. Health can be "\fBONLINE\fR", "\fBDEGRADED\fR",
+"\fBFAULTED\fR", " \fBOFFLINE\fR", "\fBREMOVED\fR", or "\fBUNAVAIL\fR".
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBguid\fR\fR
 .ad
 .RS 20n
-.rt  
 A unique identifier for the pool.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBsize\fR\fR
 .ad
 .RS 20n
-.rt  
 Total size of the storage pool.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBused\fR\fR
 .ad
 .RS 20n
-.rt  
 Amount of storage space used within the pool.
 .RE
 
 .sp
 .LP
-These space usage properties report actual physical space available to the storage pool. The physical space can be different from the total amount of space that any contained datasets can actually use. The amount of space used in a \fBraidz\fR configuration depends on the characteristics of the data being written. In addition, \fBZFS\fR reserves some space for internal accounting that the \fBzfs\fR(1M) command takes into account, but the \fBzpool\fR command does not. For non-full pools of a reasonable size, these effects should be invisible. For small pools, or pools that are close to being completely full, these discrepancies may become more noticeable.
+These space usage properties report actual physical space available to the
+storage pool. The physical space can be different from the total amount of
+space that any contained datasets can actually use. The amount of space used in
+a \fBraidz\fR configuration depends on the characteristics of the data being
+written. In addition, \fBZFS\fR reserves some space for internal accounting
+that the \fBzfs\fR(8) command takes into account, but the \fBzpool\fR command
+does not. For non-full pools of a reasonable size, these effects should be
+invisible. For small pools, or pools that are close to being completely full,
+these discrepancies may become more noticeable.
 .sp
 .LP
 The following property can be set at creation time and import time:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBaltroot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Alternate root directory. If set, this directory is prepended to any mount points within the pool. This can be used when examining an unknown pool where the mount points cannot be trusted, or in an alternate boot environment, where the typical paths are not valid. \fBaltroot\fR is not a persistent property. It is valid only while the system is up. Setting \fBaltroot\fR defaults to using \fBcachefile\fR=none, though this may be overridden	 using an explicit setting.
+Alternate root directory. If set, this directory is prepended to any mount
+points within the pool. This can be used when examining an unknown pool where
+the mount points cannot be trusted, or in an alternate boot environment, where
+the typical paths are not valid. \fBaltroot\fR is not a persistent property. It
+is valid only while the system is up. Setting \fBaltroot\fR defaults to using
+\fBcachefile\fR=none, though this may be overridden using an explicit setting.
 .RE
 
 .sp
 .LP
-The following properties can be set at creation time and import time, and later changed with the \fBzpool set\fR command:
+The following properties can be set at creation time and import time, and later
+changed with the \fBzpool set\fR command:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBautoexpand\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls automatic pool expansion when the underlying LUN is grown. If set to \fBon\fR, the pool will be resized according to the size of the expanded device. If the device is part of a mirror or \fBraidz\fR then all devices within that mirror/\fBraidz\fR group must be expanded before the new space is made available to the pool. The default behavior is \fBoff\fR. This property can also be referred to by its shortened column name, \fBexpand\fR.
+Controls automatic pool expansion when the underlying LUN is grown. If set to
+\fBon\fR, the pool will be resized according to the size of the expanded
+device. If the device is part of a mirror or \fBraidz\fR then all devices
+within that mirror/\fBraidz\fR group must be expanded before the new space is
+made available to the pool. The default behavior is \fBoff\fR. This property
+can also be referred to by its shortened column name, \fBexpand\fR.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBautoreplace\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls automatic device replacement. If set to "\fBoff\fR", device replacement must be initiated by the administrator by using the "\fBzpool replace\fR" command. If set to "\fBon\fR", any new device, found in the same physical location as a device that previously belonged to the pool, is automatically formatted and replaced. The default behavior is "\fBoff\fR". This property can also be referred to by its shortened column name, "replace".
+Controls automatic device replacement. If set to "\fBoff\fR", device
+replacement must be initiated by the administrator by using the "\fBzpool
+replace\fR" command. If set to "\fBon\fR", any new device, found in the same
+physical location as a device that previously belonged to the pool, is
+automatically formatted and replaced. The default behavior is "\fBoff\fR". This
+property can also be referred to by its shortened column name, "replace".
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBbootfs\fR=\fIpool\fR/\fIdataset\fR\fR
 .ad
 .sp .6
 .RS 4n
-Identifies the default bootable dataset for the root pool. This property is expected to be set mainly by the installation and upgrade programs.
+Identifies the default bootable dataset for the root pool. This property is
+expected to be set mainly by the installation and upgrade programs.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcachefile\fR=\fIpath\fR | \fBnone\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls the location of where the pool configuration is cached. Discovering all pools on system startup requires a cached copy of the configuration data that is stored on the root file system. All pools in this cache are automatically imported when the system boots. Some environments, such as install and clustering, need to cache this information in a different location so that pools are not automatically imported. Setting this property caches the pool configuration in a different location that can later be imported with "\fBzpool import -c\fR". Setting it to the special value "\fBnone\fR" creates a temporary pool that is never cached, and the special value \fB\&''\fR (empty string) uses the default location. 
+Controls the location of where the pool configuration is cached. Discovering
+all pools on system startup requires a cached copy of the configuration data
+that is stored on the root file system. All pools in this cache are
+automatically imported when the system boots. Some environments, such as
+install and clustering, need to cache this information in a different location
+so that pools are not automatically imported. Setting this property caches the
+pool configuration in a different location that can later be imported with
+"\fBzpool import -c\fR". Setting it to the special value "\fBnone\fR" creates a
+temporary pool that is never cached, and the special value \fB\&''\fR (empty
+string) uses the default location.
 .sp
-Multiple pools can share the same cache file. Because the kernel destroys and recreates this file when pools are added and removed, care should be taken when attempting to access this file. When the last pool using a \fBcachefile\fR is exported or destroyed, the file is removed.
+Multiple pools can share the same cache file. Because the kernel destroys and
+recreates this file when pools are added and removed, care should be taken when
+attempting to access this file. When the last pool using a \fBcachefile\fR is
+exported or destroyed, the file is removed.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBdelegation\fR=\fBon\fR | \fBoff\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether a non-privileged user is granted access based on the dataset permissions defined on the dataset. See \fBzfs\fR(1M) for more information on \fBZFS\fR delegated administration.
+Controls whether a non-privileged user is granted access based on the dataset
+permissions defined on the dataset. See \fBzfs\fR(8) for more information on
+\fBZFS\fR delegated administration.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBfailmode\fR=\fBwait\fR | \fBcontinue\fR | \fBpanic\fR\fR
 .ad
 .sp .6
 .RS 4n
-Controls the system behavior in the event of catastrophic pool failure. This condition is typically a result of a loss of connectivity to the underlying storage device(s) or a failure of all devices within the pool. The behavior of such an event is determined as follows:
+Controls the system behavior in the event of catastrophic pool failure. This
+condition is typically a result of a loss of connectivity to the underlying
+storage device(s) or a failure of all devices within the pool. The behavior of
+such an event is determined as follows:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBwait\fR\fR
 .ad
 .RS 12n
-.rt  
-Blocks all \fBI/O\fR access until the device connectivity is recovered and the errors are cleared. This is the default behavior.
+Blocks all \fBI/O\fR access until the device connectivity is recovered and the
+errors are cleared. This is the default behavior.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcontinue\fR\fR
 .ad
 .RS 12n
-.rt  
-Returns \fBEIO\fR to any new write \fBI/O\fR requests but allows reads to any of the remaining healthy devices. Any write requests that have yet to be committed to disk would be blocked.
+Returns \fBEIO\fR to any new write \fBI/O\fR requests but allows reads to any
+of the remaining healthy devices. Any write requests that have yet to be
+committed to disk would be blocked.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBpanic\fR\fR
 .ad
 .RS 12n
-.rt  
 Prints out a message to the console and generates a system crash dump.
 .RE
 
@@ -627,36 +734,42 @@ Prints out a message to the console and generates a system crash dump.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBlistsnaps\fR=on | off\fR
 .ad
 .sp .6
 .RS 4n
-Controls whether information about snapshots associated with this pool is output when "\fBzfs list\fR" is run without the \fB-t\fR option. The default value is "off".
+Controls whether information about snapshots associated with this pool is
+output when "\fBzfs list\fR" is run without the \fB-t\fR option. The default
+value is "off".
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBversion\fR=\fIversion\fR\fR
 .ad
 .sp .6
 .RS 4n
-The current on-disk version of the pool. This can be increased, but never decreased. The preferred method of updating pools is with the "\fBzpool upgrade\fR" command, though this property can be used when a specific version is needed for backwards compatibility. This property can be any number between 1 and the current version reported by "\fBzpool upgrade -v\fR".
+The current on-disk version of the pool. This can be increased, but never
+decreased. The preferred method of updating pools is with the "\fBzpool
+upgrade\fR" command, though this property can be used when a specific version
+is needed for backwards compatibility. This property can be any number between
+1 and the current version reported by "\fBzpool upgrade -v\fR".
 .RE
 
 .SS "Subcommands"
 .sp
 .LP
-All subcommands that modify state are logged persistently to the pool in their original form.
+All subcommands that modify state are logged persistently to the pool in their
+original form.
 .sp
 .LP
-The \fBzpool\fR command provides subcommands to create and destroy storage pools, add capacity to storage pools, and provide information about the storage pools. The following subcommands are supported:
+The \fBzpool\fR command provides subcommands to create and destroy storage
+pools, add capacity to storage pools, and provide information about the storage
+pools. The following subcommands are supported:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool\fR \fB-?\fR\fR
 .ad
@@ -667,122 +780,147 @@ Displays a help message.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool add\fR [\fB-fn\fR] \fIpool\fR \fIvdev\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Adds the specified virtual devices to the given pool. The \fIvdev\fR specification is described in the "Virtual Devices" section. The behavior of the \fB-f\fR option, and the device checks performed are described in the "zpool create" subcommand.
+Adds the specified virtual devices to the given pool. The \fIvdev\fR
+specification is described in the "Virtual Devices" section. The behavior of
+the \fB-f\fR option, and the device checks performed are described in the
+"zpool create" subcommand.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .RS 6n
-.rt  
-Forces use of \fBvdev\fRs, even if they appear in use or specify a conflicting replication level. Not all devices can be overridden in this manner.
+Forces use of \fBvdev\fRs, even if they appear in use or specify a conflicting
+replication level. Not all devices can be overridden in this manner.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-n\fR\fR
 .ad
 .RS 6n
-.rt  
-Displays the configuration that would be used without actually adding the \fBvdev\fRs. The actual pool creation can still fail due to insufficient privileges or device sharing.
+Displays the configuration that would be used without actually adding the
+\fBvdev\fRs. The actual pool creation can still fail due to insufficient
+privileges or device sharing.
 .RE
 
-Do not add a disk that is currently configured as a quorum device to a zpool. After a disk is in the pool, that disk can then be configured as a quorum device.
+Do not add a disk that is currently configured as a quorum device to a zpool.
+After a disk is in the pool, that disk can then be configured as a quorum
+device.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool attach\fR [\fB-f\fR] \fIpool\fR \fIdevice\fR \fInew_device\fR\fR
 .ad
 .sp .6
 .RS 4n
-Attaches \fInew_device\fR to an existing \fBzpool\fR device. The existing device cannot be part of a \fBraidz\fR configuration. If \fIdevice\fR is not currently part of a mirrored configuration, \fIdevice\fR automatically transforms into a two-way mirror of \fIdevice\fR and \fInew_device\fR. If \fIdevice\fR is part of a two-way mirror, attaching \fInew_device\fR creates a three-way mirror, and so on. In either case, \fInew_device\fR begins to resilver immediately.
+Attaches \fInew_device\fR to an existing \fBzpool\fR device. The existing
+device cannot be part of a \fBraidz\fR configuration. If \fIdevice\fR is not
+currently part of a mirrored configuration, \fIdevice\fR automatically
+transforms into a two-way mirror of \fIdevice\fR and \fInew_device\fR. If
+\fIdevice\fR is part of a two-way mirror, attaching \fInew_device\fR creates a
+three-way mirror, and so on. In either case, \fInew_device\fR begins to
+resilver immediately.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .RS 6n
-.rt  
-Forces use of \fInew_device\fR, even if its appears to be in use. Not all devices can be overridden in this manner.
+Forces use of \fInew_device\fR, even if its appears to be in use. Not all
+devices can be overridden in this manner.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool clear\fR \fIpool\fR [\fIdevice\fR] ...\fR
 .ad
 .sp .6
 .RS 4n
-Clears device errors in a pool. If no arguments are specified, all device errors within the pool are cleared. If one or more devices is specified, only those errors associated with the specified device or devices are cleared.
+Clears device errors in a pool. If no arguments are specified, all device
+errors within the pool are cleared. If one or more devices is specified, only
+those errors associated with the specified device or devices are cleared.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzpool create\fR [\fB-fn\fR] [\fB-o\fR \fIproperty=value\fR] ... [\fB-O\fR \fIfile-system-property=value\fR] ... [\fB-m\fR \fImountpoint\fR] [\fB-R\fR \fIroot\fR] \fIpool\fR \fIvdev\fR ...\fR
+\fB\fBzpool create\fR [\fB-fn\fR] [\fB-o\fR \fIproperty=value\fR] ... [\fB-O\fR
+\fIfile-system-property=value\fR] ... [\fB-m\fR \fImountpoint\fR] [\fB-R\fR
+\fIroot\fR] \fIpool\fR \fIvdev\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Creates a new storage pool containing the virtual devices specified on the command line. The pool name must begin with a letter, and can only contain alphanumeric characters as well as underscore ("_"), dash ("-"), and period ("."). The pool names "mirror", "raidz", "spare" and "log" are reserved, as are names beginning with the pattern "c[0-9]". The \fBvdev\fR specification is described in the "Virtual Devices" section.
+Creates a new storage pool containing the virtual devices specified on the
+command line. The pool name must begin with a letter, and can only contain
+alphanumeric characters as well as underscore ("_"), dash ("-"), and period
+("."). The pool names "mirror", "raidz", "spare" and "log" are reserved, as are
+names beginning with the pattern "c[0-9]". The \fBvdev\fR specification is
+described in the "Virtual Devices" section.
 .sp
-The command verifies that each device specified is accessible and not currently in use by another subsystem. There are some uses, such as being currently mounted, or specified as the dedicated dump device, that prevents a device from ever being used by \fBZFS\fR. Other uses, such as having a preexisting \fBUFS\fR file system, can be overridden with the \fB-f\fR option.
+The command verifies that each device specified is accessible and not currently
+in use by another subsystem. There are some uses, such as being currently
+mounted, or specified as the dedicated dump device, that prevents a device from
+ever being used by \fBZFS\fR. Other uses, such as having a preexisting
+\fBUFS\fR file system, can be overridden with the \fB-f\fR option.
 .sp
-The command also checks that the replication strategy for the pool is consistent. An attempt to combine redundant and non-redundant storage in a single pool, or to mix disks and files, results in an error unless \fB-f\fR is specified. The use of differently sized devices within a single \fBraidz\fR or mirror group is also flagged as an error unless \fB-f\fR is specified.
+The command also checks that the replication strategy for the pool is
+consistent. An attempt to combine redundant and non-redundant storage in a
+single pool, or to mix disks and files, results in an error unless \fB-f\fR is
+specified. The use of differently sized devices within a single \fBraidz\fR or
+mirror group is also flagged as an error unless \fB-f\fR is specified.
 .sp
-Unless the \fB-R\fR option is specified, the default mount point is "/\fIpool\fR". The mount point must not exist or must be empty, or else the root dataset cannot be mounted. This can be overridden with the \fB-m\fR option.
+Unless the \fB-R\fR option is specified, the default mount point is
+"/\fIpool\fR". The mount point must not exist or must be empty, or else the
+root dataset cannot be mounted. This can be overridden with the \fB-m\fR
+option.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .sp .6
 .RS 4n
-Forces use of \fBvdev\fRs, even if they appear in use or specify a conflicting replication level. Not all devices can be overridden in this manner.
+Forces use of \fBvdev\fRs, even if they appear in use or specify a conflicting
+replication level. Not all devices can be overridden in this manner.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-n\fR\fR
 .ad
 .sp .6
 .RS 4n
-Displays the configuration that would be used without actually creating the pool. The actual pool creation can still fail due to insufficient privileges or device sharing.
+Displays the configuration that would be used without actually creating the
+pool. The actual pool creation can still fail due to insufficient privileges or
+device sharing.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty=value\fR [\fB-o\fR \fIproperty=value\fR] ...\fR
 .ad
 .sp .6
 .RS 4n
-Sets the given pool properties. See the "Properties" section for a list of valid properties that can be set.
+Sets the given pool properties. See the "Properties" section for a list of
+valid properties that can be set.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-O\fR \fIfile-system-property=value\fR\fR
 .ad
@@ -792,12 +930,13 @@ Sets the given pool properties. See the "Properties" section for a list of valid
 .ad
 .sp .6
 .RS 4n
-Sets the given file system properties in the root file system of the pool. See the "Properties" section of \fBzfs\fR(1M) for a list of valid properties that can be set.
+Sets the given file system properties in the root file system of the pool. See
+the "Properties" section of \fBzfs\fR(8) for a list of valid properties that
+can be set.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR \fIroot\fR\fR
 .ad
@@ -808,34 +947,34 @@ Equivalent to "-o cachefile=none,altroot=\fIroot\fR"
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-m\fR \fImountpoint\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets the mount point for the root dataset. The default mount point is "/\fIpool\fR" or "\fBaltroot\fR/\fIpool\fR" if \fBaltroot\fR is specified. The mount point must be an absolute path, "\fBlegacy\fR", or "\fBnone\fR". For more information on dataset mount points, see \fBzfs\fR(1M).
+Sets the mount point for the root dataset. The default mount point is
+"/\fIpool\fR" or "\fBaltroot\fR/\fIpool\fR" if \fBaltroot\fR is specified. The
+mount point must be an absolute path, "\fBlegacy\fR", or "\fBnone\fR". For more
+information on dataset mount points, see \fBzfs\fR(8).
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool destroy\fR [\fB-f\fR] \fIpool\fR\fR
 .ad
 .sp .6
 .RS 4n
-Destroys the given pool, freeing up any devices for other use. This command tries to unmount any active datasets before destroying the pool.
+Destroys the given pool, freeing up any devices for other use. This command
+tries to unmount any active datasets before destroying the pool.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .RS 6n
-.rt  
 Forces any active datasets contained within the pool to be unmounted.
 .RE
 
@@ -843,52 +982,58 @@ Forces any active datasets contained within the pool to be unmounted.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool detach\fR \fIpool\fR \fIdevice\fR\fR
 .ad
 .sp .6
 .RS 4n
-Detaches \fIdevice\fR from a mirror. The operation is refused if there are no other valid replicas of the data.
+Detaches \fIdevice\fR from a mirror. The operation is refused if there are no
+other valid replicas of the data.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool export\fR [\fB-f\fR] \fIpool\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Exports the given pools from the system. All devices are marked as exported, but are still considered in use by other subsystems. The devices can be moved between systems (even those of different endianness) and imported as long as a sufficient number of devices are present.
+Exports the given pools from the system. All devices are marked as exported,
+but are still considered in use by other subsystems. The devices can be moved
+between systems (even those of different endianness) and imported as long as a
+sufficient number of devices are present.
 .sp
-Before exporting the pool, all datasets within the pool are unmounted. A pool can not be exported if it has a shared spare that is currently being used.
+Before exporting the pool, all datasets within the pool are unmounted. A pool
+can not be exported if it has a shared spare that is currently being used.
 .sp
-For pools to be portable, you must give the \fBzpool\fR command whole disks, not just slices, so that \fBZFS\fR can label the disks with portable \fBEFI\fR labels. Otherwise, disk drivers on platforms of different endianness will not recognize the disks.
+For pools to be portable, you must give the \fBzpool\fR command whole disks,
+not just slices, so that \fBZFS\fR can label the disks with portable \fBEFI\fR
+labels. Otherwise, disk drivers on platforms of different endianness will not
+recognize the disks.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .RS 6n
-.rt  
 Forcefully unmount all datasets, using the "\fBunmount -f\fR" command.
 .sp
-This command will forcefully export the pool even if it has a shared spare that is currently being used. This may lead to potential data corruption.
+This command will forcefully export the pool even if it has a shared spare that
+is currently being used. This may lead to potential data corruption.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool get\fR "\fIall\fR" | \fIproperty\fR[,...] \fIpool\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Retrieves the given list of properties (or all properties if "\fBall\fR" is used) for the specified storage pool(s). These properties are displayed with the following fields:
+Retrieves the given list of properties (or all properties if "\fBall\fR" is
+used) for the specified storage pool(s). These properties are displayed with
+the following fields:
 .sp
 .in +2
 .nf
@@ -900,83 +1045,88 @@ Retrieves the given list of properties (or all properties if "\fBall\fR" is used
 .in -2
 .sp
 
-See the "Properties" section for more information on the available pool properties.
+See the "Properties" section for more information on the available pool
+properties.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool history\fR [\fB-il\fR] [\fIpool\fR] ...\fR
 .ad
 .sp .6
 .RS 4n
-Displays the command history of the specified pools or all pools if no pool is specified.
+Displays the command history of the specified pools or all pools if no pool is
+specified.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-i\fR\fR
 .ad
 .RS 6n
-.rt  
-Displays internally logged \fBZFS\fR events in addition to user initiated events.
+Displays internally logged \fBZFS\fR events in addition to user initiated
+events.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-l\fR\fR
 .ad
 .RS 6n
-.rt  
-Displays log records in long format, which in addition to standard format includes, the user name, the hostname, and the zone in which the operation was performed.
+Displays log records in long format, which in addition to standard format
+includes, the user name, the hostname, and the jail in which the operation was
+performed.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzpool import\fR [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR] [\fB-D\fR]\fR
+\fB\fBzpool import\fR [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR]
+[\fB-D\fR]\fR
 .ad
 .sp .6
 .RS 4n
-Lists pools available to import. If the \fB-d\fR option is not specified, this command searches for devices in "/dev/dsk". The \fB-d\fR option can be specified multiple times, and all directories are searched. If the device appears to be part of an exported pool, this command displays a summary of the pool with the name of the pool, a numeric identifier, as well as the \fIvdev\fR layout and current health of the device for each device or file. Destroyed pools, pools that were previously destroyed with the "\fBzpool destroy\fR" command, are not listed unless the \fB-D\fR option is specified. 
+Lists pools available to import. If the \fB-d\fR option is not specified, this
+command searches for devices in "/dev/dsk". The \fB-d\fR option can be
+specified multiple times, and all directories are searched. If the device
+appears to be part of an exported pool, this command displays a summary of the
+pool with the name of the pool, a numeric identifier, as well as the \fIvdev\fR
+layout and current health of the device for each device or file. Destroyed
+pools, pools that were previously destroyed with the "\fBzpool destroy\fR"
+command, are not listed unless the \fB-D\fR option is specified.
 .sp
-The numeric identifier is unique, and can be used instead of the pool name when multiple exported pools of the same name are available.
+The numeric identifier is unique, and can be used instead of the pool name when
+multiple exported pools of the same name are available.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-c\fR \fIcachefile\fR\fR
 .ad
 .RS 16n
-.rt  
-Reads configuration from the given \fBcachefile\fR that was created with the "\fBcachefile\fR" pool property. This \fBcachefile\fR is used instead of searching for devices.
+Reads configuration from the given \fBcachefile\fR that was created with the
+"\fBcachefile\fR" pool property. This \fBcachefile\fR is used instead of
+searching for devices.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR \fIdir\fR\fR
 .ad
 .RS 16n
-.rt  
-Searches for devices or files in \fIdir\fR. The \fB-d\fR option can be specified multiple times. 
+Searches for devices or files in \fIdir\fR. The \fB-d\fR option can be
+specified multiple times.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-D\fR\fR
 .ad
 .RS 16n
-.rt  
 Lists destroyed pools only.
 .RE
 
@@ -984,161 +1134,166 @@ Lists destroyed pools only.
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzpool import\fR [\fB-o\fR \fImntopts\fR] [ \fB-o\fR \fIproperty\fR=\fIvalue\fR] ... [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR] [\fB-D\fR] [\fB-f\fR] [\fB-R\fR \fIroot\fR] \fB-a\fR\fR
+\fB\fBzpool import\fR [\fB-o\fR \fImntopts\fR] [ \fB-o\fR
+\fIproperty\fR=\fIvalue\fR] ... [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR]
+[\fB-D\fR] [\fB-f\fR] [\fB-R\fR \fIroot\fR] \fB-a\fR\fR
 .ad
 .sp .6
 .RS 4n
-Imports all pools found in the search directories. Identical to the previous command, except that all pools with a sufficient number of devices available are imported. Destroyed pools, pools that were previously destroyed with the "\fBzpool destroy\fR" command, will not be imported unless the \fB-D\fR option is specified.
+Imports all pools found in the search directories. Identical to the previous
+command, except that all pools with a sufficient number of devices available
+are imported. Destroyed pools, pools that were previously destroyed with the
+"\fBzpool destroy\fR" command, will not be imported unless the \fB-D\fR option
+is specified.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fImntopts\fR\fR
 .ad
 .RS 21n
-.rt  
-Comma-separated list of mount options to use when mounting datasets within the pool. See \fBzfs\fR(1M) for a description of dataset properties and mount options.
+Comma-separated list of mount options to use when mounting datasets within the
+pool. See \fBzfs\fR(8) for a description of dataset properties and mount
+options.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty=value\fR\fR
 .ad
 .RS 21n
-.rt  
-Sets the specified property on the imported pool. See the "Properties" section for more information on the available pool properties.
+Sets the specified property on the imported pool. See the "Properties" section
+for more information on the available pool properties.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-c\fR \fIcachefile\fR\fR
 .ad
 .RS 21n
-.rt  
-Reads configuration from the given \fBcachefile\fR that was created with the "\fBcachefile\fR" pool property. This \fBcachefile\fR is used instead of searching for devices.
+Reads configuration from the given \fBcachefile\fR that was created with the
+"\fBcachefile\fR" pool property. This \fBcachefile\fR is used instead of
+searching for devices.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR \fIdir\fR\fR
 .ad
 .RS 21n
-.rt  
-Searches for devices or files in \fIdir\fR. The \fB-d\fR option can be specified multiple times. This option is incompatible with the \fB-c\fR option.
+Searches for devices or files in \fIdir\fR. The \fB-d\fR option can be
+specified multiple times. This option is incompatible with the \fB-c\fR option.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-D\fR\fR
 .ad
 .RS 21n
-.rt  
 Imports destroyed pools only. The \fB-f\fR option is also required.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .RS 21n
-.rt  
 Forces import, even if the pool appears to be potentially active.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
 .RS 21n
-.rt  
-Searches for and imports all pools found. 
+Searches for and imports all pools found.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR \fIroot\fR\fR
 .ad
 .RS 21n
-.rt  
-Sets the "\fBcachefile\fR" property to "\fBnone\fR" and the "\fIaltroot\fR" property to "\fIroot\fR".
+Sets the "\fBcachefile\fR" property to "\fBnone\fR" and the "\fIaltroot\fR"
+property to "\fIroot\fR".
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzpool import\fR [\fB-o\fR \fImntopts\fR] [ \fB-o\fR \fIproperty\fR=\fIvalue\fR] ... [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR] [\fB-D\fR] [\fB-f\fR] [\fB-R\fR \fIroot\fR] \fIpool\fR | \fIid\fR [\fInewpool\fR]\fR
+\fB\fBzpool import\fR [\fB-o\fR \fImntopts\fR] [ \fB-o\fR
+\fIproperty\fR=\fIvalue\fR] ... [\fB-d\fR \fIdir\fR | \fB-c\fR \fIcachefile\fR]
+[\fB-D\fR] [\fB-f\fR] [\fB-R\fR \fIroot\fR] \fIpool\fR | \fIid\fR
+[\fInewpool\fR]\fR
 .ad
 .sp .6
 .RS 4n
-Imports a specific pool. A pool can be identified by its name or the numeric identifier. If \fInewpool\fR is specified, the pool is imported using the name \fInewpool\fR. Otherwise, it is imported with the same name as its exported name.
+Imports a specific pool. A pool can be identified by its name or the numeric
+identifier. If \fInewpool\fR is specified, the pool is imported using the name
+\fInewpool\fR. Otherwise, it is imported with the same name as its exported
+name.
 .sp
-If a device is removed from a system without running "\fBzpool export\fR" first, the device appears as potentially active. It cannot be determined if this was a failed export, or whether the device is really in use from another host. To import a pool in this state, the \fB-f\fR option is required.
+If a device is removed from a system without running "\fBzpool export\fR"
+first, the device appears as potentially active. It cannot be determined if
+this was a failed export, or whether the device is really in use from another
+host. To import a pool in this state, the \fB-f\fR option is required.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fImntopts\fR\fR
 .ad
 .sp .6
 .RS 4n
-Comma-separated list of mount options to use when mounting datasets within the pool. See \fBzfs\fR(1M) for a description of dataset properties and mount options.
+Comma-separated list of mount options to use when mounting datasets within the
+pool. See \fBzfs\fR(8) for a description of dataset properties and mount
+options.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIproperty=value\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets the specified property on the imported pool. See the "Properties" section for more information on the available pool properties.
+Sets the specified property on the imported pool. See the "Properties" section
+for more information on the available pool properties.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-c\fR \fIcachefile\fR\fR
 .ad
 .sp .6
 .RS 4n
-Reads configuration from the given \fBcachefile\fR that was created with the "\fBcachefile\fR" pool property. This \fBcachefile\fR is used instead of searching for devices.
+Reads configuration from the given \fBcachefile\fR that was created with the
+"\fBcachefile\fR" pool property. This \fBcachefile\fR is used instead of
+searching for devices.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-d\fR \fIdir\fR\fR
 .ad
 .sp .6
 .RS 4n
-Searches for devices or files in \fIdir\fR. The \fB-d\fR option can be specified multiple times. This option is incompatible with the \fB-c\fR option.
+Searches for devices or files in \fIdir\fR. The \fB-d\fR option can be
+specified multiple times. This option is incompatible with the \fB-c\fR option.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-D\fR\fR
 .ad
@@ -1149,7 +1304,6 @@ Imports destroyed pool. The \fB-f\fR option is also required.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
@@ -1160,61 +1314,64 @@ Forces import, even if the pool appears to be potentially active.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-R\fR \fIroot\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets the "\fBcachefile\fR" property to "\fBnone\fR" and the "\fIaltroot\fR" property to "\fIroot\fR".
+Sets the "\fBcachefile\fR" property to "\fBnone\fR" and the "\fIaltroot\fR"
+property to "\fIroot\fR".
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzpool iostat\fR [\fB-T\fR \fBu\fR | \fBd\fR] [\fB-v\fR] [\fIpool\fR] ... [\fIinterval\fR[\fIcount\fR]]\fR
+\fB\fBzpool iostat\fR [\fB-T\fR \fBu\fR | \fBd\fR] [\fB-v\fR] [\fIpool\fR] ...
+[\fIinterval\fR[\fIcount\fR]]\fR
 .ad
 .sp .6
 .RS 4n
-Displays \fBI/O\fR statistics for the given pools. When given an interval, the statistics are printed every \fIinterval\fR seconds until \fBCtrl-C\fR is pressed. If no \fIpools\fR are specified, statistics for every pool in the system is shown. If \fIcount\fR is specified, the command exits after \fIcount\fR reports are printed.
+Displays \fBI/O\fR statistics for the given pools. When given an interval, the
+statistics are printed every \fIinterval\fR seconds until \fBCtrl-C\fR is
+pressed. If no \fIpools\fR are specified, statistics for every pool in the
+system is shown. If \fIcount\fR is specified, the command exits after
+\fIcount\fR reports are printed.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-T\fR \fBu\fR | \fBd\fR\fR
 .ad
 .RS 12n
-.rt  
 Display a time stamp.
 .sp
-Specify \fBu\fR for a printed representation of the internal representation of time. See \fBtime\fR(2). Specify \fBd\fR for standard date format. See \fBdate\fR(1).
+Specify \fBu\fR for a printed representation of the internal representation of
+time. See \fBtime\fR(2). Specify \fBd\fR for standard date format. See
+\fBdate\fR(1).
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-v\fR\fR
 .ad
 .RS 12n
-.rt  
-Verbose statistics. Reports usage statistics for individual \fIvdevs\fR within the pool, in addition to the pool-wide statistics.
+Verbose statistics. Reports usage statistics for individual \fIvdevs\fR within
+the pool, in addition to the pool-wide statistics.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool labelclear\fR [\fB-f\fR] \fIdevice\fR
 .ad
 .sp .6
 .RS 4n
-Removes ZFS label information from the specified device. The device must not be part of an active pool configuration.
+Removes ZFS label information from the specified device. The device must not
+be part of an active pool configuration.
 .sp
 .ne 2
 .mk
@@ -1236,58 +1393,56 @@ Treat exported or foreign devices as inactive.
 .ad
 .sp .6
 .RS 4n
-Lists the given pools along with a health status and space usage. When given no arguments, all pools in the system are listed.
+Lists the given pools along with a health status and space usage. When given no
+arguments, all pools in the system are listed.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-H\fR\fR
 .ad
 .RS 12n
-.rt  
-Scripted mode. Do not display headers, and separate fields by a single tab instead of arbitrary space.
+Scripted mode. Do not display headers, and separate fields by a single tab
+instead of arbitrary space.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-o\fR \fIprops\fR\fR
 .ad
 .RS 12n
-.rt  
-Comma-separated list of properties to display. See the "Properties" section for a list of valid properties. The default list is "name, size, used, available, capacity, health, altroot"
+Comma-separated list of properties to display. See the "Properties" section for
+a list of valid properties. The default list is "name, size, used, available,
+capacity, health, altroot"
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool offline\fR [\fB-t\fR] \fIpool\fR \fIdevice\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Takes the specified physical device offline. While the \fIdevice\fR is offline, no attempt is made to read or write to the device.
+Takes the specified physical device offline. While the \fIdevice\fR is offline,
+no attempt is made to read or write to the device.
 .sp
 This command is not applicable to spares or cache devices.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-t\fR\fR
 .ad
 .RS 6n
-.rt  
-Temporary. Upon reboot, the specified physical device reverts to its previous state.
+Temporary. Upon reboot, the specified physical device reverts to its previous
+state.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool online\fR [\fB-e\fR] \fIpool\fR \fIdevice\fR...\fR
 .ad
@@ -1298,75 +1453,94 @@ Brings the specified physical device online.
 This command is not applicable to spares or cache devices.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-e\fR\fR
 .ad
 .RS 6n
-.rt  
-Expand the device to use all available space. If the device is part of a mirror or \fBraidz\fR then all devices must be expanded before the new space will become available to the pool.
+Expand the device to use all available space. If the device is part of a mirror
+or \fBraidz\fR then all devices must be expanded before the new space will
+become available to the pool.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool remove\fR \fIpool\fR \fIdevice\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Removes the specified device from the pool. This command currently only supports removing hot spares, cache, and log devices. A mirrored log device can be removed by specifying the top-level mirror for the log. Non-log devices that are part of a mirrored configuration can be removed using the \fBzpool detach\fR command. Non-redundant and \fBraidz\fR devices cannot be removed from a pool.
+Removes the specified device from the pool. This command currently only
+supports removing hot spares, cache, and log devices. A mirrored log device can
+be removed by specifying the top-level mirror for the log. Non-log devices that
+are part of a mirrored configuration can be removed using the \fBzpool
+detach\fR command. Non-redundant and \fBraidz\fR devices cannot be removed from
+a pool.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fB\fBzpool replace\fR [\fB-f\fR] \fIpool\fR \fIold_device\fR [\fInew_device\fR]\fR
+\fB\fBzpool replace\fR [\fB-f\fR] \fIpool\fR \fIold_device\fR
+[\fInew_device\fR]\fR
 .ad
 .sp .6
 .RS 4n
-Replaces \fIold_device\fR with \fInew_device\fR. This is equivalent to attaching \fInew_device\fR, waiting for it to resilver, and then detaching \fIold_device\fR.
+Replaces \fIold_device\fR with \fInew_device\fR. This is equivalent to
+attaching \fInew_device\fR, waiting for it to resilver, and then detaching
+\fIold_device\fR.
 .sp
-The size of \fInew_device\fR must be greater than or equal to the minimum size of all the devices in a mirror or \fBraidz\fR configuration.
+The size of \fInew_device\fR must be greater than or equal to the minimum size
+of all the devices in a mirror or \fBraidz\fR configuration.
 .sp
-\fInew_device\fR is required if the pool is not redundant. If \fInew_device\fR is not specified, it defaults to \fIold_device\fR. This form of replacement is useful after an existing disk has failed and has been physically replaced. In this case, the new disk may have the same \fB/dev/dsk\fR path as the old device, even though it is actually a different disk. \fBZFS\fR recognizes this.
+\fInew_device\fR is required if the pool is not redundant. If \fInew_device\fR
+is not specified, it defaults to \fIold_device\fR. This form of replacement is
+useful after an existing disk has failed and has been physically replaced. In
+this case, the new disk may have the same \fB/dev/dsk\fR path as the old
+device, even though it is actually a different disk. \fBZFS\fR recognizes this.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-f\fR\fR
 .ad
 .RS 6n
-.rt  
-Forces use of \fInew_device\fR, even if its appears to be in use. Not all devices can be overridden in this manner.
+Forces use of \fInew_device\fR, even if its appears to be in use. Not all
+devices can be overridden in this manner.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool scrub\fR [\fB-s\fR] \fIpool\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Begins a scrub. The scrub examines all data in the specified pools to verify that it checksums correctly. For replicated (mirror or \fBraidz\fR) devices, \fBZFS\fR automatically repairs any damage discovered during the scrub. The "\fBzpool status\fR" command reports the progress of the scrub and summarizes the results of the scrub upon completion.
+Begins a scrub. The scrub examines all data in the specified pools to verify
+that it checksums correctly. For replicated (mirror or \fBraidz\fR) devices,
+\fBZFS\fR automatically repairs any damage discovered during the scrub. The
+"\fBzpool status\fR" command reports the progress of the scrub and summarizes
+the results of the scrub upon completion.
 .sp
-Scrubbing and resilvering are very similar operations. The difference is that resilvering only examines data that \fBZFS\fR knows to be out of date (for example, when attaching a new device to a mirror or replacing an existing device), whereas scrubbing examines all data to discover silent errors due to hardware faults or disk failure.
+Scrubbing and resilvering are very similar operations. The difference is that
+resilvering only examines data that \fBZFS\fR knows to be out of date (for
+example, when attaching a new device to a mirror or replacing an existing
+device), whereas scrubbing examines all data to discover silent errors due to
+hardware faults or disk failure.
 .sp
-Because scrubbing and resilvering are \fBI/O\fR-intensive operations, \fBZFS\fR only allows one at a time. If a scrub is already in progress, the "\fBzpool scrub\fR" command terminates it and starts a new scrub. If a resilver is in progress, \fBZFS\fR does not allow a scrub to be started until the resilver completes.
+Because scrubbing and resilvering are \fBI/O\fR-intensive operations, \fBZFS\fR
+only allows one at a time. If a scrub is already in progress, the "\fBzpool
+scrub\fR" command terminates it and starts a new scrub. If a resilver is in
+progress, \fBZFS\fR does not allow a scrub to be started until the resilver
+completes.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-s\fR\fR
 .ad
 .RS 6n
-.rt  
 Stop scrubbing.
 .RE
 
@@ -1374,101 +1548,108 @@ Stop scrubbing.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool set\fR \fIproperty\fR=\fIvalue\fR \fIpool\fR\fR
 .ad
 .sp .6
 .RS 4n
-Sets the given property on the specified pool. See the "Properties" section for more information on what properties can be set and acceptable values.
+Sets the given property on the specified pool. See the "Properties" section for
+more information on what properties can be set and acceptable values.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool status\fR [\fB-xv\fR] [\fIpool\fR] ...\fR
 .ad
 .sp .6
 .RS 4n
-Displays the detailed health status for the given pools. If no \fIpool\fR is specified, then the status of each pool in the system is displayed. For more information on pool and device health, see the "Device Failure and Recovery" section.
+Displays the detailed health status for the given pools. If no \fIpool\fR is
+specified, then the status of each pool in the system is displayed. For more
+information on pool and device health, see the "Device Failure and Recovery"
+section.
 .sp
-If a scrub or resilver is in progress, this command reports the percentage done and the estimated time to completion. Both of these are only approximate, because the amount of data in the pool and the other workloads on the system can change.
+If a scrub or resilver is in progress, this command reports the percentage done
+and the estimated time to completion. Both of these are only approximate,
+because the amount of data in the pool and the other workloads on the system
+can change.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-x\fR\fR
 .ad
 .RS 6n
-.rt  
-Only display status for pools that are exhibiting errors or are otherwise unavailable.
+Only display status for pools that are exhibiting errors or are otherwise
+unavailable.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-v\fR\fR
 .ad
 .RS 6n
-.rt  
-Displays verbose data error information, printing out a complete list of all data errors since the last complete pool scrub.
+Displays verbose data error information, printing out a complete list of all
+data errors since the last complete pool scrub.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool upgrade\fR\fR
 .ad
 .sp .6
 .RS 4n
-Displays all pools formatted using a different \fBZFS\fR on-disk version. Older versions can continue to be used, but some features may not be available. These pools can be upgraded using "\fBzpool upgrade -a\fR". Pools that are formatted with a more recent version are also displayed, although these pools will be inaccessible on the system.
+Displays all pools formatted using a different \fBZFS\fR on-disk version. Older
+versions can continue to be used, but some features may not be available. These
+pools can be upgraded using "\fBzpool upgrade -a\fR". Pools that are formatted
+with a more recent version are also displayed, although these pools will be
+inaccessible on the system.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool upgrade\fR \fB-v\fR\fR
 .ad
 .sp .6
 .RS 4n
-Displays \fBZFS\fR versions supported by the current software. The current \fBZFS\fR versions and all previous supported versions are displayed, along with an explanation of the features provided with each version.
+Displays \fBZFS\fR versions supported by the current software. The current
+\fBZFS\fR versions and all previous supported versions are displayed, along
+with an explanation of the features provided with each version.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBzpool upgrade\fR [\fB-V\fR \fIversion\fR] \fB-a\fR | \fIpool\fR ...\fR
 .ad
 .sp .6
 .RS 4n
-Upgrades the given pool to the latest on-disk version. Once this is done, the pool will no longer be accessible on systems running older versions of the software.
+Upgrades the given pool to the latest on-disk version. Once this is done, the
+pool will no longer be accessible on systems running older versions of the
+software.
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-a\fR\fR
 .ad
 .RS 14n
-.rt  
 Upgrades all pools.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-V\fR \fIversion\fR\fR
 .ad
 .RS 14n
-.rt  
-Upgrade to the specified version. If the \fB-V\fR flag is not specified, the pool is upgraded to the most recent version. This option can only be used to increase the version number, and only up to the most recent version supported by this software.
+Upgrade to the specified version. If the \fB-V\fR flag is not specified, the
+pool is upgraded to the most recent version. This option can only be used to
+increase the version number, and only up to the most recent version supported
+by this software.
 .RE
 
 .RE
@@ -1478,12 +1659,13 @@ Upgrade to the specified version. If the \fB-V\fR flag is not specified, the poo
 \fBExample 1 \fRCreating a RAID-Z Storage Pool
 .sp
 .LP
-The following command creates a pool with a single \fBraidz\fR root \fIvdev\fR that consists of six disks.
+The following command creates a pool with a single \fBraidz\fR root \fIvdev\fR
+that consists of six disks.
 
 .sp
 .in +2
 .nf
-# \fBzpool create tank raidz c0t0d0 c0t1d0 c0t2d0 c0t3d0 c0t4d0 c0t5d0\fR
+# \fBzpool create tank raidz da0 da1 da2 da3 da4 da5\fR
 .fi
 .in -2
 .sp
@@ -1492,12 +1674,13 @@ The following command creates a pool with a single \fBraidz\fR root \fIvdev\fR t
 \fBExample 2 \fRCreating a Mirrored Storage Pool
 .sp
 .LP
-The following command creates a pool with two mirrors, where each mirror contains two disks.
+The following command creates a pool with two mirrors, where each mirror
+contains two disks.
 
 .sp
 .in +2
 .nf
-# \fBzpool create tank mirror c0t0d0 c0t1d0 mirror c0t2d0 c0t3d0\fR
+# \fBzpool create tank mirror da0 da1 mirror da2 da3\fR
 .fi
 .in -2
 .sp
@@ -1511,7 +1694,7 @@ The following command creates an unmirrored pool using two disk slices.
 .sp
 .in +2
 .nf
-# \fBzpool create tank /dev/dsk/c0t0d0s1 c0t1d0s4\fR
+# \fBzpool create tank /dev/da0s1 da1s4\fR
 .fi
 .in -2
 .sp
@@ -1520,7 +1703,8 @@ The following command creates an unmirrored pool using two disk slices.
 \fBExample 4 \fRCreating a ZFS Storage Pool by Using Files
 .sp
 .LP
-The following command creates an unmirrored pool using files. While not recommended, a pool based on files can be useful for experimental purposes.
+The following command creates an unmirrored pool using files. While not
+recommended, a pool based on files can be useful for experimental purposes.
 
 .sp
 .in +2
@@ -1534,12 +1718,14 @@ The following command creates an unmirrored pool using files. While not recommen
 \fBExample 5 \fRAdding a Mirror to a ZFS Storage Pool
 .sp
 .LP
-The following command adds two mirrored disks to the pool "\fItank\fR", assuming the pool is already made up of two-way mirrors. The additional space is immediately available to any datasets within the pool.
+The following command adds two mirrored disks to the pool "\fItank\fR",
+assuming the pool is already made up of two-way mirrors. The additional space
+is immediately available to any datasets within the pool.
 
 .sp
 .in +2
 .nf
-# \fBzpool add tank mirror c1t0d0 c1t1d0\fR
+# \fBzpool add tank mirror da0 da1\fR
 .fi
 .in -2
 .sp
@@ -1548,7 +1734,8 @@ The following command adds two mirrored disks to the pool "\fItank\fR", assuming
 \fBExample 6 \fRListing Available ZFS Storage Pools
 .sp
 .LP
-The following command lists all available pools on the system. In this case, the pool \fIzion\fR is faulted due to a missing device.
+The following command lists all available pools on the system. In this case,
+the pool \fIzion\fR is faulted due to a missing device.
 
 .sp
 .LP
@@ -1570,7 +1757,8 @@ The results from this command are similar to the following:
 \fBExample 7 \fRDestroying a ZFS Storage Pool
 .sp
 .LP
-The following command destroys the pool "\fItank\fR" and any datasets contained within.
+The following command destroys the pool "\fItank\fR" and any datasets contained
+within.
 
 .sp
 .in +2
@@ -1584,7 +1772,8 @@ The following command destroys the pool "\fItank\fR" and any datasets contained
 \fBExample 8 \fRExporting a ZFS Storage Pool
 .sp
 .LP
-The following command exports the devices in pool \fItank\fR so that they can be relocated or later imported.
+The following command exports the devices in pool \fItank\fR so that they can
+be relocated or later imported.
 
 .sp
 .in +2
@@ -1598,7 +1787,8 @@ The following command exports the devices in pool \fItank\fR so that they can be
 \fBExample 9 \fRImporting a ZFS Storage Pool
 .sp
 .LP
-The following command displays available pools, and then imports the pool "tank" for use on the system.
+The following command displays available pools, and then imports the pool
+"tank" for use on the system.
 
 .sp
 .LP
@@ -1616,8 +1806,8 @@ config:
 
         tank        ONLINE
           mirror    ONLINE
-            c1t2d0  ONLINE
-            c1t3d0  ONLINE
+               da0  ONLINE
+               da1  ONLINE
 
 # \fBzpool import tank\fR
 .fi
@@ -1628,7 +1818,8 @@ config:
 \fBExample 10 \fRUpgrading All ZFS Storage Pools to the Current Version
 .sp
 .LP
-The following command upgrades all ZFS Storage pools to the current version of the software.
+The following command upgrades all ZFS Storage pools to the current version of
+the software.
 
 .sp
 .in +2
@@ -1648,31 +1839,34 @@ The following command creates a new pool with an available hot spare:
 .sp
 .in +2
 .nf
-# \fBzpool create tank mirror c0t0d0 c0t1d0 spare c0t2d0\fR
+# \fBzpool create tank mirror da0 da1 spare da2\fR
 .fi
 .in -2
 .sp
 
 .sp
 .LP
-If one of the disks were to fail, the pool would be reduced to the degraded state. The failed device can be replaced using the following command:
+If one of the disks were to fail, the pool would be reduced to the degraded
+state. The failed device can be replaced using the following command:
 
 .sp
 .in +2
 .nf
-# \fBzpool replace tank c0t0d0 c0t3d0\fR
+# \fBzpool replace tank da0 da2\fR
 .fi
 .in -2
 .sp
 
 .sp
 .LP
-Once the data has been resilvered, the spare is automatically removed and is made available should another device fails. The hot spare can be permanently removed from the pool using the following command:
+Once the data has been resilvered, the spare is automatically removed and is
+made available should another device fails. The hot spare can be permanently
+removed from the pool using the following command:
 
 .sp
 .in +2
 .nf
-# \fBzpool remove tank c0t2d0\fR
+# \fBzpool remove tank da3\fR
 .fi
 .in -2
 .sp
@@ -1681,13 +1875,14 @@ Once the data has been resilvered, the spare is automatically removed and is mad
 \fBExample 12 \fRCreating a ZFS Pool with Mirrored Separate Intent Logs
 .sp
 .LP
-The following command creates a ZFS storage pool consisting of two, two-way mirrors and mirrored log devices:
+The following command creates a ZFS storage pool consisting of two, two-way
+mirrors and mirrored log devices:
 
 .sp
 .in +2
 .nf
-# \fBzpool create pool mirror c0d0 c1d0 mirror c2d0 c3d0 log mirror \e
-   c4d0 c5d0\fR
+# \fBzpool create pool mirror da0 da1 mirror da2 da3 log mirror \e
+   da4 da5\fR
 .fi
 .in -2
 .sp
@@ -1696,7 +1891,8 @@ The following command creates a ZFS storage pool consisting of two, two-way mirr
 \fBExample 13 \fRAdding Cache Devices to a ZFS Pool
 .sp
 .LP
-The following command adds two disks for use as cache devices to a ZFS storage pool:
+The following command adds two disks for use as cache devices to a ZFS storage
+pool:
 
 .sp
 .in +2
@@ -1708,7 +1904,10 @@ The following command adds two disks for use as cache devices to a ZFS storage p
 
 .sp
 .LP
-Once added, the cache devices gradually fill with content from main memory. Depending on the size of your cache devices, it could take over an hour for them to fill. Capacity and reads can be monitored using the \fBiostat\fR option as follows: 
+Once added, the cache devices gradually fill with content from main memory.
+Depending on the size of your cache devices, it could take over an hour for
+them to fill. Capacity and reads can be monitored using the \fBiostat\fR option
+as follows:
 
 .sp
 .in +2
@@ -1739,15 +1938,15 @@ config:
          NAME        STATE     READ WRITE CKSUM
          tank        ONLINE       0     0     0
            mirror-0  ONLINE       0     0     0
-             c6t0d0  ONLINE       0     0     0
-             c6t1d0  ONLINE       0     0     0
+                da0  ONLINE       0     0     0
+                da1  ONLINE       0     0     0
            mirror-1  ONLINE       0     0     0
-             c6t2d0  ONLINE       0     0     0
-             c6t3d0  ONLINE       0     0     0
+                da2  ONLINE       0     0     0
+                da3  ONLINE       0     0     0
          logs
            mirror-2  ONLINE       0     0     0
-             c4t0d0  ONLINE       0     0     0
-             c4t1d0  ONLINE       0     0     0
+                da4  ONLINE       0     0     0
+                da5  ONLINE       0     0     0
 .fi
 .in -2
 .sp
@@ -1770,57 +1969,32 @@ The command to remove the mirrored log \fBmirror-2\fR is:
 The following exit values are returned:
 .sp
 .ne 2
-.mk
 .na
 \fB\fB0\fR\fR
 .ad
 .RS 5n
-.rt  
-Successful completion. 
+Successful completion.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB1\fR\fR
 .ad
 .RS 5n
-.rt  
 An error occurred.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB2\fR\fR
 .ad
 .RS 5n
-.rt  
 Invalid command line options were specified.
 .RE
 
-.SH ATTRIBUTES
-.sp
-.LP
-See \fBattributes\fR(5) for descriptions of the following attributes:
-.sp
-
-.sp
-.TS
-tab() box;
-cw(2.75i) |cw(2.75i) 
-lw(2.75i) |lw(2.75i) 
-.
-ATTRIBUTE TYPEATTRIBUTE VALUE
-_
-AvailabilitySUNWzfsu
-_
-Interface StabilityEvolving
-.TE
-
 .SH SEE ALSO
 .sp
 .LP
-\fBzfs\fR(1M), \fBattributes\fR(5)
+zfs(8)
diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
index 849f0e0..cb09d7b 100644
--- a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
+++ b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  */
 
 #include <solaris.h>
@@ -2206,10 +2207,15 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
 		return;
 
 	for (c = 0; c < children; c++) {
-		uint64_t ishole = B_FALSE;
+		uint64_t ishole = B_FALSE, islog = B_FALSE;
 
-		if (nvlist_lookup_uint64(newchild[c],
-		    ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
+		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
+		    &ishole);
+
+		(void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
+		    &islog);
+
+		if (ishole || islog)
 			continue;
 
 		vname = zpool_vdev_name(g_zfs, zhp, newchild[c], B_FALSE);
@@ -2219,6 +2225,31 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
 	}
 
 	/*
+	 * Log device section
+	 */
+
+	if (num_logs(newnv) > 0) {
+		(void) printf("%-*s      -      -      -      -      -      "
+		    "-\n", cb->cb_namewidth, "logs");
+
+		for (c = 0; c < children; c++) {
+			uint64_t islog = B_FALSE;
+			(void) nvlist_lookup_uint64(newchild[c],
+			    ZPOOL_CONFIG_IS_LOG, &islog);
+
+			if (islog) {
+				vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
+				    B_FALSE);
+				print_vdev_stats(zhp, vname, oldnv ?
+				    oldchild[c] : NULL, newchild[c],
+				    cb, depth + 2);
+				free(vname);
+			}
+		}
+
+	}
+
+	/*
 	 * Include level 2 ARC devices in iostat output
 	 */
 	if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
@@ -3565,10 +3596,16 @@ print_dedup_stats(nvlist_t *config)
 	 * table continue processing the stats.
 	 */
 	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
-	    (uint64_t **)&ddo, &c) != 0 || ddo->ddo_count == 0)
+	    (uint64_t **)&ddo, &c) != 0)
 		return;
 
 	(void) printf("\n");
+	(void) printf(gettext(" dedup: "));
+	if (ddo->ddo_count == 0) {
+		(void) printf(gettext("no DDT entries\n"));
+		return;
+	}
+
 	(void) printf("DDT entries %llu, size %llu on disk, %llu in core\n",
 	    (u_longlong_t)ddo->ddo_count,
 	    (u_longlong_t)ddo->ddo_dspace,
diff --git a/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.1 b/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.1
index 9e11948..6e77d39 100644
--- a/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.1
+++ b/cddl/contrib/opensolaris/cmd/zstreamdump/zstreamdump.1
@@ -3,7 +3,8 @@
 .\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License").  You may not use this file except in compliance with the License. You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.
 .\"  See the License for the specific language governing permissions and limitations under the License. When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE.  If applicable, add the following below this CDDL HEADER, with
 .\" the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
-.TH zstreamdump 1M "21 Sep 2009" "SunOS 5.11" "System Administration Commands"
+.\" Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
+.TH ZSTREAMDUMP 1 "Sep 21, 2009"
 .SH NAME
 zstreamdump \- filter data in zfs send stream
 .SH SYNOPSIS
@@ -15,14 +16,15 @@ zstreamdump \- filter data in zfs send stream
 .SH DESCRIPTION
 .sp
 .LP
-The \fBzstreamdump\fR utility reads from the output of the \fBzfs send\fR command, then displays headers and some statistics from that output.  See \fBzfs\fR(1M).
+The \fBzstreamdump\fR utility reads from the output of the \fBzfs send\fR
+command, then displays headers and some statistics from that output.  See
+\fBzfs\fR(1M).
 .SH OPTIONS
 .sp
 .LP
 The following options are supported:
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-C\fR\fR
 .ad
@@ -33,7 +35,6 @@ Suppress the validation of checksums.
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fB-v\fR\fR
 .ad
@@ -42,26 +43,7 @@ Suppress the validation of checksums.
 Verbose. Dump all headers, not only begin and end headers.
 .RE
 
-.SH ATTRIBUTES
-.sp
-.LP
-See \fBattributes\fR(5) for descriptions of the following attributes:
-.sp
-
-.sp
-.TS
-tab() box;
-cw(2.75i) |cw(2.75i) 
-lw(2.75i) |lw(2.75i) 
-.
-ATTRIBUTE TYPEATTRIBUTE VALUE
-_
-AvailabilitySUNWzfsu
-_
-Interface StabilityUncommitted
-.TE
-
 .SH SEE ALSO
 .sp
 .LP
-\fBzfs\fR(1M), \fBattributes\fR(5)
+zfs(8)
diff --git a/cddl/lib/libzfs/Makefile b/cddl/lib/libzfs/Makefile
index 2235a2e..6f6414a 100644
--- a/cddl/lib/libzfs/Makefile
+++ b/cddl/lib/libzfs/Makefile
@@ -6,8 +6,8 @@
 .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libzfs/common
 
 LIB=	zfs
-DPADD=	${LIBMD} ${LIBPTHREAD} ${LIBUMEM} ${LIBUTIL}
-LDADD=	-lmd -lpthread -lumem -lutil
+DPADD=	${LIBMD} ${LIBPTHREAD} ${LIBUMEM} ${LIBUTIL} ${LIBM}
+LDADD=	-lmd -lpthread -lumem -lutil -lm
 
 SRCS=	deviceid.c \
 	fsshare.c \
diff --git a/cddl/sbin/zfs/Makefile b/cddl/sbin/zfs/Makefile
index 11f6a0f..4d3c519 100644
--- a/cddl/sbin/zfs/Makefile
+++ b/cddl/sbin/zfs/Makefile
@@ -21,8 +21,8 @@ CFLAGS+= -I${.CURDIR}/../../../sys/cddl/contrib/opensolaris/uts/common/fs/zfs
 CFLAGS+= -I${.CURDIR}/../../../sys/cddl/contrib/opensolaris/uts/common/sys
 CFLAGS+= -I${.CURDIR}/../../../sys/cddl/contrib/opensolaris/common/zfs
 
-DPADD=	${LIBBSDXML} ${LIBGEOM} ${LIBM} ${LIBNVPAIR} ${LIBSBUF} ${LIBUMEM} \
+DPADD=	${LIBGEOM} ${LIBNVPAIR} ${LIBUMEM} \
 	${LIBUTIL} ${LIBUUTIL} ${LIBZFS}
-LDADD=	-lbsdxml -lgeom -lm -lnvpair -lsbuf -lumem -lutil -luutil -lzfs
+LDADD=	-lgeom -lnvpair -lumem -lutil -luutil -lzfs
 
 .include <bsd.prog.mk>
diff --git a/cddl/sbin/zpool/Makefile b/cddl/sbin/zpool/Makefile
index f810ee1..fba9ebf 100644
--- a/cddl/sbin/zpool/Makefile
+++ b/cddl/sbin/zpool/Makefile
@@ -26,8 +26,8 @@ CFLAGS+= -I${.CURDIR}/../../../sys/cddl/contrib/opensolaris/uts/common/sys
 CFLAGS+= -I${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libzpool/common
 CFLAGS+= -I${.CURDIR}/../../../cddl/contrib/opensolaris/cmd/stat/common
 
-DPADD=	${LIBAVL} ${LIBBSDXML} ${LIBGEOM} ${LIBM} ${LIBNVPAIR} ${LIBSBUF} \
+DPADD=	${LIBAVL} ${LIBGEOM} ${LIBNVPAIR} \
 	${LIBUMEM} ${LIBUTIL} ${LIBUUTIL} ${LIBZFS}
-LDADD=	-lavl -lbsdxml -lgeom -lm -lnvpair -lsbuf -lumem -lutil -luutil -lzfs
+LDADD=	-lavl -lgeom -lnvpair -lumem -lutil -luutil -lzfs
 
 .include <bsd.prog.mk>
diff --git a/cddl/usr.bin/zinject/Makefile b/cddl/usr.bin/zinject/Makefile
index dc6de45..2ea066c2 100644
--- a/cddl/usr.bin/zinject/Makefile
+++ b/cddl/usr.bin/zinject/Makefile
@@ -19,8 +19,8 @@ CFLAGS+= -I${.CURDIR}/../../../sys/cddl/contrib/opensolaris/uts/common
 CFLAGS+= -I${.CURDIR}/../../contrib/opensolaris/head
 CFLAGS+= -I${.CURDIR}/../../lib/libumem
 
-DPADD=	${LIBAVL} ${LIBGEOM} ${LIBM} ${LIBNVPAIR} ${LIBUMEM} ${LIBUUTIL} \
+DPADD=	${LIBGEOM} ${LIBM} ${LIBNVPAIR} ${LIBUMEM} ${LIBUUTIL} \
 	${LIBZFS} ${LIBZPOOL}
-LDADD=	-lavl -lgeom -lm -lnvpair -lumem -luutil -lzfs -lzpool
+LDADD=	-lgeom -lm -lnvpair -lumem -luutil -lzfs -lzpool
 
 .include <bsd.prog.mk>
diff --git a/cddl/usr.bin/ztest/Makefile b/cddl/usr.bin/ztest/Makefile
index 979880c..7988fce 100644
--- a/cddl/usr.bin/ztest/Makefile
+++ b/cddl/usr.bin/ztest/Makefile
@@ -18,8 +18,8 @@ CFLAGS+= -I${.CURDIR}/../../contrib/opensolaris/head
 CFLAGS+= -I${.CURDIR}/../../lib/libumem
 
 DPADD=	${LIBM} ${LIBNVPAIR} ${LIBUMEM} ${LIBZPOOL} \
-	${LIBPTHREAD} ${LIBZ} ${LIBAVL}
-LDADD=	-lm -lnvpair -lumem -lzpool -lpthread -lz -lavl
+	${LIBPTHREAD} ${LIBAVL}
+LDADD=	-lm -lnvpair -lumem -lzpool -lpthread -lavl
 
 CSTD=	c99
 
diff --git a/cddl/usr.sbin/zdb/Makefile b/cddl/usr.sbin/zdb/Makefile
index 446d1c4..b59d987 100644
--- a/cddl/usr.sbin/zdb/Makefile
+++ b/cddl/usr.sbin/zdb/Makefile
@@ -23,9 +23,8 @@ CFLAGS+= -I${.CURDIR}/../../../sys/cddl/contrib/opensolaris/common/zfs
 CFLAGS+= -I${.CURDIR}/../../../cddl/contrib/opensolaris/head
 CFLAGS+= -I${.CURDIR}/../../lib/libumem
 
-DPADD=	${LIBAVL} ${LIBGEOM} ${LIBM} ${LIBNVPAIR} ${LIBPTHREAD} ${LIBUMEM} \
-	${LIBUUTIL} ${LIBZ} ${LIBZFS} ${LIBZPOOL}
-LDADD=	-lavl -lgeom -lm -lnvpair -lpthread -lumem -luutil -lz -lzfs -lzpool
-
+DPADD=	${LIBGEOM} ${LIBM} ${LIBNVPAIR} ${LIBPTHREAD} ${LIBUMEM} \
+	${LIBUUTIL} ${LIBZFS} ${LIBZPOOL}
+LDADD=	-lgeom -lm -lnvpair -lpthread -lumem -luutil -lzfs -lzpool
 
 .include <bsd.prog.mk>
diff --git a/contrib/bind9/CHANGES b/contrib/bind9/CHANGES
index 3e48dd6..dc0b2c6 100644
--- a/contrib/bind9/CHANGES
+++ b/contrib/bind9/CHANGES
@@ -1,3 +1,9 @@
+	--- 9.8.1-P1 released ---
+
+3218.	[security]	Cache lookup could return RRSIG data associated with
+			nonexistent records, leading to an assertion
+			failure. [RT #26590]
+
 	--- 9.8.1 released ---
 
 	--- 9.8.1rc1 released ---
diff --git a/contrib/bind9/bin/named/query.c b/contrib/bind9/bin/named/query.c
index 9be178b..4945f47 100644
--- a/contrib/bind9/bin/named/query.c
+++ b/contrib/bind9/bin/named/query.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: query.c,v 1.353.8.11 2011-06-09 03:14:03 marka Exp $ */
+/* $Id: query.c,v 1.353.8.11.4.1 2011-11-16 09:32:08 marka Exp $ */
 
 /*! \file */
 
@@ -1393,11 +1393,9 @@ query_addadditional(void *arg, dns_name_t *name, dns_rdatatype_t qtype) {
 			goto addname;
 		if (result == DNS_R_NCACHENXRRSET) {
 			dns_rdataset_disassociate(rdataset);
-			/*
-			 * Negative cache entries don't have sigrdatasets.
-			 */
-			INSIST(sigrdataset == NULL ||
-			       ! dns_rdataset_isassociated(sigrdataset));
+			if (sigrdataset != NULL &&
+			    dns_rdataset_isassociated(sigrdataset))
+				dns_rdataset_disassociate(sigrdataset);
 		}
 		if (result == ISC_R_SUCCESS) {
 			mname = NULL;
@@ -1438,8 +1436,9 @@ query_addadditional(void *arg, dns_name_t *name, dns_rdatatype_t qtype) {
 			goto addname;
 		if (result == DNS_R_NCACHENXRRSET) {
 			dns_rdataset_disassociate(rdataset);
-			INSIST(sigrdataset == NULL ||
-			       ! dns_rdataset_isassociated(sigrdataset));
+			if (sigrdataset != NULL &&
+			    dns_rdataset_isassociated(sigrdataset))
+				dns_rdataset_disassociate(sigrdataset);
 		}
 		if (result == ISC_R_SUCCESS) {
 			mname = NULL;
@@ -1889,10 +1888,8 @@ query_addadditional2(void *arg, dns_name_t *name, dns_rdatatype_t qtype) {
 		goto setcache;
 	if (result == DNS_R_NCACHENXRRSET) {
 		dns_rdataset_disassociate(rdataset);
-		/*
-		 * Negative cache entries don't have sigrdatasets.
-		 */
-		INSIST(! dns_rdataset_isassociated(sigrdataset));
+		if (dns_rdataset_isassociated(sigrdataset))
+			dns_rdataset_disassociate(sigrdataset);
 	}
 	if (result == ISC_R_SUCCESS) {
 		/* Remember the result as a cache */
diff --git a/contrib/bind9/lib/dns/rbtdb.c b/contrib/bind9/lib/dns/rbtdb.c
index 9d66006..10525d9 100644
--- a/contrib/bind9/lib/dns/rbtdb.c
+++ b/contrib/bind9/lib/dns/rbtdb.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: rbtdb.c,v 1.310.8.5 2011-06-08 23:02:42 each Exp $ */
+/* $Id: rbtdb.c,v 1.310.8.5.4.1 2011-11-16 09:32:08 marka Exp $ */
 
 /*! \file */
 
@@ -5053,7 +5053,7 @@ cache_find(dns_db_t *db, dns_name_t *name, dns_dbversion_t *version,
 			      rdataset);
 		if (need_headerupdate(found, search.now))
 			update = found;
-		if (foundsig != NULL) {
+		if (!NEGATIVE(found) && foundsig != NULL) {
 			bind_rdataset(search.rbtdb, node, foundsig, search.now,
 				      sigrdataset);
 			if (need_headerupdate(foundsig, search.now))
@@ -5685,7 +5685,7 @@ cache_findrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
 	}
 	if (found != NULL) {
 		bind_rdataset(rbtdb, rbtnode, found, now, rdataset);
-		if (foundsig != NULL)
+		if (!NEGATIVE(found) && foundsig != NULL)
 			bind_rdataset(rbtdb, rbtnode, foundsig, now,
 				      sigrdataset);
 	}
diff --git a/contrib/bind9/version b/contrib/bind9/version
index ede0ba0..249750e 100644
--- a/contrib/bind9/version
+++ b/contrib/bind9/version
@@ -1,4 +1,4 @@
-# $Id: version,v 1.53.8.9 2011-08-24 02:08:26 marka Exp $
+# $Id: version,v 1.53.8.9.6.1 2011-11-16 09:32:07 marka Exp $
 # 
 # This file must follow /bin/sh rules.  It is imported directly via
 # configure.
@@ -6,5 +6,5 @@
 MAJORVER=9
 MINORVER=8
 PATCHVER=1
-RELEASETYPE=
-RELEASEVER=
+RELEASETYPE=-P
+RELEASEVER=1
diff --git a/contrib/binutils/ld/emulparams/elf64ppc_fbsd.sh b/contrib/binutils/ld/emulparams/elf64ppc_fbsd.sh
new file mode 100644
index 0000000..ec2374f
--- /dev/null
+++ b/contrib/binutils/ld/emulparams/elf64ppc_fbsd.sh
@@ -0,0 +1,3 @@
+. ${srcdir}/emulparams/elf64ppc.sh
+. ${srcdir}/emulparams/elf_fbsd.sh
+
diff --git a/contrib/gcc/config/arm/arm.c b/contrib/gcc/config/arm/arm.c
index 670e791..18f6fe5 100644
--- a/contrib/gcc/config/arm/arm.c
+++ b/contrib/gcc/config/arm/arm.c
@@ -3217,7 +3217,8 @@ legitimize_pic_address (rtx orig, enum machine_mode mode, rtx reg)
 	  gcc_assert (!no_new_pseudos);
 	  if (arm_pic_register != INVALID_REGNUM)
 	    {
-	      cfun->machine->pic_reg = gen_rtx_REG (Pmode, arm_pic_register);
+	      if (!cfun->machine->pic_reg)
+		cfun->machine->pic_reg = gen_rtx_REG (Pmode, arm_pic_register);
 
 	      /* Play games to avoid marking the function as needing pic
 		 if we are being called as part of the cost-estimation
@@ -3229,7 +3230,8 @@ legitimize_pic_address (rtx orig, enum machine_mode mode, rtx reg)
 	    {
 	      rtx seq;
 
-	      cfun->machine->pic_reg = gen_reg_rtx (Pmode);
+	      if (!cfun->machine->pic_reg)
+		  cfun->machine->pic_reg = gen_reg_rtx (Pmode);
 
 	      /* Play games to avoid marking the function as needing pic
 		 if we are being called as part of the cost-estimation
diff --git a/contrib/gcc/config/rs6000/freebsd.h b/contrib/gcc/config/rs6000/freebsd.h
index 52b013e..56907a3 100644
--- a/contrib/gcc/config/rs6000/freebsd.h
+++ b/contrib/gcc/config/rs6000/freebsd.h
@@ -193,7 +193,7 @@ extern int dot_symbols;
 #undef	LINK_OS_FREEBSD_SPEC
 #define	ASM_DEFAULT_SPEC	"-mppc%{!m32:64}"
 #define	ASM_SPEC		"%{m32:-a32}%{!m32:-a64} " SVR4_ASM_SPEC
-#define	LINK_OS_FREEBSD_SPEC	"%{m32:-melf32ppc}%{!m32:-melf64ppc} " LINK_OS_FREEBSD_SPEC_DEF
+#define	LINK_OS_FREEBSD_SPEC	"%{m32:-melf32ppc_fbsd}%{!m32:-melf64ppc_fbsd} " LINK_OS_FREEBSD_SPEC_DEF
 #endif
 
 /* _init and _fini functions are built from bits spread across many
@@ -253,3 +253,22 @@ extern int dot_symbols;
 
 #undef NEED_INDICATE_EXEC_STACK
 #define NEED_INDICATE_EXEC_STACK 1
+
+/* This is how to declare the size of a function.  */
+#undef  ASM_DECLARE_FUNCTION_SIZE
+#define ASM_DECLARE_FUNCTION_SIZE(FILE, FNAME, DECL)                    \
+  do                                                                    \
+    {                                                                   \
+      if (!flag_inhibit_size_directive)                                 \
+        {                                                               \
+          fputs ("\t.size\t", (FILE));                                  \
+          if (TARGET_64BIT && DOT_SYMBOLS)                              \
+            putc ('.', (FILE));                                         \
+          assemble_name ((FILE), (FNAME));                              \
+          fputs (",.-", (FILE));                                        \
+          rs6000_output_function_entry (FILE, FNAME);                   \
+          putc ('\n', (FILE));                                          \
+        }                                                               \
+    }                                                                   \
+  while (0)
+
diff --git a/contrib/gdtoa/gdtoaimp.h b/contrib/gdtoa/gdtoaimp.h
index 75cdc4e..285f8a18 100644
--- a/contrib/gdtoa/gdtoaimp.h
+++ b/contrib/gdtoa/gdtoaimp.h
@@ -201,6 +201,7 @@ THIS SOFTWARE.
 #include "namespace.h"
 #include <pthread.h>
 #include "un-namespace.h"
+#include "xlocale_private.h"
 
 #ifdef KR_headers
 #define Char char
@@ -525,11 +526,11 @@ extern void memcpy_D2A ANSI((void*, const void*, size_t));
 #define	strtoIQ		__strtoIQ
 #define	strtoIx		__strtoIx
 #define	strtoIxL	__strtoIxL
-#define	strtord		__strtord
+#define	strtord_l		__strtord_l
 #define	strtordd	__strtordd
 #define	strtorf		__strtorf
-#define	strtorQ		__strtorQ
-#define	strtorx		__strtorx
+#define	strtorQ_l		__strtorQ_l
+#define	strtorx_l		__strtorx_l
 #define	strtorxL	__strtorxL
 #define	strtodI		__strtodI
 #define	strtopd		__strtopd
@@ -634,7 +635,7 @@ extern void memcpy_D2A ANSI((void*, const void*, size_t));
  extern Bigint *s2b ANSI((CONST char*, int, int, ULong, int));
  extern Bigint *set_ones ANSI((Bigint*, int));
  extern char *strcp ANSI((char*, const char*));
- extern int strtodg ANSI((CONST char*, char**, FPI*, Long*, ULong*));
+ extern int strtodg_l ANSI((CONST char*, char**, FPI*, Long*, ULong*, locale_t));
 
  extern int strtoId ANSI((CONST char *, char **, double *, double *));
  extern int strtoIdd ANSI((CONST char *, char **, double *, double *));
@@ -644,17 +645,18 @@ extern void memcpy_D2A ANSI((void*, const void*, size_t));
  extern int strtoIx ANSI((CONST char *, char **, void *, void *));
  extern int strtoIxL ANSI((CONST char *, char **, void *, void *));
  extern double strtod ANSI((const char *s00, char **se));
+ extern double strtod_l ANSI((const char *s00, char **se, locale_t));
  extern int strtopQ ANSI((CONST char *, char **, Void *));
  extern int strtopf ANSI((CONST char *, char **, float *));
  extern int strtopd ANSI((CONST char *, char **, double *));
  extern int strtopdd ANSI((CONST char *, char **, double *));
  extern int strtopx ANSI((CONST char *, char **, Void *));
  extern int strtopxL ANSI((CONST char *, char **, Void *));
- extern int strtord ANSI((CONST char *, char **, int, double *));
+ extern int strtord_l ANSI((CONST char *, char **, int, double *, locale_t));
  extern int strtordd ANSI((CONST char *, char **, int, double *));
  extern int strtorf ANSI((CONST char *, char **, int, float *));
- extern int strtorQ ANSI((CONST char *, char **, int, void *));
- extern int strtorx ANSI((CONST char *, char **, int, void *));
+ extern int strtorQ_l ANSI((CONST char *, char **, int, void *, locale_t));
+ extern int strtorx_l ANSI((CONST char *, char **, int, void *, locale_t));
  extern int strtorxL ANSI((CONST char *, char **, int, void *));
  extern Bigint *sum ANSI((Bigint*, Bigint*));
  extern int trailz ANSI((Bigint*));
diff --git a/contrib/gdtoa/strtod.c b/contrib/gdtoa/strtod.c
index fe8cde8..236daa7 100644
--- a/contrib/gdtoa/strtod.c
+++ b/contrib/gdtoa/strtod.c
@@ -82,11 +82,11 @@ sulp
 #endif /*}*/
 
  double
-strtod
+strtod_l
 #ifdef KR_headers
-	(s00, se) CONST char *s00; char **se;
+	(s00, se, loc) CONST char *s00; char **se; locale_t loc
 #else
-	(CONST char *s00, char **se)
+	(CONST char *s00, char **se, locale_t loc)
 #endif
 {
 #ifdef Avoid_Underflow
@@ -108,14 +108,14 @@ strtod
 #endif
 #ifdef USE_LOCALE /*{{*/
 #ifdef NO_LOCALE_CACHE
-	char *decimalpoint = localeconv()->decimal_point;
+	char *decimalpoint = localeconv_l(loc)->decimal_point;
 	int dplen = strlen(decimalpoint);
 #else
 	char *decimalpoint;
 	static char *decimalpoint_cache;
 	static int dplen;
 	if (!(s0 = decimalpoint_cache)) {
-		s0 = localeconv()->decimal_point;
+		s0 = localeconv_l(loc)->decimal_point;
 		if ((decimalpoint_cache = (char*)MALLOC(strlen(s0) + 1))) {
 			strcpy(decimalpoint_cache, s0);
 			s0 = decimalpoint_cache;
@@ -1074,3 +1074,14 @@ strtod
 	return sign ? -dval(&rv) : dval(&rv);
 	}
 
+ double
+strtod
+#ifdef KR_headers
+	(s00, se, loc) CONST char *s00; char **se; locale_t
+#else
+	(CONST char *s00, char **se)
+#endif
+{
+	return strtod_l(s00, se, __get_locale());
+}
+
diff --git a/contrib/gdtoa/strtodg.c b/contrib/gdtoa/strtodg.c
index 5059869..37832b4 100644
--- a/contrib/gdtoa/strtodg.c
+++ b/contrib/gdtoa/strtodg.c
@@ -313,12 +313,12 @@ mantbits(U *d)
 	}
 
  int
-strtodg
+strtodg_l
 #ifdef KR_headers
-	(s00, se, fpi, exp, bits)
-	CONST char *s00; char **se; FPI *fpi; Long *exp; ULong *bits;
+	(s00, se, fpi, exp, bits, loc)
+	CONST char *s00; char **se; FPI *fpi; Long *exp; ULong *bits; locale_t loc;
 #else
-	(CONST char *s00, char **se, FPI *fpi, Long *exp, ULong *bits)
+	(CONST char *s00, char **se, FPI *fpi, Long *exp, ULong *bits, locale_t loc)
 #endif
 {
 	int abe, abits, asub;
@@ -334,14 +334,14 @@ strtodg
 	Bigint *ab, *bb, *bb1, *bd, *bd0, *bs, *delta, *rvb, *rvb0;
 #ifdef USE_LOCALE /*{{*/
 #ifdef NO_LOCALE_CACHE
-	char *decimalpoint = localeconv()->decimal_point;
+	char *decimalpoint = localeconv_l(loc)->decimal_point;
 	int dplen = strlen(decimalpoint);
 #else
 	char *decimalpoint;
 	static char *decimalpoint_cache;
 	static int dplen;
 	if (!(s0 = decimalpoint_cache)) {
-		s0 = localeconv()->decimal_point;
+		s0 = localeconv_l(loc)->decimal_point;
 		if ((decimalpoint_cache = (char*)MALLOC(strlen(s0) + 1))) {
 			strcpy(decimalpoint_cache, s0);
 			s0 = decimalpoint_cache;
diff --git a/contrib/gdtoa/strtof.c b/contrib/gdtoa/strtof.c
index 84bfe24..26ab60f 100644
--- a/contrib/gdtoa/strtof.c
+++ b/contrib/gdtoa/strtof.c
@@ -35,9 +35,9 @@ THIS SOFTWARE.
 
  float
 #ifdef KR_headers
-strtof(s, sp) CONST char *s; char **sp;
+strtof_l(s, sp, loc) CONST char *s; char **sp; locale_t loc;
 #else
-strtof(CONST char *s, char **sp)
+strtof_l(CONST char *s, char **sp, locale_t loc)
 #endif
 {
 	static FPI fpi0 = { 24, 1-127-24+1,  254-127-24+1, 1, SI };
@@ -51,7 +51,7 @@ strtof(CONST char *s, char **sp)
 #define fpi &fpi0
 #endif
 
-	k = strtodg(s, sp, fpi, &exp, bits);
+	k = strtodg_l(s, sp, fpi, &exp, bits, loc);
 	switch(k & STRTOG_Retmask) {
 	  case STRTOG_NoNumber:
 	  case STRTOG_Zero:
@@ -82,3 +82,13 @@ strtof(CONST char *s, char **sp)
 		u.L[0] |= 0x80000000L;
 	return u.f;
 	}
+ float
+#ifdef KR_headers
+strtof(s, sp) CONST char *s; char **sp;
+#else
+strtof(CONST char *s, char **sp)
+#endif
+{
+	return strtof_l(s, sp, __get_locale());
+}
+
diff --git a/contrib/gdtoa/strtorQ.c b/contrib/gdtoa/strtorQ.c
index bd183bc..68f4ba4 100644
--- a/contrib/gdtoa/strtorQ.c
+++ b/contrib/gdtoa/strtorQ.c
@@ -103,9 +103,10 @@ ULtoQ(ULong *L, ULong *bits, Long exp, int k)
 
  int
 #ifdef KR_headers
-strtorQ(s, sp, rounding, L) CONST char *s; char **sp; int rounding; void *L;
+strtorQ_l(s, sp, rounding, L, locale) CONST char *s; char **sp; int rounding;
+void *L; locale_t locale;
 #else
-strtorQ(CONST char *s, char **sp, int rounding, void *L)
+strtorQ_l(CONST char *s, char **sp, int rounding, void *L, locale_t locale)
 #endif
 {
 	static FPI fpi0 = { 113, 1-16383-113+1, 32766-16383-113+1, 1, SI };
@@ -120,7 +121,7 @@ strtorQ(CONST char *s, char **sp, int rounding, void *L)
 		fpi1.rounding = rounding;
 		fpi = &fpi1;
 		}
-	k = strtodg(s, sp, fpi, &exp, bits);
+	k = strtodg_l(s, sp, fpi, &exp, bits, locale);
 	ULtoQ((ULong*)L, bits, exp, k);
 	return k;
 	}
diff --git a/contrib/gdtoa/strtord.c b/contrib/gdtoa/strtord.c
index 709af4c..490f3e1 100644
--- a/contrib/gdtoa/strtord.c
+++ b/contrib/gdtoa/strtord.c
@@ -70,9 +70,10 @@ ULtod(ULong *L, ULong *bits, Long exp, int k)
 
  int
 #ifdef KR_headers
-strtord(s, sp, rounding, d) CONST char *s; char **sp; int rounding; double *d;
+strtord_l(s, sp, rounding, d, locale) CONST char *s; char **sp; int rounding;
+double *d; locale_t locale;
 #else
-strtord(CONST char *s, char **sp, int rounding, double *d)
+strtord_l(CONST char *s, char **sp, int rounding, double *d, locale_t locale)
 #endif
 {
 	static FPI fpi0 = { 53, 1-1023-53+1, 2046-1023-53+1, 1, SI };
@@ -87,7 +88,8 @@ strtord(CONST char *s, char **sp, int rounding, double *d)
 		fpi1.rounding = rounding;
 		fpi = &fpi1;
 		}
-	k = strtodg(s, sp, fpi, &exp, bits);
+	k = strtodg_l(s, sp, fpi, &exp, bits, locale);
 	ULtod((ULong*)d, bits, exp, k);
 	return k;
 	}
+
diff --git a/contrib/gdtoa/strtorx.c b/contrib/gdtoa/strtorx.c
index cd938f1..640ba61 100644
--- a/contrib/gdtoa/strtorx.c
+++ b/contrib/gdtoa/strtorx.c
@@ -106,9 +106,10 @@ ULtox(UShort *L, ULong *bits, Long exp, int k)
 
  int
 #ifdef KR_headers
-strtorx(s, sp, rounding, L) CONST char *s; char **sp; int rounding; void *L;
+strtorx_l(s, sp, rounding, L, locale) CONST char *s; char **sp; int rounding;
+void *L; locale_t locale;
 #else
-strtorx(CONST char *s, char **sp, int rounding, void *L)
+strtorx_l(CONST char *s, char **sp, int rounding, void *L, locale_t locale)
 #endif
 {
 	static FPI fpi0 = { 64, 1-16383-64+1, 32766 - 16383 - 64 + 1, 1, SI };
@@ -123,7 +124,7 @@ strtorx(CONST char *s, char **sp, int rounding, void *L)
 		fpi1.rounding = rounding;
 		fpi = &fpi1;
 		}
-	k = strtodg(s, sp, fpi, &exp, bits);
+	k = strtodg_l(s, sp, fpi, &exp, bits, locale);
 	ULtox((UShort*)L, bits, exp, k);
 	return k;
 	}
diff --git a/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h b/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h
index c4ffac5..48d7e94 100644
--- a/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h
+++ b/contrib/llvm/tools/clang/include/clang/AST/ASTContext.h
@@ -229,6 +229,9 @@ class ASTContext : public llvm::RefCountedBase<ASTContext> {
   /// \brief The type for the C sigjmp_buf type.
   TypeDecl *sigjmp_bufDecl;
 
+  /// \brief The type for the C ucontext_t type.
+  TypeDecl *ucontext_tDecl;
+
   /// \brief Type for the Block descriptor for Blocks CodeGen.
   ///
   /// Since this is only used for generation of debug info, it is not
@@ -955,6 +958,18 @@ public:
     return QualType();
   }
 
+  /// \brief Set the type for the C ucontext_t type.
+  void setucontext_tDecl(TypeDecl *ucontext_tDecl) {
+    this->ucontext_tDecl = ucontext_tDecl;
+  }
+
+  /// \brief Retrieve the C ucontext_t type.
+  QualType getucontext_tType() const {
+    if (ucontext_tDecl)
+      return getTypeDeclType(ucontext_tDecl);
+    return QualType();
+  }
+
   /// \brief The result type of logical operations, '<', '>', '!=', etc.
   QualType getLogicalOperationType() const {
     return getLangOptions().CPlusPlus ? BoolTy : IntTy;
@@ -1099,7 +1114,8 @@ public:
   enum GetBuiltinTypeError {
     GE_None,              //< No error
     GE_Missing_stdio,     //< Missing a type from <stdio.h>
-    GE_Missing_setjmp     //< Missing a type from <setjmp.h>
+    GE_Missing_setjmp,    //< Missing a type from <setjmp.h>
+    GE_Missing_ucontext   //< Missing a type from <ucontext.h>
   };
 
   /// GetBuiltinType - Return the type for the specified builtin.  If 
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def b/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def
index e06e05b..da5aadf 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def
+++ b/contrib/llvm/tools/clang/include/clang/Basic/Builtins.def
@@ -39,6 +39,7 @@
 //  P -> FILE
 //  J -> jmp_buf
 //  SJ -> sigjmp_buf
+//  K -> ucontext_t
 //  . -> "...".  This may only occur at the end of the function list.
 //
 // Types may be prefixed with the following modifiers:
@@ -480,111 +481,111 @@ BUILTIN(__builtin_alloca, "v*z"   , "n")
 // FIXME: These assume that char -> i8, short -> i16, int -> i32,
 // long long -> i64.
 
-BUILTIN(__sync_fetch_and_add, "v.", "")
-BUILTIN(__sync_fetch_and_add_1, "ccD*c.", "n")
-BUILTIN(__sync_fetch_and_add_2, "ssD*s.", "n")
-BUILTIN(__sync_fetch_and_add_4, "iiD*i.", "n")
-BUILTIN(__sync_fetch_and_add_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_fetch_and_add_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_fetch_and_sub, "v.", "")
-BUILTIN(__sync_fetch_and_sub_1, "ccD*c.", "n")
-BUILTIN(__sync_fetch_and_sub_2, "ssD*s.", "n")
-BUILTIN(__sync_fetch_and_sub_4, "iiD*i.", "n")
-BUILTIN(__sync_fetch_and_sub_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_fetch_and_sub_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_fetch_and_or, "v.", "")
-BUILTIN(__sync_fetch_and_or_1, "ccD*c.", "n")
-BUILTIN(__sync_fetch_and_or_2, "ssD*s.", "n")
-BUILTIN(__sync_fetch_and_or_4, "iiD*i.", "n")
-BUILTIN(__sync_fetch_and_or_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_fetch_and_or_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_fetch_and_and, "v.", "")
-BUILTIN(__sync_fetch_and_and_1, "ccD*c.", "n")
-BUILTIN(__sync_fetch_and_and_2, "ssD*s.", "n")
-BUILTIN(__sync_fetch_and_and_4, "iiD*i.", "n")
-BUILTIN(__sync_fetch_and_and_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_fetch_and_and_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_fetch_and_xor, "v.", "")
-BUILTIN(__sync_fetch_and_xor_1, "ccD*c.", "n")
-BUILTIN(__sync_fetch_and_xor_2, "ssD*s.", "n")
-BUILTIN(__sync_fetch_and_xor_4, "iiD*i.", "n")
-BUILTIN(__sync_fetch_and_xor_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_fetch_and_xor_16, "LLLiLLLiD*LLLi.", "n")
-
-
-BUILTIN(__sync_add_and_fetch, "v.", "")
-BUILTIN(__sync_add_and_fetch_1, "ccD*c.", "n")
-BUILTIN(__sync_add_and_fetch_2, "ssD*s.", "n")
-BUILTIN(__sync_add_and_fetch_4, "iiD*i.", "n")
-BUILTIN(__sync_add_and_fetch_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_add_and_fetch_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_sub_and_fetch, "v.", "")
-BUILTIN(__sync_sub_and_fetch_1, "ccD*c.", "n")
-BUILTIN(__sync_sub_and_fetch_2, "ssD*s.", "n")
-BUILTIN(__sync_sub_and_fetch_4, "iiD*i.", "n")
-BUILTIN(__sync_sub_and_fetch_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_sub_and_fetch_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_or_and_fetch, "v.", "")
-BUILTIN(__sync_or_and_fetch_1, "ccD*c.", "n")
-BUILTIN(__sync_or_and_fetch_2, "ssD*s.", "n")
-BUILTIN(__sync_or_and_fetch_4, "iiD*i.", "n")
-BUILTIN(__sync_or_and_fetch_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_or_and_fetch_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_and_and_fetch, "v.", "")
-BUILTIN(__sync_and_and_fetch_1, "ccD*c.", "n")
-BUILTIN(__sync_and_and_fetch_2, "ssD*s.", "n")
-BUILTIN(__sync_and_and_fetch_4, "iiD*i.", "n")
-BUILTIN(__sync_and_and_fetch_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_and_and_fetch_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_xor_and_fetch, "v.", "")
-BUILTIN(__sync_xor_and_fetch_1, "ccD*c.", "n")
-BUILTIN(__sync_xor_and_fetch_2, "ssD*s.", "n")
-BUILTIN(__sync_xor_and_fetch_4, "iiD*i.", "n")
-BUILTIN(__sync_xor_and_fetch_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_xor_and_fetch_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_bool_compare_and_swap, "v.", "")
-BUILTIN(__sync_bool_compare_and_swap_1, "bcD*cc.", "n")
-BUILTIN(__sync_bool_compare_and_swap_2, "bsD*ss.", "n")
-BUILTIN(__sync_bool_compare_and_swap_4, "biD*ii.", "n")
-BUILTIN(__sync_bool_compare_and_swap_8, "bLLiD*LLiLLi.", "n")
-BUILTIN(__sync_bool_compare_and_swap_16, "bLLLiD*LLLiLLLi.", "n")
-
-BUILTIN(__sync_val_compare_and_swap, "v.", "")
-BUILTIN(__sync_val_compare_and_swap_1, "ccD*cc.", "n")
-BUILTIN(__sync_val_compare_and_swap_2, "ssD*ss.", "n")
-BUILTIN(__sync_val_compare_and_swap_4, "iiD*ii.", "n")
-BUILTIN(__sync_val_compare_and_swap_8, "LLiLLiD*LLiLLi.", "n")
-BUILTIN(__sync_val_compare_and_swap_16, "LLLiLLLiD*LLLiLLLi.", "n")
-
-BUILTIN(__sync_lock_test_and_set, "v.", "")
-BUILTIN(__sync_lock_test_and_set_1, "ccD*c.", "n")
-BUILTIN(__sync_lock_test_and_set_2, "ssD*s.", "n")
-BUILTIN(__sync_lock_test_and_set_4, "iiD*i.", "n")
-BUILTIN(__sync_lock_test_and_set_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_lock_test_and_set_16, "LLLiLLLiD*LLLi.", "n")
-
-BUILTIN(__sync_lock_release, "v.", "")
-BUILTIN(__sync_lock_release_1, "vcD*.", "n")
-BUILTIN(__sync_lock_release_2, "vsD*.", "n")
-BUILTIN(__sync_lock_release_4, "viD*.", "n")
-BUILTIN(__sync_lock_release_8, "vLLiD*.", "n")
-BUILTIN(__sync_lock_release_16, "vLLLiD*.", "n")
-
-BUILTIN(__sync_swap, "v.", "")
-BUILTIN(__sync_swap_1, "ccD*c.", "n")
-BUILTIN(__sync_swap_2, "ssD*s.", "n")
-BUILTIN(__sync_swap_4, "iiD*i.", "n")
-BUILTIN(__sync_swap_8, "LLiLLiD*LLi.", "n")
-BUILTIN(__sync_swap_16, "LLLiLLLiD*LLLi.", "n")
+BUILTIN(__sync_fetch_and_add, "v.", "t")
+BUILTIN(__sync_fetch_and_add_1, "ccD*c.", "nt")
+BUILTIN(__sync_fetch_and_add_2, "ssD*s.", "nt")
+BUILTIN(__sync_fetch_and_add_4, "iiD*i.", "nt")
+BUILTIN(__sync_fetch_and_add_8, "LLiLLiD*LLi.", "nt")
+BUILTIN(__sync_fetch_and_add_16, "LLLiLLLiD*LLLi.", "nt")
+
+BUILTIN(__sync_fetch_and_sub, "v.", "t")
+BUILTIN(__sync_fetch_and_sub_1, "ccD*c.", "nt")
+BUILTIN(__sync_fetch_and_sub_2, "ssD*s.", "nt")
+BUILTIN(__sync_fetch_and_sub_4, "iiD*i.", "nt")
+BUILTIN(__sync_fetch_and_sub_8, "LLiLLiD*LLi.", "nt")
+BUILTIN(__sync_fetch_and_sub_16, "LLLiLLLiD*LLLi.", "nt")
+
+BUILTIN(__sync_fetch_and_or, "v.", "t")
+BUILTIN(__sync_fetch_and_or_1, "ccD*c.", "nt")
+BUILTIN(__sync_fetch_and_or_2, "ssD*s.", "nt")
+BUILTIN(__sync_fetch_and_or_4, "iiD*i.", "nt")
+BUILTIN(__sync_fetch_and_or_8, "LLiLLiD*LLi.", "nt")
+BUILTIN(__sync_fetch_and_or_16, "LLLiLLLiD*LLLi.", "nt")
+
+BUILTIN(__sync_fetch_and_and, "v.", "t")
+BUILTIN(__sync_fetch_and_and_1, "ccD*c.", "tn")
+BUILTIN(__sync_fetch_and_and_2, "ssD*s.", "tn")
+BUILTIN(__sync_fetch_and_and_4, "iiD*i.", "tn")
+BUILTIN(__sync_fetch_and_and_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_fetch_and_and_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_fetch_and_xor, "v.", "t")
+BUILTIN(__sync_fetch_and_xor_1, "ccD*c.", "tn")
+BUILTIN(__sync_fetch_and_xor_2, "ssD*s.", "tn")
+BUILTIN(__sync_fetch_and_xor_4, "iiD*i.", "tn")
+BUILTIN(__sync_fetch_and_xor_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_fetch_and_xor_16, "LLLiLLLiD*LLLi.", "tn")
+
+
+BUILTIN(__sync_add_and_fetch, "v.", "t")
+BUILTIN(__sync_add_and_fetch_1, "ccD*c.", "tn")
+BUILTIN(__sync_add_and_fetch_2, "ssD*s.", "tn")
+BUILTIN(__sync_add_and_fetch_4, "iiD*i.", "tn")
+BUILTIN(__sync_add_and_fetch_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_add_and_fetch_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_sub_and_fetch, "v.", "t")
+BUILTIN(__sync_sub_and_fetch_1, "ccD*c.", "tn")
+BUILTIN(__sync_sub_and_fetch_2, "ssD*s.", "tn")
+BUILTIN(__sync_sub_and_fetch_4, "iiD*i.", "tn")
+BUILTIN(__sync_sub_and_fetch_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_sub_and_fetch_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_or_and_fetch, "v.", "t")
+BUILTIN(__sync_or_and_fetch_1, "ccD*c.", "tn")
+BUILTIN(__sync_or_and_fetch_2, "ssD*s.", "tn")
+BUILTIN(__sync_or_and_fetch_4, "iiD*i.", "tn")
+BUILTIN(__sync_or_and_fetch_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_or_and_fetch_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_and_and_fetch, "v.", "t")
+BUILTIN(__sync_and_and_fetch_1, "ccD*c.", "tn")
+BUILTIN(__sync_and_and_fetch_2, "ssD*s.", "tn")
+BUILTIN(__sync_and_and_fetch_4, "iiD*i.", "tn")
+BUILTIN(__sync_and_and_fetch_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_and_and_fetch_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_xor_and_fetch, "v.", "t")
+BUILTIN(__sync_xor_and_fetch_1, "ccD*c.", "tn")
+BUILTIN(__sync_xor_and_fetch_2, "ssD*s.", "tn")
+BUILTIN(__sync_xor_and_fetch_4, "iiD*i.", "tn")
+BUILTIN(__sync_xor_and_fetch_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_xor_and_fetch_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_bool_compare_and_swap, "v.", "t")
+BUILTIN(__sync_bool_compare_and_swap_1, "bcD*cc.", "tn")
+BUILTIN(__sync_bool_compare_and_swap_2, "bsD*ss.", "tn")
+BUILTIN(__sync_bool_compare_and_swap_4, "biD*ii.", "tn")
+BUILTIN(__sync_bool_compare_and_swap_8, "bLLiD*LLiLLi.", "tn")
+BUILTIN(__sync_bool_compare_and_swap_16, "bLLLiD*LLLiLLLi.", "tn")
+
+BUILTIN(__sync_val_compare_and_swap, "v.", "t")
+BUILTIN(__sync_val_compare_and_swap_1, "ccD*cc.", "tn")
+BUILTIN(__sync_val_compare_and_swap_2, "ssD*ss.", "tn")
+BUILTIN(__sync_val_compare_and_swap_4, "iiD*ii.", "tn")
+BUILTIN(__sync_val_compare_and_swap_8, "LLiLLiD*LLiLLi.", "tn")
+BUILTIN(__sync_val_compare_and_swap_16, "LLLiLLLiD*LLLiLLLi.", "tn")
+
+BUILTIN(__sync_lock_test_and_set, "v.", "t")
+BUILTIN(__sync_lock_test_and_set_1, "ccD*c.", "tn")
+BUILTIN(__sync_lock_test_and_set_2, "ssD*s.", "tn")
+BUILTIN(__sync_lock_test_and_set_4, "iiD*i.", "tn")
+BUILTIN(__sync_lock_test_and_set_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_lock_test_and_set_16, "LLLiLLLiD*LLLi.", "tn")
+
+BUILTIN(__sync_lock_release, "v.", "t")
+BUILTIN(__sync_lock_release_1, "vcD*.", "tn")
+BUILTIN(__sync_lock_release_2, "vsD*.", "tn")
+BUILTIN(__sync_lock_release_4, "viD*.", "tn")
+BUILTIN(__sync_lock_release_8, "vLLiD*.", "tn")
+BUILTIN(__sync_lock_release_16, "vLLLiD*.", "tn")
+
+BUILTIN(__sync_swap, "v.", "t")
+BUILTIN(__sync_swap_1, "ccD*c.", "tn")
+BUILTIN(__sync_swap_2, "ssD*s.", "tn")
+BUILTIN(__sync_swap_4, "iiD*i.", "tn")
+BUILTIN(__sync_swap_8, "LLiLLiD*LLi.", "tn")
+BUILTIN(__sync_swap_16, "LLLiLLLiD*LLLi.", "tn")
 
 BUILTIN(__atomic_load, "v.", "t")
 BUILTIN(__atomic_store, "v.", "t")
@@ -679,13 +680,13 @@ LIBBUILTIN(vfork, "i",            "fj",    "unistd.h", ALL_LANGUAGES)
 // it here to avoid having two identical LIBBUILTIN entries.
 #undef setjmp
 LIBBUILTIN(_setjmp, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
-LIBBUILTIN(__sigsetjmp, "iJi",    "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(__sigsetjmp, "iSJi",   "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(setjmp, "iJ",          "fj",   "setjmp.h", ALL_LANGUAGES)
-LIBBUILTIN(sigsetjmp, "iJi",      "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(sigsetjmp, "iSJi",     "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(setjmp_syscall, "iJ",  "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(savectx, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(qsetjmp, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
-LIBBUILTIN(getcontext, "iJ",      "fj",   "setjmp.h", ALL_LANGUAGES)
+LIBBUILTIN(getcontext, "iK*",     "fj",   "setjmp.h", ALL_LANGUAGES)
 
 LIBBUILTIN(_longjmp, "vJi",       "fr",    "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(siglongjmp, "vSJi",    "fr",    "setjmp.h", ALL_LANGUAGES)
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td
index 49603eb..00a6663 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticGroups.td
@@ -26,6 +26,7 @@ def : DiagGroup<"attributes">;
 def : DiagGroup<"bad-function-cast">;
 def Availability : DiagGroup<"availability">;
 def BoolConversions : DiagGroup<"bool-conversions">;
+def BuiltinRequiresHeader : DiagGroup<"builtin-requires-header">;
 def CXXCompat: DiagGroup<"c++-compat">;
 def CastAlign : DiagGroup<"cast-align">;
 def : DiagGroup<"cast-qual">;
diff --git a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 0fbf0ce..d60e69b 100644
--- a/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/contrib/llvm/tools/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -254,10 +254,16 @@ def note_please_include_header : Note<
 def note_previous_builtin_declaration : Note<"%0 is a builtin with type %1">;
 def warn_implicit_decl_requires_stdio : Warning<
   "declaration of built-in function '%0' requires inclusion of the header "
-  "<stdio.h>">;
+  "<stdio.h>">,
+  InGroup<BuiltinRequiresHeader>;
 def warn_implicit_decl_requires_setjmp : Warning<
   "declaration of built-in function '%0' requires inclusion of the header "
-  "<setjmp.h>">;
+  "<setjmp.h>">,
+  InGroup<BuiltinRequiresHeader>;
+def warn_implicit_decl_requires_ucontext : Warning<
+  "declaration of built-in function '%0' requires inclusion of the header "
+  "<ucontext.h>">,
+  InGroup<BuiltinRequiresHeader>;
 def warn_redecl_library_builtin : Warning<
   "incompatible redeclaration of library function %0">;
 def err_builtin_definition : Error<"definition of builtin function %0">;
diff --git a/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h b/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h
index dc4d05c..494835c 100644
--- a/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/contrib/llvm/tools/clang/include/clang/Serialization/ASTBitCodes.h
@@ -677,7 +677,9 @@ namespace clang {
       /// \brief Objective-C "Class" redefinition type
       SPECIAL_TYPE_OBJC_CLASS_REDEFINITION     = 7,
       /// \brief Objective-C "SEL" redefinition type
-      SPECIAL_TYPE_OBJC_SEL_REDEFINITION       = 8
+      SPECIAL_TYPE_OBJC_SEL_REDEFINITION       = 8,
+      /// \brief C ucontext_t typedef type
+      SPECIAL_TYPE_UCONTEXT_T                  = 9
     };
     
     /// \brief The number of special type IDs.
diff --git a/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp b/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp
index 4624280..0833286 100644
--- a/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp
+++ b/contrib/llvm/tools/clang/lib/AST/ASTContext.cpp
@@ -227,8 +227,9 @@ ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
     ObjCIdDecl(0), ObjCSelDecl(0), ObjCClassDecl(0),
     CFConstantStringTypeDecl(0), ObjCInstanceTypeDecl(0),
     FILEDecl(0), 
-    jmp_bufDecl(0), sigjmp_bufDecl(0), BlockDescriptorType(0), 
-    BlockDescriptorExtendedType(0), cudaConfigureCallDecl(0),
+    jmp_bufDecl(0), sigjmp_bufDecl(0), ucontext_tDecl(0),
+    BlockDescriptorType(0), BlockDescriptorExtendedType(0),
+    cudaConfigureCallDecl(0),
     NullTypeSourceInfo(QualType()),
     SourceMgr(SM), LangOpts(LOpts), 
     AddrSpaceMap(0), Target(t), PrintingPolicy(LOpts),
@@ -6293,6 +6294,15 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
       return QualType();
     }
     break;
+  case 'K':
+    assert(HowLong == 0 && !Signed && !Unsigned && "Bad modifiers for 'K'!");
+    Type = Context.getucontext_tType();
+
+    if (Type.isNull()) {
+      Error = ASTContext::GE_Missing_ucontext;
+      return QualType();
+    }
+    break;
   }
 
   // If there are modifiers and if we're allowed to parse them, go for it.
diff --git a/contrib/llvm/tools/clang/lib/Basic/Targets.cpp b/contrib/llvm/tools/clang/lib/Basic/Targets.cpp
index 6d0d7bb..0cc0122 100644
--- a/contrib/llvm/tools/clang/lib/Basic/Targets.cpp
+++ b/contrib/llvm/tools/clang/lib/Basic/Targets.cpp
@@ -1282,6 +1282,7 @@ class X86TargetInfo : public TargetInfo {
     CK_K8SSE3,
     CK_Opteron,
     CK_OpteronSSE3,
+    CK_AMDFAM10,
 
     /// This specification is deprecated and will be removed in the future.
     /// Users should prefer \see CK_K8.
@@ -1381,6 +1382,7 @@ public:
       .Case("k8-sse3", CK_K8SSE3)
       .Case("opteron", CK_Opteron)
       .Case("opteron-sse3", CK_OpteronSSE3)
+      .Case("amdfam10", CK_AMDFAM10)
       .Case("x86-64", CK_x86_64)
       .Case("geode", CK_Geode)
       .Default(CK_Generic);
@@ -1441,6 +1443,7 @@ public:
     case CK_K8SSE3:
     case CK_Opteron:
     case CK_OpteronSSE3:
+    case CK_AMDFAM10:
     case CK_x86_64:
       return true;
     }
@@ -1459,12 +1462,10 @@ void X86TargetInfo::getDefaultFeatures(llvm::StringMap<bool> &Features) const {
   Features["ssse3"] = false;
   Features["sse41"] = false;
   Features["sse42"] = false;
+  Features["sse4a"] = false;
   Features["aes"] = false;
   Features["avx"] = false;
 
-  // LLVM does not currently recognize this.
-  // Features["sse4a"] = false;
-
   // FIXME: This *really* should not be here.
 
   // X86_64 always has SSE2.
@@ -1561,6 +1562,11 @@ void X86TargetInfo::getDefaultFeatures(llvm::StringMap<bool> &Features) const {
     setFeatureEnabled(Features, "sse3", true);
     setFeatureEnabled(Features, "3dnowa", true);
     break;
+  case CK_AMDFAM10:
+    setFeatureEnabled(Features, "sse3", true);
+    setFeatureEnabled(Features, "sse4a", true);
+    setFeatureEnabled(Features, "3dnowa", true);
+    break;
   case CK_C3_2:
     setFeatureEnabled(Features, "mmx", true);
     setFeatureEnabled(Features, "sse", true);
@@ -1604,6 +1610,8 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
     else if (Name == "avx")
       Features["avx"] = Features["sse"] = Features["sse2"] = Features["sse3"] =
         Features["ssse3"] = Features["sse41"] = Features["sse42"] = true;
+    else if (Name == "sse4a")
+      Features["sse4a"] = true;
   } else {
     if (Name == "mmx")
       Features["mmx"] = Features["3dnow"] = Features["3dnowa"] = false;
@@ -1630,6 +1638,8 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
       Features["aes"] = false;
     else if (Name == "avx")
       Features["avx"] = false;
+    else if (Name == "sse4a")
+      Features["sse4a"] = false;
   }
 
   return true;
@@ -1826,6 +1836,11 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__k8__");
     Builder.defineMacro("__tune_k8__");
     break;
+  case CK_AMDFAM10:
+    Builder.defineMacro("__amdfam10");
+    Builder.defineMacro("__amdfam10__");
+    Builder.defineMacro("__tune_amdfam10__");
+    break;
   case CK_Geode:
     Builder.defineMacro("__geode");
     Builder.defineMacro("__geode__");
diff --git a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
index 94849a6..75cadc6 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
@@ -3926,7 +3926,7 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (getToolChain().getArchName() == "powerpc") {
     CmdArgs.push_back("-m");
-    CmdArgs.push_back("elf32ppc");
+    CmdArgs.push_back("elf32ppc_fbsd");
   }
 
   if (Output.isFilename()) {
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp
index eaf7bfa..a6c49bb 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaChecking.cpp
@@ -803,14 +803,6 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   for (unsigned i = 0; i != NumFixed; ++i) {
     ExprResult Arg = TheCall->getArg(i+1);
 
-    // If the argument is an implicit cast, then there was a promotion due to
-    // "...", just remove it now.
-    if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Arg.get())) {
-      Arg = ICE->getSubExpr();
-      ICE->setSubExpr(0);
-      TheCall->setArg(i+1, Arg.get());
-    }
-
     // GCC does an implicit conversion to the pointer or integer ValType.  This
     // can fail in some cases (1i -> int**), check for this error case now.
     // Initialize the argument.
diff --git a/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp b/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp
index 8d993ef..d90a821 100644
--- a/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp
+++ b/contrib/llvm/tools/clang/lib/Sema/SemaDecl.cpp
@@ -1305,6 +1305,12 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
       Diag(Loc, diag::warn_implicit_decl_requires_setjmp)
         << Context.BuiltinInfo.GetName(BID);
     return 0;
+
+  case ASTContext::GE_Missing_ucontext:
+    if (ForRedeclaration)
+      Diag(Loc, diag::warn_implicit_decl_requires_ucontext)
+        << Context.BuiltinInfo.GetName(BID);
+    return 0;
   }
 
   if (!ForRedeclaration && Context.BuiltinInfo.isPredefinedLibFunction(BID)) {
@@ -3635,6 +3641,8 @@ Sema::ActOnTypedefNameDecl(Scope *S, DeclContext *DC, TypedefNameDecl *NewTD,
         Context.setjmp_bufDecl(NewTD);
       else if (II->isStr("sigjmp_buf"))
         Context.setsigjmp_bufDecl(NewTD);
+      else if (II->isStr("ucontext_t"))
+        Context.setucontext_tDecl(NewTD);
       else if (II->isStr("__builtin_va_list"))
         Context.setBuiltinVaListType(Context.getTypedefType(NewTD));
     }
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp
index fe1cc30..de19d85 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTReader.cpp
@@ -2659,6 +2659,24 @@ void ASTReader::InitializeContext() {
       if (Context.ObjCSelRedefinitionType.isNull())
         Context.ObjCSelRedefinitionType = GetType(ObjCSelRedef);
     }
+
+    if (unsigned Ucontext_t = SpecialTypes[SPECIAL_TYPE_UCONTEXT_T]) {
+      QualType Ucontext_tType = GetType(Ucontext_t);
+      if (Ucontext_tType.isNull()) {
+        Error("ucontext_t type is NULL");
+        return;
+      }
+
+      if (!Context.ucontext_tDecl) {
+        if (const TypedefType *Typedef = Ucontext_tType->getAs<TypedefType>())
+          Context.setucontext_tDecl(Typedef->getDecl());
+        else {
+          const TagType *Tag = Ucontext_tType->getAs<TagType>();
+          assert(Tag && "Invalid ucontext_t type in AST file");
+          Context.setucontext_tDecl(Tag->getDecl());
+        }
+      }
+    }
   }
   
   ReadPragmaDiagnosticMappings(Context.getDiagnostics());
diff --git a/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp b/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp
index b31262d..a1645d7 100644
--- a/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp
+++ b/contrib/llvm/tools/clang/lib/Serialization/ASTWriter.cpp
@@ -2994,6 +2994,7 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   AddTypeRef(Context.ObjCIdRedefinitionType, SpecialTypes);
   AddTypeRef(Context.ObjCClassRedefinitionType, SpecialTypes);
   AddTypeRef(Context.ObjCSelRedefinitionType, SpecialTypes);
+  AddTypeRef(Context.getucontext_tType(), SpecialTypes);
   
   // Keep writing types and declarations until all types and
   // declarations have been written.
diff --git a/contrib/tzcode/stdtime/localtime.c b/contrib/tzcode/stdtime/localtime.c
index b1981b6..9605eeb 100644
--- a/contrib/tzcode/stdtime/localtime.c
+++ b/contrib/tzcode/stdtime/localtime.c
@@ -450,6 +450,7 @@ register const int	doextend;
 			_close(fid);
 			return -1;
 		}
+		free(fullname);
 	}
 	u = malloc(sizeof(*u));
 	if (u == NULL)
diff --git a/contrib/tzdata/australasia b/contrib/tzdata/australasia
index 930beed..d884cbc 100644
--- a/contrib/tzdata/australasia
+++ b/contrib/tzdata/australasia
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)australasia	8.28
+# @(#)australasia	8.29
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -308,6 +308,20 @@ Zone	Indian/Cocos	6:27:40	-	LMT	1900
 # advance at 2am to 3am on October 23, 2011 and one hour back at 3am to 
 # 2am on February 26 next year.
 
+# From Ken Rylander (2011-10-24)
+# Another change to the Fiji DST end date. In the TZ database the end date for
+# Fiji DST 2012, is currently Feb 26. This has been changed to Jan 22.
+#
+# <a href="http://www.fiji.gov.fj/index.php?option=com_content&view=article&id=5017:amendments-to-daylight-savings&catid=71:press-releases&Itemid=155">
+# http://www.fiji.gov.fj/index.php?option=com_content&view=article&id=5017:amendments-to-daylight-savings&catid=71:press-releases&Itemid=155
+# </a>
+# states:
+#
+# The end of daylight saving scheduled initially for the 26th of February 2012
+# has been brought forward to the 22nd of January 2012.
+# The commencement of daylight saving will remain unchanged and start
+# on the  23rd of October, 2011.
+
 # Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
 Rule	Fiji	1998	1999	-	Nov	Sun>=1	2:00	1:00	S
 Rule	Fiji	1999	2000	-	Feb	lastSun	3:00	0	-
@@ -316,7 +330,7 @@ Rule	Fiji	2010	only	-	Mar	lastSun	3:00	0	-
 Rule	Fiji	2010	only	-	Oct	24	2:00	1:00	S
 Rule	Fiji	2011	only	-	Mar	Sun>=1	3:00	0	-
 Rule	Fiji	2011	only	-	Oct	23	2:00	1:00	S
-Rule	Fiji	2012	only	-	Feb	26	3:00	0	-
+Rule	Fiji	2012	only	-	Jan	22	3:00	0	-
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Pacific/Fiji	11:53:40 -	LMT	1915 Oct 26	# Suva
 			12:00	Fiji	FJ%sT	# Fiji Time
diff --git a/contrib/tzdata/backward b/contrib/tzdata/backward
index b8e5911..b2d837d 100644
--- a/contrib/tzdata/backward
+++ b/contrib/tzdata/backward
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)backward	8.10
+# @(#)backward	8.11
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -67,6 +67,7 @@ Link	America/Havana		Cuba
 Link	Africa/Cairo		Egypt
 Link	Europe/Dublin		Eire
 Link	Europe/London		Europe/Belfast
+Link	Europe/Chisinau		Europe/Tiraspol
 Link	Europe/London		GB
 Link	Europe/London		GB-Eire
 Link	Etc/GMT			GMT+0
diff --git a/contrib/tzdata/europe b/contrib/tzdata/europe
index 22bdd36..8230b5e 100644
--- a/contrib/tzdata/europe
+++ b/contrib/tzdata/europe
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)europe	8.39
+# @(#)europe	8.40
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -1678,6 +1678,18 @@ Zone	Europe/Malta	0:58:04 -	LMT	1893 Nov  2 0:00s # Valletta
 # a pre-1880 LMT offset of 1:58:32.
 #
 # (which agrees with the earlier entry that had been removed)
+#
+# From Alexander Krivenyshev (2011-10-26)
+# NO need to divide Moldova into two timezones at this point.
+# As of today, Transnistria (Pridnestrovie)- Tiraspol reversed its own
+# decision to abolish DST this winter. 
+# Following Moldova and neighboring Ukraine- Transnistria (Pridnestrovie)-
+# Tiraspol will go back to winter time on October 30, 2011.
+# News from Moldova (in russian):
+# <a href="http://ru.publika.md/link_317061.html">
+# http://ru.publika.md/link_317061.html
+# </a>
+
 
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	Europe/Chisinau	1:55:20 -	LMT	1880
@@ -1694,21 +1706,6 @@ Zone	Europe/Chisinau	1:55:20 -	LMT	1880
 # See Romania commentary for the guessed 1997 transition to EU rules.
 			2:00	EU	EE%sT
 
-Zone	Europe/Tiraspol	1:58:32 -	LMT	1880
-			1:55	-	CMT	1918 Feb 15 # Chisinau MT
-			1:44:24	-	BMT	1931 Jul 24 # Bucharest MT
-			2:00	Romania	EE%sT	1940 Aug 15
-			2:00	1:00	EEST	1941 Jul 17
-			1:00	C-Eur	CE%sT	1944 Aug 24
-			3:00	Russia	MSK/MSD	1990
-			3:00	-	MSK	1990 May 6
-			2:00	-	EET	1991
-			2:00	Russia	EE%sT	1992
-			2:00	E-Eur	EE%sT	1997
-# See Romania commentary for the guessed 1997 transition to EU rules.
-			2:00	EU	EE%sT	2011 Mar lastSun 1:00u
-			3:00	-	FET # Further-eastern European Time
-
 # Monaco
 # Shanks & Pottenger give 0:09:20 for Paris Mean Time; go with Howse's
 # more precise 0:09:21.
diff --git a/contrib/tzdata/northamerica b/contrib/tzdata/northamerica
index 68a7cd5..a21bf89 100644
--- a/contrib/tzdata/northamerica
+++ b/contrib/tzdata/northamerica
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)northamerica	8.50
+# @(#)northamerica	8.51
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 
@@ -2690,6 +2690,20 @@ Zone America/Costa_Rica	-5:36:20 -	LMT	1890		# San Jose
 # <a href="http://www.timeanddate.com/news/time/cuba-starts-dst-2011.html">
 # http://www.timeanddate.com/news/time/cuba-starts-dst-2011.html
 # </a>
+#
+# From Steffen Thorsen (2011-10-30)
+# Cuba will end DST two weeks later this year. Instead of going back 
+# tonight, it has been delayed to 2011-11-13 at 01:00.
+#
+# One source (Spanish)
+# <a href="http://www.radioangulo.cu/noticias/cuba/17105-cuba-restablecera-el-horario-del-meridiano-de-greenwich.html">
+# http://www.radioangulo.cu/noticias/cuba/17105-cuba-restablecera-el-horario-del-meridiano-de-greenwich.html
+# </a>
+#
+# Our page:
+# <a href="http://www.timeanddate.com/news/time/cuba-time-changes-2011.html">
+# http://www.timeanddate.com/news/time/cuba-time-changes-2011.html
+# </a>
 
 # Rule	NAME	FROM	TO	TYPE	IN	ON	AT	SAVE	LETTER/S
 Rule	Cuba	1928	only	-	Jun	10	0:00	1:00	D
@@ -2721,12 +2735,14 @@ Rule	Cuba	1997	only	-	Oct	12	0:00s	0	S
 Rule	Cuba	1998	1999	-	Mar	lastSun	0:00s	1:00	D
 Rule	Cuba	1998	2003	-	Oct	lastSun	0:00s	0	S
 Rule	Cuba	2000	2004	-	Apr	Sun>=1	0:00s	1:00	D
-Rule	Cuba	2006	max	-	Oct	lastSun	0:00s	0	S
+Rule	Cuba	2006	2010	-	Oct	lastSun	0:00s	0	S
 Rule	Cuba	2007	only	-	Mar	Sun>=8	0:00s	1:00	D
 Rule	Cuba	2008	only	-	Mar	Sun>=15	0:00s	1:00	D
 Rule	Cuba	2009	2010	-	Mar	Sun>=8	0:00s	1:00	D
 Rule	Cuba	2011	only	-	Mar	Sun>=15	0:00s	1:00	D
+Rule	Cuba	2011	only	-	Nov	13	0:00s	0	S
 Rule	Cuba	2012	max	-	Mar	Sun>=8	0:00s	1:00	D
+Rule	Cuba	2012	max	-	Oct	lastSun	0:00s	0	S
 
 # Zone	NAME		GMTOFF	RULES	FORMAT	[UNTIL]
 Zone	America/Havana	-5:29:28 -	LMT	1890
diff --git a/contrib/tzdata/zone.tab b/contrib/tzdata/zone.tab
index 02166ee..76cc4ae 100644
--- a/contrib/tzdata/zone.tab
+++ b/contrib/tzdata/zone.tab
@@ -1,5 +1,5 @@
 # <pre>
-# @(#)zone.tab	8.50
+# @(#)zone.tab	8.52
 # This file is in the public domain, so clarified as of
 # 2009-05-17 by Arthur David Olson.
 #
@@ -257,8 +257,7 @@ LV	+5657+02406	Europe/Riga
 LY	+3254+01311	Africa/Tripoli
 MA	+3339-00735	Africa/Casablanca
 MC	+4342+00723	Europe/Monaco
-MD	+4700+02850	Europe/Chisinau	most locations
-MD	+4651+02938	Europe/Tiraspol	Pridnestrovie
+MD	+4700+02850	Europe/Chisinau
 ME	+4226+01916	Europe/Podgorica
 MF	+1804-06305	America/Marigot
 MG	-1855+04731	Indian/Antananarivo
diff --git a/etc/defaults/periodic.conf b/etc/defaults/periodic.conf
index 8d3d0f4..27192bc 100644
--- a/etc/defaults/periodic.conf
+++ b/etc/defaults/periodic.conf
@@ -150,8 +150,8 @@ daily_submit_queuerun="YES"				# Also submit queue
 # 800.scrub-zfs
 daily_scrub_zfs_enable="NO"
 daily_scrub_zfs_pools=""			# empty string selects all pools
-daily_scrub_zfs_default_threshold="30"		# days between scrubs
-#daily_scrub_zfs_${poolname}_threshold="30"	# pool specific threshold
+daily_scrub_zfs_default_threshold="35"		# days between scrubs
+#daily_scrub_zfs_${poolname}_threshold="35"	# pool specific threshold
 
 # 999.local
 daily_local="/etc/daily.local"				# Local scripts
diff --git a/etc/devfs.conf b/etc/devfs.conf
index f7869fe..d3d6075 100644
--- a/etc/devfs.conf
+++ b/etc/devfs.conf
@@ -31,11 +31,9 @@
 #
 # Examples:
 
-# Historically X depended on this, but version 4.3.0 doesn't seem to anymore
-#link	ttyv0	vga
-
 # Commonly used by many ports
-#link	acd0	cdrom
+#link	cd0	cdrom
+#link	cd0	dvd
 
 # Allow a user in the wheel group to query the smb0 device
 #perm	smb0	0660
diff --git a/etc/periodic/daily/310.accounting b/etc/periodic/daily/310.accounting
index b964616..d11745d 100755
--- a/etc/periodic/daily/310.accounting
+++ b/etc/periodic/daily/310.accounting
@@ -30,8 +30,13 @@ case "$daily_accounting_enable" in
 	    cd /var/account
 	    rc=0
 
-	    n=$daily_accounting_save
-	    rm -f acct.$n.gz acct.$n || rc=3
+	    n=$(( $daily_accounting_save - 1 ))
+	    for f in acct.*; do
+	    	case "$f" in acct.\*) continue ;; esac	# No files match
+	    	m=${f%.gz} ; m=${m#acct.}
+		[ $m -ge $n ] && { rm $f || rc=3; }
+	    done
+
 	    m=$n
 	    n=$(($n - 1))
 	    while [ $n -ge 0 ]
@@ -44,13 +49,14 @@ case "$daily_accounting_enable" in
 
 	    /etc/rc.d/accounting rotate_log || rc=3
 
+	    rm -f acct.merge && cp acct.0 acct.merge || rc=3
+	    sa -s $daily_accounting_flags /var/account/acct.merge || rc=3
+	    rm acct.merge
+
 	    case "$daily_accounting_compress" in
 		[Yy][Ee][Ss])
-		    gzip --keep -f acct.0 || rc=3;;
+		    gzip -f acct.0 || rc=3;;
 	    esac
-
-	    sa -s $daily_accounting_flags /var/account/acct.0 &&
-		unlink acct.0 || rc=3
 	fi;;
 
     *)  rc=0;;
diff --git a/etc/periodic/daily/800.scrub-zfs b/etc/periodic/daily/800.scrub-zfs
index b153b66..ee0e52a 100755
--- a/etc/periodic/daily/800.scrub-zfs
+++ b/etc/periodic/daily/800.scrub-zfs
@@ -15,7 +15,7 @@ then
     source_periodic_confs
 fi
 
-: ${daily_scrub_zfs_default_threshold=30}
+: ${daily_scrub_zfs_default_threshold=35}
 
 case "$daily_scrub_zfs_enable" in
     [Yy][Ee][Ss])
diff --git a/etc/rc.d/dhclient b/etc/rc.d/dhclient
index f842e12..adba369 100755
--- a/etc/rc.d/dhclient
+++ b/etc/rc.d/dhclient
@@ -15,9 +15,21 @@ name="dhclient"
 rcvar=
 pidfile="/var/run/${name}.${ifn}.pid"
 start_precmd="dhclient_prestart"
+stop_precmd="dhclient_pre_check"
+
+# rc_force check can only be done at the run_rc_command
+# time, so we're testing it in the pre* hooks.
+dhclient_pre_check()
+{
+	if [ -z "${rc_force}" ] && ! dhcpif $ifn; then
+		err 1 "'$ifn' is not a DHCP-enabled interface"
+	fi
+}
 
 dhclient_prestart()
 {
+	dhclient_pre_check
+
 	# Interface-specific flags (see rc.subr for $flags setting)
 	specific=$(get_if_var $ifn dhclient_flags_IF)
 	if [ -z "$flags" -a -n "$specific" ]; then
@@ -38,11 +50,8 @@ load_rc_config network
 if [ -z $ifn ] ; then
 	# only complain if a command was specified but no interface
 	if [ -n "$1" ] ; then
-		echo 1>&2 "$0: no interface specified"
-		return 1
+		err 1 "$0: no interface specified"
 	fi
-elif [ -z "${rc_force}" ] && ! dhcpif $ifn; then
-	return 1
 fi
 
 run_rc_command "$1"
diff --git a/etc/rc.d/nfsd b/etc/rc.d/nfsd
index 8dae57c..e1fd968 100755
--- a/etc/rc.d/nfsd
+++ b/etc/rc.d/nfsd
@@ -29,6 +29,8 @@ nfsd_precmd()
 		if checkyesno nfs_reserved_port_only; then
 			echo 'NFS on reserved port only=YES'
 			sysctl vfs.nfsrv.nfs_privport=1 > /dev/null
+		else
+			sysctl vfs.nfsrv.nfs_privport=0 > /dev/null
 		fi
 	else
 		rc_flags="${nfs_server_flags}"
@@ -40,9 +42,12 @@ nfsd_precmd()
 		if checkyesno nfs_reserved_port_only; then
 			echo 'NFS on reserved port only=YES'
 			sysctl vfs.nfsd.nfs_privport=1 > /dev/null
+		else
+			sysctl vfs.nfsd.nfs_privport=0 > /dev/null
 		fi
 
 		if checkyesno nfsv4_server_enable; then
+			sysctl vfs.nfsd.server_max_nfsvers=4 > /dev/null
 			if ! checkyesno nfsuserd_enable  && \
 			    ! /etc/rc.d/nfsuserd forcestatus 1>/dev/null 2>&1
 			then
diff --git a/etc/rc.d/routing b/etc/rc.d/routing
index 7d18a7c..616a19c 100755
--- a/etc/rc.d/routing
+++ b/etc/rc.d/routing
@@ -247,6 +247,7 @@ static_atm()
 
 static_ipx()
 {
+	:
 }
 
 ropts_init()
diff --git a/etc/rc.d/syscons b/etc/rc.d/syscons
index 815f0b2..f611e3b 100755
--- a/etc/rc.d/syscons
+++ b/etc/rc.d/syscons
@@ -99,7 +99,7 @@ syscons_configure_keyboard()
 	*)
 		sc_init
 		echo -n ' keychange'
-		set - ${keychange}
+		set -- ${keychange}
 		while [ $# -gt 0 ]; do
 			kbdcontrol <${kbddev} -f "$1" "$2"
 			shift; shift
diff --git a/etc/rc.resume b/etc/rc.resume
index 2cf1a60..3b2e970 100755
--- a/etc/rc.resume
+++ b/etc/rc.resume
@@ -49,6 +49,10 @@ if [ -r /var/run/rc.suspend.tch ]; then
 	/bin/rm -f /var/run/rc.suspend.tch
 fi
 
+if [ -r /var/run/moused.pid ]; then
+	pkill -HUP -F /var/run/moused.pid
+fi
+
 # Turns on a power supply of a card in the slot inactivated.
 # See also contrib/pccardq.c (only for PAO users).
 # pccardq | awk -F '~' '$5 == "inactive" \
diff --git a/etc/rc.subr b/etc/rc.subr
index 36cba6d..29ed3dd 100644
--- a/etc/rc.subr
+++ b/etc/rc.subr
@@ -1638,7 +1638,7 @@ find_local_scripts_old () {
 				    continue
 				zlist="$zlist $file"
 			done
-			for file in ${dir}/[^0-9]*.sh; do
+			for file in ${dir}/[!0-9]*.sh; do
 				grep '^# PROVIDE:' $file >/dev/null 2>&1 &&
 				    continue
 				slist="$slist $file"
diff --git a/games/bcd/bcd.c b/games/bcd/bcd.c
index a294ac3..056d238 100644
--- a/games/bcd/bcd.c
+++ b/games/bcd/bcd.c
@@ -82,7 +82,7 @@ static const char rcsid[] =
 #include <string.h>
 #include <unistd.h>
 
-u_short holes[256] = {
+static u_short holes[256] = {
     0x0,	 0x0,	  0x0,	   0x0,	    0x0,     0x0,     0x0,     0x0,
     0x0,	 0x0,	  0x0,	   0x0,	    0x0,     0x0,     0x0,     0x0,
     0x0,	 0x0,	  0x0,	   0x0,	    0x0,     0x0,     0x0,     0x0,
@@ -117,7 +117,7 @@ u_short holes[256] = {
     0x202,	 0x201,	  0x082,   0x806,   0x822,   0x600,   0x282,   0x0
 };
 
-void printcard(char *);
+static void printcard(char *);
 
 /*
  * i'th bit of w.
@@ -148,7 +148,7 @@ main(int argc, char **argv)
 
 #define	COLUMNS	48
 
-void
+static void
 printcard(char *str)
 {
 	static char rowchars[] = "   123456789";
diff --git a/games/caesar/caesar.c b/games/caesar/caesar.c
index a12f641..2415ca7 100644
--- a/games/caesar/caesar.c
+++ b/games/caesar/caesar.c
@@ -67,13 +67,13 @@ __FBSDID("$FreeBSD$");
  * letter frequencies (taken from some unix(tm) documentation)
  * (unix is a trademark of Bell Laboratories)
  */
-double stdf[26] = {
+static double stdf[26] = {
 	7.97, 1.35, 3.61, 4.78, 12.37, 2.01, 1.46, 4.49, 6.39, 0.04,
 	0.42, 3.81, 2.69, 5.92,  6.96, 2.91, 0.08, 6.63, 8.77, 9.68,
 	2.62, 0.81, 1.88, 0.23,  2.07, 0.06,
 };
 
-void printit(char *);
+static void printit(char *);
 
 int
 main(int argc, char **argv)
@@ -147,7 +147,7 @@ main(int argc, char **argv)
 	exit(0);
 }
 
-void
+static void
 printit(char *arg)
 {
 	int ch, rot;
diff --git a/games/fortune/datfiles/fortunes b/games/fortune/datfiles/fortunes
index 5a52b18..1d0ccf4 100644
--- a/games/fortune/datfiles/fortunes
+++ b/games/fortune/datfiles/fortunes
@@ -4942,25 +4942,37 @@ cats on the dinette table, etc.
 	"That stop him?"
 	"No, but it sure slowed him up."
 %
-	Youth is not a time of life, it is a state of mind; it is a temper of
-the will, a quality of the imagination, a vigor of the emotions, a predominance
-of courage over timidity, of the appetite for adventure over love of ease.
-	Nobody grows old by merely living a number of years; people grow
-old only by deserting their ideals.  Years wrinkle the skin, but to give up
-enthusiasm wrinkles the soul.  Worry, doubt, self-distrust, fear, and despair
--- these are the long, long years that bow the head and turn the growing spirit
-back to dust.
-	Whether seventy or sixteen, there is in every being's heart the love
-of wonder, the sweet amazement at the stars and the starlike things and
-thoughts, the undaunted challenge of events, the unfailing childlike appetite
-for what next, and the joy and the game of life.
-	You are as young as your faith, as old as your doubt; as young as your
-self-confidence, as old as your fear, as young as your hope, as old as your
-despair.
-	So long as your heart receives messages of beauty, cheer, courage,
-grandeur and power from the earth, from man, and from the Infinite, so long
-you are young.
-		-- Samuel Ullman
+	Youth is not a time of life--it is a state of mind. It is not a
+matter of red cheeks, red lips and supple knees. It is a temper of the
+will; a quality of the imagination; a vigor of the emotions; it is a
+freshness of the deep springs of life.  Youth means a tempermental
+predominance of courage over timidity, of the appetite for adventure
+over a life of ease.  This often exists in a man of fifty, more than in
+a boy of twenty.  Nobody grows old by merely living a number of years;
+people grow old by deserting their ideals.
+
+	Years may wrinkle the skin, but to give up enthusiasm wrinkles
+the soul.  Worry, doubt, self-distrust, fear and despair--these are the
+long, long years that bow the head and turn the growing spirit back to
+dust.
+
+	Whether seventy or sixteen, there is in every being’s heart a
+love of wonder; the sweet amazement at the stars and starlike things and
+thoughts; the undaunted challenge of events, the unfailing childlike
+appetite for what comes next, and the joy in the game of life.
+
+	You are as young as your faith, as old as your doubt; as young
+as your self-confidence, as old as your fear, as young as your hope, as
+old as your despair.
+
+	In the central place of your heart there is a wireless station.
+So long as it receives messages of beauty, hope, cheer, grandeur,
+courage, and power from the earth, from men and from the Infinite--so
+long are you young.  When the wires are all down and the central places
+of your heart are covered with the snows of pessimism and the ice of
+cynicism, then are you grown old, indeed!
+		-- Samuel Ullman, "Youth" (1934), as published in
+		   The Silver Treasury, Prose and Verse for Every Mood
 %
 " "
 		-- Charlie Chaplin
diff --git a/games/fortune/fortune/fortune.c b/games/fortune/fortune/fortune.c
index 65ae35e..e93b0e7 100644
--- a/games/fortune/fortune/fortune.c
+++ b/games/fortune/fortune/fortune.c
@@ -97,73 +97,71 @@ typedef struct fd {
 	struct fd	*next, *prev;
 } FILEDESC;
 
-bool	Found_one;			/* did we find a match? */
-bool	Find_files	= FALSE;	/* just find a list of proper fortune files */
-bool	Fortunes_only   = FALSE;	/* check only "fortunes" files */
-bool	Wait		= FALSE;	/* wait desired after fortune */
-bool	Short_only	= FALSE;	/* short fortune desired */
-bool	Long_only	= FALSE;	/* long fortune desired */
-bool	Offend		= FALSE;	/* offensive fortunes only */
-bool	All_forts	= FALSE;	/* any fortune allowed */
-bool	Equal_probs	= FALSE;	/* scatter un-allocted prob equally */
-bool	Match		= FALSE;	/* dump fortunes matching a pattern */
+static bool	Found_one;		/* did we find a match? */
+static bool	Find_files = FALSE;	/* just find a list of proper fortune files */
+static bool	Fortunes_only = FALSE;	/* check only "fortunes" files */
+static bool	Wait = FALSE;		/* wait desired after fortune */
+static bool	Short_only = FALSE;	/* short fortune desired */
+static bool	Long_only = FALSE;	/* long fortune desired */
+static bool	Offend = FALSE;		/* offensive fortunes only */
+static bool	All_forts = FALSE;	/* any fortune allowed */
+static bool	Equal_probs = FALSE;	/* scatter un-allocted prob equally */
+static bool	Match = FALSE;		/* dump fortunes matching a pattern */
 #ifdef DEBUG
-bool	Debug = FALSE;			/* print debug messages */
+static bool	Debug = FALSE;		/* print debug messages */
 #endif
 
-char	*Fortbuf = NULL;			/* fortune buffer for -m */
+static char	*Fortbuf = NULL;	/* fortune buffer for -m */
 
-int	Fort_len = 0;
+static int	Fort_len = 0;
 
-off_t	Seekpts[2];                     /* seek pointers to fortunes */
+static off_t	Seekpts[2];		/* seek pointers to fortunes */
 
-FILEDESC	*File_list = NULL,	/* Head of file list */
+static FILEDESC	*File_list = NULL,	/* Head of file list */
 		*File_tail = NULL;	/* Tail of file list */
-FILEDESC	*Fortfile;		/* Fortune file to use */
-
-STRFILE		Noprob_tbl;		/* sum of data for all no prob files */
-
-const char	*Fortune_path;
-char	**Fortune_path_arr;
-
-int	 add_dir(FILEDESC *);
-int	 add_file(int, const char *, const char *, FILEDESC **, FILEDESC **,
-	    FILEDESC *);
-void	 all_forts(FILEDESC *, char *);
-char	*copy(const char *, u_int);
-void	 display(FILEDESC *);
-void	 do_free(void *);
-void	*do_malloc(u_int);
-int	 form_file_list(char **, int);
-int	 fortlen(void);
-void	 get_fort(void);
-void	 get_pos(FILEDESC *);
-void	 get_tbl(FILEDESC *);
-void	 getargs(int, char *[]);
-void	 getpath(void);
-void	 init_prob(void);
-int	 is_dir(const char *);
-int	 is_fortfile(const char *, char **, char **, int);
-int	 is_off_name(const char *);
-int	 max(int, int);
-FILEDESC *
-	 new_fp(void);
-char	*off_name(const char *);
-void	 open_dat(FILEDESC *);
-void	 open_fp(FILEDESC *);
-FILEDESC *
-	 pick_child(FILEDESC *);
-void	 print_file_list(void);
-void	 print_list(FILEDESC *, int);
-void	 sum_noprobs(FILEDESC *);
-void	 sum_tbl(STRFILE *, STRFILE *);
-void	 usage(void);
-void	 zero_tbl(STRFILE *);
-
-char	*conv_pat(char *);
-int	 find_matches(void);
-void	 matches_in_list(FILEDESC *);
-int	 maxlen_in_list(FILEDESC *);
+static FILEDESC	*Fortfile;		/* Fortune file to use */
+
+static STRFILE	Noprob_tbl;		/* sum of data for all no prob files */
+
+static const char *Fortune_path;
+static char	**Fortune_path_arr;
+
+static int	 add_dir(FILEDESC *);
+static int	 add_file(int, const char *, const char *, FILEDESC **,
+		     FILEDESC **, FILEDESC *);
+static void	 all_forts(FILEDESC *, char *);
+static char	*copy(const char *, u_int);
+static void	 display(FILEDESC *);
+static void	 do_free(void *);
+static void	*do_malloc(u_int);
+static int	 form_file_list(char **, int);
+static int	 fortlen(void);
+static void	 get_fort(void);
+static void	 get_pos(FILEDESC *);
+static void	 get_tbl(FILEDESC *);
+static void	 getargs(int, char *[]);
+static void	 getpath(void);
+static void	 init_prob(void);
+static int	 is_dir(const char *);
+static int	 is_fortfile(const char *, char **, char **, int);
+static int	 is_off_name(const char *);
+static int	 max(int, int);
+static FILEDESC *new_fp(void);
+static char	*off_name(const char *);
+static void	 open_dat(FILEDESC *);
+static void	 open_fp(FILEDESC *);
+static FILEDESC *pick_child(FILEDESC *);
+static void	 print_file_list(void);
+static void	 print_list(FILEDESC *, int);
+static void	 sum_noprobs(FILEDESC *);
+static void	 sum_tbl(STRFILE *, STRFILE *);
+static void	 usage(void);
+static void	 zero_tbl(STRFILE *);
+
+static char	*conv_pat(char *);
+static int	 find_matches(void);
+static void	 matches_in_list(FILEDESC *);
+static int	 maxlen_in_list(FILEDESC *);
 
 static regex_t Re_pat;
 
@@ -219,7 +217,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 display(FILEDESC *fp)
 {
 	char   *p;
@@ -252,7 +250,7 @@ display(FILEDESC *fp)
  * fortlen:
  *	Return the length of the fortune.
  */
-int
+static int
 fortlen(void)
 {
 	int	nchar;
@@ -276,7 +274,7 @@ fortlen(void)
 /*
  *	This routine evaluates the arguments on the command line
  */
-void
+static void
 getargs(int argc, char *argv[])
 {
 	int	ignore_case;
@@ -362,7 +360,7 @@ getargs(int argc, char *argv[])
  * form_file_list:
  *	Form the file list from the file specifications.
  */
-int
+static int
 form_file_list(char **files, int file_cnt)
 {
 	int	i, percent;
@@ -456,7 +454,7 @@ form_file_list(char **files, int file_cnt)
  * add_file:
  *	Add a file to the file list.
  */
-int
+static int
 add_file(int percent, const char *file, const char *dir, FILEDESC **head,
     FILEDESC **tail, FILEDESC *parent)
 {
@@ -598,7 +596,7 @@ over:
  * new_fp:
  *	Return a pointer to an initialized new FILEDESC.
  */
-FILEDESC *
+static FILEDESC *
 new_fp(void)
 {
 	FILEDESC	*fp;
@@ -624,7 +622,7 @@ new_fp(void)
  * off_name:
  *	Return a pointer to the offensive version of a file of this name.
  */
-char *
+static char *
 off_name(const char *file)
 {
 	char	*new;
@@ -638,7 +636,7 @@ off_name(const char *file)
  * is_off_name:
  *	Is the file an offensive-style name?
  */
-int
+static int
 is_off_name(const char *file)
 {
 	int	len;
@@ -653,7 +651,7 @@ is_off_name(const char *file)
  *	Modify a FILEDESC element to be the parent of two children if
  *	there are two children to be a parent of.
  */
-void
+static void
 all_forts(FILEDESC *fp, char *offensive)
 {
 	char		*sp;
@@ -701,7 +699,7 @@ all_forts(FILEDESC *fp, char *offensive)
  * add_dir:
  *	Add the contents of an entire directory.
  */
-int
+static int
 add_dir(FILEDESC *fp)
 {
 	DIR		*dir;
@@ -740,7 +738,7 @@ add_dir(FILEDESC *fp)
  * is_dir:
  *	Return TRUE if the file is a directory, FALSE otherwise.
  */
-int
+static int
 is_dir(const char *file)
 {
 	struct stat	sbuf;
@@ -759,7 +757,7 @@ is_dir(const char *file)
  *	suffixes, as contained in suflist[], are ruled out.
  */
 /* ARGSUSED */
-int
+static int
 is_fortfile(const char *file, char **datp, char **posp, int check_for_offend)
 {
 	int	i;
@@ -840,7 +838,7 @@ is_fortfile(const char *file, char **datp, char **posp, int check_for_offend)
  * copy:
  *	Return a malloc()'ed copy of the string
  */
-char *
+static char *
 copy(const char *str, unsigned int len)
 {
 	char *new, *sp;
@@ -858,7 +856,7 @@ copy(const char *str, unsigned int len)
  * do_malloc:
  *	Do a malloc, checking for NULL return.
  */
-void *
+static void *
 do_malloc(unsigned int size)
 {
 	void *new;
@@ -875,7 +873,7 @@ do_malloc(unsigned int size)
  * do_free:
  *	Free malloc'ed space, if any.
  */
-void
+static void
 do_free(void *ptr)
 {
 	if (ptr != NULL)
@@ -886,7 +884,7 @@ do_free(void *ptr)
  * init_prob:
  *	Initialize the fortune probabilities.
  */
-void
+static void
 init_prob(void)
 {
 	FILEDESC       *fp, *last = NULL;
@@ -955,7 +953,7 @@ init_prob(void)
  * get_fort:
  *	Get the fortune data file's seek pointer for the next fortune.
  */
-void
+static void
 get_fort(void)
 {
 	FILEDESC	*fp;
@@ -1018,7 +1016,7 @@ get_fort(void)
  * pick_child
  *	Pick a child from a chosen parent.
  */
-FILEDESC *
+static FILEDESC *
 pick_child(FILEDESC *parent)
 {
 	FILEDESC	*fp;
@@ -1054,7 +1052,7 @@ pick_child(FILEDESC *parent)
  * sum_noprobs:
  *	Sum up all the noprob probabilities, starting with fp.
  */
-void
+static void
 sum_noprobs(FILEDESC *fp)
 {
 	static bool	did_noprobs = FALSE;
@@ -1070,7 +1068,7 @@ sum_noprobs(FILEDESC *fp)
 	did_noprobs = TRUE;
 }
 
-int
+static int
 max(int i, int j)
 {
 	return (i >= j ? i : j);
@@ -1080,7 +1078,7 @@ max(int i, int j)
  * open_fp:
  *	Assocatiate a FILE * with the given FILEDESC.
  */
-void
+static void
 open_fp(FILEDESC *fp)
 {
 	if (fp->inf == NULL && (fp->inf = fdopen(fp->fd, "r")) == NULL) {
@@ -1093,7 +1091,7 @@ open_fp(FILEDESC *fp)
  * open_dat:
  *	Open up the dat file if we need to.
  */
-void
+static void
 open_dat(FILEDESC *fp)
 {
 	if (fp->datfd < 0 && (fp->datfd = open(fp->datfile, O_RDONLY)) < 0) {
@@ -1107,7 +1105,7 @@ open_dat(FILEDESC *fp)
  *	Get the position from the pos file, if there is one.  If not,
  *	return a random number.
  */
-void
+static void
 get_pos(FILEDESC *fp)
 {
 #ifdef OK_TO_WRITE_DISK
@@ -1137,7 +1135,7 @@ get_pos(FILEDESC *fp)
  * get_tbl:
  *	Get the tbl data file the datfile.
  */
-void
+static void
 get_tbl(FILEDESC *fp)
 {
 	int		fd;
@@ -1176,7 +1174,7 @@ get_tbl(FILEDESC *fp)
  * zero_tbl:
  *	Zero out the fields we care about in a tbl structure.
  */
-void
+static void
 zero_tbl(STRFILE *tp)
 {
 	tp->str_numstr = 0;
@@ -1188,7 +1186,7 @@ zero_tbl(STRFILE *tp)
  * sum_tbl:
  *	Merge the tbl data of t2 into t1.
  */
-void
+static void
 sum_tbl(STRFILE *t1, STRFILE *t2)
 {
 	t1->str_numstr += t2->str_numstr;
@@ -1204,7 +1202,7 @@ sum_tbl(STRFILE *t1, STRFILE *t2)
  * print_file_list:
  *	Print out the file list
  */
-void
+static void
 print_file_list(void)
 {
 	print_list(File_list, 0);
@@ -1214,7 +1212,7 @@ print_file_list(void)
  * print_list:
  *	Print out the actual list, recursively.
  */
-void
+static void
 print_list(FILEDESC *list, int lev)
 {
 	while (list != NULL) {
@@ -1237,7 +1235,7 @@ print_list(FILEDESC *list, int lev)
  * conv_pat:
  *	Convert the pattern to an ignore-case equivalent.
  */
-char *
+static char *
 conv_pat(char *orig)
 {
 	char		*sp;
@@ -1280,7 +1278,7 @@ conv_pat(char *orig)
  * find_matches:
  *	Find all the fortunes which match the pattern we've been given.
  */
-int
+static int
 find_matches(void)
 {
 	Fort_len = maxlen_in_list(File_list);
@@ -1298,7 +1296,7 @@ find_matches(void)
  * maxlen_in_list
  *	Return the maximum fortune len in the file list.
  */
-int
+static int
 maxlen_in_list(FILEDESC *list)
 {
 	FILEDESC	*fp;
@@ -1324,7 +1322,7 @@ maxlen_in_list(FILEDESC *list)
  * matches_in_list
  *	Print out the matches from the files in the list.
  */
-void
+static void
 matches_in_list(FILEDESC *list)
 {
 	char           *sp, *p;
@@ -1375,7 +1373,7 @@ matches_in_list(FILEDESC *list)
 	}
 }
 
-void
+static void
 usage(void)
 {
 	(void) fprintf(stderr, "fortune [-a");
@@ -1394,7 +1392,7 @@ usage(void)
  *	if not set, use the compiled in FORTDIR.
  */
 
-void
+static void
 getpath(void)
 {
 	int	nstr, foundenv;
diff --git a/games/fortune/unstr/unstr.c b/games/fortune/unstr/unstr.c
index da799a0..f79a0ec 100644
--- a/games/fortune/unstr/unstr.c
+++ b/games/fortune/unstr/unstr.c
@@ -67,13 +67,13 @@ __FBSDID("$FreeBSD$");
 
 #include "strfile.h"
 
-char	*Infile,			/* name of input file */
-	Datafile[MAXPATHLEN],		/* name of data file */
-	Delimch;			/* delimiter character */
+static char	*Infile,		/* name of input file */
+		Datafile[MAXPATHLEN],	/* name of data file */
+		Delimch;		/* delimiter character */
 
-FILE	*Inf, *Dataf;
+static FILE	*Inf, *Dataf;
 
-void order_unstr(STRFILE *);
+static void order_unstr(STRFILE *);
 
 /* ARGSUSED */
 int
@@ -107,7 +107,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 order_unstr(STRFILE *tbl)
 {
 	uint32_t i;
diff --git a/games/grdc/grdc.c b/games/grdc/grdc.c
index 6881cf1..4632c79 100644
--- a/games/grdc/grdc.c
+++ b/games/grdc/grdc.c
@@ -26,29 +26,30 @@
 #define XLENGTH 58
 #define YDEPTH  7
 
-struct timespec now;
-struct tm *tm;
+static struct timespec now;
+static struct tm *tm;
 
-short disp[11] = {
+static short disp[11] = {
 	075557, 011111, 071747, 071717, 055711,
 	074717, 074757, 071111, 075757, 075717, 002020
 };
-long old[6], next[6], new[6], mask;
+static long old[6], next[6], new[6], mask;
 
-volatile sig_atomic_t sigtermed;
+static volatile sig_atomic_t sigtermed;
 
-int hascolor = 0;
+static int hascolor = 0;
 
-void set(int, int);
-void standt(int);
-void movto(int, int);
-void sighndl(int);
-void usage(void);
+static void set(int, int);
+static void standt(int);
+static void movto(int, int);
+static void sighndl(int);
+static void usage(void);
 
-void
+static void
 sighndl(int signo)
 {
-	sigtermed=signo;
+
+	sigtermed = signo;
 }
 
 int
@@ -225,7 +226,7 @@ main(int argc, char *argv[])
 	return(0);
 }
 
-void
+static void
 set(int t, int n)
 {
 	int i, m;
@@ -239,7 +240,7 @@ set(int t, int n)
 		mask |= m;
 }
 
-void
+static void
 standt(int on)
 {
 	if (on) {
@@ -257,13 +258,13 @@ standt(int on)
 	}
 }
 
-void
+static void
 movto(int line, int col)
 {
 	move(line, col);
 }
 
-void
+static void
 usage(void)
 {
 
diff --git a/games/morse/morse.c b/games/morse/morse.c
index f0abae39..f6c95ee 100644
--- a/games/morse/morse.c
+++ b/games/morse/morse.c
@@ -261,33 +261,33 @@ static const struct morsetab koi8rtab[] = {
 	{'\0', ""}
 };
 
-void            show(const char *), play(const char *), morse(char);
-void		ttyout(const char *);
-void		sighandler(int);
+static void	show(const char *), play(const char *), morse(char);
+static void	ttyout(const char *);
+static void	sighandler(int);
 
 #define GETOPTOPTS "c:d:ef:lsw:"
 #define USAGE \
 "usage: morse [-els] [-d device] [-w speed] [-c speed] [-f frequency] [string ...]\n"
 
-static int      pflag, lflag, sflag, eflag;
-static int      wpm = 20;	/* effective words per minute */
-static int      cpm;		/* effective words per minute between
+static int	pflag, lflag, sflag, eflag;
+static int	wpm = 20;	/* effective words per minute */
+static int	cpm;		/* effective words per minute between
 				 * characters */
 #define FREQUENCY 600
-static int      freq = FREQUENCY;
+static int	freq = FREQUENCY;
 static char	*device;	/* for tty-controlled generator */
 
 #define DASH_LEN 3
 #define CHAR_SPACE 3
 #define WORD_SPACE (7 - CHAR_SPACE - 1)
-static float    dot_clock;
-static float    cdot_clock;
-int             spkr, line;
-struct termios	otty, ntty;
-int		olflags;
+static float	dot_clock;
+static float	cdot_clock;
+static int	spkr, line;
+static struct termios otty, ntty;
+static int	olflags;
 
 #ifdef SPEAKER
-tone_t          sound;
+static tone_t	sound;
 #undef GETOPTOPTS
 #define GETOPTOPTS "c:d:ef:lpsw:"
 #undef USAGE
@@ -439,7 +439,7 @@ main(int argc, char **argv)
 	exit(0);
 }
 
-void
+static void
 morse(char c)
 {
 	const struct morsetab *m;
@@ -473,7 +473,7 @@ morse(char c)
 	}
 }
 
-void
+static void
 show(const char *s)
 {
 	if (lflag) {
@@ -488,7 +488,7 @@ show(const char *s)
 	}
 }
 
-void
+static void
 play(const char *s)
 {
 #ifdef SPEAKER
@@ -530,7 +530,7 @@ play(const char *s)
 #endif
 }
 
-void
+static void
 ttyout(const char *s)
 {
 	const char *c;
@@ -572,7 +572,7 @@ ttyout(const char *s)
 	usleep(duration);
 }
 
-void
+static void
 sighandler(int signo)
 {
 
diff --git a/games/number/number.c b/games/number/number.c
index 2ffc87c..e8cf181 100644
--- a/games/number/number.c
+++ b/games/number/number.c
@@ -74,14 +74,13 @@ static const char	*name1[] = {
 	"novemdecillion",		"vigintillion",
 };
 
-void	convert(char *);
-int	number(char *, int);
-void	pfract(int);
-void	toobig(void);
-int	unit(int, char *);
-void	usage(void);
+static void	convert(char *);
+static int	number(char *, int);
+static void	pfract(int);
+static int	unit(int, char *);
+static void	usage(void);
 
-int lflag;
+static int lflag;
 
 int
 main(int argc, char *argv[])
@@ -120,7 +119,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 convert(char *line)
 {
 	int flen, len, rval;
@@ -188,7 +187,7 @@ badnum:			errx(1, "illegal number: %s", line);
 		(void)printf("\n");
 }
 
-int
+static int
 unit(int len, char *p)
 {
 	int off, rval;
@@ -222,7 +221,7 @@ unit(int len, char *p)
 	return (rval);
 }
 
-int
+static int
 number(char *p, int len)
 {
 	int val, rval;
@@ -260,7 +259,7 @@ number(char *p, int len)
 	return (rval);
 }
 
-void
+static void
 pfract(int len)
 {
 	static char const * const pref[] = { "", "ten-", "hundred-" };
@@ -278,7 +277,7 @@ pfract(int len)
 	}
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr, "usage: number [-l] [# ...]\n");
diff --git a/gnu/lib/libstdc++/config.h b/gnu/lib/libstdc++/config.h
index 4109752..a4afbdb 100644
--- a/gnu/lib/libstdc++/config.h
+++ b/gnu/lib/libstdc++/config.h
@@ -671,7 +671,7 @@
 /* #undef VERSION */
 
 /* Define if builtin atomic operations are supported on this host. */
-#if defined(__amd64__) || defined(__i386__)
+#if defined(__amd64__) || (defined(__i386__) && !defined(__tune_i386__))
 #define _GLIBCXX_ATOMIC_BUILTINS 1
 #endif
 
diff --git a/gnu/usr.bin/binutils/ld/Makefile.powerpc b/gnu/usr.bin/binutils/ld/Makefile.powerpc
index 921b88c..fed5234 100644
--- a/gnu/usr.bin/binutils/ld/Makefile.powerpc
+++ b/gnu/usr.bin/binutils/ld/Makefile.powerpc
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-NATIVE_EMULATION=	elf32ppc
+NATIVE_EMULATION=	elf32ppc_fbsd
 
 SRCS+=		e${NATIVE_EMULATION}.c
 CLEANFILES+=	e${NATIVE_EMULATION}.c
diff --git a/gnu/usr.bin/binutils/ld/Makefile.powerpc64 b/gnu/usr.bin/binutils/ld/Makefile.powerpc64
index 8c607d7..29fba2a 100644
--- a/gnu/usr.bin/binutils/ld/Makefile.powerpc64
+++ b/gnu/usr.bin/binutils/ld/Makefile.powerpc64
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-NATIVE_EMULATION=	elf64ppc
+NATIVE_EMULATION=	elf64ppc_fbsd
 
 SRCS+=		e${NATIVE_EMULATION}.c
 CLEANFILES+=	e${NATIVE_EMULATION}.c
@@ -11,7 +11,7 @@ e${NATIVE_EMULATION}.c: emulparams/${NATIVE_EMULATION}.sh emultempl/elf32.em \
 	    ${HOST} ${TARGET_TUPLE} ${TARGET_TUPLE} \
 	    ${NATIVE_EMULATION} ""  no ${NATIVE_EMULATION} ${TARGET_TUPLE}
 
-PPC32_EMULATION= elf32ppc
+PPC32_EMULATION= elf32ppc_fbsd
 _ppc32_path=	\"${TOOLS_PREFIX}/usr/lib32\"
 EMS+=		${PPC32_EMULATION}
 .for ext in ${ELF_SCR_EXT}
diff --git a/include/Makefile b/include/Makefile
index 3e6705c..c6fb526 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -24,7 +24,7 @@ INCS=	a.out.h ar.h assert.h bitstring.h complex.h cpio.h _ctype.h ctype.h \
 	strings.h sysexits.h tar.h termios.h tgmath.h \
 	time.h timeconv.h timers.h ttyent.h \
 	ulimit.h unistd.h utime.h utmpx.h uuid.h varargs.h vis.h \
-	wchar.h wctype.h wordexp.h
+	wchar.h wctype.h wordexp.h xlocale.h _xlocale_ctype.h
 
 MHDRS=	float.h floatingpoint.h stdarg.h
 
diff --git a/include/_xlocale_ctype.h b/include/_xlocale_ctype.h
new file mode 100644
index 0000000..1b2e21b
--- /dev/null
+++ b/include/_xlocale_ctype.h
@@ -0,0 +1,162 @@
+/*-
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by David Chisnall under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions * are met:
+ * 1.  Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#ifndef _XLOCALE_H_
+#error This header should only be included by <xlocale.h>, never directly.
+#endif
+
+#ifndef _XLOCALE_CTYPE_H_
+__BEGIN_DECLS
+unsigned long	___runetype_l(__ct_rune_t, locale_t) __pure;
+__ct_rune_t	___tolower_l(__ct_rune_t, locale_t) __pure;
+__ct_rune_t	___toupper_l(__ct_rune_t, locale_t) __pure;
+_RuneLocale	*__runes_for_locale(locale_t, int*);
+__END_DECLS
+#endif
+
+#ifndef _XLOCALE_INLINE
+#if __GNUC__ && !__GNUC_STDC_INLINE__
+#define _XLOCALE_INLINE extern inline
+#else
+#define _XLOCALE_INLINE inline
+#endif
+#endif
+
+#ifdef XLOCALE_WCTYPES
+static __inline int
+__maskrune_l(__ct_rune_t _c, unsigned long _f, locale_t locale)
+{
+	int mb_sb_limit;
+	_RuneLocale *runes = __runes_for_locale(locale, &mb_sb_limit);
+	return (_c < 0 || _c >= _CACHED_RUNES) ? ___runetype_l(_c, locale) :
+	       runes->__runetype[_c] & _f;
+}
+
+static __inline int
+__istype_l(__ct_rune_t _c, unsigned long _f, locale_t locale)
+{
+	return (!!__maskrune_l(_c, _f, locale));
+}
+
+#define XLOCALE_ISCTYPE(fname, cat) \
+		_XLOCALE_INLINE int isw##fname##_l(int c, locale_t l)\
+		{ return __istype_l(c, cat, l); }
+#else
+static __inline int
+__sbmaskrune_l(__ct_rune_t _c, unsigned long _f, locale_t locale)
+{
+	int mb_sb_limit;
+	_RuneLocale *runes = __runes_for_locale(locale, &mb_sb_limit);
+	return (_c < 0 || _c >= mb_sb_limit) ? 0 :
+	       runes->__runetype[_c] & _f;
+}
+
+static __inline int
+__sbistype_l(__ct_rune_t _c, unsigned long _f, locale_t locale)
+{
+	return (!!__sbmaskrune_l(_c, _f, locale));
+}
+
+#define XLOCALE_ISCTYPE(fname, cat) \
+		_XLOCALE_INLINE int is##fname##_l(int c, locale_t l)\
+		{ return __sbistype_l(c, cat, l); }
+#endif
+
+XLOCALE_ISCTYPE(alnum, _CTYPE_A|_CTYPE_D)
+XLOCALE_ISCTYPE(alpha, _CTYPE_A)
+XLOCALE_ISCTYPE(blank, _CTYPE_B)
+XLOCALE_ISCTYPE(cntrl, _CTYPE_C)
+XLOCALE_ISCTYPE(digit, _CTYPE_D)
+XLOCALE_ISCTYPE(graph, _CTYPE_G)
+XLOCALE_ISCTYPE(hexnumber, _CTYPE_X)
+XLOCALE_ISCTYPE(ideogram, _CTYPE_I)
+XLOCALE_ISCTYPE(lower, _CTYPE_L)
+XLOCALE_ISCTYPE(number, _CTYPE_D)
+XLOCALE_ISCTYPE(phonogram, _CTYPE_Q)
+XLOCALE_ISCTYPE(print, _CTYPE_R)
+XLOCALE_ISCTYPE(punct, _CTYPE_P)
+XLOCALE_ISCTYPE(rune, 0xFFFFFF00L)
+XLOCALE_ISCTYPE(space, _CTYPE_S)
+XLOCALE_ISCTYPE(special, _CTYPE_T)
+XLOCALE_ISCTYPE(upper, _CTYPE_U)
+XLOCALE_ISCTYPE(xdigit, _CTYPE_X)
+#undef XLOCALE_ISCTYPE
+
+#ifdef XLOCALE_WCTYPES
+_XLOCALE_INLINE int towlower_l(int c, locale_t locale)
+{
+	int mb_sb_limit;
+	_RuneLocale *runes = __runes_for_locale(locale, &mb_sb_limit);
+	return (c < 0 || c >= _CACHED_RUNES) ? ___tolower_l(c, locale) :
+	       runes->__maplower[c];
+}
+_XLOCALE_INLINE int towupper_l(int c, locale_t locale)
+{
+	int mb_sb_limit;
+	_RuneLocale *runes = __runes_for_locale(locale, &mb_sb_limit);
+	return (c < 0 || c >= _CACHED_RUNES) ? ___toupper_l(c, locale) :
+	       runes->__mapupper[c];
+}
+_XLOCALE_INLINE int
+__wcwidth_l(__ct_rune_t _c, locale_t locale)
+{
+	unsigned int _x;
+
+	if (_c == 0)
+		return (0);
+	_x = (unsigned int)__maskrune_l(_c, _CTYPE_SWM|_CTYPE_R, locale);
+	if ((_x & _CTYPE_SWM) != 0)
+		return ((_x & _CTYPE_SWM) >> _CTYPE_SWS);
+	return ((_x & _CTYPE_R) != 0 ? 1 : -1);
+}
+int iswctype_l(wint_t wc, wctype_t charclass, locale_t locale);
+wctype_t wctype_l(const char *property, locale_t locale);
+wint_t towctrans_l(wint_t wc, wctrans_t desc, locale_t locale);
+wint_t nextwctype_l(wint_t wc, wctype_t wct, locale_t locale);
+wctrans_t wctrans_l(const char *charclass, locale_t locale);
+#undef XLOCALE_WCTYPES
+#else
+_XLOCALE_INLINE int digittoint_l(int c, locale_t locale)
+{ return __sbmaskrune_l((c), 0xFF, locale); }
+
+_XLOCALE_INLINE int tolower_l(int c, locale_t locale)
+{
+	int mb_sb_limit;
+	_RuneLocale *runes = __runes_for_locale(locale, &mb_sb_limit);
+	return (c < 0 || c >= mb_sb_limit) ? c :
+	       runes->__maplower[c];
+}
+_XLOCALE_INLINE int toupper_l(int c, locale_t locale)
+{
+	int mb_sb_limit;
+	_RuneLocale *runes = __runes_for_locale(locale, &mb_sb_limit);
+	return (c < 0 || c >= mb_sb_limit) ? c :
+	       runes->__mapupper[c];
+}
+#endif
diff --git a/include/ctype.h b/include/ctype.h
index 94bc8ab..c3f80f3 100644
--- a/include/ctype.h
+++ b/include/ctype.h
@@ -80,6 +80,7 @@ int	isspecial(int);
 #endif
 __END_DECLS
 
+#ifndef __cplusplus
 #define	isalnum(c)	__sbistype((c), _CTYPE_A|_CTYPE_D)
 #define	isalpha(c)	__sbistype((c), _CTYPE_A)
 #define	iscntrl(c)	__sbistype((c), _CTYPE_C)
@@ -93,6 +94,7 @@ __END_DECLS
 #define	isxdigit(c)	__isctype((c), _CTYPE_X) /* ANSI -- locale independent */
 #define	tolower(c)	__sbtolower(c)
 #define	toupper(c)	__sbtoupper(c)
+#endif /* !__cplusplus */
 
 #if __XSI_VISIBLE
 /*
@@ -112,7 +114,7 @@ __END_DECLS
 #define	toascii(c)	((c) & 0x7F)
 #endif
 
-#if __ISO_C_VISIBLE >= 1999
+#if __ISO_C_VISIBLE >= 1999 && !defined(__cplusplus)
 #define	isblank(c)	__sbistype((c), _CTYPE_B)
 #endif
 
diff --git a/include/locale.h b/include/locale.h
index 83620f8..8df0a6a 100644
--- a/include/locale.h
+++ b/include/locale.h
@@ -79,4 +79,52 @@ struct lconv	*localeconv(void);
 char		*setlocale(int, const char *);
 __END_DECLS
 
+#if __POSIX_VISIBLE >= 200809
+
+#define LC_COLLATE_MASK  (1<<0)
+#define LC_CTYPE_MASK    (1<<1)
+#define LC_MESSAGES_MASK (1<<2)
+#define LC_MONETARY_MASK (1<<3)
+#define LC_NUMERIC_MASK  (1<<4)
+#define LC_TIME_MASK     (1<<5)
+#define LC_ALL_MASK      (LC_COLLATE_MASK | LC_CTYPE_MASK | LC_MESSAGES_MASK | \
+		LC_MONETARY_MASK | LC_NUMERIC_MASK | LC_TIME_MASK)
+
+#define LC_GLOBAL_LOCALE ((locale_t)-1)
+
+__BEGIN_DECLS
+
+typedef struct _xlocale *locale_t;
+/**
+ * Creates a new locale.  
+ */
+locale_t	 newlocale(int mask, const char *locale, locale_t base);
+
+/**
+ * Returns an identical duplicate of the passed locale.  The returned locale
+ * must be freed with freelocale().  The returned locale will share components
+ * with the original.
+ */
+locale_t	 duplocale(locale_t base);
+/*
+ * Free a locale_t.  This is quite a poorly named function.  It actually
+ * disclaims a reference to a locale_t, rather than freeing it.  
+ */
+int	 freelocale(locale_t loc);
+
+/*
+ * Returns the name of the locale for a particular component of a locale_t.
+ */
+const char	*querylocale(int mask, locale_t loc);
+
+/*
+ * Installs the specified locale_t as this thread's locale.
+ */
+locale_t	 uselocale(locale_t loc);
+
+__END_DECLS
+
+#endif /* __POSIX_VISIBLE >= 200809 */
+
+
 #endif /* _LOCALE_H_ */
diff --git a/include/runetype.h b/include/runetype.h
index a63d909..c33ea2b 100644
--- a/include/runetype.h
+++ b/include/runetype.h
@@ -83,8 +83,14 @@ typedef struct {
 } _RuneLocale;
 
 #define	_RUNE_MAGIC_1	"RuneMagi"	/* Indicates version 0 of RuneLocale */
-
-extern _RuneLocale _DefaultRuneLocale;
+__BEGIN_DECLS
+extern const _RuneLocale _DefaultRuneLocale;
+__attribute__((deprecated))
 extern _RuneLocale *_CurrentRuneLocale;
+/* TODO: This is called quite a lot, so we should use a __thread variable when
+ * it's available. */
+extern _RuneLocale *__getCurrentRuneLocale(void);
+#define _CurrentRuneLocale (__getCurrentRuneLocale())
+__END_DECLS
 
 #endif	/* !_RUNETYPE_H_ */
diff --git a/include/stdio.h b/include/stdio.h
index ea3dea8e..bbeb0de 100644
--- a/include/stdio.h
+++ b/include/stdio.h
@@ -107,7 +107,7 @@ struct __sbuf {
  * inline functions.  To preserve ABI compat, these members must not
  * be disturbed.  These members are marked below with (*).
  */
-typedef	struct __sFILE {
+struct __sFILE {
 	unsigned char *_p;	/* (*) current position in (some) buffer */
 	int	_r;		/* (*) read space left for getc() */
 	int	_w;		/* (*) write space left for putc() */
@@ -144,8 +144,11 @@ typedef	struct __sFILE {
 	int	_fl_count;	/* recursive lock count */
 	int	_orientation;	/* orientation for fwide() */
 	__mbstate_t _mbstate;	/* multibyte conversion state */
-} FILE;
-
+};
+#ifndef _STDFILE_DECLARED
+#define _STDFILE_DECLARED
+typedef struct __sFILE FILE;
+#endif
 #ifndef _STDSTREAM_DECLARED
 __BEGIN_DECLS
 extern FILE *__stdinp;
@@ -467,6 +470,7 @@ static __inline int __sputc(int _c, FILE *_p) {
 			__swbuf((int)(c), p) : \
 		(*(p)->_p = (c), (int)*(p)->_p++))
 #endif
+#ifndef __cplusplus
 
 #define	__sfeof(p)	(((p)->_flags & __SEOF) != 0)
 #define	__sferror(p)	(((p)->_flags & __SERR) != 0)
@@ -506,6 +510,7 @@ extern int __isthreaded;
 #define	getchar_unlocked()	getc_unlocked(stdin)
 #define	putchar_unlocked(x)	putc_unlocked(x, stdout)
 #endif
+#endif /* __cplusplus */
 
 __END_DECLS
 #endif /* !_STDIO_H_ */
diff --git a/include/stdlib.h b/include/stdlib.h
index 3c48f17..3c26dfb 100644
--- a/include/stdlib.h
+++ b/include/stdlib.h
@@ -71,10 +71,11 @@ typedef struct {
 
 #define	RAND_MAX	0x7fffffff
 
+__BEGIN_DECLS
 extern int __mb_cur_max;
-#define	MB_CUR_MAX	__mb_cur_max
+extern int ___mb_cur_max(void);
+#define	MB_CUR_MAX	(___mb_cur_max())
 
-__BEGIN_DECLS
 void	 abort(void) __dead2;
 int	 abs(int) __pure2;
 int	 atexit(void (*)(void));
diff --git a/include/wchar.h b/include/wchar.h
index 9b59494..c641165 100644
--- a/include/wchar.h
+++ b/include/wchar.h
@@ -97,20 +97,23 @@ typedef	__wint_t	wint_t;
 #define	WEOF 	((wint_t)-1)
 #endif
 
-struct __sFILE;
+#ifndef _STDFILE_DECLARED
+#define _STDFILE_DECLARED
+typedef struct __sFILE FILE;
+#endif
 struct tm;
 
 __BEGIN_DECLS
 wint_t	btowc(int);
-wint_t	fgetwc(struct __sFILE *);
+wint_t	fgetwc(FILE *);
 wchar_t	*
-	fgetws(wchar_t * __restrict, int, struct __sFILE * __restrict);
-wint_t	fputwc(wchar_t, struct __sFILE *);
-int	fputws(const wchar_t * __restrict, struct __sFILE * __restrict);
-int	fwide(struct __sFILE *, int);
-int	fwprintf(struct __sFILE * __restrict, const wchar_t * __restrict, ...);
-int	fwscanf(struct __sFILE * __restrict, const wchar_t * __restrict, ...);
-wint_t	getwc(struct __sFILE *);
+	fgetws(wchar_t * __restrict, int, FILE * __restrict);
+wint_t	fputwc(wchar_t, FILE *);
+int	fputws(const wchar_t * __restrict, FILE * __restrict);
+int	fwide(FILE *, int);
+int	fwprintf(FILE * __restrict, const wchar_t * __restrict, ...);
+int	fwscanf(FILE * __restrict, const wchar_t * __restrict, ...);
+wint_t	getwc(FILE *);
 wint_t	getwchar(void);
 size_t	mbrlen(const char * __restrict, size_t, mbstate_t * __restrict);
 size_t	mbrtowc(wchar_t * __restrict, const char * __restrict, size_t,
@@ -118,13 +121,13 @@ size_t	mbrtowc(wchar_t * __restrict, const char * __restrict, size_t,
 int	mbsinit(const mbstate_t *);
 size_t	mbsrtowcs(wchar_t * __restrict, const char ** __restrict, size_t,
 	    mbstate_t * __restrict);
-wint_t	putwc(wchar_t, struct __sFILE *);
+wint_t	putwc(wchar_t, FILE *);
 wint_t	putwchar(wchar_t);
 int	swprintf(wchar_t * __restrict, size_t n, const wchar_t * __restrict,
 	    ...);
 int	swscanf(const wchar_t * __restrict, const wchar_t * __restrict, ...);
-wint_t	ungetwc(wint_t, struct __sFILE *);
-int	vfwprintf(struct __sFILE * __restrict, const wchar_t * __restrict,
+wint_t	ungetwc(wint_t, FILE *);
+int	vfwprintf(FILE * __restrict, const wchar_t * __restrict,
 	    __va_list);
 int	vswprintf(wchar_t * __restrict, size_t n, const wchar_t * __restrict,
 	    __va_list);
@@ -167,9 +170,9 @@ int	wprintf(const wchar_t * __restrict, ...);
 int	wscanf(const wchar_t * __restrict, ...);
 
 #ifndef _STDSTREAM_DECLARED
-extern struct __sFILE *__stdinp;
-extern struct __sFILE *__stdoutp;
-extern struct __sFILE *__stderrp;
+extern FILE *__stdinp;
+extern FILE *__stdoutp;
+extern FILE *__stderrp;
 #define	_STDSTREAM_DECLARED
 #endif
 
@@ -179,7 +182,7 @@ extern struct __sFILE *__stderrp;
 #define	putwchar(wc)	fputwc(wc, __stdoutp)
 
 #if __ISO_C_VISIBLE >= 1999
-int	vfwscanf(struct __sFILE * __restrict, const wchar_t * __restrict,
+int	vfwscanf(FILE * __restrict, const wchar_t * __restrict,
 	    __va_list);
 int	vswscanf(const wchar_t * __restrict, const wchar_t * __restrict,
 	    __va_list);
@@ -217,7 +220,7 @@ size_t	wcsnrtombs(char * __restrict, const wchar_t ** __restrict, size_t,
 #endif
 
 #if __BSD_VISIBLE
-wchar_t	*fgetwln(struct __sFILE * __restrict, size_t * __restrict);
+wchar_t	*fgetwln(FILE * __restrict, size_t * __restrict);
 size_t	wcslcat(wchar_t *, const wchar_t *, size_t);
 size_t	wcslcpy(wchar_t *, const wchar_t *, size_t);
 #endif
diff --git a/include/wctype.h b/include/wctype.h
index 098045f..183a2cd 100644
--- a/include/wctype.h
+++ b/include/wctype.h
@@ -89,6 +89,7 @@ wint_t	nextwctype(wint_t, wctype_t);
 #endif
 __END_DECLS
 
+#ifndef __cplusplus
 #define	iswalnum(wc)		__istype((wc), _CTYPE_A|_CTYPE_D)
 #define	iswalpha(wc)		__istype((wc), _CTYPE_A)
 #define	iswblank(wc)		__istype((wc), _CTYPE_B)
@@ -113,6 +114,7 @@ __END_DECLS
 #define	iswphonogram(wc)	__istype((wc), _CTYPE_Q)
 #define	iswrune(wc)		__istype((wc), 0xFFFFFF00L)
 #define	iswspecial(wc)		__istype((wc), _CTYPE_T)
-#endif
+#endif /* __BSD_VISIBLE */
+#endif /* __cplusplus */
 
 #endif		/* _WCTYPE_H_ */
diff --git a/include/xlocale.h b/include/xlocale.h
new file mode 100644
index 0000000..30fe87e
--- /dev/null
+++ b/include/xlocale.h
@@ -0,0 +1,258 @@
+/*-
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by David Chisnall under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1.  Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _XLOCALE_H_
+#define _XLOCALE_H_
+
+#include <locale.h>
+
+__BEGIN_DECLS
+
+/*
+ * Extended locale versions of the locale-aware functions from string.h.
+ *
+ * Include <string.h> before <xlocale.h> to expose these.
+ */
+#ifdef _STRING_H_
+int	 strcoll_l(const char *, const char *, locale_t);
+size_t	 strxfrm_l(char *, const char *, size_t, locale_t);
+int	 strcasecmp_l(const char *, const char *, locale_t);
+char	*strcasestr_l(const char *, const char *, locale_t);
+int	 strncasecmp_l(const char *, const char *, size_t, locale_t);
+#endif
+/*
+ * Extended locale versions of the locale-aware functions from inttypes.h.
+ *
+ * Include <inttypes.h> before <xlocale.h> to expose these.
+ */
+#ifdef _INTTYPES_H_
+intmax_t 
+strtoimax_l(const char * __restrict, char ** __restrict, int, locale_t);
+uintmax_t
+strtoumax_l(const char * __restrict, char ** __restrict, int, locale_t);
+intmax_t 
+wcstoimax_l(const wchar_t * __restrict, wchar_t ** __restrict, int , locale_t);
+uintmax_t
+wcstoumax_l(const wchar_t * __restrict, wchar_t ** __restrict, int, locale_t);
+#endif
+/*
+ * Extended locale versions of the locale-aware functions from monetary.h.
+ *
+ * Include <monetary.h> before <xlocale.h> to expose these.
+ */
+#ifdef _MONETARY_H_
+ssize_t strfmon_l(char *, size_t, locale_t, const char *, ...)
+#	if __GNUC__ > 2 || __GNUC__ == 2 && __GNUC_MINOR__ >= 7
+	__attribute__((__format__ (__strfmon__, 4, 5)))
+#	endif
+	;
+#endif
+
+/*
+ * Extended locale versions of the locale-aware functions from stdlib.h.
+ *
+ * Include <stdlib.h> before <xlocale.h> to expose these.
+ */
+#ifdef _STDLIB_H_
+double	 atof_l(const char *, locale_t);
+int	 atoi_l(const char *, locale_t);
+long	 atol_l(const char *, locale_t);
+long long	 atoll_l(const char *, locale_t);
+int	 mblen_l(const char *, size_t, locale_t);
+size_t
+mbstowcs_l(wchar_t * __restrict, const char * __restrict, size_t, locale_t);
+int
+mbtowc_l(wchar_t * __restrict, const char * __restrict, size_t, locale_t);
+double	 strtod_l(const char *, char **, locale_t);
+float	 strtof_l(const char *, char **, locale_t);
+long	 strtol_l(const char *, char **, int, locale_t);
+long	 double strtold_l(const char *, char **, locale_t);
+long long	 strtoll_l(const char *, char **, int, locale_t);
+unsigned long	 strtoul_l(const char *, char **, int, locale_t);
+unsigned long long	 strtoull_l(const char *, char **, int, locale_t);
+size_t
+wcstombs_l(char * __restrict, const wchar_t * __restrict, size_t, locale_t);
+int	 wctomb_l(char *, wchar_t, locale_t);
+
+int	 ___mb_cur_max_l(locale_t);
+#define MB_CUR_MAX_L(x) (___mb_cur_max_l(x))
+
+#endif
+/*
+ * Extended locale versions of the locale-aware functions from time.h.
+ *
+ * Include <time.h> before <xlocale.h> to expose these.
+ */
+#ifdef _TIME_H_
+size_t
+strftime_l(char * __restrict, size_t, const char * __restrict, const
+           struct tm * __restrict, locale_t)
+#	if __GNUC__ > 2 || __GNUC__ == 2 && __GNUC_MINOR__ >= 7
+	__attribute__((__format__ (__strftime__, 3, 0)))
+#	endif
+	;
+char *
+strptime_l(const char * __restrict, const char * __restrict,
+           struct tm * __restrict, locale_t);
+#endif
+#ifdef _LANGINFO_H_
+char	*nl_langinfo_l(nl_item, locale_t);
+#endif
+#ifdef _CTYPE_H_
+#include <_xlocale_ctype.h>
+#endif
+#ifdef _WCTYPE_H_
+#define XLOCALE_WCTYPES 1
+#include <_xlocale_ctype.h>
+#endif
+
+#ifdef _STDIO_H_
+int	 fprintf_l(FILE * __restrict, locale_t, const char * __restrict, ...)
+		__printflike(3, 4);
+int	 fscanf_l(FILE * __restrict, locale_t, const char * __restrict, ...)
+		__scanflike(3, 4);
+int	 printf_l(locale_t, const char * __restrict, ...) __printflike(2, 3);
+int	 scanf_l(locale_t, const char * __restrict, ...) __scanflike(2, 3);
+int	 sprintf_l(char * __restrict, locale_t, const char * __restrict, ...)
+		__printflike(3, 4);
+int	 sscanf_l(const char * __restrict, locale_t, const char * __restrict, ...)
+		__scanflike(3, 4);
+int	 vfprintf_l(FILE * __restrict, locale_t, const char * __restrict, __va_list)
+		__printflike(3, 0);
+int	 vprintf_l(locale_t, const char * __restrict, __va_list) __printflike(2, 0);
+int	 vsprintf_l(char * __restrict, locale_t, const char * __restrict, __va_list)
+		__printflike(3, 0);
+
+int	 snprintf_l(char * __restrict, size_t, locale_t, const char * __restrict,
+		...) __printflike(4, 5);
+int	 vfscanf_l(FILE * __restrict, locale_t, const char * __restrict, __va_list)
+		__scanflike(3, 0);
+int	 vscanf_l(locale_t, const char * __restrict, __va_list) __scanflike(2, 0);
+int	 vsnprintf_l(char * __restrict, size_t, locale_t, const char * __restrict,
+		va_list) __printflike(4, 0);
+int	 vsscanf_l(const char * __restrict, locale_t, const char * __restrict,
+		va_list) __scanflike(3, 0);
+int	 dprintf_l(int, locale_t, const char * __restrict, ...) __printflike(3, 4);
+int	 vdprintf_l(int, locale_t, const char * __restrict, __va_list)
+		__printflike(3, 0);
+int	 asprintf_l(char **, locale_t, const char *, ...) __printflike(3, 4);
+int	 vasprintf_l(char **, locale_t, const char *, __va_list) __printflike(3, 0);
+#endif
+#ifdef _WCHAR_H_
+wint_t	 btowc_l(int, locale_t);
+wint_t	 fgetwc_l(FILE *, locale_t);
+wchar_t *
+fgetws_l(wchar_t * __restrict, int, FILE * __restrict, locale_t);
+wint_t	 fputwc_l(wchar_t, FILE *, locale_t);
+int
+fputws_l(const wchar_t * __restrict, FILE * __restrict, locale_t);
+int
+fwprintf_l(FILE * __restrict, locale_t, const wchar_t * __restrict,
+		...);
+int
+fwscanf_l(FILE * __restrict, locale_t, const wchar_t * __restrict, ...);
+wint_t	 getwc_l(FILE *, locale_t);
+wint_t	 getwchar_l(locale_t);
+size_t
+mbrlen_l(const char * __restrict, size_t, mbstate_t * __restrict, locale_t);
+size_t
+mbrtowc_l(wchar_t * __restrict, const char * __restrict, size_t,
+		mbstate_t * __restrict, locale_t);
+int	 mbsinit_l(const mbstate_t *, locale_t);
+size_t
+mbsrtowcs_l(wchar_t * __restrict, const char ** __restrict, size_t,
+		mbstate_t * __restrict, locale_t);
+wint_t	 putwc_l(wchar_t, FILE *, locale_t);
+wint_t	 putwchar_l(wchar_t, locale_t);
+int
+swprintf_l(wchar_t * __restrict, size_t n, locale_t,
+		const wchar_t * __restrict, ...);
+int
+swscanf_l(const wchar_t * __restrict, locale_t, const wchar_t * __restrict,
+		...);
+wint_t	 ungetwc_l(wint_t, FILE *, locale_t);
+int
+vfwprintf_l(FILE * __restrict, locale_t, const wchar_t * __restrict,
+		__va_list);
+int
+vswprintf_l(wchar_t * __restrict, size_t n, locale_t,
+		const wchar_t * __restrict, __va_list);
+int	 vwprintf_l(locale_t, const wchar_t * __restrict, __va_list);
+size_t
+wcrtomb_l(char * __restrict, wchar_t, mbstate_t * __restrict, locale_t);
+int	 wcscoll_l(const wchar_t *, const wchar_t *, locale_t);
+size_t
+wcsftime_l(wchar_t * __restrict, size_t, const wchar_t * __restrict,
+		const struct tm * __restrict, locale_t);
+size_t 
+wcsrtombs_l(char * __restrict, const wchar_t ** __restrict, size_t,
+		mbstate_t * __restrict, locale_t);
+double	 wcstod_l(const wchar_t * __restrict, wchar_t ** __restrict, locale_t);
+long
+wcstol_l(const wchar_t * __restrict, wchar_t ** __restrict, int, locale_t);
+unsigned long
+wcstoul_l(const wchar_t * __restrict, wchar_t ** __restrict, int, locale_t);
+int	 wcswidth_l(const wchar_t *, size_t, locale_t);
+size_t
+wcsxfrm_l(wchar_t * __restrict, const wchar_t * __restrict, size_t, locale_t);
+int	 wctob_l(wint_t, locale_t);
+int	 wcwidth_l(wchar_t, locale_t);
+int	 wprintf_l(locale_t, const wchar_t * __restrict, ...);
+int	 wscanf_l(locale_t, const wchar_t * __restrict, ...);
+
+int
+vfwscanf_l(FILE * __restrict, locale_t, const wchar_t * __restrict,
+		__va_list);
+int	 vswscanf_l(const wchar_t * __restrict, locale_t,
+const wchar_t	*__restrict, __va_list);
+int	 vwscanf_l(locale_t, const wchar_t * __restrict, __va_list);
+float 	wcstof_l(const wchar_t * __restrict, wchar_t ** __restrict, locale_t);
+long double
+wcstold_l(const wchar_t * __restrict, wchar_t ** __restrict, locale_t);
+long long
+wcstoll_l(const wchar_t * __restrict, wchar_t ** __restrict, int, locale_t);
+unsigned long long
+wcstoull_l(const wchar_t * __restrict, wchar_t ** __restrict, int, locale_t);
+size_t
+mbsnrtowcs_l(wchar_t * __restrict, const char ** __restrict, size_t, size_t,
+		mbstate_t * __restrict, locale_t);
+int	 wcscasecmp_l(const wchar_t *, const wchar_t *, locale_t);
+int	 wcsncasecmp_l(const wchar_t *, const wchar_t *, size_t, locale_t);
+size_t
+wcsnrtombs_l(char * __restrict, const wchar_t ** __restrict, size_t, size_t,
+		mbstate_t * __restrict, locale_t);
+
+#endif
+
+struct lconv	*localeconv_l(locale_t);
+__END_DECLS
+
+#endif
diff --git a/lib/clang/clang.build.mk b/lib/clang/clang.build.mk
index ceaeb05..47b6795 100644
--- a/lib/clang/clang.build.mk
+++ b/lib/clang/clang.build.mk
@@ -9,7 +9,7 @@ CFLAGS+=-I${LLVM_SRCS}/include -I${CLANG_SRCS}/include \
 	-D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS #-DNDEBUG
 
 # Correct for gcc miscompilation when compiling on PPC with -O2
-.if ${MACHINE_ARCH} == "powerpc"
+.if ${MACHINE_CPUARCH} == "powerpc"
 CFLAGS+= -O1
 .endif
 
diff --git a/lib/clang/include/llvm/Config/config.h b/lib/clang/include/llvm/Config/config.h
index 5b6761c..92c960b 100644
--- a/lib/clang/include/llvm/Config/config.h
+++ b/lib/clang/include/llvm/Config/config.h
@@ -552,7 +552,7 @@
 /* #undef LLVM_ETCDIR */
 
 /* Has gcc/MSVC atomic intrinsics */
-#define LLVM_HAS_ATOMICS 1
+#define LLVM_HAS_ATOMICS 0
 
 /* Host triple we were built on */
 /* #undef LLVM_HOSTTRIPLE */
diff --git a/lib/clang/include/llvm/Config/llvm-config.h b/lib/clang/include/llvm/Config/llvm-config.h
index 05e17ca..1b50ab8 100644
--- a/lib/clang/include/llvm/Config/llvm-config.h
+++ b/lib/clang/include/llvm/Config/llvm-config.h
@@ -34,7 +34,7 @@
 /* #undef LLVM_ETCDIR */
 
 /* Has gcc/MSVC atomic intrinsics */
-#define LLVM_HAS_ATOMICS 1
+#define LLVM_HAS_ATOMICS 0
 
 /* Host triple we were built on */
 /* #undef LLVM_HOSTTRIPLE */
diff --git a/lib/libc/amd64/gen/setjmp.S b/lib/libc/amd64/gen/setjmp.S
index 1409f4c..47772be 100644
--- a/lib/libc/amd64/gen/setjmp.S
+++ b/lib/libc/amd64/gen/setjmp.S
@@ -54,6 +54,7 @@ ENTRY(setjmp)
 	movq	$1,%rdi			/* SIG_BLOCK       */
 	movq	$0,%rsi			/* (sigset_t*)set  */
 	leaq	72(%rcx),%rdx		/* 9,10; (sigset_t*)oset */
+	/* stack is 16-byte aligned */
 	call	PIC_PLT(CNAME(_sigprocmask))
 	popq	%rdi
 	movq	%rdi,%rcx
@@ -81,7 +82,9 @@ ENTRY(__longjmp)
 	movq	$3,%rdi			/* SIG_SETMASK     */
 	leaq	72(%rdx),%rsi		/* (sigset_t*)set  */
 	movq	$0,%rdx			/* (sigset_t*)oset */
+	subq	$0x8,%rsp		/* make the stack 16-byte aligned */
 	call	PIC_PLT(CNAME(_sigprocmask))
+	addq	$0x8,%rsp
 	popq	%rsi
 	popq	%rdi			/* jmpbuf */
 	movq	%rdi,%rdx
diff --git a/lib/libc/amd64/gen/sigsetjmp.S b/lib/libc/amd64/gen/sigsetjmp.S
index 438d440..ef90bc6 100644
--- a/lib/libc/amd64/gen/sigsetjmp.S
+++ b/lib/libc/amd64/gen/sigsetjmp.S
@@ -62,6 +62,7 @@ ENTRY(sigsetjmp)
 	movq	$1,%rdi			/* SIG_BLOCK       */
 	movq	$0,%rsi			/* (sigset_t*)set  */
 	leaq	72(%rcx),%rdx		/* 9,10 (sigset_t*)oset */
+	/* stack is 16-byte aligned */
 	call	PIC_PLT(CNAME(_sigprocmask))
 	popq	%rdi
 2:	movq	%rdi,%rcx
@@ -90,7 +91,9 @@ ENTRY(__siglongjmp)
 	movq	$3,%rdi			/* SIG_SETMASK     */
 	leaq	72(%rdx),%rsi		/* (sigset_t*)set  */
 	movq	$0,%rdx			/* (sigset_t*)oset */
+	subq	$0x8,%rsp		/* make the stack 16-byte aligned */
 	call	PIC_PLT(CNAME(_sigprocmask))
+	addq	$0x8,%rsp
 	popq	%rsi
 	popq	%rdi			/* jmpbuf */
 2:	movq	%rdi,%rdx
diff --git a/lib/libc/gdtoa/machdep_ldisQ.c b/lib/libc/gdtoa/machdep_ldisQ.c
index ebcb37e..7b5c7af 100644
--- a/lib/libc/gdtoa/machdep_ldisQ.c
+++ b/lib/libc/gdtoa/machdep_ldisQ.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2003 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,10 +43,10 @@ __FBSDID("$FreeBSD$");
 #include "gdtoaimp.h"
 
 long double
-strtold(const char * __restrict s, char ** __restrict sp)
+strtold_l(const char * __restrict s, char ** __restrict sp, locale_t locale)
 {
 	long double result;
 
-	strtorQ(s, sp, FLT_ROUNDS, &result);
+	strtorQ_l(s, sp, FLT_ROUNDS, &result, locale);
 	return result;
 }
diff --git a/lib/libc/gdtoa/machdep_ldisd.c b/lib/libc/gdtoa/machdep_ldisd.c
index e2dbb60..d5c7329 100644
--- a/lib/libc/gdtoa/machdep_ldisd.c
+++ b/lib/libc/gdtoa/machdep_ldisd.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2003 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,10 +39,11 @@
 __FBSDID("$FreeBSD$");
 
 #include "gdtoaimp.h"
+#undef strtold_l
 
 long double
-strtold(const char * __restrict s, char ** __restrict sp)
+strtold_l(const char * __restrict s, char ** __restrict sp, locale_t locale)
 {
 
-	return strtod(s, sp);
+	return strtod_l(s, sp, locale);
 }
diff --git a/lib/libc/gdtoa/machdep_ldisx.c b/lib/libc/gdtoa/machdep_ldisx.c
index c08c85e..b8e2a05 100644
--- a/lib/libc/gdtoa/machdep_ldisx.c
+++ b/lib/libc/gdtoa/machdep_ldisx.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2003 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,10 +43,11 @@ __FBSDID("$FreeBSD$");
 #include "gdtoaimp.h"
 
 long double
-strtold(const char * __restrict s, char ** __restrict sp)
+strtold_l(const char * __restrict s, char ** __restrict sp, locale_t locale)
 {
 	long double result;
+	FIX_LOCALE(locale);
 
-	strtorx(s, sp, FLT_ROUNDS, &result);
+	strtorx_l(s, sp, FLT_ROUNDS, &result, locale);
 	return result;
 }
diff --git a/lib/libc/gen/arc4random.c b/lib/libc/gen/arc4random.c
index 56dfba9..a2b235f 100644
--- a/lib/libc/gen/arc4random.c
+++ b/lib/libc/gen/arc4random.c
@@ -1,3 +1,5 @@
+/*	$OpenBSD: arc4random.c,v 1.22 2010/12/22 08:23:42 otto Exp $	*/
+
 /*
  * Copyright (c) 1996, David Mazieres <dm@uun.org>
  * Copyright (c) 2008, Damien Miller <djm@openbsd.org>
@@ -24,11 +26,6 @@
  * which is a trade secret).  The same algorithm is used as a stream
  * cipher called "arcfour" in Tatu Ylonen's ssh package.
  *
- * Here the stream cipher has been modified always to include the time
- * when initializing the state.  That makes it impossible to
- * regenerate the same random sequence twice, so this can't be used
- * for encryption, but will generate good random numbers.
- *
  * RC4 is a registered trademark of RSA Laboratories.
  */
 
@@ -36,16 +33,24 @@
 __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
-#include <sys/types.h>
-#include <sys/time.h>
-#include <stdlib.h>
 #include <fcntl.h>
+#include <limits.h>
+#include <stdlib.h>
 #include <unistd.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/time.h>
 #include <pthread.h>
 
 #include "libc_private.h"
 #include "un-namespace.h"
 
+#ifdef __GNUC__
+#define inline __inline
+#else				/* !__GNUC__ */
+#define inline
+#endif				/* !__GNUC__ */
+
 struct arc4_stream {
 	u_int8_t i;
 	u_int8_t j;
@@ -55,22 +60,22 @@ struct arc4_stream {
 static pthread_mutex_t	arc4random_mtx = PTHREAD_MUTEX_INITIALIZER;
 
 #define	RANDOMDEV	"/dev/random"
-#define KEYSIZE		128
-#define	THREAD_LOCK()						\
+#define	KEYSIZE		128
+#define	_ARC4_LOCK()						\
 	do {							\
 		if (__isthreaded)				\
 			_pthread_mutex_lock(&arc4random_mtx);	\
 	} while (0)
 
-#define	THREAD_UNLOCK()						\
+#define	_ARC4_UNLOCK()						\
 	do {							\
 		if (__isthreaded)				\
 			_pthread_mutex_unlock(&arc4random_mtx);	\
 	} while (0)
 
-static struct arc4_stream rs;
 static int rs_initialized;
-static int rs_stired;
+static struct arc4_stream rs;
+static pid_t arc4_stir_pid;
 static int arc4_count;
 
 static inline u_int8_t arc4_getbyte(void);
@@ -107,20 +112,24 @@ arc4_addrandom(u_char *dat, int datlen)
 static void
 arc4_stir(void)
 {
-	int done, fd, n;
+	int done, fd, i;
 	struct {
 		struct timeval	tv;
-		pid_t 		pid;
-		u_int8_t 	rnd[KEYSIZE];
+		pid_t		pid;
+		u_char	 	rnd[KEYSIZE];
 	} rdat;
 
+	if (!rs_initialized) {
+		arc4_init();
+		rs_initialized = 1;
+	}
 	fd = _open(RANDOMDEV, O_RDONLY, 0);
 	done = 0;
 	if (fd >= 0) {
 		if (_read(fd, &rdat, KEYSIZE) == KEYSIZE)
 			done = 1;
 		(void)_close(fd);
-	} 
+	}
 	if (!done) {
 		(void)gettimeofday(&rdat.tv, NULL);
 		rdat.pid = getpid();
@@ -130,17 +139,26 @@ arc4_stir(void)
 	arc4_addrandom((u_char *)&rdat, KEYSIZE);
 
 	/*
-	 * Throw away the first N bytes of output, as suggested in the
-	 * paper "Weaknesses in the Key Scheduling Algorithm of RC4"
-	 * by Fluher, Mantin, and Shamir.  N=1024 is based on
-	 * suggestions in the paper "(Not So) Random Shuffles of RC4"
-	 * by Ilya Mironov.
+	 * Discard early keystream, as per recommendations in:
+	 * "(Not So) Random Shuffles of RC4" by Ilya Mironov.
 	 */
-	for (n = 0; n < 1024; n++)
-		(void) arc4_getbyte();
+	for (i = 0; i < 1024; i++)
+		(void)arc4_getbyte();
 	arc4_count = 1600000;
 }
 
+static void
+arc4_stir_if_needed(void)
+{
+	pid_t pid = getpid();
+
+	if (arc4_count <= 0 || !rs_initialized || arc4_stir_pid != pid)
+	{
+		arc4_stir_pid = pid;
+		arc4_stir();
+	}
+}
+
 static inline u_int8_t
 arc4_getbyte(void)
 {
@@ -152,7 +170,6 @@ arc4_getbyte(void)
 	sj = rs.s[rs.j];
 	rs.s[rs.i] = sj;
 	rs.s[rs.j] = si;
-
 	return (rs.s[(si + sj) & 0xff]);
 }
 
@@ -160,81 +177,55 @@ static inline u_int32_t
 arc4_getword(void)
 {
 	u_int32_t val;
-
 	val = arc4_getbyte() << 24;
 	val |= arc4_getbyte() << 16;
 	val |= arc4_getbyte() << 8;
 	val |= arc4_getbyte();
-
-	return (val);
-}
-
-static void
-arc4_check_init(void)
-{
-	if (!rs_initialized) {
-		arc4_init();
-		rs_initialized = 1;
-	}
-}
-
-static inline void
-arc4_check_stir(void)
-{
-	if (!rs_stired || arc4_count <= 0) {
-		arc4_stir();
-		rs_stired = 1;
-	}
+	return val;
 }
 
 void
 arc4random_stir(void)
 {
-	THREAD_LOCK();
-	arc4_check_init();
+	_ARC4_LOCK();
 	arc4_stir();
-	rs_stired = 1;
-	THREAD_UNLOCK();
+	_ARC4_UNLOCK();
 }
 
 void
 arc4random_addrandom(u_char *dat, int datlen)
 {
-	THREAD_LOCK();
-	arc4_check_init();
-	arc4_check_stir();
+	_ARC4_LOCK();
+	if (!rs_initialized)
+		arc4_stir();
 	arc4_addrandom(dat, datlen);
-	THREAD_UNLOCK();
+	_ARC4_UNLOCK();
 }
 
 u_int32_t
 arc4random(void)
 {
-	u_int32_t rnd;
-
-	THREAD_LOCK();
-	arc4_check_init();
-	arc4_check_stir();
-	rnd = arc4_getword();
+	u_int32_t val;
+	_ARC4_LOCK();
 	arc4_count -= 4;
-	THREAD_UNLOCK();
-
-	return (rnd);
+	arc4_stir_if_needed();
+	val = arc4_getword();
+	_ARC4_UNLOCK();
+	return val;
 }
 
 void
 arc4random_buf(void *_buf, size_t n)
 {
 	u_char *buf = (u_char *)_buf;
-
-	THREAD_LOCK();
-	arc4_check_init();
+	_ARC4_LOCK();
+	arc4_stir_if_needed();
 	while (n--) {
-		arc4_check_stir();
+		if (--arc4_count <= 0)
+			arc4_stir();
 		buf[n] = arc4_getbyte();
-		arc4_count--;
 	}
-	THREAD_UNLOCK();
+	_ARC4_UNLOCK();
 }
 
 /*
@@ -253,7 +244,7 @@ arc4random_uniform(u_int32_t upper_bound)
 	u_int32_t r, min;
 
 	if (upper_bound < 2)
-		return (0);
+		return 0;
 
 #if (ULONG_MAX > 0xffffffffUL)
 	min = 0x100000000UL % upper_bound;
@@ -279,7 +270,7 @@ arc4random_uniform(u_int32_t upper_bound)
 			break;
 	}
 
-	return (r % upper_bound);
+	return r % upper_bound;
 }
 
 #if 0
diff --git a/lib/libc/gen/fnmatch.c b/lib/libc/gen/fnmatch.c
index 3acbc3b..c38aafc 100644
--- a/lib/libc/gen/fnmatch.c
+++ b/lib/libc/gen/fnmatch.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Guido van Rossum.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -222,6 +227,8 @@ rangematch(pattern, test, flags, newp, patmbs)
 	wchar_t c, c2;
 	size_t pclen;
 	const char *origpat;
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)__get_locale()->components[XLC_COLLATE];
 
 	/*
 	 * A bracket expression starting with an unquoted circumflex
@@ -276,10 +283,10 @@ rangematch(pattern, test, flags, newp, patmbs)
 			if (flags & FNM_CASEFOLD)
 				c2 = towlower(c2);
 
-			if (__collate_load_error ?
+			if (table->__collate_load_error ?
 			    c <= test && test <= c2 :
-			       __collate_range_cmp(c, test) <= 0
-			    && __collate_range_cmp(test, c2) <= 0
+			       __collate_range_cmp(table, c, test) <= 0
+			    && __collate_range_cmp(table, test, c2) <= 0
 			   )
 				ok = 1;
 		} else if (c == test)
diff --git a/lib/libc/gen/getutxent.3 b/lib/libc/gen/getutxent.3
index d25aacf..e7bfae4 100644
--- a/lib/libc/gen/getutxent.3
+++ b/lib/libc/gen/getutxent.3
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 19, 2011
+.Dd October 27, 2011
 .Dt GETUTXENT 3
 .Os
 .Sh NAME
@@ -301,7 +301,6 @@ The value of
 determines which databases are modified.
 .Pp
 Entries of type
-.Dv BOOT_TIME ,
 .Dv SHUTDOWN_TIME ,
 .Dv OLD_TIME
 and
@@ -335,7 +334,7 @@ In addition, entries of type
 .Dv BOOT_TIME
 and
 .Dv SHUTDOWN_TIME
-will cause all entries in
+will cause all existing entries in
 .Pa /var/run/utx.active
 to be discarded.
 .Pp
@@ -386,10 +385,10 @@ Otherwise, -1 is returned and the global variable
 is set to indicate the error.
 .Sh ERRORS
 In addition to the error conditions described in
+.Xr open 2 ,
 .Xr fdopen 3 ,
 .Xr fopen 3 ,
 .Xr fseek 3 ,
-.Xr open 3 ,
 the
 .Fn pututxline
 function can generate the following errors:
diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c
index c3d9f08..59663d9 100644
--- a/lib/libc/gen/glob.c
+++ b/lib/libc/gen/glob.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Guido van Rossum.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -751,6 +756,8 @@ match(Char *name, Char *pat, Char *patend)
 {
 	int ok, negate_range;
 	Char c, k;
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)__get_locale()->components[XLC_COLLATE];
 
 	while (pat < patend) {
 		c = *pat++;
@@ -775,10 +782,10 @@ match(Char *name, Char *pat, Char *patend)
 				++pat;
 			while (((c = *pat++) & M_MASK) != M_END)
 				if ((*pat & M_MASK) == M_RNG) {
-					if (__collate_load_error ?
+					if (table->__collate_load_error ?
 					    CHAR(c) <= CHAR(k) && CHAR(k) <= CHAR(pat[1]) :
-					       __collate_range_cmp(CHAR(c), CHAR(k)) <= 0
-					    && __collate_range_cmp(CHAR(k), CHAR(pat[1])) <= 0
+					       __collate_range_cmp(table, CHAR(c), CHAR(k)) <= 0
+					    && __collate_range_cmp(table, CHAR(k), CHAR(pat[1])) <= 0
 					   )
 						ok = 1;
 					pat += 2;
diff --git a/lib/libc/gen/pututxline.c b/lib/libc/gen/pututxline.c
index 0cc7a01..98addee 100644
--- a/lib/libc/gen/pututxline.c
+++ b/lib/libc/gen/pututxline.c
@@ -86,6 +86,9 @@ utx_active_add(const struct futx *fu)
 		return (-1);
 	while (fread(&fe, sizeof(fe), 1, fp) == 1) {
 		switch (fe.fu_type) {
+		case BOOT_TIME:
+			/* Leave these intact. */
+			break;
 		case USER_PROCESS:
 		case INIT_PROCESS:
 		case LOGIN_PROCESS:
@@ -171,6 +174,19 @@ utx_active_remove(struct futx *fu)
 }
 
 static void
+utx_active_init(const struct futx *fu)
+{
+	int fd;
+
+	/* Initialize utx.active with a single BOOT_TIME record. */
+	fd = _open(_PATH_UTX_ACTIVE, O_CREAT|O_RDWR|O_TRUNC, 0644);
+	if (fd < 0)
+		return;
+	_write(fd, fu, sizeof(*fu));
+	_close(fd);
+}
+
+static void
 utx_active_purge(void)
 {
 
@@ -277,9 +293,11 @@ pututxline(const struct utmpx *utmpx)
 
 	switch (fu.fu_type) {
 	case BOOT_TIME:
+		utx_active_init(&fu);
+		utx_lastlogin_upgrade();
+		break;
 	case SHUTDOWN_TIME:
 		utx_active_purge();
-		utx_lastlogin_upgrade();
 		break;
 	case OLD_TIME:
 	case NEW_TIME:
diff --git a/lib/libc/i386/gen/setjmp.S b/lib/libc/i386/gen/setjmp.S
index 5d0ddc4..a409e35 100644
--- a/lib/libc/i386/gen/setjmp.S
+++ b/lib/libc/i386/gen/setjmp.S
@@ -51,12 +51,19 @@ __FBSDID("$FreeBSD$");
 ENTRY(setjmp)
 	movl	4(%esp),%ecx
 	PIC_PROLOGUE
+#ifdef PIC
+	subl	$12,%esp		/* make the stack 16-byte aligned */
+#endif
 	leal	28(%ecx), %eax
 	pushl	%eax			/* (sigset_t*)oset */
 	pushl	$0			/* (sigset_t*)set  */
 	pushl	$1			/* SIG_BLOCK       */
 	call	PIC_PLT(CNAME(_sigprocmask))
+#ifdef PIC
+	addl	$24,%esp
+#else
 	addl	$12,%esp
+#endif
 	PIC_EPILOGUE
 	movl	4(%esp),%ecx
 	movl	0(%esp),%edx
@@ -76,12 +83,19 @@ END(setjmp)
 ENTRY(__longjmp)
 	movl	4(%esp),%edx
 	PIC_PROLOGUE
+#ifdef PIC
+	subl	$12,%esp		/* make the stack 16-byte aligned */
+#endif
 	pushl	$0			/* (sigset_t*)oset */
 	leal	28(%edx), %eax
 	pushl	%eax			/* (sigset_t*)set  */
 	pushl	$3			/* SIG_SETMASK     */
 	call	PIC_PLT(CNAME(_sigprocmask))
+#ifdef PIC
+	addl	$24,%esp
+#else
 	addl	$12,%esp
+#endif
 	PIC_EPILOGUE
 	movl	4(%esp),%edx
 	movl	8(%esp),%eax
diff --git a/lib/libc/i386/gen/sigsetjmp.S b/lib/libc/i386/gen/sigsetjmp.S
index 6487745..936edba 100644
--- a/lib/libc/i386/gen/sigsetjmp.S
+++ b/lib/libc/i386/gen/sigsetjmp.S
@@ -60,12 +60,19 @@ ENTRY(sigsetjmp)
 	testl	%eax,%eax
 	jz	2f
 	PIC_PROLOGUE
+#ifdef PIC
+	subl	$12,%esp		/* make the stack 16-byte aligned */
+#endif
 	leal	28(%ecx), %eax
 	pushl	%eax			/* (sigset_t*)oset */
 	pushl	$0			/* (sigset_t*)set  */
 	pushl	$1			/* SIG_BLOCK       */
 	call	PIC_PLT(CNAME(_sigprocmask))
+#ifdef PIC
+	addl	$24,%esp
+#else
 	addl	$12,%esp
+#endif
 	PIC_EPILOGUE
 	movl	4(%esp),%ecx
 2:	movl	0(%esp),%edx
@@ -87,12 +94,19 @@ ENTRY(__siglongjmp)
 	cmpl	$0,44(%edx)
 	jz	2f
 	PIC_PROLOGUE
+#ifdef PIC
+	subl	$12,%esp		/* make the stack 16-byte aligned */
+#endif
 	pushl	$0			/* (sigset_t*)oset */
 	leal	28(%edx), %eax
 	pushl	%eax			/* (sigset_t*)set  */
 	pushl	$3			/* SIG_SETMASK     */
 	call	PIC_PLT(CNAME(_sigprocmask))
+#ifdef PIC
+	addl	$24,%esp
+#else
 	addl	$12,%esp
+#endif
 	PIC_EPILOGUE
 	movl	4(%esp),%edx
 2:	movl	8(%esp),%eax
diff --git a/lib/libc/locale/DESIGN.xlocale b/lib/libc/locale/DESIGN.xlocale
new file mode 100644
index 0000000..5d998d3
--- /dev/null
+++ b/lib/libc/locale/DESIGN.xlocale
@@ -0,0 +1,159 @@
+$FreeBSD$
+
+Design of xlocale
+=================
+
+The xlocale APIs come from Darwin, although a subset is now part of POSIX 2008.
+They fall into two broad categories:
+
+- Manipulation of per-thread locales (POSIX)
+- Locale-aware functions taking an explicit locale argument (Darwin)
+
+This document describes the implementation of these APIs for FreeBSD.
+
+Goals
+-----
+
+The overall goal of this implementation is to be compatible with the Darwin
+version.  Additionally, it should include minimal changes to the existing
+locale code.  A lot of the existing locale code originates with 4BSD or earlier
+and has had over a decade of testing.  Replacing this code, unless absolutely
+necessary, gives us the potential for more bugs without much benefit.
+
+With this in mind, various libc-private functions have been modified to take a
+locale_t parameter.  This causes a compiler error if they are accidentally
+called without a locale.  This approach was taken, rather than adding _l
+variants of these functions, to make it harder for accidental uses of the
+global-locale versions to slip in.
+
+Locale Objects
+--------------
+
+A locale is encapsulated in a `locale_t`, which is an opaque type: a pointer to
+a `struct _xlocale`.  The name `_xlocale` is unfortunate, as it does not fit
+well with existing conventions, but is used because this is the name the Darwin
+implementation gives to this structure and so may be used by existing (bad) code.
+
+This structure should include all of the information corresponding to a locale.
+A locale_t is almost immutable after creation.  There are no functions that modify it,
+and it can therefore be used without locking.  It is the responsibility of the
+caller to ensure that a locale is not deallocated during a call that uses it.
+
+Each locale contains a number of components, one for each of the categories
+supported by `setlocale()`.  These are likewise immutable after creation.  This
+differs from the Darwin implementation, which includes a deprecated
+`setinvalidrune()` function that can modify the rune locale.
+
+The exception to these mutability rules is a set of `mbstate_t` flags stored
+with each locale.  These are used by various functions that previously had a
+static local `mbstate_t` variable.  
+
+The components are reference counted, and so can be aliased between locale
+objects.  This makes copying locales very cheap.
+
+The Global Locale
+-----------------
+
+All locales and locale components are reference counted.  The global locale,
+however, is special.  It, and all of its components, are static and so no
+malloc() memory is required when using a single locale.
+
+This means that threads using the global locale are subject to the same
+constraints as with the pre-xlocale libc.  Calls to any locale-aware functions
+in threads using the global locale, while modifying the global locale, have
+undefined behaviour.
+
+Because of this, we have to ensure that we always copy the components of the
+global locale, rather than alias them.  
+
+It would be cleaner to simply remove the special treatment of the global locale
+and have a locale_t lazily allocated for the global context.  This would cost a
+little more `malloc()` memory, so is not done in the initial version.
+
+Caching
+-------
+
+The existing locale implementation included several ad-hoc caching layers.
+None of these were thread safe.  Caching is only really of use for supporting
+the pattern where the locale is briefly changed to something and then changed
+back.
+
+The current xlocale implementation removes the caching entirely.  This pattern
+is not one that should be encouraged.  If you need to make some calls with a
+modified locale, then you should use the _l suffix versions of the calls,
+rather than switch the global locale.  If you do need to temporarily switch the
+locale and then switch it back, `uselocale()` provides a way of doing this very
+easily: It returns the old locale, which can then be passed to a subsequent
+call to `uselocale()` to restore it, without the need to load any locale data
+from the disk.
+
+If, in the future, it is determined that caching is beneficial, it can be added
+quite easily in xlocale.c.  Given, however, that any locale-aware call is going
+to be a preparation for presenting data to the user, and so is invariably going
+to be part of an I/O operation, this seems like a case of premature
+optimisation.
+
+localeconv
+----------
+
+The `localeconv()` function is an exception to the immutable-after-creation
+rule.  In the classic implementation, this function returns a pointer to some
+global storage, which is initialised with the data from the current locale.
+This is not possible in a multithreaded environment, with multiple locales.  
+
+Instead, each locale contains a `struct lconv` that is lazily initialised on
+calls to `localeconv()`.  This is not protected by any locking, however this is
+still safe on any machine where word-sized stores are atomic: two concurrent
+calls will write the same data into the structure.
+
+Explicit Locale Calls
+---------------------
+
+A large number of functions have been modified to take an explicit `locale_t`
+parameter.  The old APIs are then reimplemented with a call to `__get_locale()`
+to supply the `locale_t` parameter.  This is in line with the Darwin public
+APIs, but also simplifies the modifications to these functions.  The
+`__get_locale()` function is now the only way to access the current locale
+within libc.  All of the old globals have gone, so there is now a linker error
+if any functions attempt to use them.  
+
+The ctype.h functions are a little different.  These are not implemented in
+terms of their locale-aware versions, for performance reasons.  Each of these
+is implemented as a short inline function.
+
+Differences to Darwin APIs
+--------------------------
+
+`strtoq_l()` and `strtouq_l() `are not provided.  These are extensions to
+deprecated functions - we should not be encouraging people to use deprecated
+interfaces.
+
+Locale Placeholders
+-------------------
+
+The pointer values 0 and -1 have special meanings as `locale_t` values.  Any
+public function that accepts a `locale_t` parameter must use the `FIX_LOCALE()`
+macro on it before using it.  For efficiency, this can be emitted in functions
+which *only* use their locale parameter as an argument to another public
+function, as the callee will do the `FIX_LOCALE()` itself.
+
+Potential Improvements
+----------------------
+
+Currently, the current rune set is accessed via a function call.  This makes it
+fairly expensive to use any of the ctype.h functions.  We could improve this
+quite a lot by storing the rune locale data in a __thread-qualified variable.
+
+Several of the existing FreeBSD locale-aware functions appear to be wrong.  For
+example, most of the `strto*()` family should probably use `digittoint_l()`,
+but instead they assume ASCII.  These will break if using a character encoding
+that does not put numbers and the letters A-F in the same location as ASCII.
+Some functions, like `strcoll()` only work on single-byte encodings.  No
+attempt has been made to fix existing limitations in the libc functions other
+than to add support for xlocale.
+
+Intuitively, setting a thread-local locale should ensure that all locale-aware
+functions can be used safely from that thread.  In fact, this is not the case
+in either this implementation or the Darwin one.  You must call `duplocale()`
+or `newlocale()` before calling `uselocale()`.  This is a bit ugly, and it
+would be better if libc ensure that every thread had its own locale object.
diff --git a/lib/libc/locale/Makefile.inc b/lib/libc/locale/Makefile.inc
index 8fff0589..55800d0 100644
--- a/lib/libc/locale/Makefile.inc
+++ b/lib/libc/locale/Makefile.inc
@@ -5,7 +5,7 @@
 .PATH: ${.CURDIR}/${LIBC_ARCH}/locale ${.CURDIR}/locale
 
 SRCS+=	ascii.c big5.c btowc.c collate.c collcmp.c euc.c fix_grouping.c \
-	gb18030.c gb2312.c gbk.c isctype.c iswctype.c \
+	gb18030.c gb2312.c gbk.c ctype.c isctype.c iswctype.c \
 	ldpart.c lmessages.c lmonetary.c lnumeric.c localeconv.c mblen.c \
 	mbrlen.c \
 	mbrtowc.c mbsinit.c mbsnrtowcs.c \
@@ -20,7 +20,8 @@ SRCS+=	ascii.c big5.c btowc.c collate.c collcmp.c euc.c fix_grouping.c \
 	wcstoimax.c wcstol.c wcstold.c wcstoll.c \
 	wcstombs.c \
 	wcstoul.c wcstoull.c wcstoumax.c wctob.c wctomb.c wctrans.c wctype.c \
-	wcwidth.c
+	wcwidth.c\
+	xlocale.c
 
 SYM_MAPS+=${.CURDIR}/locale/Symbol.map
 
@@ -37,7 +38,9 @@ MAN+=	btowc.3 \
 	wcsftime.3 \
 	wcrtomb.3 \
 	wcsrtombs.3 wcstod.3 wcstol.3 wcstombs.3 wctomb.3 \
-	wctrans.3 wctype.3 wcwidth.3
+	wctrans.3 wctype.3 wcwidth.3 \
+	duplocale.3 freelocale.3 newlocale.3 querylocale.3 uselocale.3 xlocale.3
+
 MAN+=	big5.5 euc.5 gb18030.5 gb2312.5 gbk.5 mskanji.5 utf8.5
 
 MLINKS+=btowc.3 wctob.3
diff --git a/lib/libc/locale/Symbol.map b/lib/libc/locale/Symbol.map
index 20d092b..01242b6 100644
--- a/lib/libc/locale/Symbol.map
+++ b/lib/libc/locale/Symbol.map
@@ -101,6 +101,101 @@ FBSD_1.0 {
 	wcwidth;
 };
 
+FBSD_1.3 {
+	newlocale;
+	duplocale;
+	freelocale;
+	querylocale;
+	uselocale;
+	__getCurrentRuneLocale;
+	btowc_l;
+	localeconv_l;
+	mblen_l;
+	mbrlen_l;
+	mbrtowc_l;
+	mbsinit_l;
+	mbsnrtowcs_l;
+	mbsrtowcs_l;
+	mbstowcs_l;
+	mbtowc_l;
+	nl_langinfo_l;
+	strcoll_l;
+	strfmon_l;
+	strftime_l;
+	strptime_l;
+	strxfrm_l;
+	wcrtomb_l;
+	wcscoll_l;
+	wcsnrtombs_l;
+	wcsrtombs_l;
+	wcstombs_l;
+	wcsxfrm_l;
+	wctob_l;
+	wctomb_l;
+	___tolower_l;
+	___toupper_l;
+	___runetype_l;
+	digittoint_l;
+	isalnum_l;
+	isalpha_l;
+	isblank_l;
+	iscntrl_l;
+	isdigit_l;
+	isgraph_l;
+	ishexnumber_l;
+	isideogram_l;
+	islower_l;
+	isnumber_l;
+	isphonogram_l;
+	isprint_l;
+	ispunct_l;
+	isrune_l;
+	isspace_l;
+	isspecial_l;
+	isupper_l;
+	isxdigit_l;
+	tolower_l;
+	toupper_l;
+	iswalnum_l;
+	iswalpha_l;
+	iswblank_l;
+	iswcntrl_l;
+	iswdigit_l;
+	iswgraph_l;
+	iswhexnumber_l;
+	iswideogram_l;
+	iswlower_l;
+	iswnumber_l;
+	iswphonogram_l;
+	iswprint_l;
+	iswpunct_l;
+	iswrune_l;
+	iswspace_l;
+	iswspecial_l;
+	iswupper_l;
+	iswxdigit_l;
+	towlower_l;
+	towupper_l;
+	iswctype_l;
+	wctype_l;
+	nextwctype_l;
+	___mb_cur_max;
+	___mb_cur_max_l;
+	towctrans_l;
+	wctrans_l;
+	wcsftime_l;
+	wcstod_l;
+	wcstof_l;
+	wcstoimax_l;
+	wcstol_l;
+	wcstold_l;
+	wcstoll_l;
+	wcstoul_l;
+	wcstoull_l;
+	wcstoumax_l;
+	__runes_for_locale;
+};
+
 FBSDprivate_1.0 {
 	_PathLocale;
 	__detect_path_locale;
diff --git a/lib/libc/locale/ascii.c b/lib/libc/locale/ascii.c
index 493bf42..784814d 100644
--- a/lib/libc/locale/ascii.c
+++ b/lib/libc/locale/ascii.c
@@ -6,6 +6,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -56,17 +61,17 @@ static size_t	_ascii_wcsnrtombs(char * __restrict, const wchar_t ** __restrict,
 		    size_t, size_t, mbstate_t * __restrict);
 
 int
-_ascii_init(_RuneLocale *rl)
+_ascii_init(struct xlocale_ctype *l,_RuneLocale *rl)
 {
 
-	__mbrtowc = _ascii_mbrtowc;
-	__mbsinit = _ascii_mbsinit;
-	__mbsnrtowcs = _ascii_mbsnrtowcs;
-	__wcrtomb = _ascii_wcrtomb;
-	__wcsnrtombs = _ascii_wcsnrtombs;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 1;
-	__mb_sb_limit = 128;
+	l->__mbrtowc = _ascii_mbrtowc;
+	l->__mbsinit = _ascii_mbsinit;
+	l->__mbsnrtowcs = _ascii_mbsnrtowcs;
+	l->__wcrtomb = _ascii_wcrtomb;
+	l->__wcsnrtombs = _ascii_wcsnrtombs;
+	l->runes = rl;
+	l->__mb_cur_max = 1;
+	l->__mb_sb_limit = 128;
 	return(0);
 }
 
diff --git a/lib/libc/locale/big5.c b/lib/libc/locale/big5.c
index 19977d0..4b37265 100644
--- a/lib/libc/locale/big5.c
+++ b/lib/libc/locale/big5.c
@@ -6,6 +6,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -62,15 +67,15 @@ typedef struct {
 } _BIG5State;
 
 int
-_BIG5_init(_RuneLocale *rl)
+_BIG5_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	__mbrtowc = _BIG5_mbrtowc;
-	__wcrtomb = _BIG5_wcrtomb;
-	__mbsinit = _BIG5_mbsinit;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 2;
-	__mb_sb_limit = 128;
+	l->__mbrtowc = _BIG5_mbrtowc;
+	l->__wcrtomb = _BIG5_wcrtomb;
+	l->__mbsinit = _BIG5_mbsinit;
+	l->runes = rl;
+	l->__mb_cur_max = 2;
+	l->__mb_sb_limit = 128;
 	return (0);
 }
 
diff --git a/lib/libc/locale/btowc.c b/lib/libc/locale/btowc.c
index 2c4d493..72507bc 100644
--- a/lib/libc/locale/btowc.c
+++ b/lib/libc/locale/btowc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002, 2003 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,12 +37,13 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 wint_t
-btowc(int c)
+btowc_l(int c, locale_t l)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs = initial;
 	char cc;
 	wchar_t wc;
+	FIX_LOCALE(l);
 
 	if (c == EOF)
 		return (WEOF);
@@ -47,7 +53,12 @@ btowc(int c)
 	 * counts.
 	 */
 	cc = (char)c;
-	if (__mbrtowc(&wc, &cc, 1, &mbs) > 1)
+	if (XLOCALE_CTYPE(l)->__mbrtowc(&wc, &cc, 1, &mbs) > 1)
 		return (WEOF);
 	return (wc);
 }
+wint_t
+btowc(int c)
+{
+	return btowc_l(c, __get_locale());
+}
diff --git a/lib/libc/locale/collate.c b/lib/libc/locale/collate.c
index 6833598..e87b6dc 100644
--- a/lib/libc/locale/collate.c
+++ b/lib/libc/locale/collate.c
@@ -3,6 +3,16 @@
  *		at Electronni Visti IA, Kiev, Ukraine.
  *			All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -44,24 +54,77 @@ __FBSDID("$FreeBSD$");
 
 #include "libc_private.h"
 
-int __collate_load_error = 1;
-int __collate_substitute_nontrivial;
+/*
+ * To avoid modifying the original (single-threaded) code too much, we'll just
+ * define the old globals as fields inside the table.
+ *
+ * We also modify the collation table test functions to search the thread-local
+ * table first and the global table second.  
+ */
+#define __collate_load_error (table->__collate_load_error)
+#define __collate_substitute_nontrivial (table->__collate_substitute_nontrivial)
+#define __collate_substitute_table_ptr (table->__collate_substitute_table_ptr)
+#define __collate_char_pri_table_ptr (table->__collate_char_pri_table_ptr)
+#define __collate_chain_pri_table (table->__collate_chain_pri_table)
 
-u_char __collate_substitute_table[UCHAR_MAX + 1][STR_LEN];
-struct __collate_st_char_pri __collate_char_pri_table[UCHAR_MAX + 1];
-struct __collate_st_chain_pri *__collate_chain_pri_table;
+
+struct xlocale_collate __xlocale_global_collate = {
+	{{0}, "C"}, 1, 0
+};
+
+ struct xlocale_collate __xlocale_C_collate = {
+	{{0}, "C"}, 1, 0
+};
 
 void __collate_err(int ex, const char *f) __dead2;
 
 int
+__collate_load_tables_l(const char *encoding, struct xlocale_collate *table);
+
+static void
+destruct_collate(void *t)
+{
+	struct xlocale_collate *table = t;
+	if (__collate_chain_pri_table) {
+		free(__collate_chain_pri_table);
+	}
+	free(t);
+}
+
+void *
+__collate_load(const char *encoding, locale_t unused)
+{
+	if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0) {
+		return &__xlocale_C_collate;
+	}
+	struct xlocale_collate *table = calloc(sizeof(struct xlocale_collate), 1);
+	table->header.header.destructor = destruct_collate;
+	// FIXME: Make sure that _LDP_CACHE is never returned.  We should be doing
+	// the caching outside of this section
+	if (__collate_load_tables_l(encoding, table) != _LDP_LOADED) {
+		xlocale_release(table);
+		return NULL;
+	}
+	return table;
+}
+
+/**
+ * Load the collation tables for the specified encoding into the global table.
+ */
+int
 __collate_load_tables(const char *encoding)
 {
+	return __collate_load_tables_l(encoding, &__xlocale_global_collate);
+}
+
+int
+__collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
+{
 	FILE *fp;
 	int i, saverr, chains;
 	uint32_t u32;
 	char strbuf[STR_LEN], buf[PATH_MAX];
 	void *TMP_substitute_table, *TMP_char_pri_table, *TMP_chain_pri_table;
-	static char collate_encoding[ENCODING_LEN + 1];
 
 	/* 'encoding' must be already checked. */
 	if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0) {
@@ -69,18 +132,6 @@ __collate_load_tables(const char *encoding)
 		return (_LDP_CACHE);
 	}
 
-	/*
-	 * If the locale name is the same as our cache, use the cache.
-	 */
-	if (strcmp(encoding, collate_encoding) == 0) {
-		__collate_load_error = 0;
-		return (_LDP_CACHE);
-	}
-
-	/*
-	 * Slurp the locale file into the cache.
-	 */
-
 	/* 'PathLocale' must be already set & checked. */
 	/* Range checking not needed, encoding has fixed size */
 	(void)strcpy(buf, _PathLocale);
@@ -165,7 +216,6 @@ __collate_load_tables(const char *encoding)
 	      sizeof(*__collate_chain_pri_table), chains, fp);
 	(void)fclose(fp);
 
-	(void)strcpy(collate_encoding, encoding);
 	if (__collate_substitute_table_ptr != NULL)
 		free(__collate_substitute_table_ptr);
 	__collate_substitute_table_ptr = TMP_substitute_table;
@@ -201,7 +251,7 @@ __collate_load_tables(const char *encoding)
 }
 
 u_char *
-__collate_substitute(const u_char *s)
+__collate_substitute(struct xlocale_collate *table, const u_char *s)
 {
 	int dest_len, len, nlen;
 	int delta = strlen(s);
@@ -228,7 +278,7 @@ __collate_substitute(const u_char *s)
 }
 
 void
-__collate_lookup(const u_char *t, int *len, int *prim, int *sec)
+__collate_lookup(struct xlocale_collate *table, const u_char *t, int *len, int *prim, int *sec)
 {
 	struct __collate_st_chain_pri *p2;
 
diff --git a/lib/libc/locale/collate.h b/lib/libc/locale/collate.h
index 2f5f5d4..ad034d4 100644
--- a/lib/libc/locale/collate.h
+++ b/lib/libc/locale/collate.h
@@ -3,6 +3,11 @@
  *		at Electronni Visti IA, Kiev, Ukraine.
  *			All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,6 +38,7 @@
 #include <sys/cdefs.h>
 #include <sys/types.h>
 #include <limits.h>
+#include "xlocale_private.h"
 
 #define STR_LEN 10
 #define TABLE_SIZE 100
@@ -47,20 +53,26 @@ struct __collate_st_chain_pri {
 	int prim, sec;
 };
 
-extern int __collate_load_error;
-extern int __collate_substitute_nontrivial;
 #define __collate_substitute_table (*__collate_substitute_table_ptr)
-extern u_char __collate_substitute_table[UCHAR_MAX + 1][STR_LEN];
 #define __collate_char_pri_table (*__collate_char_pri_table_ptr)
-extern struct __collate_st_char_pri __collate_char_pri_table[UCHAR_MAX + 1];
-extern struct __collate_st_chain_pri *__collate_chain_pri_table;
+
+struct xlocale_collate {
+	struct xlocale_component header;
+	int __collate_load_error;
+	int __collate_substitute_nontrivial;
+
+	u_char (*__collate_substitute_table_ptr)[UCHAR_MAX + 1][STR_LEN];
+	struct __collate_st_char_pri (*__collate_char_pri_table_ptr)[UCHAR_MAX + 1];
+	struct __collate_st_chain_pri *__collate_chain_pri_table;
+};
+
 
 __BEGIN_DECLS
 u_char	*__collate_strdup(u_char *);
-u_char	*__collate_substitute(const u_char *);
+u_char	*__collate_substitute(struct xlocale_collate *, const u_char *);
 int	__collate_load_tables(const char *);
-void	__collate_lookup(const u_char *, int *, int *, int *);
-int	__collate_range_cmp(int, int);
+void	__collate_lookup(struct xlocale_collate *, const u_char *, int *, int *, int *);
+int	__collate_range_cmp(struct xlocale_collate *, int, int);
 #ifdef COLLATE_DEBUG
 void	__collate_print_tables(void);
 #endif
diff --git a/lib/libc/locale/collcmp.c b/lib/libc/locale/collcmp.c
index 313e043..aa17afd 100644
--- a/lib/libc/locale/collcmp.c
+++ b/lib/libc/locale/collcmp.c
@@ -2,6 +2,11 @@
  * Copyright (C) 1996 by Andrey A. Chernov, Moscow, Russia.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -28,17 +33,20 @@
 __FBSDID("$FreeBSD$");
 
 #include <string.h>
+#include <xlocale.h>
 #include "collate.h"
 
 /*
  * Compare two characters using collate
  */
 
-int __collate_range_cmp(int c1, int c2)
+int __collate_range_cmp(struct xlocale_collate *table, int c1, int c2)
 {
 	static char s1[2], s2[2];
 
 	s1[0] = c1;
 	s2[0] = c2;
-	return (strcoll(s1, s2));
+	struct _xlocale l = {{0}};
+	l.components[XLC_COLLATE] = (struct xlocale_component *)table;
+	return (strcoll_l(s1, s2, &l));
 }
diff --git a/lib/libc/locale/ctype.c b/lib/libc/locale/ctype.c
new file mode 100644
index 0000000..f0ec3f6
--- /dev/null
+++ b/lib/libc/locale/ctype.c
@@ -0,0 +1,33 @@
+/*-
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by David Chisnall under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions * are met:
+ * 1.  Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#define _XLOCALE_INLINE
+#include <ctype.h>
+#include <wctype.h>
+#include <xlocale.h>
diff --git a/lib/libc/locale/duplocale.3 b/lib/libc/locale/duplocale.3
new file mode 100644
index 0000000..19d2569
--- /dev/null
+++ b/lib/libc/locale/duplocale.3
@@ -0,0 +1,78 @@
+.\" Copyright (c) 2011 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This documentation was written by David Chisnall under sponsorship from
+.\" the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd September 17 2011
+.Dt DUPLOCALE 3
+.Os
+.Sh NAME
+.Nm duplocale
+.Nd duplicate an locale
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In xlocale.h 
+.Ft locale_t 
+.Fn duplocale "locale_t locale"
+.Sh DESCRIPTION
+Duplicates an existing
+.Fa locale_t
+returning a new 
+.Fa locale_t
+that refers to the same locale values but has independent internal state.
+Various functions, such as
+.Xr mblen 3
+require presistent state.  These functions formerly used static variables and
+calls to them from multiple threads had undefined behavior.  They now use
+fields in the
+.Fa locale_t
+associated with the current thread by
+.Xr uselocale 3 .
+These calls are therefore only thread safe on threads with a unique per-thread
+locale.  
+.Pt
+The locale returned by this call must be freed with
+.Xr freelocale 3 .
+.Sh BUGS
+Ideally,
+.Xr uselocale 3
+should make a copy of the
+.Fa locale_t
+implicitly to ensure thread safety, and a copy of the global locale should be
+installed lazily on each thread.  The FreeBSD implementation does not do this,
+for compatibility with Darwin.
+.Sh SEE ALSO
+.Xr freelocale 3 ,
+.Xr localeconv 3 ,
+.Xr newlocale 3 ,
+.Xr querylocale 3 ,
+.Xr uselocale 3 ,
+.Xr xlocale 3
+.Sh STANDARDS
+This function, conforms to
+.St -p1003.1-2008
diff --git a/lib/libc/locale/euc.c b/lib/libc/locale/euc.c
index 188073e..26ad413 100644
--- a/lib/libc/locale/euc.c
+++ b/lib/libc/locale/euc.c
@@ -6,6 +6,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -70,7 +75,7 @@ typedef struct {
 } _EucState;
 
 int
-_EUC_init(_RuneLocale *rl)
+_EUC_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 	_EucInfo *ei;
 	int x, new__mb_cur_max;
@@ -113,12 +118,12 @@ _EUC_init(_RuneLocale *rl)
 	}
 	rl->__variable = ei;
 	rl->__variable_len = sizeof(_EucInfo);
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = new__mb_cur_max;
-	__mbrtowc = _EUC_mbrtowc;
-	__wcrtomb = _EUC_wcrtomb;
-	__mbsinit = _EUC_mbsinit;
-	__mb_sb_limit = 256;
+	l->runes = rl;
+	l->__mb_cur_max = new__mb_cur_max;
+	l->__mbrtowc = _EUC_mbrtowc;
+	l->__wcrtomb = _EUC_wcrtomb;
+	l->__mbsinit = _EUC_mbsinit;
+	l->__mb_sb_limit = 256;
 	return (0);
 }
 
diff --git a/lib/libc/locale/freelocale.3 b/lib/libc/locale/freelocale.3
new file mode 100644
index 0000000..cf7174d
--- /dev/null
+++ b/lib/libc/locale/freelocale.3
@@ -0,0 +1,61 @@
+.\" Copyright (c) 2011 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This documentation was written by David Chisnall under sponsorship from
+.\" the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd September 17 2011
+.Dt FREELOCALE 3
+.Os
+.Sh NAME
+.Nm freelocale
+.Nd Frees a locale created with
+.Xr duplocale 3
+or 
+.Xr newlocale 3 .
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In xlocale.h 
+.Ft int
+.Fn freelocale "locale_t locale"
+.Sh DESCRIPTION
+Frees a
+.Fa locale_t .
+This relinquishes any resources held exclusively by this locale.  Note that
+locales share reference-counted components, so a call to this function is not
+guaranteed to free all of the components.
+.Sh RETURN VALUES
+Returns 0 on success or -1 on error.  
+.Sh SEE ALSO
+.Xr duplocale 3 ,
+.Xr localeconv 3 ,
+.Xr newlocale 3 ,
+.Xr querylocale 3 ,
+.Xr uselocale 3 ,
+.Xr xlocale 3
+.Sh STANDARDS
+This function, conforms to
+.St -p1003.1-2008 .
diff --git a/lib/libc/locale/gb18030.c b/lib/libc/locale/gb18030.c
index 1457d3e..9214385 100644
--- a/lib/libc/locale/gb18030.c
+++ b/lib/libc/locale/gb18030.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,8 +44,6 @@ __FBSDID("$FreeBSD$");
 #include <wchar.h>
 #include "mblocal.h"
 
-extern int __mb_sb_limit;
-
 static size_t	_GB18030_mbrtowc(wchar_t * __restrict, const char * __restrict,
 		    size_t, mbstate_t * __restrict);
 static int	_GB18030_mbsinit(const mbstate_t *);
@@ -53,15 +56,15 @@ typedef struct {
 } _GB18030State;
 
 int
-_GB18030_init(_RuneLocale *rl)
+_GB18030_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	__mbrtowc = _GB18030_mbrtowc;
-	__wcrtomb = _GB18030_wcrtomb;
-	__mbsinit = _GB18030_mbsinit;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 4;
-	__mb_sb_limit = 128;
+	l->__mbrtowc = _GB18030_mbrtowc;
+	l->__wcrtomb = _GB18030_wcrtomb;
+	l->__mbsinit = _GB18030_mbsinit;
+	l->runes = rl;
+	l->__mb_cur_max = 4;
+	l->__mb_sb_limit = 128;
 
 	return (0);
 }
diff --git a/lib/libc/locale/gb2312.c b/lib/libc/locale/gb2312.c
index 74a7bdc..5fbc07d 100644
--- a/lib/libc/locale/gb2312.c
+++ b/lib/libc/locale/gb2312.c
@@ -3,6 +3,11 @@
  * Copyright (c) 2003 David Xu <davidxu@freebsd.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,8 +40,6 @@ __FBSDID("$FreeBSD$");
 #include <wchar.h>
 #include "mblocal.h"
 
-extern int __mb_sb_limit;
-
 static size_t	_GB2312_mbrtowc(wchar_t * __restrict, const char * __restrict,
 		    size_t, mbstate_t * __restrict);
 static int	_GB2312_mbsinit(const mbstate_t *);
@@ -49,15 +52,15 @@ typedef struct {
 } _GB2312State;
 
 int
-_GB2312_init(_RuneLocale *rl)
+_GB2312_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	_CurrentRuneLocale = rl;
-	__mbrtowc = _GB2312_mbrtowc;
-	__wcrtomb = _GB2312_wcrtomb;
-	__mbsinit = _GB2312_mbsinit;
-	__mb_cur_max = 2;
-	__mb_sb_limit = 128;
+	l->runes = rl;
+	l->__mbrtowc = _GB2312_mbrtowc;
+	l->__wcrtomb = _GB2312_wcrtomb;
+	l->__mbsinit = _GB2312_mbsinit;
+	l->__mb_cur_max = 2;
+	l->__mb_sb_limit = 128;
 	return (0);
 }
 
diff --git a/lib/libc/locale/gbk.c b/lib/libc/locale/gbk.c
index 802f78e..43269c7 100644
--- a/lib/libc/locale/gbk.c
+++ b/lib/libc/locale/gbk.c
@@ -6,6 +6,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -55,15 +60,15 @@ typedef struct {
 } _GBKState;
 
 int
-_GBK_init(_RuneLocale *rl)
+_GBK_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	__mbrtowc = _GBK_mbrtowc;
-	__wcrtomb = _GBK_wcrtomb;
-	__mbsinit = _GBK_mbsinit;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 2;
-	__mb_sb_limit = 128;
+	l->__mbrtowc = _GBK_mbrtowc;
+	l->__wcrtomb = _GBK_wcrtomb;
+	l->__mbsinit = _GBK_mbsinit;
+	l->runes = rl;
+	l->__mb_cur_max = 2;
+	l->__mb_sb_limit = 128;
 	return (0);
 }
 
diff --git a/lib/libc/locale/lmessages.c b/lib/libc/locale/lmessages.c
index 3498be2..a0664d0 100644
--- a/lib/libc/locale/lmessages.c
+++ b/lib/libc/locale/lmessages.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -36,6 +41,14 @@ __FBSDID("$FreeBSD$");
 #define LCMESSAGES_SIZE_MIN \
 		(offsetof(struct lc_messages_T, yesstr) / sizeof(char *))
 
+struct xlocale_messages {
+	struct xlocale_component header;
+	char *buffer;
+	struct lc_messages_T locale;
+};
+
+struct xlocale_messages __xlocale_global_messages;
+
 static char empty[] = "";
 
 static const struct lc_messages_T _C_messages_locale = {
@@ -45,33 +58,55 @@ static const struct lc_messages_T _C_messages_locale = {
 	"no"		/* nostr */
 };
 
-static struct lc_messages_T _messages_locale;
-static int	_messages_using_locale;
-static char	*_messages_locale_buf;
+static void destruct_messages(void *v)
+{
+	struct xlocale_messages *l = v;
+	if (l->buffer)
+		free(l->buffer);
+	free(l);
+}
 
-int
-__messages_load_locale(const char *name)
+static int
+messages_load_locale(struct xlocale_messages *loc, int *using_locale, const char *name)
 {
 	int ret;
+	struct lc_messages_T *l = &loc->locale;
 
-	ret = __part_load_locale(name, &_messages_using_locale,
-		  &_messages_locale_buf, "LC_MESSAGES",
+	ret = __part_load_locale(name, using_locale,
+		  &loc->buffer, "LC_MESSAGES",
 		  LCMESSAGES_SIZE_FULL, LCMESSAGES_SIZE_MIN,
-		  (const char **)&_messages_locale);
+		  (const char **)l);
 	if (ret == _LDP_LOADED) {
-		if (_messages_locale.yesstr == NULL)
-			_messages_locale.yesstr = empty;
-		if (_messages_locale.nostr == NULL)
-			_messages_locale.nostr = empty;
+		if (l->yesstr == NULL)
+			l->yesstr = empty;
+		if (l->nostr == NULL)
+			l->nostr = empty;
 	}
 	return (ret);
 }
+int
+__messages_load_locale(const char *name)
+{
+	return messages_load_locale(&__xlocale_global_messages,
+			&__xlocale_global_locale.using_messages_locale, name);
+}
+void *
+__messages_load(const char *name, locale_t l)
+{
+	struct xlocale_messages *new = calloc(sizeof(struct xlocale_messages), 1);
+	new->header.header.destructor = destruct_messages;
+	if (messages_load_locale(new, &l->using_messages_locale, name) == _LDP_ERROR) {
+		xlocale_release(new);
+		return NULL;
+	}
+	return new;
+}
 
 struct lc_messages_T *
-__get_current_messages_locale(void)
+__get_current_messages_locale(locale_t loc)
 {
-	return (_messages_using_locale
-		? &_messages_locale
+	return (loc->using_messages_locale
+		? &((struct xlocale_messages *)loc->components[XLC_MESSAGES])->locale
 		: (struct lc_messages_T *)&_C_messages_locale);
 }
 
diff --git a/lib/libc/locale/lmessages.h b/lib/libc/locale/lmessages.h
index 41dcfa9..ea9c57e 100644
--- a/lib/libc/locale/lmessages.h
+++ b/lib/libc/locale/lmessages.h
@@ -2,6 +2,11 @@
  * Copyright (c) 2000, 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -29,6 +34,8 @@
 #ifndef _LMESSAGES_H_
 #define	_LMESSAGES_H_
 
+#include "xlocale_private.h"
+
 struct	lc_messages_T {
 	const char	*yesexpr;
 	const char	*noexpr;
@@ -36,7 +43,7 @@ struct	lc_messages_T {
 	const char	*nostr;
 };
 
-struct lc_messages_T *__get_current_messages_locale(void);
+struct lc_messages_T *__get_current_messages_locale(locale_t);
 int	__messages_load_locale(const char *);
 
 #endif /* !_LMESSAGES_H_ */
diff --git a/lib/libc/locale/lmonetary.c b/lib/libc/locale/lmonetary.c
index 8d55ab2..a9d67d3 100644
--- a/lib/libc/locale/lmonetary.c
+++ b/lib/libc/locale/lmonetary.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2000, 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,7 +39,6 @@ __FBSDID("$FreeBSD$");
 #include "ldpart.h"
 #include "lmonetary.h"
 
-extern int __mlocale_changed;
 extern const char * __fix_locale_grouping_str(const char *);
 
 #define LCMONETARY_SIZE_FULL (sizeof(struct lc_monetary_T) / sizeof(char *))
@@ -69,9 +73,7 @@ static const struct lc_monetary_T _C_monetary_locale = {
 	numempty	/* int_n_sign_posn */
 };
 
-static struct lc_monetary_T _monetary_locale;
-static int	_monetary_using_locale;
-static char	*_monetary_locale_buf;
+struct xlocale_monetary __xlocale_global_monetary;
 
 static char
 cnv(const char *str)
@@ -83,23 +85,34 @@ cnv(const char *str)
 	return ((char)i);
 }
 
-int
-__monetary_load_locale(const char *name)
+static void
+destruct_monetary(void *v)
+{
+	struct xlocale_monetary *l = v;
+	if (l->buffer)
+		free(l->buffer);
+	free(l);
+}
+
+static int
+monetary_load_locale_l(struct xlocale_monetary *loc, int *using_locale,
+		int *changed, const char *name)
 {
 	int ret;
+	struct lc_monetary_T *l = &loc->locale;
 
-	ret = __part_load_locale(name, &_monetary_using_locale,
-		&_monetary_locale_buf, "LC_MONETARY",
+	ret = __part_load_locale(name, using_locale,
+		&loc->buffer, "LC_MONETARY",
 		LCMONETARY_SIZE_FULL, LCMONETARY_SIZE_MIN,
-		(const char **)&_monetary_locale);
+		(const char **)l);
 	if (ret != _LDP_ERROR)
-		__mlocale_changed = 1;
+		*changed = 1;
 	if (ret == _LDP_LOADED) {
-		_monetary_locale.mon_grouping =
-		     __fix_locale_grouping_str(_monetary_locale.mon_grouping);
+		l->mon_grouping =
+		     __fix_locale_grouping_str(l->mon_grouping);
 
-#define M_ASSIGN_CHAR(NAME) (((char *)_monetary_locale.NAME)[0] = \
-			     cnv(_monetary_locale.NAME))
+#define M_ASSIGN_CHAR(NAME) (((char *)l->NAME)[0] = \
+			     cnv(l->NAME))
 
 		M_ASSIGN_CHAR(int_frac_digits);
 		M_ASSIGN_CHAR(frac_digits);
@@ -117,9 +130,9 @@ __monetary_load_locale(const char *name)
 		 */
 #define	M_ASSIGN_ICHAR(NAME)						\
 		do {							\
-			if (_monetary_locale.int_##NAME == NULL)	\
-				_monetary_locale.int_##NAME =		\
-				    _monetary_locale.NAME;		\
+			if (l->int_##NAME == NULL)	\
+				l->int_##NAME =		\
+				    l->NAME;		\
 			else						\
 				M_ASSIGN_CHAR(int_##NAME);		\
 		} while (0)
@@ -133,12 +146,32 @@ __monetary_load_locale(const char *name)
 	}
 	return (ret);
 }
+int
+__monetary_load_locale(const char *name)
+{
+	return monetary_load_locale_l(&__xlocale_global_monetary,
+			&__xlocale_global_locale.using_monetary_locale,
+			&__xlocale_global_locale.monetary_locale_changed, name);
+}
+void* __monetary_load(const char *name, locale_t l)
+{
+	struct xlocale_monetary *new = calloc(sizeof(struct xlocale_monetary), 1);
+	new->header.header.destructor = destruct_monetary;
+	if (monetary_load_locale_l(new, &l->using_monetary_locale,
+				&l->monetary_locale_changed, name) == _LDP_ERROR)
+	{
+		xlocale_release(new);
+		return NULL;
+	}
+	return new;
+}
+
 
 struct lc_monetary_T *
-__get_current_monetary_locale(void)
+__get_current_monetary_locale(locale_t loc)
 {
-	return (_monetary_using_locale
-		? &_monetary_locale
+	return (loc->using_monetary_locale
+		? &((struct xlocale_monetary*)loc->components[XLC_MONETARY])->locale
 		: (struct lc_monetary_T *)&_C_monetary_locale);
 }
 
diff --git a/lib/libc/locale/lmonetary.h b/lib/libc/locale/lmonetary.h
index 45ec323..b606862 100644
--- a/lib/libc/locale/lmonetary.h
+++ b/lib/libc/locale/lmonetary.h
@@ -2,6 +2,11 @@
  * Copyright (c) 2000, 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -28,6 +33,7 @@
 
 #ifndef _LMONETARY_H_
 #define	_LMONETARY_H_
+#include "xlocale_private.h"
 
 struct lc_monetary_T {
 	const char	*int_curr_symbol;
@@ -52,8 +58,13 @@ struct lc_monetary_T {
 	const char	*int_p_sign_posn;
 	const char	*int_n_sign_posn;
 };
+struct xlocale_monetary {
+	struct xlocale_component header;
+	char *buffer;
+	struct lc_monetary_T locale;
+};
 
-struct lc_monetary_T *__get_current_monetary_locale(void);
+struct lc_monetary_T *__get_current_monetary_locale(locale_t loc);
 int	__monetary_load_locale(const char *);
 
 #endif /* !_LMONETARY_H_ */
diff --git a/lib/libc/locale/lnumeric.c b/lib/libc/locale/lnumeric.c
index d4ed04f..a2468f4 100644
--- a/lib/libc/locale/lnumeric.c
+++ b/lib/libc/locale/lnumeric.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2000, 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include "ldpart.h"
 #include "lnumeric.h"
 
-extern int __nlocale_changed;
 extern const char *__fix_locale_grouping_str(const char *);
 
 #define LCNUMERIC_SIZE (sizeof(struct lc_numeric_T) / sizeof(char *))
@@ -45,37 +49,67 @@ static const struct lc_numeric_T _C_numeric_locale = {
 	numempty	/* grouping */
 };
 
-static struct lc_numeric_T _numeric_locale;
-static int	_numeric_using_locale;
-static char	*_numeric_locale_buf;
+static void
+destruct_numeric(void *v)
+{
+	struct xlocale_numeric *l = v;
+	if (l->buffer)
+		free(l->buffer);
+	free(l);
+}
+
+struct xlocale_numeric __xlocale_global_numeric;
 
-int
-__numeric_load_locale(const char *name)
+static int
+numeric_load_locale(struct xlocale_numeric *loc, int *using_locale, int *changed,
+		const char *name)
 {
 	int ret;
+	struct lc_numeric_T *l = &loc->locale;
 
-	ret = __part_load_locale(name, &_numeric_using_locale,
-		&_numeric_locale_buf, "LC_NUMERIC",
+	ret = __part_load_locale(name, using_locale,
+		&loc->buffer, "LC_NUMERIC",
 		LCNUMERIC_SIZE, LCNUMERIC_SIZE,
-		(const char **)&_numeric_locale);
+		(const char**)l);
 	if (ret != _LDP_ERROR)
-		__nlocale_changed = 1;
+		*changed= 1;
 	if (ret == _LDP_LOADED) {
 		/* Can't be empty according to C99 */
-		if (*_numeric_locale.decimal_point == '\0')
-			_numeric_locale.decimal_point =
+		if (*l->decimal_point == '\0')
+			l->decimal_point =
 			    _C_numeric_locale.decimal_point;
-		_numeric_locale.grouping =
-		    __fix_locale_grouping_str(_numeric_locale.grouping);
+		l->grouping =
+		    __fix_locale_grouping_str(l->grouping);
 	}
 	return (ret);
 }
 
+int
+__numeric_load_locale(const char *name)
+{
+	return numeric_load_locale(&__xlocale_global_numeric,
+			&__xlocale_global_locale.using_numeric_locale,
+			&__xlocale_global_locale.numeric_locale_changed, name);
+}
+void *
+__numeric_load(const char *name, locale_t l)
+{
+	struct xlocale_numeric *new = calloc(sizeof(struct xlocale_numeric), 1);
+	new->header.header.destructor = destruct_numeric;
+	if (numeric_load_locale(new, &l->using_numeric_locale,
+				&l->numeric_locale_changed, name) == _LDP_ERROR)
+	{
+		xlocale_release(new);
+		return NULL;
+	}
+	return new;
+}
+
 struct lc_numeric_T *
-__get_current_numeric_locale(void)
+__get_current_numeric_locale(locale_t loc)
 {
-	return (_numeric_using_locale
-		? &_numeric_locale
+	return (loc->using_numeric_locale
+		? &((struct xlocale_numeric *)loc->components[XLC_NUMERIC])->locale
 		: (struct lc_numeric_T *)&_C_numeric_locale);
 }
 
diff --git a/lib/libc/locale/lnumeric.h b/lib/libc/locale/lnumeric.h
index cc6965b..5088dd4 100644
--- a/lib/libc/locale/lnumeric.h
+++ b/lib/libc/locale/lnumeric.h
@@ -2,6 +2,11 @@
  * Copyright (c) 2000, 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -28,14 +33,20 @@
 
 #ifndef _LNUMERIC_H_
 #define	_LNUMERIC_H_
+#include "xlocale_private.h"
 
 struct lc_numeric_T {
 	const char	*decimal_point;
 	const char	*thousands_sep;
 	const char	*grouping;
 };
+struct xlocale_numeric {
+	struct xlocale_component header;
+	char *buffer;
+	struct lc_numeric_T locale;
+};
 
-struct lc_numeric_T *__get_current_numeric_locale(void);
+struct lc_numeric_T *__get_current_numeric_locale(locale_t loc);
 int	__numeric_load_locale(const char *);
 
 #endif /* !_LNUMERIC_H_ */
diff --git a/lib/libc/locale/localeconv.3 b/lib/libc/locale/localeconv.3
index c4b5746..6ebb878 100644
--- a/lib/libc/locale/localeconv.3
+++ b/lib/libc/locale/localeconv.3
@@ -44,6 +44,9 @@
 .In locale.h
 .Ft struct lconv *
 .Fn localeconv "void"
+.In xlocale.h
+.Ft struct lconv *
+.Fn localeconv_l "locale_t locale"
 .Sh DESCRIPTION
 The
 .Fn localeconv
@@ -196,6 +199,11 @@ a value that is not in the current locale.
 A
 .Dv CHAR_MAX
 result similarly denotes an unavailable value.
+.Pp
+The
+.Fn localeconv_l
+function takes an explicit locale parameter.  For more information, see
+.Xr xlocale 3 .  
 .Sh RETURN VALUES
 The
 .Fn localeconv
@@ -204,6 +212,13 @@ which may be altered by later calls to
 .Xr setlocale 3
 or
 .Fn localeconv .
+The return value for 
+.Fn localeconv_l
+is stored with the locale.  It will remain valid until a subsequent call to
+.Xr freelocale 3 .
+If a thread-local locale is in effect then the return value from
+.Fn localeconv
+will remain valid until the locale is destroyed.
 .Sh ERRORS
 No errors are defined.
 .Sh SEE ALSO
diff --git a/lib/libc/locale/localeconv.c b/lib/libc/locale/localeconv.c
index fed8d35..4b19160 100644
--- a/lib/libc/locale/localeconv.c
+++ b/lib/libc/locale/localeconv.c
@@ -3,6 +3,11 @@
  * Copyright (c) 1991, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -48,25 +53,24 @@ __FBSDID("$FreeBSD$");
  * lconv structure are computed only when the monetary or numeric 
  * locale has been changed.
  */
-int __mlocale_changed = 1;
-int __nlocale_changed = 1;
 
 /*
  * Return the current locale conversion.
  */
 struct lconv *
-localeconv()
+localeconv_l(locale_t loc)
 {
-    static struct lconv ret;
+	FIX_LOCALE(loc);
+    struct lconv *ret = &loc->lconv;
 
-    if (__mlocale_changed) {
+    if (loc->monetary_locale_changed) {
 	/* LC_MONETARY part */
         struct lc_monetary_T * mptr; 
 
-#define M_ASSIGN_STR(NAME) (ret.NAME = (char*)mptr->NAME)
-#define M_ASSIGN_CHAR(NAME) (ret.NAME = mptr->NAME[0])
+#define M_ASSIGN_STR(NAME) (ret->NAME = (char*)mptr->NAME)
+#define M_ASSIGN_CHAR(NAME) (ret->NAME = mptr->NAME[0])
 
-	mptr = __get_current_monetary_locale();
+	mptr = __get_current_monetary_locale(loc);
 	M_ASSIGN_STR(int_curr_symbol);
 	M_ASSIGN_STR(currency_symbol);
 	M_ASSIGN_STR(mon_decimal_point);
@@ -88,21 +92,26 @@ localeconv()
 	M_ASSIGN_CHAR(int_n_sep_by_space);
 	M_ASSIGN_CHAR(int_p_sign_posn);
 	M_ASSIGN_CHAR(int_n_sign_posn);
-	__mlocale_changed = 0;
+	loc->monetary_locale_changed = 0;
     }
 
-    if (__nlocale_changed) {
+    if (loc->numeric_locale_changed) {
 	/* LC_NUMERIC part */
         struct lc_numeric_T * nptr; 
 
-#define N_ASSIGN_STR(NAME) (ret.NAME = (char*)nptr->NAME)
+#define N_ASSIGN_STR(NAME) (ret->NAME = (char*)nptr->NAME)
 
-	nptr = __get_current_numeric_locale();
+	nptr = __get_current_numeric_locale(loc);
 	N_ASSIGN_STR(decimal_point);
 	N_ASSIGN_STR(thousands_sep);
 	N_ASSIGN_STR(grouping);
-	__nlocale_changed = 0;
+	loc->numeric_locale_changed = 0;
     }
 
-    return (&ret);
+    return ret;
+}
+struct lconv *
+localeconv(void)
+{
+	return localeconv_l(__get_locale());
 }
diff --git a/lib/libc/locale/mblen.c b/lib/libc/locale/mblen.c
index 4a2cc5c..1bacf3e 100644
--- a/lib/libc/locale/mblen.c
+++ b/lib/libc/locale/mblen.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,19 +37,25 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 int
-mblen(const char *s, size_t n)
+mblen_l(const char *s, size_t n, locale_t locale)
 {
 	static const mbstate_t initial;
-	static mbstate_t mbs;
 	size_t rval;
+	FIX_LOCALE(locale);
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		mbs = initial;
+		locale->mblen = initial;
 		return (0);
 	}
-	rval = __mbrtowc(NULL, s, n, &mbs);
+	rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, &locale->mblen);
 	if (rval == (size_t)-1 || rval == (size_t)-2)
 		return (-1);
 	return ((int)rval);
 }
+
+int
+mblen(const char *s, size_t n)
+{
+	return mblen_l(s, n, __get_locale());
+}
diff --git a/lib/libc/locale/mblocal.h b/lib/libc/locale/mblocal.h
index 0faebe5..d172764 100644
--- a/lib/libc/locale/mblocal.h
+++ b/lib/libc/locale/mblocal.h
@@ -2,6 +2,11 @@
  * Copyright (c) 2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,35 +35,45 @@
 #define	_MBLOCAL_H_
 
 #include <runetype.h>
+#include "xlocale_private.h"
+
 
 /*
- * Rune initialization function prototypes.
+ * Conversion function pointers for current encoding.
  */
-int	_none_init(_RuneLocale *);
-int	_ascii_init(_RuneLocale *);
-int	_UTF8_init(_RuneLocale *);
-int	_EUC_init(_RuneLocale *);
-int	_GB18030_init(_RuneLocale *);
-int	_GB2312_init(_RuneLocale *);
-int	_GBK_init(_RuneLocale *);
-int	_BIG5_init(_RuneLocale *);
-int	_MSKanji_init(_RuneLocale *);
+struct xlocale_ctype {
+	struct xlocale_component header;
+	_RuneLocale *runes;
+	size_t (*__mbrtowc)(wchar_t * __restrict, const char * __restrict,
+		size_t, mbstate_t * __restrict);
+	int (*__mbsinit)(const mbstate_t *);
+	size_t (*__mbsnrtowcs)(wchar_t * __restrict, const char ** __restrict,
+		size_t, size_t, mbstate_t * __restrict);
+	size_t (*__wcrtomb)(char * __restrict, wchar_t, mbstate_t * __restrict);
+	size_t (*__wcsnrtombs)(char * __restrict, const wchar_t ** __restrict,
+		size_t, size_t, mbstate_t * __restrict);
+	int __mb_cur_max;
+	int __mb_sb_limit;
+};
+#define XLOCALE_CTYPE(x) ((struct xlocale_ctype*)(x)->components[XLC_CTYPE])
+extern struct xlocale_ctype __xlocale_global_ctype;
 
 /*
- * Conversion function pointers for current encoding.
+ * Rune initialization function prototypes.
  */
-extern size_t (*__mbrtowc)(wchar_t * __restrict, const char * __restrict,
-    size_t, mbstate_t * __restrict);
-extern int (*__mbsinit)(const mbstate_t *);
-extern size_t (*__mbsnrtowcs)(wchar_t * __restrict, const char ** __restrict,
-    size_t, size_t, mbstate_t * __restrict);
-extern size_t (*__wcrtomb)(char * __restrict, wchar_t, mbstate_t * __restrict);
-extern size_t (*__wcsnrtombs)(char * __restrict, const wchar_t ** __restrict,
-    size_t, size_t, mbstate_t * __restrict);
+int	_none_init(struct xlocale_ctype *, _RuneLocale *);
+int	_ascii_init(struct xlocale_ctype *, _RuneLocale *);
+int	_UTF8_init(struct xlocale_ctype *, _RuneLocale *);
+int	_EUC_init(struct xlocale_ctype *, _RuneLocale *);
+int	_GB18030_init(struct xlocale_ctype *, _RuneLocale *);
+int	_GB2312_init(struct xlocale_ctype *, _RuneLocale *);
+int	_GBK_init(struct xlocale_ctype *, _RuneLocale *);
+int	_BIG5_init(struct xlocale_ctype *, _RuneLocale *);
+int	_MSKanji_init(struct xlocale_ctype *, _RuneLocale *);
 
 extern size_t __mbsnrtowcs_std(wchar_t * __restrict, const char ** __restrict,
-    size_t, size_t, mbstate_t * __restrict);
+	size_t, size_t, mbstate_t * __restrict);
 extern size_t __wcsnrtombs_std(char * __restrict, const wchar_t ** __restrict,
-    size_t, size_t, mbstate_t * __restrict);
+	size_t, size_t, mbstate_t * __restrict);
 
 #endif	/* _MBLOCAL_H_ */
diff --git a/lib/libc/locale/mbrlen.c b/lib/libc/locale/mbrlen.c
index b4c3a04..9ce0bc0 100644
--- a/lib/libc/locale/mbrlen.c
+++ b/lib/libc/locale/mbrlen.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -31,11 +36,16 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
-mbrlen(const char * __restrict s, size_t n, mbstate_t * __restrict ps)
+mbrlen_l(const char * __restrict s, size_t n, mbstate_t * __restrict ps, locale_t locale)
 {
-	static mbstate_t mbs;
-
+	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &mbs;
-	return (__mbrtowc(NULL, s, n, ps));
+		ps = &locale->mbrlen;
+	return (XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, ps));
+}
+
+size_t
+mbrlen(const char * __restrict s, size_t n, mbstate_t * __restrict ps)
+{
+	return mbrlen_l(s, n, ps, __get_locale());
 }
diff --git a/lib/libc/locale/mbrtowc.c b/lib/libc/locale/mbrtowc.c
index 42d5f7a..8f1aebd 100644
--- a/lib/libc/locale/mbrtowc.c
+++ b/lib/libc/locale/mbrtowc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -31,12 +36,18 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
+mbrtowc_l(wchar_t * __restrict pwc, const char * __restrict s,
+    size_t n, mbstate_t * __restrict ps, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	if (ps == NULL)
+		ps = &locale->mbrtowc;
+	return (XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, ps));
+}
+
+size_t
 mbrtowc(wchar_t * __restrict pwc, const char * __restrict s,
     size_t n, mbstate_t * __restrict ps)
 {
-	static mbstate_t mbs;
-
-	if (ps == NULL)
-		ps = &mbs;
-	return (__mbrtowc(pwc, s, n, ps));
+	return mbrtowc_l(pwc, s, n, ps, __get_locale());
 }
diff --git a/lib/libc/locale/mbsinit.c b/lib/libc/locale/mbsinit.c
index 24408c7..be44fe5 100644
--- a/lib/libc/locale/mbsinit.c
+++ b/lib/libc/locale/mbsinit.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -31,8 +36,13 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 int
+mbsinit_l(const mbstate_t *ps, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	return (XLOCALE_CTYPE(locale)->__mbsinit(ps));
+}
+int
 mbsinit(const mbstate_t *ps)
 {
-
-	return (__mbsinit(ps));
+	return mbsinit_l(ps, __get_locale());
 }
diff --git a/lib/libc/locale/mbsnrtowcs.c b/lib/libc/locale/mbsnrtowcs.c
index 5284087..15b48dd7 100644
--- a/lib/libc/locale/mbsnrtowcs.c
+++ b/lib/libc/locale/mbsnrtowcs.c
@@ -1,5 +1,10 @@
 /*-
  * Copyright (c) 2002-2004 Tim J. Robbins.
+ *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,14 +39,19 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
+mbsnrtowcs_l(wchar_t * __restrict dst, const char ** __restrict src,
+    size_t nms, size_t len, mbstate_t * __restrict ps, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	if (ps == NULL)
+		ps = &locale->mbsnrtowcs;
+	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, nms, len, ps));
+}
+size_t
 mbsnrtowcs(wchar_t * __restrict dst, const char ** __restrict src,
     size_t nms, size_t len, mbstate_t * __restrict ps)
 {
-	static mbstate_t mbs;
-
-	if (ps == NULL)
-		ps = &mbs;
-	return (__mbsnrtowcs(dst, src, nms, len, ps));
+	return mbsnrtowcs_l(dst, src, nms, len, ps, __get_locale());
 }
 
 size_t
@@ -52,13 +62,14 @@ __mbsnrtowcs_std(wchar_t * __restrict dst, const char ** __restrict src,
 	size_t nchr;
 	wchar_t wc;
 	size_t nb;
+	struct xlocale_ctype *ct = XLOCALE_CTYPE(__get_locale());
 
 	s = *src;
 	nchr = 0;
 
 	if (dst == NULL) {
 		for (;;) {
-			if ((nb = __mbrtowc(&wc, s, nms, ps)) == (size_t)-1)
+			if ((nb = ct->__mbrtowc(&wc, s, nms, ps)) == (size_t)-1)
 				/* Invalid sequence - mbrtowc() sets errno. */
 				return ((size_t)-1);
 			else if (nb == 0 || nb == (size_t)-2)
@@ -71,7 +82,7 @@ __mbsnrtowcs_std(wchar_t * __restrict dst, const char ** __restrict src,
 	}
 
 	while (len-- > 0) {
-		if ((nb = __mbrtowc(dst, s, nms, ps)) == (size_t)-1) {
+		if ((nb = ct->__mbrtowc(dst, s, nms, ps)) == (size_t)-1) {
 			*src = s;
 			return ((size_t)-1);
 		} else if (nb == (size_t)-2) {
diff --git a/lib/libc/locale/mbsrtowcs.c b/lib/libc/locale/mbsrtowcs.c
index 1239c82..9032b94 100644
--- a/lib/libc/locale/mbsrtowcs.c
+++ b/lib/libc/locale/mbsrtowcs.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,12 +39,17 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
+mbsrtowcs_l(wchar_t * __restrict dst, const char ** __restrict src, size_t len,
+    mbstate_t * __restrict ps, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	if (ps == NULL)
+		ps = &locale->mbsrtowcs;
+	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, SIZE_T_MAX, len, ps));
+}
+size_t
 mbsrtowcs(wchar_t * __restrict dst, const char ** __restrict src, size_t len,
     mbstate_t * __restrict ps)
 {
-	static mbstate_t mbs;
-
-	if (ps == NULL)
-		ps = &mbs;
-	return (__mbsnrtowcs(dst, src, SIZE_T_MAX, len, ps));
+	return mbsrtowcs_l(dst, src, len, ps, __get_locale());
 }
diff --git a/lib/libc/locale/mbstowcs.c b/lib/libc/locale/mbstowcs.c
index 194ec2e..6905d2e 100644
--- a/lib/libc/locale/mbstowcs.c
+++ b/lib/libc/locale/mbstowcs.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,13 +38,19 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
-mbstowcs(wchar_t * __restrict pwcs, const char * __restrict s, size_t n)
+mbstowcs_l(wchar_t * __restrict pwcs, const char * __restrict s, size_t n, locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
 	const char *sp;
+	FIX_LOCALE(locale);
 
 	mbs = initial;
 	sp = s;
-	return (__mbsnrtowcs(pwcs, &sp, SIZE_T_MAX, n, &mbs));
+	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(pwcs, &sp, SIZE_T_MAX, n, &mbs));
+}
+size_t
+mbstowcs(wchar_t * __restrict pwcs, const char * __restrict s, size_t n)
+{
+	return mbstowcs_l(pwcs, s, n, __get_locale());
 }
diff --git a/lib/libc/locale/mbtowc.c b/lib/libc/locale/mbtowc.c
index ad5b735..70fc19e 100644
--- a/lib/libc/locale/mbtowc.c
+++ b/lib/libc/locale/mbtowc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,19 +37,24 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 int
-mbtowc(wchar_t * __restrict pwc, const char * __restrict s, size_t n)
+mbtowc_l(wchar_t * __restrict pwc, const char * __restrict s, size_t n, locale_t locale)
 {
 	static const mbstate_t initial;
-	static mbstate_t mbs;
 	size_t rval;
+	FIX_LOCALE(locale);
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		mbs = initial;
+		locale->mbtowc = initial;
 		return (0);
 	}
-	rval = __mbrtowc(pwc, s, n, &mbs);
+	rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, &locale->mbtowc);
 	if (rval == (size_t)-1 || rval == (size_t)-2)
 		return (-1);
 	return ((int)rval);
 }
+int
+mbtowc(wchar_t * __restrict pwc, const char * __restrict s, size_t n)
+{
+	return mbtowc_l(pwc, s, n, __get_locale());
+}
diff --git a/lib/libc/locale/mskanji.c b/lib/libc/locale/mskanji.c
index 9ee91de..9fdd080 100644
--- a/lib/libc/locale/mskanji.c
+++ b/lib/libc/locale/mskanji.c
@@ -6,6 +6,11 @@
  *    (C) Sin'ichiro MIYATANI / Phase One, Inc
  *    May 12, 1995
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -60,15 +65,15 @@ typedef struct {
 } _MSKanjiState;
 
 int
-_MSKanji_init(_RuneLocale *rl)
+_MSKanji_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	__mbrtowc = _MSKanji_mbrtowc;
-	__wcrtomb = _MSKanji_wcrtomb;
-	__mbsinit = _MSKanji_mbsinit;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 2;
-	__mb_sb_limit = 256;
+	l->__mbrtowc = _MSKanji_mbrtowc;
+	l->__wcrtomb = _MSKanji_wcrtomb;
+	l->__mbsinit = _MSKanji_mbsinit;
+	l->runes = rl;
+	l->__mb_cur_max = 2;
+	l->__mb_sb_limit = 256;
 	return (0);
 }
 
diff --git a/lib/libc/locale/newlocale.3 b/lib/libc/locale/newlocale.3
new file mode 100644
index 0000000..b351f99
--- /dev/null
+++ b/lib/libc/locale/newlocale.3
@@ -0,0 +1,109 @@
+.\" Copyright (c) 2011 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This documentation was written by David Chisnall under sponsorship from
+.\" the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd September 17 2011
+.Dt newlocale 3
+.Os
+.Sh NAME
+.Nm newlocale
+.Nd Creates a new locale 
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In xlocale
+.Ft 
+.Fn newlocale "int mask" "const char * locale" "locale_t base"
+.Sh DESCRIPTION
+Creates a new locale, inheriting some properties from an existing locale.  The
+.Fa mask 
+defines the components that the new locale will have set to the locale with the
+name specified in the 
+.Fa locale
+parameter.  Any other components will be inherited from
+.Fa base .
+.Pt
+The
+.Fa mask
+is either
+.Fa LC_ALL_MASK,
+indicating all possible locale components, or the logical OR of some
+combination of the following:
+.Bl -tag -width "LC_MESSAGES_MASK" -offset indent
+.It LC_COLLATE_MASK
+The locale for string collation routines.  This controls alphabetic ordering in
+.Xr strcoll 3
+ and 
+.Xr strxfrm 3 .
+.It LC_CTYPE_MASK
+The locale for the
+.Xr ctype 3
+and
+.Xr multibyte 3
+functions.  This controls recognition of upper and lower case, alpha- betic or
+non-alphabetic characters, and so on.
+.It LC_MESSAGES_MASK
+Set a locale for message catalogs, see
+.Xr catopen 3
+function.
+.It LC_MONETARY_MASK
+Set a locale for formatting monetary values; this affects
+the
+.Xr localeconv 3
+function.
+.It LC_NUMERIC_MASK
+Set a locale for formatting numbers.  This controls the for-
+matting of decimal points in input and output of floating
+point numbers in functions such as
+.Xr printf 3
+and
+.Xr scanf 3 ,
+as well as values returned by
+.Xr localeconv 3 .
+.It LC_TIME_MASK
+Set a locale for formatting dates and times using the
+.Xr strftime 3
+function.
+.El
+
+This function uses the same rules for loading locale components as
+.Xr setlocale 3 .
+.Sh RETURN VALUES
+Returns a new, valid,
+.Fa locale_t
+or NULL if an error occurs.  You must free the returned locale with
+.Xr freelocale 3 .
+.Sh SEE ALSO
+.Xr duplocale 3 ,
+.Xr freelocale 3 ,
+.Xr localeconv 3 ,
+.Xr querylocale 3 ,
+.Xr uselocale 3 ,
+.Xr xlocale 3
+.Sh STANDARDS
+This function, conforms to
+.St -p1003.1-2008
diff --git a/lib/libc/locale/nextwctype.c b/lib/libc/locale/nextwctype.c
index 9363b0a..9fc8ce0 100644
--- a/lib/libc/locale/nextwctype.c
+++ b/lib/libc/locale/nextwctype.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,12 +35,15 @@ __FBSDID("$FreeBSD$");
 #include <runetype.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "mblocal.h"
 
 wint_t
-nextwctype(wint_t wc, wctype_t wct)
+nextwctype_l(wint_t wc, wctype_t wct, locale_t locale)
 {
 	size_t lim;
-	_RuneRange *rr = &_CurrentRuneLocale->__runetype_ext;
+	FIX_LOCALE(locale);
+	_RuneLocale *runes = XLOCALE_CTYPE(locale)->runes;
+	_RuneRange *rr = &runes->__runetype_ext;
 	_RuneEntry *base, *re;
 	int noinc;
 
@@ -43,7 +51,7 @@ nextwctype(wint_t wc, wctype_t wct)
 	if (wc < _CACHED_RUNES) {
 		wc++;
 		while (wc < _CACHED_RUNES) {
-			if (_CurrentRuneLocale->__runetype[wc] & wct)
+			if (runes->__runetype[wc] & wct)
 				return (wc);
 			wc++;
 		}
@@ -88,3 +96,8 @@ found:
 	}
 	return (-1);
 }
+wint_t
+nextwctype(wint_t wc, wctype_t wct)
+{
+	return nextwctype_l(wc, wct, __get_locale());
+}
diff --git a/lib/libc/locale/nl_langinfo.c b/lib/libc/locale/nl_langinfo.c
index 26ca025..3e8fe7c 100644
--- a/lib/libc/locale/nl_langinfo.c
+++ b/lib/libc/locale/nl_langinfo.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2001, 2003 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,15 +46,16 @@ __FBSDID("$FreeBSD$");
 #define _REL(BASE) ((int)item-BASE)
 
 char *
-nl_langinfo(nl_item item)
+nl_langinfo_l(nl_item item, locale_t loc)
 {
-   char *ret, *s, *cs;
-   static char *csym = NULL;
+   char *ret, *cs;
+   const char *s;
+   FIX_LOCALE(loc);
 
    switch (item) {
 	case CODESET:
 		ret = "";
-		if ((s = setlocale(LC_CTYPE, NULL)) != NULL) {
+		if ((s = querylocale(LC_CTYPE_MASK, loc)) != NULL) {
 			if ((cs = strchr(s, '.')) != NULL)
 				ret = cs + 1;
 			else if (strcmp(s, "C") == 0 ||
@@ -58,46 +64,46 @@ nl_langinfo(nl_item item)
 		}
 		break;
 	case D_T_FMT:
-		ret = (char *) __get_current_time_locale()->c_fmt;
+		ret = (char *) __get_current_time_locale(loc)->c_fmt;
 		break;
 	case D_FMT:
-		ret = (char *) __get_current_time_locale()->x_fmt;
+		ret = (char *) __get_current_time_locale(loc)->x_fmt;
 		break;
 	case T_FMT:
-		ret = (char *) __get_current_time_locale()->X_fmt;
+		ret = (char *) __get_current_time_locale(loc)->X_fmt;
 		break;
 	case T_FMT_AMPM:
-		ret = (char *) __get_current_time_locale()->ampm_fmt;
+		ret = (char *) __get_current_time_locale(loc)->ampm_fmt;
 		break;
 	case AM_STR:
-		ret = (char *) __get_current_time_locale()->am;
+		ret = (char *) __get_current_time_locale(loc)->am;
 		break;
 	case PM_STR:
-		ret = (char *) __get_current_time_locale()->pm;
+		ret = (char *) __get_current_time_locale(loc)->pm;
 		break;
 	case DAY_1: case DAY_2: case DAY_3:
 	case DAY_4: case DAY_5: case DAY_6: case DAY_7:
-		ret = (char*) __get_current_time_locale()->weekday[_REL(DAY_1)];
+		ret = (char*) __get_current_time_locale(loc)->weekday[_REL(DAY_1)];
 		break;
 	case ABDAY_1: case ABDAY_2: case ABDAY_3:
 	case ABDAY_4: case ABDAY_5: case ABDAY_6: case ABDAY_7:
-		ret = (char*) __get_current_time_locale()->wday[_REL(ABDAY_1)];
+		ret = (char*) __get_current_time_locale(loc)->wday[_REL(ABDAY_1)];
 		break;
 	case MON_1: case MON_2: case MON_3: case MON_4:
 	case MON_5: case MON_6: case MON_7: case MON_8:
 	case MON_9: case MON_10: case MON_11: case MON_12:
-		ret = (char*) __get_current_time_locale()->month[_REL(MON_1)];
+		ret = (char*) __get_current_time_locale(loc)->month[_REL(MON_1)];
 		break;
 	case ABMON_1: case ABMON_2: case ABMON_3: case ABMON_4:
 	case ABMON_5: case ABMON_6: case ABMON_7: case ABMON_8:
 	case ABMON_9: case ABMON_10: case ABMON_11: case ABMON_12:
-		ret = (char*) __get_current_time_locale()->mon[_REL(ABMON_1)];
+		ret = (char*) __get_current_time_locale(loc)->mon[_REL(ABMON_1)];
 		break;
 	case ALTMON_1: case ALTMON_2: case ALTMON_3: case ALTMON_4:
 	case ALTMON_5: case ALTMON_6: case ALTMON_7: case ALTMON_8:
 	case ALTMON_9: case ALTMON_10: case ALTMON_11: case ALTMON_12:
 		ret = (char*)
-		    __get_current_time_locale()->alt_month[_REL(ALTMON_1)];
+		    __get_current_time_locale(loc)->alt_month[_REL(ALTMON_1)];
 		break;
 	case ERA:
 		/* XXX: need to be implemented  */
@@ -120,16 +126,16 @@ nl_langinfo(nl_item item)
 		ret = "";
 		break;
 	case RADIXCHAR:
-		ret = (char*) __get_current_numeric_locale()->decimal_point;
+		ret = (char*) __get_current_numeric_locale(loc)->decimal_point;
 		break;
 	case THOUSEP:
-		ret = (char*) __get_current_numeric_locale()->thousands_sep;
+		ret = (char*) __get_current_numeric_locale(loc)->thousands_sep;
 		break;
 	case YESEXPR:
-		ret = (char*) __get_current_messages_locale()->yesexpr;
+		ret = (char*) __get_current_messages_locale(loc)->yesexpr;
 		break;
 	case NOEXPR:
-		ret = (char*) __get_current_messages_locale()->noexpr;
+		ret = (char*) __get_current_messages_locale(loc)->noexpr;
 		break;
 	/*
 	 * YESSTR and NOSTR items marked with LEGACY are available, but not
@@ -137,45 +143,51 @@ nl_langinfo(nl_item item)
 	 * they're subject to remove in future specification editions.
 	 */
 	case YESSTR:            /* LEGACY  */
-		ret = (char*) __get_current_messages_locale()->yesstr;
+		ret = (char*) __get_current_messages_locale(loc)->yesstr;
 		break;
 	case NOSTR:             /* LEGACY  */
-		ret = (char*) __get_current_messages_locale()->nostr;
+		ret = (char*) __get_current_messages_locale(loc)->nostr;
 		break;
 	/*
 	 * SUSv2 special formatted currency string 
 	 */
 	case CRNCYSTR:
 		ret = "";
-		cs = (char*) __get_current_monetary_locale()->currency_symbol;
+		cs = (char*) __get_current_monetary_locale(loc)->currency_symbol;
 		if (*cs != '\0') {
-			char pos = localeconv()->p_cs_precedes;
+			char pos = localeconv_l(loc)->p_cs_precedes;
 
-			if (pos == localeconv()->n_cs_precedes) {
+			if (pos == localeconv_l(loc)->n_cs_precedes) {
 				char psn = '\0';
 
 				if (pos == CHAR_MAX) {
-					if (strcmp(cs, __get_current_monetary_locale()->mon_decimal_point) == 0)
+					if (strcmp(cs, __get_current_monetary_locale(loc)->mon_decimal_point) == 0)
 						psn = '.';
 				} else
 					psn = pos ? '-' : '+';
 				if (psn != '\0') {
 					int clen = strlen(cs);
 
-					if ((csym = reallocf(csym, clen + 2)) != NULL) {
-						*csym = psn;
-						strcpy(csym + 1, cs);
-						ret = csym;
+					if ((loc->csym = reallocf(loc->csym, clen + 2)) != NULL) {
+						*loc->csym = psn;
+						strcpy(loc->csym + 1, cs);
+						ret = loc->csym;
 					}
 				}
 			}
 		}
 		break;
 	case D_MD_ORDER:        /* FreeBSD local extension */
-		ret = (char *) __get_current_time_locale()->md_order;
+		ret = (char *) __get_current_time_locale(loc)->md_order;
 		break;
 	default:
 		ret = "";
    }
    return (ret);
 }
+
+char *
+nl_langinfo(nl_item item)
+{
+	return nl_langinfo_l(item, __get_locale());
+}
diff --git a/lib/libc/locale/none.c b/lib/libc/locale/none.c
index 22fcd20..75adffa 100644
--- a/lib/libc/locale/none.c
+++ b/lib/libc/locale/none.c
@@ -6,6 +6,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -64,17 +69,17 @@ int __mb_cur_max = 1;
 int __mb_sb_limit = 256; /* Expected to be <= _CACHED_RUNES */
 
 int
-_none_init(_RuneLocale *rl)
+_none_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	__mbrtowc = _none_mbrtowc;
-	__mbsinit = _none_mbsinit;
-	__mbsnrtowcs = _none_mbsnrtowcs;
-	__wcrtomb = _none_wcrtomb;
-	__wcsnrtombs = _none_wcsnrtombs;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 1;
-	__mb_sb_limit = 256;
+	l->__mbrtowc = _none_mbrtowc;
+	l->__mbsinit = _none_mbsinit;
+	l->__mbsnrtowcs = _none_mbsnrtowcs;
+	l->__wcrtomb = _none_wcrtomb;
+	l->__wcsnrtombs = _none_wcsnrtombs;
+	l->runes = rl;
+	l->__mb_cur_max = 1;
+	l->__mb_sb_limit = 256;
 	return(0);
 }
 
@@ -192,3 +197,26 @@ size_t (*__wcrtomb)(char * __restrict, wchar_t, mbstate_t * __restrict) =
 size_t (*__wcsnrtombs)(char * __restrict, const wchar_t ** __restrict,
     size_t, size_t, mbstate_t * __restrict) = _none_wcsnrtombs;
 
+struct xlocale_ctype __xlocale_global_ctype = {
+	{{0}, "C"},
+	(_RuneLocale*)&_DefaultRuneLocale,
+	_none_mbrtowc,
+	_none_mbsinit,
+	_none_mbsnrtowcs,
+	_none_wcrtomb,
+	_none_wcsnrtombs,
+	1, /* __mb_cur_max, */
+	256 /* __mb_sb_limit */
+};
+
+const struct xlocale_ctype __xlocale_C_ctype = {
+	{{0}, "C"},
+	(_RuneLocale*)&_DefaultRuneLocale,
+	_none_mbrtowc,
+	_none_mbsinit,
+	_none_mbsnrtowcs,
+	_none_wcrtomb,
+	_none_wcsnrtombs,
+	1, /* __mb_cur_max, */
+	256 /* __mb_sb_limit */
+};
diff --git a/lib/libc/locale/querylocale.3 b/lib/libc/locale/querylocale.3
new file mode 100644
index 0000000..ceb4da0
--- /dev/null
+++ b/lib/libc/locale/querylocale.3
@@ -0,0 +1,57 @@
+.\" Copyright (c) 2011 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This documentation was written by David Chisnall under sponsorship from
+.\" the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd September 17 2011
+.Dt QUERYLOCALE 3
+.Os
+.Sh NAME
+.Nm querylocale
+.Nd Look up the locale name for a specified category.
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In xlocale.h 
+.Ft const char *
+.Fn querylocale "int mask" "locale_t locale"
+.Sh DESCRIPTION
+Returns the name of the locale for the category specified by
+.Fa mask.
+This possible values for the mask are the same as those in
+.Xr newlocale 3 .  If more than one bit in the mask is set, the returned value
+is undefined.
+.Sh SEE ALSO
+.Xr duplocale 3 ,
+.Xr freelocale 3 ,
+.Xr localeconv 3 ,
+.Xr newlocale 3 ,
+.Xr uselocale 3 ,
+.Xr xlocale 3
+.Sh STANDARDS
+This function, conforms to
+.St -p1003.1-2008
diff --git a/lib/libc/locale/runetype.c b/lib/libc/locale/runetype.c
index 5765b85..a0da47b 100644
--- a/lib/libc/locale/runetype.c
+++ b/lib/libc/locale/runetype.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -36,12 +41,15 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <stdio.h>
 #include <runetype.h>
+#include <wchar.h>
+#include "mblocal.h"
 
 unsigned long
-___runetype(__ct_rune_t c)
+___runetype_l(__ct_rune_t c, locale_t locale)
 {
 	size_t lim;
-	_RuneRange *rr = &_CurrentRuneLocale->__runetype_ext;
+	FIX_LOCALE(locale);
+	_RuneRange *rr = &(XLOCALE_CTYPE(locale)->runes->__runetype_ext);
 	_RuneEntry *base, *re;
 
 	if (c < 0 || c == EOF)
@@ -64,3 +72,18 @@ ___runetype(__ct_rune_t c)
 
 	return(0L);
 }
+unsigned long
+___runetype(__ct_rune_t c)
+{
+	return ___runetype_l(c, __get_locale());
+}
+
+int ___mb_cur_max(void)
+{
+	return XLOCALE_CTYPE(__get_locale())->__mb_cur_max;
+}
+int ___mb_cur_max_l(locale_t locale)
+{
+	FIX_LOCALE(locale);
+	return XLOCALE_CTYPE(locale)->__mb_cur_max;
+}
diff --git a/lib/libc/locale/setlocale.c b/lib/libc/locale/setlocale.c
index d6db181..8cf8fd4 100644
--- a/lib/libc/locale/setlocale.c
+++ b/lib/libc/locale/setlocale.c
@@ -95,7 +95,7 @@ static char current_locale_string[_LC_LAST * (ENCODING_LEN + 1/*"/"*/ + 1)];
 
 static char	*currentlocale(void);
 static char	*loadlocale(int);
-static const char *__get_locale_env(int);
+const char *__get_locale_env(int);
 
 char *
 setlocale(category, locale)
@@ -278,13 +278,14 @@ loadlocale(category)
 
 	if (func(new) != _LDP_ERROR) {
 		(void)strcpy(old, new);
+		(void)strcpy(__xlocale_global_locale.components[category-1]->locale, new);
 		return (old);
 	}
 
 	return (NULL);
 }
 
-static const char *
+const char *
 __get_locale_env(category)
         int category;
 {
diff --git a/lib/libc/locale/setrunelocale.c b/lib/libc/locale/setrunelocale.c
index 36d2894..61ce5d9 100644
--- a/lib/libc/locale/setrunelocale.c
+++ b/lib/libc/locale/setrunelocale.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -49,68 +54,46 @@ extern int __mb_sb_limit;
 
 extern _RuneLocale	*_Read_RuneMagi(FILE *);
 
-static int		__setrunelocale(const char *);
+static int		__setrunelocale(struct xlocale_ctype *l, const char *);
+
+#define __collate_load_error (table->__collate_load_error)
+#define __collate_substitute_nontrivial (table->__collate_substitute_nontrivial)
+#define __collate_substitute_table_ptr (table->__collate_substitute_table_ptr)
+#define __collate_char_pri_table_ptr (table->__collate_char_pri_table_ptr)
+#define __collate_chain_pri_table (table->__collate_chain_pri_table)
+
+
+static void destruct_ctype(void *v)
+{
+	struct xlocale_ctype *l = v;
+	if (strcmp(l->runes->__encoding, "EUC") == 0)
+		free(l->runes->__variable);
+	if (&_DefaultRuneLocale != l->runes) 
+		free(l->runes);
+	free(l);
+}
+_RuneLocale *__getCurrentRuneLocale(void)
+{
+	return XLOCALE_CTYPE(__get_locale())->runes;
+}
 
 static int
-__setrunelocale(const char *encoding)
+__setrunelocale(struct xlocale_ctype *l, const char *encoding)
 {
 	FILE *fp;
 	char name[PATH_MAX];
 	_RuneLocale *rl;
 	int saverr, ret;
-	size_t (*old__mbrtowc)(wchar_t * __restrict,
-	    const char * __restrict, size_t, mbstate_t * __restrict);
-	size_t (*old__wcrtomb)(char * __restrict, wchar_t,
-	    mbstate_t * __restrict);
-	int (*old__mbsinit)(const mbstate_t *);
-	size_t (*old__mbsnrtowcs)(wchar_t * __restrict,
-	    const char ** __restrict, size_t, size_t, mbstate_t * __restrict);
-	size_t (*old__wcsnrtombs)(char * __restrict,
-	    const wchar_t ** __restrict, size_t, size_t,
-	    mbstate_t * __restrict);
-	static char ctype_encoding[ENCODING_LEN + 1];
-	static _RuneLocale *CachedRuneLocale;
-	static int Cached__mb_cur_max;
-	static int Cached__mb_sb_limit;
-	static size_t (*Cached__mbrtowc)(wchar_t * __restrict,
-	    const char * __restrict, size_t, mbstate_t * __restrict);
-	static size_t (*Cached__wcrtomb)(char * __restrict, wchar_t,
-	    mbstate_t * __restrict);
-	static int (*Cached__mbsinit)(const mbstate_t *);
-	static size_t (*Cached__mbsnrtowcs)(wchar_t * __restrict,
-	    const char ** __restrict, size_t, size_t, mbstate_t * __restrict);
-	static size_t (*Cached__wcsnrtombs)(char * __restrict,
-	    const wchar_t ** __restrict, size_t, size_t,
-	    mbstate_t * __restrict);
+	struct xlocale_ctype saved = *l;
 
 	/*
 	 * The "C" and "POSIX" locale are always here.
 	 */
 	if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0) {
-		(void) _none_init(&_DefaultRuneLocale);
+		(void) _none_init(l, (_RuneLocale*)&_DefaultRuneLocale);
 		return (0);
 	}
 
-	/*
-	 * If the locale name is the same as our cache, use the cache.
-	 */
-	if (CachedRuneLocale != NULL &&
-	    strcmp(encoding, ctype_encoding) == 0) {
-		_CurrentRuneLocale = CachedRuneLocale;
-		__mb_cur_max = Cached__mb_cur_max;
-		__mb_sb_limit = Cached__mb_sb_limit;
-		__mbrtowc = Cached__mbrtowc;
-		__mbsinit = Cached__mbsinit;
-		__mbsnrtowcs = Cached__mbsnrtowcs;
-		__wcrtomb = Cached__wcrtomb;
-		__wcsnrtombs = Cached__wcsnrtombs;
-		return (0);
-	}
-
-	/*
-	 * Slurp the locale file into the cache.
-	 */
-
 	/* Range checking not needed, encoding length already checked before */
 	(void) strcpy(name, _PathLocale);
 	(void) strcat(name, "/");
@@ -127,63 +110,47 @@ __setrunelocale(const char *encoding)
 	}
 	(void)fclose(fp);
 
-	old__mbrtowc = __mbrtowc;
-	old__mbsinit = __mbsinit;
-	old__mbsnrtowcs = __mbsnrtowcs;
-	old__wcrtomb = __wcrtomb;
-	old__wcsnrtombs = __wcsnrtombs;
-
-	__mbrtowc = NULL;
-	__mbsinit = NULL;
-	__mbsnrtowcs = __mbsnrtowcs_std;
-	__wcrtomb = NULL;
-	__wcsnrtombs = __wcsnrtombs_std;
+	l->__mbrtowc = NULL;
+	l->__mbsinit = NULL;
+	l->__mbsnrtowcs = __mbsnrtowcs_std;
+	l->__wcrtomb = NULL;
+	l->__wcsnrtombs = __wcsnrtombs_std;
 
 	rl->__sputrune = NULL;
 	rl->__sgetrune = NULL;
 	if (strcmp(rl->__encoding, "NONE") == 0)
-		ret = _none_init(rl);
+		ret = _none_init(l, rl);
 	else if (strcmp(rl->__encoding, "ASCII") == 0)
-		ret = _ascii_init(rl);
+		ret = _ascii_init(l, rl);
 	else if (strcmp(rl->__encoding, "UTF-8") == 0)
-		ret = _UTF8_init(rl);
+		ret = _UTF8_init(l, rl);
 	else if (strcmp(rl->__encoding, "EUC") == 0)
-		ret = _EUC_init(rl);
+		ret = _EUC_init(l, rl);
 	else if (strcmp(rl->__encoding, "GB18030") == 0)
- 		ret = _GB18030_init(rl);
+ 		ret = _GB18030_init(l, rl);
 	else if (strcmp(rl->__encoding, "GB2312") == 0)
-		ret = _GB2312_init(rl);
+		ret = _GB2312_init(l, rl);
 	else if (strcmp(rl->__encoding, "GBK") == 0)
-		ret = _GBK_init(rl);
+		ret = _GBK_init(l, rl);
 	else if (strcmp(rl->__encoding, "BIG5") == 0)
-		ret = _BIG5_init(rl);
+		ret = _BIG5_init(l, rl);
 	else if (strcmp(rl->__encoding, "MSKanji") == 0)
-		ret = _MSKanji_init(rl);
+		ret = _MSKanji_init(l, rl);
 	else
 		ret = EFTYPE;
 
 	if (ret == 0) {
-		if (CachedRuneLocale != NULL) {
-			/* See euc.c */
-			if (strcmp(CachedRuneLocale->__encoding, "EUC") == 0)
-				free(CachedRuneLocale->__variable);
-			free(CachedRuneLocale);
+		/* Free the old runes if it exists. */
+		/* FIXME: The "EUC" check here is a hideous abstraction violation. */
+		if ((saved.runes != &_DefaultRuneLocale) && (saved.runes)) {
+			if (strcmp(saved.runes->__encoding, "EUC") == 0) {
+				free(saved.runes->__variable);
+			}
+			free(saved.runes);
 		}
-		CachedRuneLocale = _CurrentRuneLocale;
-		Cached__mb_cur_max = __mb_cur_max;
-		Cached__mb_sb_limit = __mb_sb_limit;
-		Cached__mbrtowc = __mbrtowc;
-		Cached__mbsinit = __mbsinit;
-		Cached__mbsnrtowcs = __mbsnrtowcs;
-		Cached__wcrtomb = __wcrtomb;
-		Cached__wcsnrtombs = __wcsnrtombs;
-		(void)strcpy(ctype_encoding, encoding);
 	} else {
-		__mbrtowc = old__mbrtowc;
-		__mbsinit = old__mbsinit;
-		__mbsnrtowcs = old__mbsnrtowcs;
-		__wcrtomb = old__wcrtomb;
-		__wcsnrtombs = old__wcsnrtombs;
+		/* Restore the saved version if this failed. */
+		memcpy(l, &saved, sizeof(struct xlocale_ctype));
 		free(rl);
 	}
 
@@ -193,12 +160,24 @@ __setrunelocale(const char *encoding)
 int
 __wrap_setrunelocale(const char *locale)
 {
-	int ret = __setrunelocale(locale);
+	int ret = __setrunelocale(&__xlocale_global_ctype, locale);
 
 	if (ret != 0) {
 		errno = ret;
 		return (_LDP_ERROR);
 	}
+	__mb_cur_max = __xlocale_global_ctype.__mb_cur_max;
+	__mb_sb_limit = __xlocale_global_ctype.__mb_sb_limit;
 	return (_LDP_LOADED);
 }
-
+void *__ctype_load(const char *locale, locale_t unused)
+{
+	struct xlocale_ctype *l = calloc(sizeof(struct xlocale_ctype), 1);
+	l->header.header.destructor = destruct_ctype;
+	if (__setrunelocale(l, locale))
+	{
+		free(l);
+		return NULL;
+	}
+	return l;
+}
diff --git a/lib/libc/locale/table.c b/lib/libc/locale/table.c
index e1f49d1..8c876e9 100644
--- a/lib/libc/locale/table.c
+++ b/lib/libc/locale/table.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,7 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <wchar.h>
 #include "mblocal.h"
 
-_RuneLocale _DefaultRuneLocale = {
+const _RuneLocale _DefaultRuneLocale = {
     _RUNE_MAGIC_1,
     "NONE",
     NULL,
@@ -245,5 +250,14 @@ _RuneLocale _DefaultRuneLocale = {
     },
 };
 
-_RuneLocale *_CurrentRuneLocale = &_DefaultRuneLocale;
+#undef _CurrentRuneLocale
+_RuneLocale *_CurrentRuneLocale = (_RuneLocale*)&_DefaultRuneLocale;
 
+_RuneLocale *
+__runes_for_locale(locale_t locale, int *mb_sb_limit)
+{
+	FIX_LOCALE(locale);
+	struct xlocale_ctype *c = XLOCALE_CTYPE(locale);
+	*mb_sb_limit = c->__mb_sb_limit;
+	return c->runes;
+}
diff --git a/lib/libc/locale/tolower.c b/lib/libc/locale/tolower.c
index 82a78cf..f2eb483 100644
--- a/lib/libc/locale/tolower.c
+++ b/lib/libc/locale/tolower.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -36,13 +41,17 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <stdio.h>
 #include <runetype.h>
+#include <wchar.h>
+#include "mblocal.h"
 
 __ct_rune_t
-___tolower(c)
+___tolower_l(c, l)
 	__ct_rune_t c;
+	locale_t l;
 {
 	size_t lim;
-	_RuneRange *rr = &_CurrentRuneLocale->__maplower_ext;
+	FIX_LOCALE(l);
+	_RuneRange *rr = &XLOCALE_CTYPE(l)->runes->__maplower_ext;
 	_RuneEntry *base, *re;
 
 	if (c < 0 || c == EOF)
@@ -62,3 +71,9 @@ ___tolower(c)
 
 	return(c);
 }
+__ct_rune_t
+___tolower(c)
+	__ct_rune_t c;
+{
+	return ___tolower_l(c, __get_locale());
+}
diff --git a/lib/libc/locale/toupper.c b/lib/libc/locale/toupper.c
index 4870e8e..d3a4fa3 100644
--- a/lib/libc/locale/toupper.c
+++ b/lib/libc/locale/toupper.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -36,13 +41,17 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <stdio.h>
 #include <runetype.h>
+#include <wchar.h>
+#include "mblocal.h"
 
 __ct_rune_t
-___toupper(c)
+___toupper_l(c, l)
 	__ct_rune_t c;
+	locale_t l;
 {
 	size_t lim;
-	_RuneRange *rr = &_CurrentRuneLocale->__mapupper_ext;
+	FIX_LOCALE(l);
+	_RuneRange *rr = &XLOCALE_CTYPE(l)->runes->__maplower_ext;
 	_RuneEntry *base, *re;
 
 	if (c < 0 || c == EOF)
@@ -53,7 +62,9 @@ ___toupper(c)
 	for (lim = rr->__nranges; lim != 0; lim >>= 1) {
 		re = base + (lim >> 1);
 		if (re->__min <= c && c <= re->__max)
+		{
 			return (re->__map + c - re->__min);
+		}
 		else if (c > re->__max) {
 			base = re + 1;
 			lim--;
@@ -62,3 +73,9 @@ ___toupper(c)
 
 	return(c);
 }
+__ct_rune_t
+___toupper(c)
+	__ct_rune_t c;
+{
+	return ___toupper_l(c, __get_locale());
+}
diff --git a/lib/libc/locale/uselocale.3 b/lib/libc/locale/uselocale.3
new file mode 100644
index 0000000..a7068b4
--- /dev/null
+++ b/lib/libc/locale/uselocale.3
@@ -0,0 +1,59 @@
+.\" Copyright (c) 2011 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This documentation was written by David Chisnall under sponsorship from
+.\" the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd September 17 2011
+.Dt USELOCALE 3
+.Os
+.Sh NAME
+.Nm uselocale
+.Nd Sets a thread-local locale.
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In xlocale.h 
+.Ft locale_t
+.Fn uselocale "locale_t locale"
+.Sh DESCRIPTION
+Specifies the locale for this thread to use.  Specifying
+.Fa LC_GLOBAL_LOCALE
+disables the per-thread locale, while NULL returns the current locale without
+setting a new one.
+.Sh RETURN VALUES
+Returns the previous locale, or LC_GLOBAL_LOCALE if this thread has no locale
+associated with it.
+.Sh SEE ALSO
+.Xr duplocale 3 ,
+.Xr freelocale 3 ,
+.Xr localeconv 3 ,
+.Xr newlocale 3 ,
+.Xr querylocale 3 ,
+.Xr xlocale 3
+.Sh STANDARDS
+This function, conforms to
+.St -p1003.1-2008
diff --git a/lib/libc/locale/utf8.c b/lib/libc/locale/utf8.c
index 0ccadd3..40f0e17 100644
--- a/lib/libc/locale/utf8.c
+++ b/lib/libc/locale/utf8.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -55,22 +60,22 @@ typedef struct {
 } _UTF8State;
 
 int
-_UTF8_init(_RuneLocale *rl)
+_UTF8_init(struct xlocale_ctype *l, _RuneLocale *rl)
 {
 
-	__mbrtowc = _UTF8_mbrtowc;
-	__wcrtomb = _UTF8_wcrtomb;
-	__mbsinit = _UTF8_mbsinit;
-	__mbsnrtowcs = _UTF8_mbsnrtowcs;
-	__wcsnrtombs = _UTF8_wcsnrtombs;
-	_CurrentRuneLocale = rl;
-	__mb_cur_max = 6;
+	l->__mbrtowc = _UTF8_mbrtowc;
+	l->__wcrtomb = _UTF8_wcrtomb;
+	l->__mbsinit = _UTF8_mbsinit;
+	l->__mbsnrtowcs = _UTF8_mbsnrtowcs;
+	l->__wcsnrtombs = _UTF8_wcsnrtombs;
+	l->runes = rl;
+	l->__mb_cur_max = 6;
 	/*
 	 * UCS-4 encoding used as the internal representation, so
 	 * slots 0x0080-0x00FF are occuped and must be excluded
 	 * from the single byte ctype by setting the limit.
 	 */
-	__mb_sb_limit = 128;
+	l->__mb_sb_limit = 128;
 
 	return (0);
 }
diff --git a/lib/libc/locale/wcrtomb.c b/lib/libc/locale/wcrtomb.c
index ef75b78..32d4df2 100644
--- a/lib/libc/locale/wcrtomb.c
+++ b/lib/libc/locale/wcrtomb.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -31,11 +36,17 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
-wcrtomb(char * __restrict s, wchar_t wc, mbstate_t * __restrict ps)
+wcrtomb_l(char * __restrict s, wchar_t wc, mbstate_t * __restrict ps,
+		locale_t locale)
 {
-	static mbstate_t mbs;
-
+	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &mbs;
-	return (__wcrtomb(s, wc, ps));
+		ps = &locale->wcrtomb;
+	return (XLOCALE_CTYPE(locale)->__wcrtomb(s, wc, ps));
+}
+
+size_t
+wcrtomb(char * __restrict s, wchar_t wc, mbstate_t * __restrict ps)
+{
+	return wcrtomb_l(s, wc, ps, __get_locale());
 }
diff --git a/lib/libc/locale/wcsftime.c b/lib/libc/locale/wcsftime.c
index a546dc6..c4f6b2e 100644
--- a/lib/libc/locale/wcsftime.c
+++ b/lib/libc/locale/wcsftime.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <time.h>
 #include <wchar.h>
+#include "xlocale_private.h"
 
 /*
  * Convert date and time to a wide-character string.
@@ -47,8 +53,9 @@ __FBSDID("$FreeBSD$");
  * format specifications in the format string.
  */
 size_t
-wcsftime(wchar_t * __restrict wcs, size_t maxsize,
-    const wchar_t * __restrict format, const struct tm * __restrict timeptr)
+wcsftime_l(wchar_t * __restrict wcs, size_t maxsize,
+	const wchar_t * __restrict format, const struct tm * __restrict timeptr,
+	locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
@@ -57,6 +64,7 @@ wcsftime(wchar_t * __restrict wcs, size_t maxsize,
 	const wchar_t *formatp;
 	size_t n, sflen;
 	int sverrno;
+	FIX_LOCALE(locale);
 
 	sformat = dst = NULL;
 
@@ -66,13 +74,13 @@ wcsftime(wchar_t * __restrict wcs, size_t maxsize,
 	 */
 	mbs = initial;
 	formatp = format;
-	sflen = wcsrtombs(NULL, &formatp, 0, &mbs);
+	sflen = wcsrtombs_l(NULL, &formatp, 0, &mbs, locale);
 	if (sflen == (size_t)-1)
 		goto error;
 	if ((sformat = malloc(sflen + 1)) == NULL)
 		goto error;
 	mbs = initial;
-	wcsrtombs(sformat, &formatp, sflen + 1, &mbs);
+	wcsrtombs_l(sformat, &formatp, sflen + 1, &mbs, locale);
 
 	/*
 	 * Allocate memory for longest multibyte sequence that will fit
@@ -87,11 +95,11 @@ wcsftime(wchar_t * __restrict wcs, size_t maxsize,
 	}
 	if ((dst = malloc(maxsize * MB_CUR_MAX)) == NULL)
 		goto error;
-	if (strftime(dst, maxsize, sformat, timeptr) == 0)
+	if (strftime_l(dst, maxsize, sformat, timeptr, locale) == 0)
 		goto error;
 	dstp = dst;
 	mbs = initial;
-	n = mbsrtowcs(wcs, &dstp, maxsize, &mbs);
+	n = mbsrtowcs_l(wcs, &dstp, maxsize, &mbs, locale);
 	if (n == (size_t)-2 || n == (size_t)-1 || dstp != NULL)
 		goto error;
 
@@ -106,3 +114,9 @@ error:
 	errno = sverrno;
 	return (0);
 }
+size_t
+wcsftime(wchar_t * __restrict wcs, size_t maxsize,
+	const wchar_t * __restrict format, const struct tm * __restrict timeptr)
+{
+	return wcsftime_l(wcs, maxsize, format, timeptr, __get_locale());
+}
diff --git a/lib/libc/locale/wcsnrtombs.c b/lib/libc/locale/wcsnrtombs.c
index e899698..2f3bf1e 100644
--- a/lib/libc/locale/wcsnrtombs.c
+++ b/lib/libc/locale/wcsnrtombs.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,16 +39,22 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
+wcsnrtombs_l(char * __restrict dst, const wchar_t ** __restrict src, size_t nwc,
+    size_t len, mbstate_t * __restrict ps, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	if (ps == NULL)
+		ps = &locale->wcsnrtombs;
+	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, nwc, len, ps));
+}
+size_t
 wcsnrtombs(char * __restrict dst, const wchar_t ** __restrict src, size_t nwc,
     size_t len, mbstate_t * __restrict ps)
 {
-	static mbstate_t mbs;
-
-	if (ps == NULL)
-		ps = &mbs;
-	return (__wcsnrtombs(dst, src, nwc, len, ps));
+	return wcsnrtombs_l(dst, src, nwc, len, ps, __get_locale());
 }
 
+
 size_t
 __wcsnrtombs_std(char * __restrict dst, const wchar_t ** __restrict src,
     size_t nwc, size_t len, mbstate_t * __restrict ps)
@@ -53,13 +64,14 @@ __wcsnrtombs_std(char * __restrict dst, const wchar_t ** __restrict src,
 	const wchar_t *s;
 	size_t nbytes;
 	size_t nb;
+	struct xlocale_ctype *l = XLOCALE_CTYPE(__get_locale());
 
 	s = *src;
 	nbytes = 0;
 
 	if (dst == NULL) {
 		while (nwc-- > 0) {
-			if ((nb = __wcrtomb(buf, *s, ps)) == (size_t)-1)
+			if ((nb = l->__wcrtomb(buf, *s, ps)) == (size_t)-1)
 				/* Invalid character - wcrtomb() sets errno. */
 				return ((size_t)-1);
 			else if (*s == L'\0')
@@ -73,7 +85,7 @@ __wcsnrtombs_std(char * __restrict dst, const wchar_t ** __restrict src,
 	while (len > 0 && nwc-- > 0) {
 		if (len > (size_t)MB_CUR_MAX) {
 			/* Enough space to translate in-place. */
-			if ((nb = __wcrtomb(dst, *s, ps)) == (size_t)-1) {
+			if ((nb = l->__wcrtomb(dst, *s, ps)) == (size_t)-1) {
 				*src = s;
 				return ((size_t)-1);
 			}
@@ -86,7 +98,7 @@ __wcsnrtombs_std(char * __restrict dst, const wchar_t ** __restrict src,
 			 * character is too long for the buffer.
 			 */
 			mbsbak = *ps;
-			if ((nb = __wcrtomb(buf, *s, ps)) == (size_t)-1) {
+			if ((nb = l->__wcrtomb(buf, *s, ps)) == (size_t)-1) {
 				*src = s;
 				return ((size_t)-1);
 			}
diff --git a/lib/libc/locale/wcsrtombs.c b/lib/libc/locale/wcsrtombs.c
index f3b38b7..f6d1ca0 100644
--- a/lib/libc/locale/wcsrtombs.c
+++ b/lib/libc/locale/wcsrtombs.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,12 +39,18 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
+wcsrtombs_l(char * __restrict dst, const wchar_t ** __restrict src, size_t len,
+    mbstate_t * __restrict ps, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	if (ps == NULL)
+		ps = &locale->wcsrtombs;
+	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, SIZE_T_MAX, len, ps));
+}
+
+size_t
 wcsrtombs(char * __restrict dst, const wchar_t ** __restrict src, size_t len,
     mbstate_t * __restrict ps)
 {
-	static mbstate_t mbs;
-
-	if (ps == NULL)
-		ps = &mbs;
-	return (__wcsnrtombs(dst, src, SIZE_T_MAX, len, ps));
+	return wcsrtombs_l(dst, src, len, ps, __get_locale());
 }
diff --git a/lib/libc/locale/wcstod.c b/lib/libc/locale/wcstod.c
index 68df1ed..8bc46a7 100644
--- a/lib/libc/locale/wcstod.c
+++ b/lib/libc/locale/wcstod.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to a double-precision number.
@@ -41,17 +47,22 @@ __FBSDID("$FreeBSD$");
  * for at least the digits, radix character and letters.
  */
 double
-wcstod(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
+wcstod_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+		locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
 	double val;
 	char *buf, *end;
-	const wchar_t *wcp;
+	const wchar_t *wcp = nptr;
 	size_t len;
+	size_t spaces = 0;
+	FIX_LOCALE(locale);
 
-	while (iswspace(*nptr))
-		nptr++;
+	while (iswspace_l(*wcp, locale)) {
+		wcp++;
+		spaces++;
+	}
 
 	/*
 	 * Convert the supplied numeric wide char. string to multibyte.
@@ -63,9 +74,8 @@ wcstod(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 	 * duplicates a lot of strtod()'s functionality and slows down the
 	 * most common cases.
 	 */
-	wcp = nptr;
 	mbs = initial;
-	if ((len = wcsrtombs(NULL, &wcp, 0, &mbs)) == (size_t)-1) {
+	if ((len = wcsrtombs_l(NULL, &wcp, 0, &mbs, locale)) == (size_t)-1) {
 		if (endptr != NULL)
 			*endptr = (wchar_t *)nptr;
 		return (0.0);
@@ -73,10 +83,10 @@ wcstod(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 	if ((buf = malloc(len + 1)) == NULL)
 		return (0.0);
 	mbs = initial;
-	wcsrtombs(buf, &wcp, len + 1, &mbs);
+	wcsrtombs_l(buf, &wcp, len + 1, &mbs, locale);
 
 	/* Let strtod() do most of the work for us. */
-	val = strtod(buf, &end);
+	val = strtod_l(buf, &end, locale);
 
 	/*
 	 * We only know where the number ended in the _multibyte_
@@ -84,11 +94,20 @@ wcstod(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 	 * where it ended, count multibyte characters to find the
 	 * corresponding position in the wide char string.
 	 */
-	if (endptr != NULL)
+	if (endptr != NULL) {
 		/* XXX Assume each wide char is one byte. */
 		*endptr = (wchar_t *)nptr + (end - buf);
+		if (buf != end)
+			*endptr += spaces;
+	}
+
 
 	free(buf);
 
 	return (val);
 }
+double
+wcstod(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
+{
+	return wcstod_l(nptr, endptr, __get_locale());
+}
diff --git a/lib/libc/locale/wcstof.c b/lib/libc/locale/wcstof.c
index ba238d4..93a5af8 100644
--- a/lib/libc/locale/wcstof.c
+++ b/lib/libc/locale/wcstof.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002, 2003 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,12 +35,14 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * See wcstod() for comments as to the logic used.
  */
 float
-wcstof(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
+wcstof_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+		locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
@@ -43,13 +50,14 @@ wcstof(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 	char *buf, *end;
 	const wchar_t *wcp;
 	size_t len;
+	FIX_LOCALE(locale);
 
-	while (iswspace(*nptr))
+	while (iswspace_l(*nptr, locale))
 		nptr++;
 
 	wcp = nptr;
 	mbs = initial;
-	if ((len = wcsrtombs(NULL, &wcp, 0, &mbs)) == (size_t)-1) {
+	if ((len = wcsrtombs_l(NULL, &wcp, 0, &mbs, locale)) == (size_t)-1) {
 		if (endptr != NULL)
 			*endptr = (wchar_t *)nptr;
 		return (0.0);
@@ -57,9 +65,9 @@ wcstof(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 	if ((buf = malloc(len + 1)) == NULL)
 		return (0.0);
 	mbs = initial;
-	wcsrtombs(buf, &wcp, len + 1, &mbs);
+	wcsrtombs_l(buf, &wcp, len + 1, &mbs, locale);
 
-	val = strtof(buf, &end);
+	val = strtof_l(buf, &end, locale);
 
 	if (endptr != NULL)
 		*endptr = (wchar_t *)nptr + (end - buf);
@@ -68,3 +76,8 @@ wcstof(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 
 	return (val);
 }
+float
+wcstof(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
+{
+	return wcstof_l(nptr, endptr, __get_locale());
+}
diff --git a/lib/libc/locale/wcstoimax.c b/lib/libc/locale/wcstoimax.c
index 66e692a..ce066a5 100644
--- a/lib/libc/locale/wcstoimax.c
+++ b/lib/libc/locale/wcstoimax.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,19 +46,21 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a wide character string to an intmax_t integer.
  */
 intmax_t
-wcstoimax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
-    int base)
+wcstoimax_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+    int base, locale_t locale)
 {
 	const wchar_t *s;
 	uintmax_t acc;
 	wchar_t c;
 	uintmax_t cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtoimax for comments as to the logic used.
@@ -61,7 +68,7 @@ wcstoimax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 	s = nptr;
 	do {
 		c = *s++;
-	} while (iswspace(c));
+	} while (iswspace_l(c, locale));
 	if (c == L'-') {
 		neg = 1;
 		c = *s++;
@@ -88,8 +95,8 @@ wcstoimax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 	cutoff /= base;
 	for ( ; ; c = *s++) {
 #ifdef notyet
-		if (iswdigit(c))
-			c = digittoint(c);
+		if (iswdigit_l(c, locale))
+			c = digittoint_l(c, locale);
 		else
 #endif
 		if (c >= L'0' && c <= L'9')
@@ -122,3 +129,9 @@ noconv:
 		*endptr = (wchar_t *)(any ? s - 1 : nptr);
 	return (acc);
 }
+intmax_t
+wcstoimax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+    int base)
+{
+	return wcstoimax_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/locale/wcstol.c b/lib/libc/locale/wcstol.c
index 294b6e0..2b21d12 100644
--- a/lib/libc/locale/wcstol.c
+++ b/lib/libc/locale/wcstol.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,18 +40,21 @@ __FBSDID("$FreeBSD$");
 #include <limits.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to a long integer.
  */
 long
-wcstol(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
+wcstol_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int
+		base, locale_t locale)
 {
 	const wchar_t *s;
 	unsigned long acc;
 	wchar_t c;
 	unsigned long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtol for comments as to the logic used.
@@ -54,7 +62,7 @@ wcstol(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (iswspace(c));
+	} while (iswspace_l(c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -81,8 +89,8 @@ wcstol(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
 	cutoff /= base;
 	for ( ; ; c = *s++) {
 #ifdef notyet
-		if (iswdigit(c))
-			c = digittoint(c);
+		if (iswdigit_l(c, locale))
+			c = digittoint_l(c, locale);
 		else
 #endif
 		if (c >= L'0' && c <= L'9')
@@ -115,3 +123,8 @@ noconv:
 		*endptr = (wchar_t *)(any ? s - 1 : nptr);
 	return (acc);
 }
+long
+wcstol(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
+{
+	return wcstol_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/locale/wcstold.c b/lib/libc/locale/wcstold.c
index cf9d874..fcfd48f 100644
--- a/lib/libc/locale/wcstold.c
+++ b/lib/libc/locale/wcstold.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002, 2003 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,26 +35,32 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * See wcstod() for comments as to the logic used.
  */
 long double
-wcstold(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
+wcstold_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+		locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
 	long double val;
 	char *buf, *end;
-	const wchar_t *wcp;
+	const wchar_t *wcp = nptr;
 	size_t len;
+	size_t spaces = 0;
+	FIX_LOCALE(locale);
 
-	while (iswspace(*nptr))
-		nptr++;
+	while (iswspace_l(*wcp, locale)) {
+		wcp++;
+		spaces++;
+	}
 
 	wcp = nptr;
 	mbs = initial;
-	if ((len = wcsrtombs(NULL, &wcp, 0, &mbs)) == (size_t)-1) {
+	if ((len = wcsrtombs_l(NULL, &wcp, 0, &mbs, locale)) == (size_t)-1) {
 		if (endptr != NULL)
 			*endptr = (wchar_t *)nptr;
 		return (0.0);
@@ -57,14 +68,23 @@ wcstold(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
 	if ((buf = malloc(len + 1)) == NULL)
 		return (0.0);
 	mbs = initial;
-	wcsrtombs(buf, &wcp, len + 1, &mbs);
+	wcsrtombs_l(buf, &wcp, len + 1, &mbs, locale);
 
-	val = strtold(buf, &end);
+	val = strtold_l(buf, &end, locale);
 
-	if (endptr != NULL)
+	if (endptr != NULL) {
+		/* XXX Assume each wide char is one byte. */
 		*endptr = (wchar_t *)nptr + (end - buf);
+		if (buf != end)
+			*endptr += spaces;
+	}
 
 	free(buf);
 
 	return (val);
 }
+long double
+wcstold(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr)
+{
+	return wcstold_l(nptr, endptr, __get_locale());
+}
diff --git a/lib/libc/locale/wcstoll.c b/lib/libc/locale/wcstoll.c
index 19b285f..f246502 100644
--- a/lib/libc/locale/wcstoll.c
+++ b/lib/libc/locale/wcstoll.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,18 +46,21 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a wide character string to a long long integer.
  */
 long long
-wcstoll(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
+wcstoll_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+		int base, locale_t locale)
 {
 	const wchar_t *s;
 	unsigned long long acc;
 	wchar_t c;
 	unsigned long long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtoll for comments as to the logic used.
@@ -60,7 +68,7 @@ wcstoll(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (iswspace(c));
+	} while (iswspace_l(c, locale));
 	if (c == L'-') {
 		neg = 1;
 		c = *s++;
@@ -87,8 +95,8 @@ wcstoll(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
 	cutoff /= base;
 	for ( ; ; c = *s++) {
 #ifdef notyet
-		if (iswdigit(c))
-			c = digittoint(c);
+		if (iswdigit_l(c, locale))
+			c = digittoint_l(c, locale);
 		else
 #endif
 		if (c >= L'0' && c <= L'9')
@@ -121,3 +129,8 @@ noconv:
 		*endptr = (wchar_t *)(any ? s - 1 : nptr);
 	return (acc);
 }
+long long
+wcstoll(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
+{
+	return wcstoll_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/locale/wcstombs.c b/lib/libc/locale/wcstombs.c
index 250d2b9..8d4ae1c 100644
--- a/lib/libc/locale/wcstombs.c
+++ b/lib/libc/locale/wcstombs.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,13 +38,21 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 size_t
-wcstombs(char * __restrict s, const wchar_t * __restrict pwcs, size_t n)
+wcstombs_l(char * __restrict s, const wchar_t * __restrict pwcs, size_t n,
+		locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
 	const wchar_t *pwcsp;
+	FIX_LOCALE(locale);
 
 	mbs = initial;
 	pwcsp = pwcs;
-	return (__wcsnrtombs(s, &pwcsp, SIZE_T_MAX, n, &mbs));
+	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(s, &pwcsp, SIZE_T_MAX, n, &mbs));
+}
+size_t
+wcstombs(char * __restrict s, const wchar_t * __restrict pwcs, size_t n)
+{
+	return wcstombs_l(s, pwcs, n, __get_locale());
 }
+
diff --git a/lib/libc/locale/wcstoul.c b/lib/libc/locale/wcstoul.c
index f50e7a7..54e6b4f 100644
--- a/lib/libc/locale/wcstoul.c
+++ b/lib/libc/locale/wcstoul.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,18 +40,21 @@ __FBSDID("$FreeBSD$");
 #include <limits.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a wide character string to an unsigned long integer.
  */
 unsigned long
-wcstoul(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
+wcstoul_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+		int base, locale_t locale)
 {
 	const wchar_t *s;
 	unsigned long acc;
 	wchar_t c;
 	unsigned long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtol for comments as to the logic used.
@@ -54,7 +62,7 @@ wcstoul(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (iswspace(c));
+	} while (iswspace_l(c, locale));
 	if (c == L'-') {
 		neg = 1;
 		c = *s++;
@@ -79,8 +87,8 @@ wcstoul(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
 	cutlim = ULONG_MAX % base;
 	for ( ; ; c = *s++) {
 #ifdef notyet
-		if (iswdigit(c))
-			c = digittoint(c);
+		if (iswdigit_l(c, locale))
+			c = digittoint_l(c, locale);
 		else
 #endif
 		if (c >= L'0' && c <= L'9')
@@ -113,3 +121,8 @@ noconv:
 		*endptr = (wchar_t *)(any ? s - 1 : nptr);
 	return (acc);
 }
+unsigned long
+wcstoul(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int base)
+{
+	return wcstoul_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/locale/wcstoull.c b/lib/libc/locale/wcstoull.c
index d840424..c56ee77 100644
--- a/lib/libc/locale/wcstoull.c
+++ b/lib/libc/locale/wcstoull.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,19 +46,21 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a wide character string to an unsigned long long integer.
  */
 unsigned long long
-wcstoull(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
-    int base)
+wcstoull_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+    int base, locale_t locale)
 {
 	const wchar_t *s;
 	unsigned long long acc;
 	wchar_t c;
 	unsigned long long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtoull for comments as to the logic used.
@@ -61,7 +68,7 @@ wcstoull(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 	s = nptr;
 	do {
 		c = *s++;
-	} while (iswspace(c));
+	} while (iswspace_l(c, locale));
 	if (c == L'-') {
 		neg = 1;
 		c = *s++;
@@ -86,8 +93,8 @@ wcstoull(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 	cutlim = ULLONG_MAX % base;
 	for ( ; ; c = *s++) {
 #ifdef notyet
-		if (iswdigit(c))
-			c = digittoint(c);
+		if (iswdigit_l(c, locale))
+			c = digittoint_l(c, locale);
 		else
 #endif
 		if (c >= L'0' && c <= L'9')
@@ -120,3 +127,9 @@ noconv:
 		*endptr = (wchar_t *)(any ? s - 1 : nptr);
 	return (acc);
 }
+unsigned long long
+wcstoull(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+    int base)
+{
+	return wcstoull_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/locale/wcstoumax.c b/lib/libc/locale/wcstoumax.c
index d643b0b..397c1e3 100644
--- a/lib/libc/locale/wcstoumax.c
+++ b/lib/libc/locale/wcstoumax.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,19 +46,21 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a wide character string to a uintmax_t integer.
  */
 uintmax_t
-wcstoumax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
-    int base)
+wcstoumax_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+    int base, locale_t locale)
 {
 	const wchar_t *s;
 	uintmax_t acc;
 	wchar_t c;
 	uintmax_t cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtoimax for comments as to the logic used.
@@ -61,7 +68,7 @@ wcstoumax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 	s = nptr;
 	do {
 		c = *s++;
-	} while (iswspace(c));
+	} while (iswspace_l(c, locale));
 	if (c == L'-') {
 		neg = 1;
 		c = *s++;
@@ -86,8 +93,8 @@ wcstoumax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 	cutlim = UINTMAX_MAX % base;
 	for ( ; ; c = *s++) {
 #ifdef notyet
-		if (iswdigit(c))
-			c = digittoint(c);
+		if (iswdigit_l(c, locale))
+			c = digittoint_l(c, locale);
 		else
 #endif
 		if (c >= L'0' && c <= L'9')
@@ -120,3 +127,9 @@ noconv:
 		*endptr = (wchar_t *)(any ? s - 1 : nptr);
 	return (acc);
 }
+uintmax_t
+wcstoumax(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
+    int base)
+{
+	return wcstoumax_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/locale/wctob.c b/lib/libc/locale/wctob.c
index cb39adc..96e8b73 100644
--- a/lib/libc/locale/wctob.c
+++ b/lib/libc/locale/wctob.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,13 +38,19 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 int
-wctob(wint_t c)
+wctob_l(wint_t c, locale_t locale)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs = initial;
 	char buf[MB_LEN_MAX];
+	FIX_LOCALE(locale);
 
-	if (c == WEOF || __wcrtomb(buf, c, &mbs) != 1)
+	if (c == WEOF || XLOCALE_CTYPE(locale)->__wcrtomb(buf, c, &mbs) != 1)
 		return (EOF);
 	return ((unsigned char)*buf);
 }
+int
+wctob(wint_t c)
+{
+	return wctob_l(c, __get_locale());
+}
diff --git a/lib/libc/locale/wctomb.c b/lib/libc/locale/wctomb.c
index 77b9043..da3d263 100644
--- a/lib/libc/locale/wctomb.c
+++ b/lib/libc/locale/wctomb.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,18 +37,23 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 int
-wctomb(char *s, wchar_t wchar)
+wctomb_l(char *s, wchar_t wchar, locale_t locale)
 {
 	static const mbstate_t initial;
-	static mbstate_t mbs;
 	size_t rval;
+	FIX_LOCALE(locale);
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		mbs = initial;
+		locale->wctomb = initial;
 		return (0);
 	}
-	if ((rval = __wcrtomb(s, wchar, &mbs)) == (size_t)-1)
+	if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar, &locale->wctomb)) == (size_t)-1)
 		return (-1);
 	return ((int)rval);
 }
+int
+wctomb(char *s, wchar_t wchar)
+{
+	return wctomb_l(s, wchar, __get_locale());
+}
diff --git a/lib/libc/locale/wctrans.c b/lib/libc/locale/wctrans.c
index 6813e33..0831146 100644
--- a/lib/libc/locale/wctrans.c
+++ b/lib/libc/locale/wctrans.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <errno.h>
 #include <string.h>
 #include <wctype.h>
+#include "xlocale_private.h"
 
 enum {
 	_WCT_ERROR	= 0,
@@ -38,15 +44,14 @@ enum {
 };
 
 wint_t
-towctrans(wint_t wc, wctrans_t desc)
+towctrans_l(wint_t wc, wctrans_t desc, locale_t locale)
 {
-
 	switch (desc) {
 	case _WCT_TOLOWER:
-		wc = towlower(wc);
+		wc = towlower_l(wc, locale);
 		break;
 	case _WCT_TOUPPER:
-		wc = towupper(wc);
+		wc = towupper_l(wc, locale);
 		break;
 	case _WCT_ERROR:
 	default:
@@ -56,9 +61,18 @@ towctrans(wint_t wc, wctrans_t desc)
 
 	return (wc);
 }
+wint_t
+towctrans(wint_t wc, wctrans_t desc)
+{
+	return towctrans_l(wc, desc, __get_locale());
+}
 
+/*
+ * wctrans() calls this will a 0 locale.  If this is ever modified to actually
+ * use the locale, wctrans() must be modified to call __get_locale().
+ */
 wctrans_t
-wctrans(const char *charclass)
+wctrans_l(const char *charclass, locale_t locale)
 {
 	struct {
 		const char	*name;
@@ -78,3 +92,10 @@ wctrans(const char *charclass)
 		errno = EINVAL;
 	return (ccls[i].trans);
 }
+
+wctrans_t
+wctrans(const char *charclass)
+{
+	return wctrans_l(charclass, 0);
+}
+
diff --git a/lib/libc/locale/wctype.c b/lib/libc/locale/wctype.c
index f94a735..5ffbb79 100644
--- a/lib/libc/locale/wctype.c
+++ b/lib/libc/locale/wctype.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,17 +35,27 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <string.h>
 #include <wctype.h>
+#include <xlocale.h>
 
 #undef iswctype
 int
 iswctype(wint_t wc, wctype_t charclass)
 {
-
 	return (__istype(wc, charclass));
 }
+int
+iswctype_l(wint_t wc, wctype_t charclass, locale_t locale)
+{
+	return __istype_l(wc, charclass, locale);
+}
 
+/*
+ * IMPORTANT: The 0 in the call to this function in wctype() must be changed to
+ * __get_locale() if wctype_l() is ever modified to actually use the locale
+ * parameter.
+ */
 wctype_t
-wctype(const char *property)
+wctype_l(const char *property, locale_t locale)
 {
 	static const struct {
 		const char	*name;
@@ -72,3 +87,8 @@ wctype(const char *property)
 
 	return (props[i].mask);
 }
+
+wctype_t wctype(const char *property)
+{
+	return wctype_l(property, 0);
+}
diff --git a/lib/libc/locale/wcwidth.c b/lib/libc/locale/wcwidth.c
index 7a5d2ed..9e74217 100644
--- a/lib/libc/locale/wcwidth.c
+++ b/lib/libc/locale/wcwidth.c
@@ -10,6 +10,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,12 +44,18 @@
 __FBSDID("$FreeBSD$");
 
 #include <wchar.h>
+#include <wctype.h>
+#include <xlocale.h>
 
 #undef wcwidth
 
 int
 wcwidth(wchar_t wc)
 {
-
 	return (__wcwidth(wc));
 }
+int
+wcwidth_l(wchar_t wc, locale_t locale)
+{
+	return (__wcwidth_l(wc, locale));
+}
diff --git a/lib/libc/locale/xlocale.3 b/lib/libc/locale/xlocale.3
new file mode 100644
index 0000000..ecfbb4c
--- /dev/null
+++ b/lib/libc/locale/xlocale.3
@@ -0,0 +1,270 @@
+.\" Copyright (c) 2011 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This documentation was written by David Chisnall under sponsorship from
+.\" the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd September 17 2011
+.Dt XLOCALE 3
+.Os
+.Sh NAME
+.Nm xlocale
+.Nd Thread-safe extended locale support.
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In xlocale.h
+.Sh DESCRIPTION
+The extended locale support includes a set of functions for setting
+thread-local locales, as well convenience functions for performing locale-aware
+calls with a specified locale.
+.Pp
+The core of the xlocale API is the
+.Fa locale_t
+type.  This is an opaque type encapsulating a locale.  Instances of this can be
+either set as the locale for a specific thread or passed directly to the
+.Fa _l
+suffixed variants of various standard C functions.  Two special
+.Fa locale_t
+values are available:
+.Bl -bullet -offset indent 
+.It
+NULL refers to the current locale for the thread, or to the global locale if no
+locale has been set for this thread.
+.It
+LC_GLOBAL_LOCALE refers to the global locale.
+.El
+.Pp
+The global locale is the locale set with the 
+.Xr setlocale 3
+function.
+.Sh CAVEATS
+The 
+.Xr setlocale 3
+function, and others in the family, refer to the global locale.  Other
+functions that depend on the locale, however, will take the thread-local locale
+if one has been set.  This means that the idiom of setting the locale using
+.Xr setlocale 3 ,
+calling a locale-dependent function, and then restoring the locale will not
+have the expected behavior if the current thread has had a locale set using
+.Xr uselocale 3 .
+You should avoid this idiom and prefer to use the 
+.Fa _l
+suffixed versions instead.
+.Sh SEE ALSO
+.Xr duplocale 3 ,
+.Xr freelocale 3 ,
+.Xr localeconv 3 ,
+.Xr newlocale 3 ,
+.Xr querylocale 3 ,
+.Xr uselocale 3 ,
+.Sh CONVENIENCE FUNCTIONS
+The xlocale API includes a number of
+.Fa _l
+suffixed convenience functions.  These are variants of standard C functions
+that have been modified to take an explicit 
+.Fa locale_t
+parameter as the final argument or, in the case of variadic functions, as an
+additional argument directly before the format string.  Each of these functions
+accepts either NULL or LC_GLOBAL_LOCALE.  In these functions, NULL refers to
+the C locale, rather than the thread's current locale.  If you wish to use the
+thread's current locale, then use the unsuffixed version of the function.
+.Pp
+These functions are exposed by including 
+.In xlocale.h
+.Em after
+including the relevant headers for the standard variant.  For example, the 
+.Xr strtol_l 3
+function is exposed by including 
+.In xlocale.h
+after 
+.In stdlib.h ,
+which defines
+.Xr strtol 3 .
+.Pp
+For reference, a complete list of the locale-aware functions that are available
+in this form, along with the headers that expose them, is provided here:
+.Pp
+.Bl -tag -width "<monetary.h> "
+.It In wctype.h
+.Xr iswalnum_l 3 ,
+.Xr iswalpha_l 3 ,
+.Xr iswcntrl_l 3 ,
+.Xr iswctype_l 3 ,
+.Xr iswdigit_l 3 ,
+.Xr iswgraph_l 3 ,
+.Xr iswlower_l 3 ,
+.Xr iswprint_l 3 , 
+.Xr iswpunct_l 3 ,
+.Xr iswspace_l 3 ,
+.Xr iswupper_l 3 ,
+.Xr iswxdigit_l 3 ,
+.Xr towlower_l 3 ,
+.Xr towupper_l 3 ,
+.Xr wctype_l 3 ,
+.It In ctype.h
+.Xr digittoint_l 3 ,
+.Xr isalnum_l 3 ,
+.Xr isalpha_l 3 ,
+.Xr isblank_l 3 ,
+.Xr iscntrl_l 3 ,
+.Xr isdigit_l 3 ,
+.Xr isgraph_l 3 ,
+.Xr ishexnumber_l 3 ,
+.Xr isideogram_l 3 ,
+.Xr islower_l 3 ,
+.Xr isnumber_l 3 ,
+.Xr isphonogram_l 3 ,
+.Xr isprint_l 3 ,
+.Xr ispunct_l 3 ,
+.Xr isrune_l 3 ,
+.Xr isspace_l 3 ,
+.Xr isspecial_l 3 ,
+.Xr isupper_l 3 ,
+.Xr isxdigit_l 3 ,
+.Xr tolower_l 3 ,
+.Xr toupper_l 3
+.It In inttypes.h
+.Xr strtoimax_l 3 ,
+.Xr strtoumax_l 3 ,
+.Xr wcstoimax_l 3 ,
+.Xr wcstoumax_l 3
+.It In langinfo.h
+.Xr nl_langinfo_l 3
+.It In monetary.h
+.Xr strfmon_l 3
+.It In stdio.h
+.Xr asprintf_l 3 ,
+.Xr fprintf_l 3 ,
+.Xr fscanf_l 3 ,
+.Xr printf_l 3 ,
+.Xr scanf_l 3 ,
+.Xr snprintf_l 3 ,
+.Xr sprintf_l 3 ,
+.Xr sscanf_l 3 ,
+.Xr vasprintf_l 3 ,
+.Xr vfprintf_l 3 ,
+.Xr vfscanf_l 3 ,
+.Xr vprintf_l 3 ,
+.Xr vscanf_l 3 ,
+.Xr vsnprintf_l 3 ,
+.Xr vsprintf_l 3 ,
+.Xr vsscanf_l 3
+.It In stdlib.h
+.Xr atof_l 3 ,
+.Xr atoi_l 3 ,
+.Xr atol_l 3 ,
+.Xr atoll_l 3 ,
+.Xr mblen_l 3 ,
+.Xr mbstowcs_l 3 ,
+.Xr mbtowc_l 3 ,
+.Xr strtod_l 3 ,
+.Xr strtof_l 3 ,
+.Xr strtol_l 3 ,
+.Xr strtold_l 3 ,
+.Xr strtoll_l 3 ,
+.Xr strtoq_l 3 ,
+.Xr strtoul_l 3 ,
+.Xr strtoull_l 3 ,
+.Xr strtouq_l 3 ,
+.Xr wcstombs_l 3 ,
+.Xr wctomb_l 3
+.It In string.h
+.Xr strcoll_l 3 ,
+.Xr strxfrm_l 3 ,
+.Xr strcasecmp_l 3 ,
+.Xr strcasestr_l 3 ,
+.Xr strncasecmp_l 3
+.It In time.h
+.Xr strftime_l 3
+.Xr strptime_l 3
+.It In wchar.h
+.Xr btowc_l 3 ,
+.Xr fgetwc_l 3 ,
+.Xr fgetws_l 3 ,
+.Xr fputwc_l 3 ,
+.Xr fputws_l 3 ,
+.Xr fwprintf_l 3 ,
+.Xr fwscanf_l 3 ,
+.Xr getwc_l 3 ,
+.Xr getwchar_l 3 ,
+.Xr mbrlen_l 3 ,
+.Xr mbrtowc_l 3 ,
+.Xr mbsinit_l 3 ,
+.Xr mbsnrtowcs_l 3 ,
+.Xr mbsrtowcs_l 3 ,
+.Xr putwc_l 3 ,
+.Xr putwchar_l 3 ,
+.Xr swprintf_l 3 ,
+.Xr swscanf_l 3 ,
+.Xr ungetwc_l 3 ,
+.Xr vfwprintf_l 3 ,
+.Xr vfwscanf_l 3 ,
+.Xr vswprintf_l 3 ,
+.Xr vswscanf_l 3 ,
+.Xr vwprintf_l 3 ,
+.Xr vwscanf_l 3 ,
+.Xr wcrtomb_l 3 ,
+.Xr wcscoll_l 3 ,
+.Xr wcsftime_l 3 ,
+.Xr wcsnrtombs_l 3 ,
+.Xr wcsrtombs_l 3 ,
+.Xr wcstod_l 3 ,
+.Xr wcstof_l 3 ,
+.Xr wcstol_l 3 ,
+.Xr wcstold_l 3 ,
+.Xr wcstoll_l 3 ,
+.Xr wcstoul_l 3 ,
+.Xr wcstoull_l 3 ,
+.Xr wcswidth_l 3 ,
+.Xr wcsxfrm_l 3 ,
+.Xr wctob_l 3 ,
+.Xr wcwidth_l 3 ,
+.Xr wprintf_l 3 ,
+.Xr wscanf_l 3
+.It In wctype.h
+.Xr iswblank_l 3 ,
+.Xr iswhexnumber_l 3 ,
+.Xr iswideogram_l 3 ,
+.Xr iswnumber_l 3 ,
+.Xr iswphonogram_l 3 ,
+.Xr iswrune_l 3 ,
+.Xr iswspecial_l 3 ,
+.Xr nextwctype_l 3 ,
+.Xr towctrans_l 3 ,
+.Xr wctrans_l 3
+.It In xlocale.h
+.Xr localeconv_l 3
+.El
+.Sh STANDARDS
+The functions
+conform to
+.St -p1003.1-2008 .
+.Sh HISTORY
+The xlocale APIs first appeared in Darwin 8.0.  This implementation was
+written by David Chisnall, under sponsorship from the FreeBSD Foundation and
+first appeared in
+.Fx 9.1 .
diff --git a/lib/libc/locale/xlocale.c b/lib/libc/locale/xlocale.c
new file mode 100644
index 0000000..f14f952
--- /dev/null
+++ b/lib/libc/locale/xlocale.c
@@ -0,0 +1,334 @@
+/*-
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by David Chisnall under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions * are met:
+ * 1.  Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <string.h>
+#include "libc_private.h"
+#include "xlocale_private.h"
+
+/**
+ * Each locale loader declares a global component.  This is used by setlocale()
+ * and also by xlocale with LC_GLOBAL_LOCALE..
+ */
+extern struct xlocale_component __xlocale_global_collate;
+extern struct xlocale_component __xlocale_global_ctype;
+extern struct xlocale_component __xlocale_global_monetary;
+extern struct xlocale_component __xlocale_global_numeric;
+extern struct xlocale_component __xlocale_global_time;
+extern struct xlocale_component __xlocale_global_messages;
+/*
+ * And another version for the statically-allocated C locale.  We only have
+ * components for the parts that are expected to be sensible.
+ */
+extern struct xlocale_component __xlocale_C_collate;
+extern struct xlocale_component __xlocale_C_ctype;
+/*
+ * Private functions in setlocale.c.
+ */
+const char *
+__get_locale_env(int category);
+int
+__detect_path_locale(void);
+
+struct _xlocale __xlocale_global_locale = {
+	{0},
+	{
+		&__xlocale_global_collate,
+		&__xlocale_global_ctype,
+		&__xlocale_global_monetary,
+		&__xlocale_global_numeric,
+		&__xlocale_global_time,
+		&__xlocale_global_messages
+	},
+	1,
+	0,
+	1,
+	0
+};
+
+struct _xlocale __xlocale_C_locale = {
+	{0},
+	{
+		&__xlocale_C_collate,
+		&__xlocale_C_ctype,
+		0, 0, 0, 0
+	},
+	1,
+	0,
+	1,
+	0
+};
+
+static void*(*constructors[])(const char*, locale_t) =
+{
+	__collate_load,
+	__ctype_load,
+	__monetary_load,
+	__numeric_load,
+	__time_load,
+	__messages_load
+};
+
+static pthread_key_t locale_info_key;
+static int fake_tls;
+static locale_t thread_local_locale;
+
+static void init_key(void)
+{
+	pthread_key_create(&locale_info_key, xlocale_release);
+	pthread_setspecific(locale_info_key, (void*)42);
+	if (pthread_getspecific(locale_info_key) == (void*)42) {
+		pthread_setspecific(locale_info_key, 0);
+	} else {
+		fake_tls = 1;
+	}
+	__detect_path_locale();
+}
+
+static pthread_once_t once_control = PTHREAD_ONCE_INIT;
+
+static locale_t
+get_thread_locale(void)
+{
+	_once(&once_control, init_key);
+	
+	return (fake_tls ? thread_local_locale :
+		pthread_getspecific(locale_info_key));
+}
+
+locale_t
+__get_locale(void)
+{
+	locale_t l = get_thread_locale();
+	return (l ? l : &__xlocale_global_locale);
+
+}
+
+static void
+set_thread_locale(locale_t loc)
+{
+	pthread_once(&once_control, init_key);
+	
+	if (NULL != loc) {
+		xlocale_retain((struct xlocale_refcounted*)loc);
+	}
+	locale_t old = pthread_getspecific(locale_info_key);
+	if ((NULL != old) && (loc != old)) {
+		xlocale_release((struct xlocale_refcounted*)old);
+	}
+	if (fake_tls) {
+		thread_local_locale = loc;
+	} else {
+		pthread_setspecific(locale_info_key, loc);
+	}
+}
+
+/**
+ * Clean up a locale, once its reference count reaches zero.  This function is
+ * called by xlocale_release(), it should not be called directly.
+ */
+static void
+destruct_locale(void *l)
+{
+	locale_t loc = l;
+	for (int type=0 ; type<XLC_LAST ; type++) {
+		if (loc->components[type]) {
+			xlocale_release(loc->components[type]);
+		}
+	}
+	if (loc->csym) {
+		free(loc->csym);
+	}
+	free(l);
+}
+
+/**
+ * Allocates a new, uninitialised, locale.
+ */
+static locale_t
+alloc_locale(void)
+{
+	locale_t new = calloc(sizeof(struct _xlocale), 1);
+	new->header.destructor = destruct_locale;
+	new->monetary_locale_changed = 1;
+	new->numeric_locale_changed = 1;
+	return (new);
+}
+static void
+copyflags(locale_t new, locale_t old)
+{
+	new->using_monetary_locale = old->using_monetary_locale;
+	new->using_numeric_locale = old->using_numeric_locale;
+	new->using_time_locale = old->using_time_locale;
+	new->using_messages_locale = old->using_messages_locale;
+}
+
+static int dupcomponent(int type, locale_t base, locale_t new) 
+{
+	/* Always copy from the global locale, since it has mutable components. */
+	struct xlocale_component *src = base->components[type];
+	if (&__xlocale_global_locale == base) {
+		new->components[type] = constructors[type](src->locale, new);
+		if (new->components[type]) {
+			strncpy(new->components[type]->locale, src->locale, ENCODING_LEN);
+		}
+	} else {
+		new->components[type] = xlocale_retain(base->components[type]);
+	}
+	return (0 != new->components[type]);
+}
+
+/*
+ * Public interfaces.  These are the five public functions described by the
+ * xlocale interface.  
+ */
+
+locale_t newlocale(int mask, const char *locale, locale_t base)
+{
+	int type;
+	const char *realLocale = locale;
+	int useenv = 0;
+	int success = 1;
+
+	_once(&once_control, init_key);
+
+	locale_t new = alloc_locale();
+	if (NULL == new) {
+		return (NULL);
+	}
+
+	FIX_LOCALE(base);
+	copyflags(new, base);
+
+	if (NULL == locale) {
+		realLocale = "C";
+	} else if ('\0' == locale[0]) {
+		useenv = 1;
+	}
+
+	for (type=0 ; type<XLC_LAST ; type++) {
+		if (mask & 1) {
+			if (useenv) {
+				realLocale = __get_locale_env(type);
+			}
+			new->components[type] = constructors[type](realLocale, new);
+			if (new->components[type]) {
+				strncpy(new->components[type]->locale, realLocale, ENCODING_LEN);
+			} else {
+				success = 0;
+				break;
+			}
+		} else {
+			if (!dupcomponent(type, base, new)) {
+				success = 0;
+				break;
+			}
+		}
+		mask >>= 1;
+	}
+	if (0 == success) {
+		xlocale_release(new);
+		new = NULL;
+	}
+
+	return (new);
+}
+
+locale_t duplocale(locale_t base)
+{
+	locale_t new = alloc_locale();
+	int type;
+
+	_once(&once_control, init_key);
+
+	if (NULL == new) {
+		return (NULL);
+	}
+	
+	FIX_LOCALE(base);
+	copyflags(new, base);
+
+	for (type=0 ; type<XLC_LAST ; type++) {
+		dupcomponent(type, base, new);
+	}
+
+	return (new);
+}
+
+/*
+ * Free a locale_t.  This is quite a poorly named function.  It actually
+ * disclaims a reference to a locale_t, rather than freeing it.  
+ */
+int
+freelocale(locale_t loc)
+{
+	/* Fail if we're passed something that isn't a locale. */
+	if ((NULL == loc) || (LC_GLOBAL_LOCALE == loc)) {
+		return (-1);
+	}
+	/* If we're passed the global locale, pretend that we freed it but don't
+	 * actually do anything. */
+	if (&__xlocale_global_locale == loc) {
+		return (0);
+	}
+	xlocale_release(loc);
+	return (0);
+}
+
+/*
+ * Returns the name of the locale for a particular component of a locale_t.
+ */
+const char *querylocale(int mask, locale_t loc)
+{
+	int type = ffs(mask) - 1;
+	FIX_LOCALE(loc);
+	if (type >= XLC_LAST)
+		return (NULL);
+	if (loc->components[type])
+		return (loc->components[type]->locale);
+	return "C";
+}
+
+/*
+ * Installs the specified locale_t as this thread's locale.
+ */
+locale_t uselocale(locale_t loc)
+{
+	locale_t old = get_thread_locale();
+	if (NULL != loc) {
+		if (LC_GLOBAL_LOCALE == loc) {
+			loc = NULL;
+		}
+		set_thread_locale(loc);
+	}
+	return (old ? old : LC_GLOBAL_LOCALE);
+}
+
diff --git a/lib/libc/locale/xlocale_private.h b/lib/libc/locale/xlocale_private.h
new file mode 100644
index 0000000..272d15e
--- /dev/null
+++ b/lib/libc/locale/xlocale_private.h
@@ -0,0 +1,198 @@
+/*-
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by David Chisnall under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1.  Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _XLOCALE_PRIVATE__H_
+#define _XLOCALE_PRIVATE__H_
+
+#include <xlocale.h>
+#include <locale.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <machine/atomic.h>
+#include "setlocale.h"
+
+enum {
+	XLC_COLLATE = 0,
+	XLC_CTYPE,
+	XLC_MONETARY,
+	XLC_NUMERIC,
+	XLC_TIME,
+	XLC_MESSAGES,
+	XLC_LAST
+};
+
+
+/**
+ * Header used for objects that are reference counted.  Objects may optionally
+ * have a destructor associated, which is responsible for destroying the
+ * structure.  Global / static versions of the structure should have no
+ * destructor set - they can then have their reference counts manipulated as
+ * normal, but will not do anything with them.
+ *
+ * The header stores a retain count - objects are assumed to have a reference
+ * count of 1 when they are created, but the retain count is 0.  When the
+ * retain count is less than 0, they are freed.
+ */
+struct xlocale_refcounted {
+	/** Number of references to this component.  */
+	long retain_count;
+	/** Function used to destroy this component, if one is required*/
+	void(*destructor)(void*);
+};
+/**
+ * Header for a locale component.  All locale components must begin with this
+ * header.
+ */
+struct xlocale_component {
+	struct xlocale_refcounted header;
+	/** Name of the locale used for this component. */
+	char locale[ENCODING_LEN+1];
+};
+
+/**
+ * xlocale structure, stores per-thread locale information.  
+ */
+struct _xlocale {
+	struct xlocale_refcounted header;
+	/** Components for the locale.  */
+	struct xlocale_component *components[XLC_LAST];
+	/** Flag indicating if components[XLC_MONETARY] has changed since the last
+	 * call to localeconv_l() with this locale. */
+	int monetary_locale_changed;
+	/** Flag indicating whether this locale is actually using a locale for
+	 * LC_MONETARY (1), or if it should use the C default instead (0). */
+	int using_monetary_locale;
+	/** Flag indicating if components[XLC_NUMERIC] has changed since the last
+	 * call to localeconv_l() with this locale. */
+	int numeric_locale_changed;
+	/** Flag indicating whether this locale is actually using a locale for
+	 * LC_NUMERIC (1), or if it should use the C default instead (0). */
+	int using_numeric_locale;
+	/** Flag indicating whether this locale is actually using a locale for
+	 * LC_TIME (1), or if it should use the C default instead (0). */
+	int using_time_locale;
+	/** Flag indicating whether this locale is actually using a locale for
+	 * LC_MESSAGES (1), or if it should use the C default instead (0). */
+	int using_messages_locale;
+	/** The structure to be returned from localeconv_l() for this locale. */
+	struct lconv lconv;
+	/** Persistent state used by mblen() calls. */
+	__mbstate_t mblen;
+	/** Persistent state used by mbrlen() calls. */
+	__mbstate_t mbrlen;
+	/** Persistent state used by mbrtowc() calls. */
+	__mbstate_t mbrtowc;
+	/** Persistent state used by mbsnrtowcs() calls. */
+	__mbstate_t mbsnrtowcs;
+	/** Persistent state used by mbsrtowcs() calls. */
+	__mbstate_t mbsrtowcs;
+	/** Persistent state used by mbtowc() calls. */
+	__mbstate_t mbtowc;
+	/** Persistent state used by wcrtomb() calls. */
+	__mbstate_t wcrtomb;
+	/** Persistent state used by wcsnrtombs() calls. */
+	__mbstate_t wcsnrtombs;
+	/** Persistent state used by wcsrtombs() calls. */
+	__mbstate_t wcsrtombs;
+	/** Persistent state used by wctomb() calls. */
+	__mbstate_t wctomb;
+	/** Buffer used by nl_langinfo_l() */
+	char *csym;
+};
+
+/**
+ * Increments the reference count of a reference-counted structure.
+ */
+__attribute__((unused)) static void*
+xlocale_retain(void *val)
+{
+	struct xlocale_refcounted *obj = val;
+	atomic_add_long(&(obj->retain_count), 1);
+	return (val);
+}
+/**
+ * Decrements the reference count of a reference-counted structure, freeing it
+ * if this is the last reference, calling its destructor if it has one.
+ */
+__attribute__((unused)) static void
+xlocale_release(void *val)
+{
+	struct xlocale_refcounted *obj = val;
+	long count = atomic_fetchadd_long(&(obj->retain_count), -1) - 1;
+	if (count < 0) {
+		if (0 != obj->destructor) {
+			obj->destructor(obj);
+		}
+	}
+}
+
+/**
+ * Load functions.  Each takes the name of a locale and a pointer to the data
+ * to be initialised as arguments.  Two special values are allowed for the 
+ */
+extern void* __collate_load(const char*, locale_t);
+extern void* __ctype_load(const char*, locale_t);
+extern void* __messages_load(const char*, locale_t);
+extern void* __monetary_load(const char*, locale_t);
+extern void* __numeric_load(const char*, locale_t);
+extern void* __time_load(const char*, locale_t);
+
+extern struct _xlocale __xlocale_global_locale;
+extern struct _xlocale __xlocale_C_locale;
+
+/**
+ * Returns the current locale for this thread, or the global locale if none is
+ * set.  The caller does not have to free the locale.  The return value from
+ * this call is not guaranteed to remain valid after the locale changes.  As
+ * such, this should only be called within libc functions.
+ */
+locale_t __get_locale(void);
+
+/**
+ * Two magic values are allowed for locale_t objects.  NULL and -1.  This
+ * function maps those to the real locales that they represent.
+ */
+static inline locale_t get_real_locale(locale_t locale)
+{
+	switch ((intptr_t)locale) {
+		case 0: return (&__xlocale_C_locale);
+		case -1: return (&__xlocale_global_locale);
+		default: return (locale);
+	}
+}
+
+/**
+ * Replace a placeholder locale with the real global or thread-local locale_t.
+ */
+#define FIX_LOCALE(l) (l = get_real_locale(l))
+
+#endif
diff --git a/lib/libc/net/sctp_sys_calls.c b/lib/libc/net/sctp_sys_calls.c
index 0d721d5..72e9883 100644
--- a/lib/libc/net/sctp_sys_calls.c
+++ b/lib/libc/net/sctp_sys_calls.c
@@ -413,6 +413,9 @@ sctp_opt_info(int sd, sctp_assoc_t id, int opt, void *arg, socklen_t * size)
 	case SCTP_PEER_ADDR_THLDS:
 		((struct sctp_paddrthlds *)arg)->spt_assoc_id = id;
 		break;
+	case SCTP_REMOTE_UDP_ENCAPS_PORT:
+		((struct sctp_udpencaps *)arg)->sue_assoc_id = id;
+		break;
 	case SCTP_MAX_BURST:
 		((struct sctp_assoc_value *)arg)->assoc_id = id;
 		break;
diff --git a/lib/libc/regex/regcomp.c b/lib/libc/regex/regcomp.c
index 39ae1d4..c24b382 100644
--- a/lib/libc/regex/regcomp.c
+++ b/lib/libc/regex/regcomp.c
@@ -3,9 +3,19 @@
  * Copyright (c) 1992, 1993, 1994
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * This code is derived from software contributed to Berkeley by
  * Henry Spencer.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -81,10 +91,10 @@ extern "C" {
 #endif
 
 /* === regcomp.c === */
-static void p_ere(struct parse *p, wint_t stop);
+static void p_ere(struct parse *p, int stop);
 static void p_ere_exp(struct parse *p);
 static void p_str(struct parse *p);
-static void p_bre(struct parse *p, wint_t end1, wint_t end2);
+static void p_bre(struct parse *p, int end1, int end2);
 static int p_simp_re(struct parse *p, int starordinary);
 static int p_count(struct parse *p);
 static void p_bracket(struct parse *p);
@@ -109,7 +119,7 @@ static sopno dupl(struct parse *p, sopno start, sopno finish);
 static void doemit(struct parse *p, sop op, size_t opnd);
 static void doinsert(struct parse *p, sop op, size_t opnd, sopno pos);
 static void dofwd(struct parse *p, sopno pos, sop value);
-static void enlarge(struct parse *p, sopno size);
+static int enlarge(struct parse *p, sopno size);
 static void stripsnug(struct parse *p, struct re_guts *g);
 static void findmust(struct parse *p, struct re_guts *g);
 static int altoffset(sop *scan, int offset);
@@ -285,7 +295,7 @@ regcomp(regex_t * __restrict preg,
 
 /*
  - p_ere - ERE parser top level, concatenation and alternation
- == static void p_ere(struct parse *p, int stop);
+ == static void p_ere(struct parse *p, int_t stop);
  */
 static void
 p_ere(struct parse *p,
@@ -493,7 +503,7 @@ p_str(struct parse *p)
 
 /*
  - p_bre - BRE parser top level, anchoring and concatenation
- == static void p_bre(struct parse *p, int end1, \
+ == static void p_bre(struct parse *p,  int end1, \
  ==	int end2);
  * Giving end1 as OUT essentially eliminates the end1/end2 check.
  *
@@ -730,6 +740,8 @@ p_b_term(struct parse *p, cset *cs)
 	char c;
 	wint_t start, finish;
 	wint_t i;
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)__get_locale()->components[XLC_COLLATE];
 
 	/* classify what we've got */
 	switch ((MORE()) ? PEEK() : '\0') {
@@ -778,14 +790,14 @@ p_b_term(struct parse *p, cset *cs)
 		if (start == finish)
 			CHadd(p, cs, start);
 		else {
-			if (__collate_load_error) {
+			if (table->__collate_load_error) {
 				(void)REQUIRE((uch)start <= (uch)finish, REG_ERANGE);
 				CHaddrange(p, cs, start, finish);
 			} else {
-				(void)REQUIRE(__collate_range_cmp(start, finish) <= 0, REG_ERANGE);
+				(void)REQUIRE(__collate_range_cmp(table, start, finish) <= 0, REG_ERANGE);
 				for (i = 0; i <= UCHAR_MAX; i++) {
-					if (   __collate_range_cmp(start, i) <= 0
-					    && __collate_range_cmp(i, finish) <= 0
+					if (   __collate_range_cmp(table, start, i) <= 0
+					    && __collate_range_cmp(table, i, finish) <= 0
 					   )
 						CHadd(p, cs, i);
 				}
@@ -840,7 +852,7 @@ p_b_eclass(struct parse *p, cset *cs)
 
 /*
  - p_b_symbol - parse a character or [..]ed multicharacter collating symbol
- == static char p_b_symbol(struct parse *p);
+ == static wint_t p_b_symbol(struct parse *p);
  */
 static wint_t			/* value of symbol */
 p_b_symbol(struct parse *p)
@@ -859,7 +871,7 @@ p_b_symbol(struct parse *p)
 
 /*
  - p_b_coll_elem - parse a collating-element name and look it up
- == static char p_b_coll_elem(struct parse *p, int endc);
+ == static wint_t p_b_coll_elem(struct parse *p, wint_t endc);
  */
 static wint_t			/* value of collating element */
 p_b_coll_elem(struct parse *p,
@@ -894,7 +906,7 @@ p_b_coll_elem(struct parse *p,
 
 /*
  - othercase - return the case counterpart of an alphabetic
- == static char othercase(int ch);
+ == static wint_t othercase(wint_t ch);
  */
 static wint_t			/* if no counterpart, return ch */
 othercase(wint_t ch)
@@ -910,7 +922,7 @@ othercase(wint_t ch)
 
 /*
  - bothcases - emit a dualcase version of a two-case character
- == static void bothcases(struct parse *p, int ch);
+ == static void bothcases(struct parse *p, wint_t ch);
  *
  * Boy, is this implementation ever a kludge...
  */
@@ -939,7 +951,7 @@ bothcases(struct parse *p, wint_t ch)
 
 /*
  - ordinary - emit an ordinary character
- == static void ordinary(struct parse *p, int ch);
+ == static void ordinary(struct parse *p, wint_t ch);
  */
 static void
 ordinary(struct parse *p, wint_t ch)
@@ -1246,8 +1258,8 @@ dupl(struct parse *p,
 	assert(finish >= start);
 	if (len == 0)
 		return(ret);
-	enlarge(p, p->ssize + len);	/* this many unexpected additions */
-	assert(p->ssize >= p->slen + len);
+	if (!enlarge(p, p->ssize + len)) /* this many unexpected additions */
+		return(ret);
 	(void) memcpy((char *)(p->strip + p->slen),
 		(char *)(p->strip + start), (size_t)len*sizeof(sop));
 	p->slen += len;
@@ -1274,8 +1286,8 @@ doemit(struct parse *p, sop op, size_t opnd)
 
 	/* deal with undersized strip */
 	if (p->slen >= p->ssize)
-		enlarge(p, (p->ssize+1) / 2 * 3);	/* +50% */
-	assert(p->slen < p->ssize);
+		if (!enlarge(p, (p->ssize+1) / 2 * 3))	/* +50% */
+			return;
 
 	/* finally, it's all reduced to the easy case */
 	p->strip[p->slen++] = SOP(op, opnd);
@@ -1334,23 +1346,24 @@ dofwd(struct parse *p, sopno pos, sop value)
 
 /*
  - enlarge - enlarge the strip
- == static void enlarge(struct parse *p, sopno size);
+ == static int enlarge(struct parse *p, sopno size);
  */
-static void
+static int
 enlarge(struct parse *p, sopno size)
 {
 	sop *sp;
 
 	if (p->ssize >= size)
-		return;
+		return 1;
 
 	sp = (sop *)realloc(p->strip, size*sizeof(sop));
 	if (sp == NULL) {
 		SETERROR(REG_ESPACE);
-		return;
+		return 0;
 	}
 	p->strip = sp;
 	p->ssize = size;
+	return 1;
 }
 
 /*
diff --git a/lib/libc/stdio/Symbol.map b/lib/libc/stdio/Symbol.map
index df66ab3..cafb0c6 100644
--- a/lib/libc/stdio/Symbol.map
+++ b/lib/libc/stdio/Symbol.map
@@ -117,6 +117,46 @@ FBSD_1.1 {
 	 vdprintf;
 };
 
+FBSD_1.3 {
+	asprintf_l;
+	fprintf_l;
+	fwprintf_l;
+	printf_l;
+	snprintf_l;
+	sprintf_l;
+	swprintf_l;
+	vasprintf_l;
+	vfprintf_l;
+	vfwprintf_l;
+	vprintf_l;
+	vsnprintf_l;
+	vsprintf_l;
+	vswprintf_l;
+	vwprintf_l;
+	wprintf_l;
+	fgetwc_l;
+	fputwc_l;
+	ungetwc_l;
+	vfwscanf_l;
+	vswscanf_l;
+	fscanf_l;
+	fwscanf_l;
+	scanf_l;
+	sscanf_l;
+	swscanf_l;
+	vfscanf_l;
+	vscanf_l;
+	vsscanf_l;
+	vwscanf_l;
+	wscanf_l;
+	fgetws_l;
+	fputws_l;
+	getwc_l;
+	getwchar_l;
+	putwc_l;
+	putwchar_l;
+};
+
 FBSDprivate_1.0 {
 	_flockfile;
 	_flockfile_debug_stub;
diff --git a/lib/libc/stdio/asprintf.c b/lib/libc/stdio/asprintf.c
index 90c62bc..cd475cd 100644
--- a/lib/libc/stdio/asprintf.c
+++ b/lib/libc/stdio/asprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,6 +40,7 @@ __FBSDID("$FreeBSD$");
 
 #include <stdio.h>
 #include <stdarg.h>
+#include <xlocale.h>
 
 int
 asprintf(char ** __restrict s, char const * __restrict fmt, ...)
@@ -47,3 +53,15 @@ asprintf(char ** __restrict s, char const * __restrict fmt, ...)
 	va_end(ap);
 	return (ret);
 }
+int
+asprintf_l(char ** __restrict s, locale_t locale, char const * __restrict fmt,
+		...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vasprintf_l(s, locale, fmt, ap);
+	va_end(ap);
+	return (ret);
+}
diff --git a/lib/libc/stdio/fgetwc.c b/lib/libc/stdio/fgetwc.c
index 3a5b09b..0e87753 100644
--- a/lib/libc/stdio/fgetwc.c
+++ b/lib/libc/stdio/fgetwc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -36,31 +41,39 @@ __FBSDID("$FreeBSD$");
 #include "libc_private.h"
 #include "local.h"
 #include "mblocal.h"
+#include "xlocale_private.h"
 
 /*
  * MT-safe version.
  */
 wint_t
-fgetwc(FILE *fp)
+fgetwc_l(FILE *fp, locale_t locale)
 {
 	wint_t r;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
-	r = __fgetwc(fp);
+	r = __fgetwc(fp, locale);
 	FUNLOCKFILE(fp);
 
 	return (r);
 }
+wint_t
+fgetwc(FILE *fp)
+{
+	return fgetwc_l(fp, __get_locale());
+}
 
 /*
  * Non-MT-safe version.
  */
 wint_t
-__fgetwc(FILE *fp)
+__fgetwc(FILE *fp, locale_t locale)
 {
 	wchar_t wc;
 	size_t nconv;
+	struct xlocale_ctype *l = XLOCALE_CTYPE(locale);
 
 	if (fp->_r <= 0 && __srefill(fp))
 		return (WEOF);
@@ -71,7 +84,7 @@ __fgetwc(FILE *fp)
 		return (wc);
 	}
 	do {
-		nconv = __mbrtowc(&wc, fp->_p, fp->_r, &fp->_mbstate);
+		nconv = l->__mbrtowc(&wc, fp->_p, fp->_r, &fp->_mbstate);
 		if (nconv == (size_t)-1)
 			break;
 		else if (nconv == (size_t)-2)
diff --git a/lib/libc/stdio/fgetwln.c b/lib/libc/stdio/fgetwln.c
index 1a1ad2d..6d9087b 100644
--- a/lib/libc/stdio/fgetwln.c
+++ b/lib/libc/stdio/fgetwln.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,18 +38,20 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 wchar_t *
-fgetwln(FILE * __restrict fp, size_t *lenp)
+fgetwln_l(FILE * __restrict fp, size_t *lenp, locale_t locale)
 {
 	wint_t wc;
 	size_t len;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
 
 	len = 0;
-	while ((wc = __fgetwc(fp)) != WEOF) {
+	while ((wc = __fgetwc(fp, locale)) != WEOF) {
 #define	GROW	512
 		if (len * sizeof(wchar_t) >= fp->_lb._size &&
 		    __slbexpand(fp, (len + GROW) * sizeof(wchar_t)))
@@ -65,3 +72,8 @@ error:
 	*lenp = 0;
 	return (NULL);
 }
+wchar_t *
+fgetwln(FILE * __restrict fp, size_t *lenp)
+{
+	return fgetwln_l(fp, lenp, __get_locale());
+}
diff --git a/lib/libc/stdio/fgetws.c b/lib/libc/stdio/fgetws.c
index 550843d..f7a63fe 100644
--- a/lib/libc/stdio/fgetws.c
+++ b/lib/libc/stdio/fgetws.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,12 +43,14 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 wchar_t *
-fgetws(wchar_t * __restrict ws, int n, FILE * __restrict fp)
+fgetws_l(wchar_t * __restrict ws, int n, FILE * __restrict fp, locale_t locale)
 {
 	wchar_t *wsp;
 	size_t nconv;
 	const char *src;
 	unsigned char *nl;
+	FIX_LOCALE(locale);
+	struct xlocale_ctype *l = XLOCALE_CTYPE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
@@ -60,7 +67,7 @@ fgetws(wchar_t * __restrict ws, int n, FILE * __restrict fp)
 	do {
 		src = fp->_p;
 		nl = memchr(fp->_p, '\n', fp->_r);
-		nconv = __mbsnrtowcs(wsp, &src,
+		nconv = l->__mbsnrtowcs(wsp, &src,
 		    nl != NULL ? (nl - fp->_p + 1) : fp->_r,
 		    n - 1, &fp->_mbstate);
 		if (nconv == (size_t)-1)
@@ -86,7 +93,7 @@ fgetws(wchar_t * __restrict ws, int n, FILE * __restrict fp)
 	if (wsp == ws)
 		/* EOF */
 		goto error;
-	if (!__mbsinit(&fp->_mbstate))
+	if (!l->__mbsinit(&fp->_mbstate))
 		/* Incomplete character */
 		goto error;
 	*wsp = L'\0';
@@ -98,3 +105,8 @@ error:
 	FUNLOCKFILE(fp);
 	return (NULL);
 }
+wchar_t *
+fgetws(wchar_t * __restrict ws, int n, FILE * __restrict fp)
+{
+	return fgetws_l(ws, n, fp, __get_locale());
+}
diff --git a/lib/libc/stdio/fprintf.c b/lib/libc/stdio/fprintf.c
index bebc182..169532d 100644
--- a/lib/libc/stdio/fprintf.c
+++ b/lib/libc/stdio/fprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,6 +43,7 @@ __FBSDID("$FreeBSD$");
 
 #include <stdio.h>
 #include <stdarg.h>
+#include "xlocale_private.h"
 
 int
 fprintf(FILE * __restrict fp, const char * __restrict fmt, ...)
@@ -46,7 +52,19 @@ fprintf(FILE * __restrict fp, const char * __restrict fmt, ...)
 	va_list ap;
 
 	va_start(ap, fmt);
-	ret = vfprintf(fp, fmt, ap);
+	ret = vfprintf_l(fp, __get_locale(), fmt, ap);
+	va_end(ap);
+	return (ret);
+}
+int
+fprintf_l(FILE * __restrict fp, locale_t locale, const char * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+	FIX_LOCALE(locale);
+
+	va_start(ap, fmt);
+	ret = vfprintf_l(fp, locale, fmt, ap);
 	va_end(ap);
 	return (ret);
 }
diff --git a/lib/libc/stdio/fputwc.c b/lib/libc/stdio/fputwc.c
index ae92f66..7b05d4a 100644
--- a/lib/libc/stdio/fputwc.c
+++ b/lib/libc/stdio/fputwc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -42,10 +47,11 @@ __FBSDID("$FreeBSD$");
  * Non-MT-safe version.
  */
 wint_t
-__fputwc(wchar_t wc, FILE *fp)
+__fputwc(wchar_t wc, FILE *fp, locale_t locale)
 {
 	char buf[MB_LEN_MAX];
 	size_t i, len;
+	struct xlocale_ctype *l = XLOCALE_CTYPE(locale);
 
 	if (MB_CUR_MAX == 1 && wc > 0 && wc <= UCHAR_MAX) {
 		/*
@@ -56,7 +62,7 @@ __fputwc(wchar_t wc, FILE *fp)
 		*buf = (unsigned char)wc;
 		len = 1;
 	} else {
-		if ((len = __wcrtomb(buf, wc, &fp->_mbstate)) == (size_t)-1) {
+		if ((len = l->__wcrtomb(buf, wc, &fp->_mbstate)) == (size_t)-1) {
 			fp->_flags |= __SERR;
 			return (WEOF);
 		}
@@ -73,14 +79,20 @@ __fputwc(wchar_t wc, FILE *fp)
  * MT-safe version.
  */
 wint_t
-fputwc(wchar_t wc, FILE *fp)
+fputwc_l(wchar_t wc, FILE *fp, locale_t locale)
 {
 	wint_t r;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
-	r = __fputwc(wc, fp);
+	r = __fputwc(wc, fp, locale);
 	FUNLOCKFILE(fp);
 
 	return (r);
 }
+wint_t
+fputwc(wchar_t wc, FILE *fp)
+{
+	return fputwc_l(wc, fp, __get_locale());
+}
diff --git a/lib/libc/stdio/fputws.c b/lib/libc/stdio/fputws.c
index b4462b7..a318e2d 100644
--- a/lib/libc/stdio/fputws.c
+++ b/lib/libc/stdio/fputws.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,13 +44,15 @@ __FBSDID("$FreeBSD$");
 #include "mblocal.h"
 
 int
-fputws(const wchar_t * __restrict ws, FILE * __restrict fp)
+fputws_l(const wchar_t * __restrict ws, FILE * __restrict fp, locale_t locale)
 {
 	size_t nbytes;
 	char buf[BUFSIZ];
 	struct __suio uio;
 	struct __siov iov;
 	const wchar_t *wsp;
+	FIX_LOCALE(locale);
+	struct xlocale_ctype *l = XLOCALE_CTYPE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
@@ -56,7 +63,7 @@ fputws(const wchar_t * __restrict ws, FILE * __restrict fp)
 	iov.iov_base = buf;
 	do {
 		wsp = ws;
-		nbytes = __wcsnrtombs(buf, &wsp, SIZE_T_MAX, sizeof(buf),
+		nbytes = l->__wcsnrtombs(buf, &wsp, SIZE_T_MAX, sizeof(buf),
 		    &fp->_mbstate);
 		if (nbytes == (size_t)-1)
 			goto error;
@@ -71,3 +78,9 @@ error:
 	FUNLOCKFILE(fp);
 	return (-1);
 }
+
+int
+fputws(const wchar_t * __restrict ws, FILE * __restrict fp)
+{
+	return fputws_l(ws, fp, __get_locale());
+}
diff --git a/lib/libc/stdio/fscanf.c b/lib/libc/stdio/fscanf.c
index ce3001a..c5fa85f 100644
--- a/lib/libc/stdio/fscanf.c
+++ b/lib/libc/stdio/fscanf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -42,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 int
 fscanf(FILE * __restrict fp, char const * __restrict fmt, ...)
@@ -51,7 +57,21 @@ fscanf(FILE * __restrict fp, char const * __restrict fmt, ...)
 
 	va_start(ap, fmt);
 	FLOCKFILE(fp);
-	ret = __svfscanf(fp, fmt, ap);
+	ret = __svfscanf(fp, __get_locale(), fmt, ap);
+	va_end(ap);
+	FUNLOCKFILE(fp);
+	return (ret);
+}
+int
+fscanf_l(FILE * __restrict fp, locale_t locale, char const * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+	FIX_LOCALE(locale);
+
+	va_start(ap, fmt);
+	FLOCKFILE(fp);
+	ret = __svfscanf(fp, locale, fmt, ap);
 	va_end(ap);
 	FUNLOCKFILE(fp);
 	return (ret);
diff --git a/lib/libc/stdio/fwprintf.c b/lib/libc/stdio/fwprintf.c
index a22edae..f4342ab 100644
--- a/lib/libc/stdio/fwprintf.c
+++ b/lib/libc/stdio/fwprintf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 fwprintf(FILE * __restrict fp, const wchar_t * __restrict fmt, ...)
@@ -43,3 +49,15 @@ fwprintf(FILE * __restrict fp, const wchar_t * __restrict fmt, ...)
 
 	return (ret);
 }
+int
+fwprintf_l(FILE * __restrict fp, locale_t locale, const wchar_t * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vfwprintf_l(fp, locale, fmt, ap);
+	va_end(ap);
+
+	return (ret);
+}
diff --git a/lib/libc/stdio/fwscanf.c b/lib/libc/stdio/fwscanf.c
index f779c53..1756077 100644
--- a/lib/libc/stdio/fwscanf.c
+++ b/lib/libc/stdio/fwscanf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 fwscanf(FILE * __restrict fp, const wchar_t * __restrict fmt, ...)
@@ -43,3 +49,15 @@ fwscanf(FILE * __restrict fp, const wchar_t * __restrict fmt, ...)
 
 	return (r);
 }
+int
+fwscanf_l(FILE * __restrict fp, locale_t locale, const wchar_t * __restrict fmt, ...)
+{
+	va_list ap;
+	int r;
+
+	va_start(ap, fmt);
+	r = vfwscanf_l(fp, locale, fmt, ap);
+	va_end(ap);
+
+	return (r);
+}
diff --git a/lib/libc/stdio/getwc.c b/lib/libc/stdio/getwc.c
index ba5ab60..666350b 100644
--- a/lib/libc/stdio/getwc.c
+++ b/lib/libc/stdio/getwc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include "namespace.h"
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
@@ -46,3 +52,9 @@ getwc(FILE *fp)
 
 	return (fgetwc(fp));
 }
+wint_t
+getwc_l(FILE *fp, locale_t locale)
+{
+
+	return (fgetwc_l(fp, locale));
+}
diff --git a/lib/libc/stdio/getwchar.c b/lib/libc/stdio/getwchar.c
index 79dd7bc..bc3b8a6 100644
--- a/lib/libc/stdio/getwchar.c
+++ b/lib/libc/stdio/getwchar.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include "namespace.h"
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
@@ -42,6 +48,10 @@ __FBSDID("$FreeBSD$");
 wint_t
 getwchar(void)
 {
-
 	return (fgetwc(stdin));
 }
+wint_t
+getwchar_l(locale_t locale)
+{
+	return (fgetwc_l(stdin, locale));
+}
diff --git a/lib/libc/stdio/local.h b/lib/libc/stdio/local.h
index 6380b83..8b37f0d 100644
--- a/lib/libc/stdio/local.h
+++ b/lib/libc/stdio/local.h
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@
 #include <pthread.h>
 #include <string.h>
 #include <wchar.h>
+#include <locale.h>
 
 /*
  * Information local to this implementation of stdio,
@@ -50,8 +56,8 @@ extern int	_ftello(FILE *, fpos_t *);
 extern int	_fseeko(FILE *, off_t, int, int);
 extern int	__fflush(FILE *fp);
 extern void	__fcloseall(void);
-extern wint_t	__fgetwc(FILE *);
-extern wint_t	__fputwc(wchar_t, FILE *);
+extern wint_t	__fgetwc(FILE *, locale_t);
+extern wint_t	__fputwc(wchar_t, FILE *, locale_t);
 extern int	__sflush(FILE *);
 extern FILE	*__sfp(void);
 extern int	__slbexpand(FILE *, size_t);
@@ -65,15 +71,15 @@ extern void	_cleanup(void);
 extern void	__smakebuf(FILE *);
 extern int	__swhatbuf(FILE *, size_t *, int *);
 extern int	_fwalk(int (*)(FILE *));
-extern int	__svfscanf(FILE *, const char *, __va_list);
+extern int	__svfscanf(FILE *, locale_t, const char *, __va_list);
 extern int	__swsetup(FILE *);
 extern int	__sflags(const char *, int *);
 extern int	__ungetc(int, FILE *);
-extern wint_t	__ungetwc(wint_t, FILE *);
-extern int	__vfprintf(FILE *, const char *, __va_list);
+extern wint_t	__ungetwc(wint_t, FILE *, locale_t);
+extern int	__vfprintf(FILE *, locale_t, const char *, __va_list);
 extern int	__vfscanf(FILE *, const char *, __va_list);
-extern int	__vfwprintf(FILE *, const wchar_t *, __va_list);
-extern int	__vfwscanf(FILE * __restrict, const wchar_t * __restrict,
+extern int	__vfwprintf(FILE *, locale_t, const wchar_t *, __va_list);
+extern int	__vfwscanf(FILE * __restrict, locale_t, const wchar_t * __restrict,
 		    __va_list);
 extern size_t	__fread(void * __restrict buf, size_t size, size_t count,
 		FILE * __restrict fp);
diff --git a/lib/libc/stdio/printf.c b/lib/libc/stdio/printf.c
index df290fa..f623f2f 100644
--- a/lib/libc/stdio/printf.c
+++ b/lib/libc/stdio/printf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,6 +43,7 @@ __FBSDID("$FreeBSD$");
 
 #include <stdio.h>
 #include <stdarg.h>
+#include <xlocale.h>
 
 int
 printf(char const * __restrict fmt, ...)
@@ -50,3 +56,14 @@ printf(char const * __restrict fmt, ...)
 	va_end(ap);
 	return (ret);
 }
+int
+printf_l(locale_t locale, char const * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vfprintf_l(stdout, locale, fmt, ap);
+	va_end(ap);
+	return (ret);
+}
diff --git a/lib/libc/stdio/printfcommon.h b/lib/libc/stdio/printfcommon.h
index 39b0003..97acc53 100644
--- a/lib/libc/stdio/printfcommon.h
+++ b/lib/libc/stdio/printfcommon.h
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -79,14 +84,14 @@ io_init(struct io_state *iop, FILE *fp)
  * remain valid until io_flush() is called.
  */
 static inline int
-io_print(struct io_state *iop, const CHAR * __restrict ptr, int len)
+io_print(struct io_state *iop, const CHAR * __restrict ptr, int len, locale_t locale)
 {
 
 	iop->iov[iop->uio.uio_iovcnt].iov_base = (char *)ptr;
 	iop->iov[iop->uio.uio_iovcnt].iov_len = len;
 	iop->uio.uio_resid += len;
 	if (++iop->uio.uio_iovcnt >= NIOV)
-		return (__sprint(iop->fp, &iop->uio));
+		return (__sprint(iop->fp, &iop->uio, locale));
 	else
 		return (0);
 }
@@ -107,13 +112,14 @@ static const CHAR zeroes[PADSIZE] =
  * or the zeroes array.
  */
 static inline int
-io_pad(struct io_state *iop, int howmany, const CHAR * __restrict with)
+io_pad(struct io_state *iop, int howmany, const CHAR * __restrict with,
+		locale_t locale)
 {
 	int n;
 
 	while (howmany > 0) {
 		n = (howmany >= PADSIZE) ? PADSIZE : howmany;
-		if (io_print(iop, with, n))
+		if (io_print(iop, with, n, locale))
 			return (-1);
 		howmany -= n;
 	}
@@ -126,7 +132,7 @@ io_pad(struct io_state *iop, int howmany, const CHAR * __restrict with)
  */
 static inline int
 io_printandpad(struct io_state *iop, const CHAR *p, const CHAR *ep,
-	       int len, const CHAR * __restrict with)
+	       int len, const CHAR * __restrict with, locale_t locale)
 {
 	int p_len;
 
@@ -134,19 +140,19 @@ io_printandpad(struct io_state *iop, const CHAR *p, const CHAR *ep,
 	if (p_len > len)
 		p_len = len;
 	if (p_len > 0) {
-		if (io_print(iop, p, p_len))
+		if (io_print(iop, p, p_len, locale))
 			return (-1);
 	} else {
 		p_len = 0;
 	}
-	return (io_pad(iop, len - p_len, with));
+	return (io_pad(iop, len - p_len, with, locale));
 }
 
 static inline int
-io_flush(struct io_state *iop)
+io_flush(struct io_state *iop, locale_t locale)
 {
 
-	return (__sprint(iop->fp, &iop->uio));
+	return (__sprint(iop->fp, &iop->uio, locale));
 }
 
 /*
diff --git a/lib/libc/stdio/putwc.c b/lib/libc/stdio/putwc.c
index 8fe065b..56acf50 100644
--- a/lib/libc/stdio/putwc.c
+++ b/lib/libc/stdio/putwc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 #undef putwc
 
@@ -41,8 +47,13 @@ __FBSDID("$FreeBSD$");
  * macro, may evaluate `fp' more than once.
  */
 wint_t
+putwc_l(wchar_t wc, FILE *fp, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	return (fputwc_l(wc, fp, locale));
+}
+wint_t
 putwc(wchar_t wc, FILE *fp)
 {
-
-	return (fputwc(wc, fp));
+	return putwc_l(wc, fp, __get_locale());
 }
diff --git a/lib/libc/stdio/putwchar.c b/lib/libc/stdio/putwchar.c
index 5503071..8673bd5 100644
--- a/lib/libc/stdio/putwchar.c
+++ b/lib/libc/stdio/putwchar.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 #undef putwchar
 
@@ -40,8 +46,13 @@ __FBSDID("$FreeBSD$");
  * Synonym for fputwc(wc, stdout).
  */
 wint_t
+putwchar_l(wchar_t wc, locale_t locale)
+{
+	FIX_LOCALE(locale);
+	return (fputwc_l(wc, stdout, locale));
+}
+wint_t
 putwchar(wchar_t wc)
 {
-
-	return (fputwc(wc, stdout));
+	return putwchar_l(wc, __get_locale());
 }
diff --git a/lib/libc/stdio/scanf.c b/lib/libc/stdio/scanf.c
index 4ae88da..ba33caa 100644
--- a/lib/libc/stdio/scanf.c
+++ b/lib/libc/stdio/scanf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -42,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 int
 scanf(char const * __restrict fmt, ...)
@@ -51,7 +57,21 @@ scanf(char const * __restrict fmt, ...)
 
 	va_start(ap, fmt);
 	FLOCKFILE(stdin);
-	ret = __svfscanf(stdin, fmt, ap);
+	ret = __svfscanf(stdin, __get_locale(), fmt, ap);
+	FUNLOCKFILE(stdin);
+	va_end(ap);
+	return (ret);
+}
+int
+scanf_l(locale_t locale, char const * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+	FIX_LOCALE(locale);
+
+	va_start(ap, fmt);
+	FLOCKFILE(stdin);
+	ret = __svfscanf(stdin, locale, fmt, ap);
 	FUNLOCKFILE(stdin);
 	va_end(ap);
 	return (ret);
diff --git a/lib/libc/stdio/snprintf.c b/lib/libc/stdio/snprintf.c
index e6d7115..74af42d 100644
--- a/lib/libc/stdio/snprintf.c
+++ b/lib/libc/stdio/snprintf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
@@ -39,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <limits.h>
 #include <stdio.h>
 #include <stdarg.h>
+#include "xlocale_private.h"
 
 #include "local.h"
 
@@ -59,7 +65,32 @@ snprintf(char * __restrict str, size_t n, char const * __restrict fmt, ...)
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;
 	f._bf._size = f._w = n;
-	ret = __vfprintf(&f, fmt, ap);
+	ret = __vfprintf(&f, __get_locale(), fmt, ap);
+	if (on > 0)
+		*f._p = '\0';
+	va_end(ap);
+	return (ret);
+}
+int
+snprintf_l(char * __restrict str, size_t n, locale_t locale,
+		char const * __restrict fmt, ...)
+{
+	size_t on;
+	int ret;
+	va_list ap;
+	FILE f = FAKE_FILE;
+	FIX_LOCALE(locale);
+
+	on = n;
+	if (n != 0)
+		n--;
+	if (n > INT_MAX)
+		n = INT_MAX;
+	va_start(ap, fmt);
+	f._flags = __SWR | __SSTR;
+	f._bf._base = f._p = (unsigned char *)str;
+	f._bf._size = f._w = n;
+	ret = __vfprintf(&f, locale, fmt, ap);
 	if (on > 0)
 		*f._p = '\0';
 	va_end(ap);
diff --git a/lib/libc/stdio/sprintf.c b/lib/libc/stdio/sprintf.c
index b55bd6c..6e20e04 100644
--- a/lib/libc/stdio/sprintf.c
+++ b/lib/libc/stdio/sprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -40,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <limits.h>
 #include "local.h"
+#include "xlocale_private.h"
 
 int
 sprintf(char * __restrict str, char const * __restrict fmt, ...)
@@ -52,3 +58,16 @@ sprintf(char * __restrict str, char const * __restrict fmt, ...)
 	va_end(ap);
 	return (ret);
 }
+int
+sprintf_l(char * __restrict str, locale_t locale, char const * __restrict fmt,
+		...)
+{
+	int ret;
+	va_list ap;
+	FIX_LOCALE(locale);
+
+	va_start(ap, fmt);
+	ret = vsprintf_l(str, locale, fmt, ap);
+	va_end(ap);
+	return (ret);
+}
diff --git a/lib/libc/stdio/sscanf.c b/lib/libc/stdio/sscanf.c
index c793b86..1c339eb 100644
--- a/lib/libc/stdio/sscanf.c
+++ b/lib/libc/stdio/sscanf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <string.h>
 #include <stdarg.h>
+#include <xlocale.h>
 #include "local.h"
 
 int
@@ -52,3 +58,15 @@ sscanf(const char * __restrict str, char const * __restrict fmt, ...)
 	va_end(ap);
 	return (ret);
 }
+int
+sscanf_l(const char * __restrict str, locale_t locale,
+		char const * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vsscanf_l(str, locale, fmt, ap);
+	va_end(ap);
+	return (ret);
+}
diff --git a/lib/libc/stdio/swprintf.c b/lib/libc/stdio/swprintf.c
index d665318..1f6ecc6 100644
--- a/lib/libc/stdio/swprintf.c
+++ b/lib/libc/stdio/swprintf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 swprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt, ...)
@@ -43,3 +49,16 @@ swprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt, ...)
 
 	return (ret);
 }
+int
+swprintf_l(wchar_t * __restrict s, size_t n, locale_t locale,
+		const wchar_t * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vswprintf_l(s, n, locale, fmt, ap);
+	va_end(ap);
+
+	return (ret);
+}
diff --git a/lib/libc/stdio/swscanf.c b/lib/libc/stdio/swscanf.c
index 728a3d6..c305ac1 100644
--- a/lib/libc/stdio/swscanf.c
+++ b/lib/libc/stdio/swscanf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 swscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt, ...)
@@ -43,3 +49,16 @@ swscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt, ...)
 
 	return (r);
 }
+int
+swscanf_l(const wchar_t * __restrict str, locale_t locale,
+		const wchar_t * __restrict fmt, ...)
+{
+	va_list ap;
+	int r;
+
+	va_start(ap, fmt);
+	r = vswscanf_l(str, locale, fmt, ap);
+	va_end(ap);
+
+	return (r);
+}
diff --git a/lib/libc/stdio/ungetwc.c b/lib/libc/stdio/ungetwc.c
index 10dbbbf..78bc38d 100644
--- a/lib/libc/stdio/ungetwc.c
+++ b/lib/libc/stdio/ungetwc.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002-2004 Tim J. Robbins.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,19 +42,21 @@ __FBSDID("$FreeBSD$");
 #include "libc_private.h"
 #include "local.h"
 #include "mblocal.h"
+#include "xlocale_private.h"
 
 /*
  * Non-MT-safe version.
  */
 wint_t
-__ungetwc(wint_t wc, FILE *fp)
+__ungetwc(wint_t wc, FILE *fp, locale_t locale)
 {
 	char buf[MB_LEN_MAX];
 	size_t len;
+	struct xlocale_ctype *l = XLOCALE_CTYPE(locale);
 
 	if (wc == WEOF)
 		return (WEOF);
-	if ((len = __wcrtomb(buf, wc, &fp->_mbstate)) == (size_t)-1) {
+	if ((len = l->__wcrtomb(buf, wc, &fp->_mbstate)) == (size_t)-1) {
 		fp->_flags |= __SERR;
 		return (WEOF);
 	}
@@ -64,14 +71,20 @@ __ungetwc(wint_t wc, FILE *fp)
  * MT-safe version.
  */
 wint_t
-ungetwc(wint_t wc, FILE *fp)
+ungetwc_l(wint_t wc, FILE *fp, locale_t locale)
 {
 	wint_t r;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
-	r = __ungetwc(wc, fp);
+	r = __ungetwc(wc, fp, locale);
 	FUNLOCKFILE(fp);
 
 	return (r);
 }
+wint_t
+ungetwc(wint_t wc, FILE *fp)
+{
+	return ungetwc_l(wc, fp, __get_locale());
+}
diff --git a/lib/libc/stdio/vasprintf.c b/lib/libc/stdio/vasprintf.c
index a1b600a..8feb23d 100644
--- a/lib/libc/stdio/vasprintf.c
+++ b/lib/libc/stdio/vasprintf.c
@@ -4,6 +4,11 @@
  * Copyright (c) 1997 Todd C. Miller <Todd.Miller@courtesan.com>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,13 +38,15 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
+#include "xlocale_private.h"
 #include "local.h"
 
 int
-vasprintf(char **str, const char *fmt, __va_list ap)
+vasprintf_l(char **str, locale_t locale, const char *fmt, __va_list ap)
 {
 	FILE f = FAKE_FILE;
 	int ret;
+	FIX_LOCALE(locale);
 
 	f._flags = __SWR | __SSTR | __SALC;
 	f._bf._base = f._p = malloc(128);
@@ -49,7 +56,7 @@ vasprintf(char **str, const char *fmt, __va_list ap)
 		return (-1);
 	}
 	f._bf._size = f._w = 127;		/* Leave room for the NUL */
-	ret = __vfprintf(&f, fmt, ap);
+	ret = __vfprintf(&f, locale, fmt, ap);
 	if (ret < 0) {
 		free(f._bf._base);
 		*str = NULL;
@@ -60,3 +67,8 @@ vasprintf(char **str, const char *fmt, __va_list ap)
 	*str = (char *)f._bf._base;
 	return (ret);
 }
+int
+vasprintf(char **str, const char *fmt, __va_list ap)
+{
+	return vasprintf_l(str, __get_locale(), fmt, ap);
+}
diff --git a/lib/libc/stdio/vdprintf.c b/lib/libc/stdio/vdprintf.c
index 3ad273e..454b0f9 100644
--- a/lib/libc/stdio/vdprintf.c
+++ b/lib/libc/stdio/vdprintf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2009 David Schultz <das@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 
 #include "local.h"
+#include "xlocale_private.h"
 
 int
 vdprintf(int fd, const char * __restrict fmt, va_list ap)
@@ -57,7 +63,7 @@ vdprintf(int fd, const char * __restrict fmt, va_list ap)
 	f._bf._base = buf;
 	f._bf._size = sizeof(buf);
 
-	if ((ret = __vfprintf(&f, fmt, ap)) < 0)
+	if ((ret = __vfprintf(&f, __get_locale(), fmt, ap)) < 0)
 		return (ret);
 
 	return (__fflush(&f) ? EOF : ret);
diff --git a/lib/libc/stdio/vfprintf.c b/lib/libc/stdio/vfprintf.c
index 17ad824..bfbdc13 100644
--- a/lib/libc/stdio/vfprintf.c
+++ b/lib/libc/stdio/vfprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -57,6 +62,7 @@ __FBSDID("$FreeBSD$");
 #include <printf.h>
 
 #include <stdarg.h>
+#include "xlocale_private.h"
 #include "un-namespace.h"
 
 #include "libc_private.h"
@@ -64,8 +70,8 @@ __FBSDID("$FreeBSD$");
 #include "fvwrite.h"
 #include "printflocal.h"
 
-static int	__sprint(FILE *, struct __suio *);
-static int	__sbprintf(FILE *, const char *, va_list) __printflike(2, 0)
+static int	__sprint(FILE *, struct __suio *, locale_t);
+static int	__sbprintf(FILE *, locale_t, const char *, va_list) __printflike(3, 0)
 	__noinline;
 static char	*__wcsconv(wchar_t *, int);
 
@@ -87,11 +93,11 @@ struct grouping_state {
  * of bytes that will be needed.
  */
 static int
-grouping_init(struct grouping_state *gs, int ndigits)
+grouping_init(struct grouping_state *gs, int ndigits, locale_t loc)
 {
 	struct lconv *locale;
 
-	locale = localeconv();
+	locale = localeconv_l(loc);
 	gs->grouping = locale->grouping;
 	gs->thousands_sep = locale->thousands_sep;
 	gs->thousep_len = strlen(gs->thousands_sep);
@@ -116,11 +122,11 @@ grouping_init(struct grouping_state *gs, int ndigits)
  */
 static int
 grouping_print(struct grouping_state *gs, struct io_state *iop,
-	       const CHAR *cp, const CHAR *ep)
+	       const CHAR *cp, const CHAR *ep, locale_t locale)
 {
 	const CHAR *cp0 = cp;
 
-	if (io_printandpad(iop, cp, ep, gs->lead, zeroes))
+	if (io_printandpad(iop, cp, ep, gs->lead, zeroes, locale))
 		return (-1);
 	cp += gs->lead;
 	while (gs->nseps > 0 || gs->nrepeats > 0) {
@@ -130,9 +136,9 @@ grouping_print(struct grouping_state *gs, struct io_state *iop,
 			gs->grouping--;
 			gs->nseps--;
 		}
-		if (io_print(iop, gs->thousands_sep, gs->thousep_len))
+		if (io_print(iop, gs->thousands_sep, gs->thousep_len, locale))
 			return (-1);
-		if (io_printandpad(iop, cp, ep, *gs->grouping, zeroes))
+		if (io_printandpad(iop, cp, ep, *gs->grouping, zeroes, locale))
 			return (-1);
 		cp += *gs->grouping;
 	}
@@ -146,7 +152,7 @@ grouping_print(struct grouping_state *gs, struct io_state *iop,
  * then reset it so that it can be reused.
  */
 static int
-__sprint(FILE *fp, struct __suio *uio)
+__sprint(FILE *fp, struct __suio *uio, locale_t locale)
 {
 	int err;
 
@@ -166,7 +172,7 @@ __sprint(FILE *fp, struct __suio *uio)
  * worries about ungetc buffers and so forth.
  */
 static int
-__sbprintf(FILE *fp, const char *fmt, va_list ap)
+__sbprintf(FILE *fp, locale_t locale, const char *fmt, va_list ap)
 {
 	int ret;
 	FILE fake = FAKE_FILE;
@@ -190,7 +196,7 @@ __sbprintf(FILE *fp, const char *fmt, va_list ap)
 	fake._lbfsize = 0;	/* not actually used, but Just In Case */
 
 	/* do the work, then copy any error status */
-	ret = __vfprintf(&fake, fmt, ap);
+	ret = __vfprintf(&fake, locale, fmt, ap);
 	if (ret >= 0 && __fflush(&fake))
 		ret = EOF;
 	if (fake._flags & __SERR)
@@ -261,21 +267,27 @@ __wcsconv(wchar_t *wcsarg, int prec)
  * MT-safe version
  */
 int
-vfprintf(FILE * __restrict fp, const char * __restrict fmt0, va_list ap)
-
+vfprintf_l(FILE * __restrict fp, locale_t locale, const char * __restrict fmt0,
+		va_list ap)
 {
 	int ret;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(fp);
 	/* optimise fprintf(stderr) (and other unbuffered Unix files) */
 	if ((fp->_flags & (__SNBF|__SWR|__SRW)) == (__SNBF|__SWR) &&
 	    fp->_file >= 0)
-		ret = __sbprintf(fp, fmt0, ap);
+		ret = __sbprintf(fp, locale, fmt0, ap);
 	else
-		ret = __vfprintf(fp, fmt0, ap);
+		ret = __vfprintf(fp, locale, fmt0, ap);
 	FUNLOCKFILE(fp);
 	return (ret);
 }
+int
+vfprintf(FILE * __restrict fp, const char * __restrict fmt0, va_list ap)
+{
+	return vfprintf_l(fp, __get_locale(), fmt0, ap);
+}
 
 /*
  * The size of the buffer we use as scratch space for integer
@@ -292,7 +304,7 @@ vfprintf(FILE * __restrict fp, const char * __restrict fmt0, va_list ap)
  * Non-MT-safe version
  */
 int
-__vfprintf(FILE *fp, const char *fmt0, va_list ap)
+__vfprintf(FILE *fp, locale_t locale, const char *fmt0, va_list ap)
 {
 	char *fmt;		/* format string */
 	int ch;			/* character from fmt */
@@ -357,19 +369,19 @@ __vfprintf(FILE *fp, const char *fmt0, va_list ap)
 
 	/* BEWARE, these `goto error' on error. */
 #define	PRINT(ptr, len) { \
-	if (io_print(&io, (ptr), (len)))	\
+	if (io_print(&io, (ptr), (len), locale))	\
 		goto error; \
 }
 #define	PAD(howmany, with) { \
-	if (io_pad(&io, (howmany), (with))) \
+	if (io_pad(&io, (howmany), (with), locale)) \
 		goto error; \
 }
 #define	PRINTANDPAD(p, ep, len, with) {	\
-	if (io_printandpad(&io, (p), (ep), (len), (with))) \
+	if (io_printandpad(&io, (p), (ep), (len), (with), locale)) \
 		goto error; \
 }
 #define	FLUSH() { \
-	if (io_flush(&io)) \
+	if (io_flush(&io, locale)) \
 		goto error; \
 }
 
@@ -454,7 +466,7 @@ __vfprintf(FILE *fp, const char *fmt0, va_list ap)
 	ret = 0;
 #ifndef NO_FLOATING_POINT
 	dtoaresult = NULL;
-	decimal_point = localeconv()->decimal_point;
+	decimal_point = localeconv_l(locale)->decimal_point;
 	/* The overwhelmingly common case is decpt_len == 1. */
 	decpt_len = (decimal_point[1] == '\0' ? 1 : strlen(decimal_point));
 #endif
@@ -750,7 +762,7 @@ fp_common:
 				if (prec || flags & ALT)
 					size += prec + decpt_len;
 				if ((flags & GROUPING) && expt > 0)
-					size += grouping_init(&gs, expt);
+					size += grouping_init(&gs, expt, locale);
 			}
 			break;
 #endif /* !NO_FLOATING_POINT */
@@ -887,7 +899,7 @@ number:			if ((dprec = prec) >= 0)
 			if (size > BUF)	/* should never happen */
 				abort();
 			if ((flags & GROUPING) && size != 0)
-				size += grouping_init(&gs, size);
+				size += grouping_init(&gs, size, locale);
 			break;
 		default:	/* "%?" prints ?, unless ? is NUL */
 			if (ch == '\0')
@@ -950,7 +962,7 @@ number:			if ((dprec = prec) >= 0)
 			/* leading zeroes from decimal precision */
 			PAD(dprec - size, zeroes);
 			if (gs.grouping) {
-				if (grouping_print(&gs, &io, cp, buf+BUF) < 0)
+				if (grouping_print(&gs, &io, cp, buf+BUF, locale) < 0)
 					goto error;
 			} else {
 				PRINT(cp, size);
@@ -968,7 +980,7 @@ number:			if ((dprec = prec) >= 0)
 				} else {
 					if (gs.grouping) {
 						n = grouping_print(&gs, &io,
-						    cp, dtoaend);
+						    cp, dtoaend, locale);
 						if (n < 0)
 							goto error;
 						cp += n;
diff --git a/lib/libc/stdio/vfscanf.c b/lib/libc/stdio/vfscanf.c
index bce4481..0e78949 100644
--- a/lib/libc/stdio/vfscanf.c
+++ b/lib/libc/stdio/vfscanf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
@@ -51,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include "collate.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 #ifndef NO_FLOATING_POINT
 #include <locale.h>
@@ -95,7 +101,7 @@ __FBSDID("$FreeBSD$");
 
 static const u_char *__sccl(char *, const u_char *);
 #ifndef NO_FLOATING_POINT
-static int parsefloat(FILE *, char *, char *);
+static int parsefloat(FILE *, char *, char *, locale_t);
 #endif
 
 __weak_reference(__vfscanf, vfscanf);
@@ -109,7 +115,18 @@ __vfscanf(FILE *fp, char const *fmt0, va_list ap)
 	int ret;
 
 	FLOCKFILE(fp);
-	ret = __svfscanf(fp, fmt0, ap);
+	ret = __svfscanf(fp, __get_locale(), fmt0, ap);
+	FUNLOCKFILE(fp);
+	return (ret);
+}
+int
+vfscanf_l(FILE *fp, locale_t locale, char const *fmt0, va_list ap)
+{
+	int ret;
+	FIX_LOCALE(locale);
+
+	FLOCKFILE(fp);
+	ret = __svfscanf(fp, locale, fmt0, ap);
 	FUNLOCKFILE(fp);
 	return (ret);
 }
@@ -118,7 +135,7 @@ __vfscanf(FILE *fp, char const *fmt0, va_list ap)
  * __svfscanf - non-MT-safe version of __vfscanf
  */
 int
-__svfscanf(FILE *fp, const char *fmt0, va_list ap)
+__svfscanf(FILE *fp, locale_t locale, const char *fmt0, va_list ap)
 {
 	const u_char *fmt = (const u_char *)fmt0;
 	int c;			/* character from format, or conversion */
@@ -733,9 +750,9 @@ literal:
 
 				*p = 0;
 				if ((flags & UNSIGNED) == 0)
-				    res = strtoimax(buf, (char **)NULL, base);
+				    res = strtoimax_l(buf, (char **)NULL, base, locale);
 				else
-				    res = strtoumax(buf, (char **)NULL, base);
+				    res = strtoumax_l(buf, (char **)NULL, base, locale);
 				if (flags & POINTER)
 					*va_arg(ap, void **) =
 							(void *)(uintptr_t)res;
@@ -766,17 +783,17 @@ literal:
 			/* scan a floating point number as if by strtod */
 			if (width == 0 || width > sizeof(buf) - 1)
 				width = sizeof(buf) - 1;
-			if ((width = parsefloat(fp, buf, buf + width)) == 0)
+			if ((width = parsefloat(fp, buf, buf + width, locale)) == 0)
 				goto match_failure;
 			if ((flags & SUPPRESS) == 0) {
 				if (flags & LONGDBL) {
-					long double res = strtold(buf, &p);
+					long double res = strtold_l(buf, &p, locale);
 					*va_arg(ap, long double *) = res;
 				} else if (flags & LONG) {
-					double res = strtod(buf, &p);
+					double res = strtod_l(buf, &p, locale);
 					*va_arg(ap, double *) = res;
 				} else {
-					float res = strtof(buf, &p);
+					float res = strtof_l(buf, &p, locale);
 					*va_arg(ap, float *) = res;
 				}
 				nassigned++;
@@ -805,6 +822,8 @@ __sccl(tab, fmt)
 	const u_char *fmt;
 {
 	int c, n, v, i;
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)__get_locale()->components[XLC_COLLATE];
 
 	/* first `clear' the whole table */
 	c = *fmt++;		/* first char hat => negated scanset */
@@ -858,8 +877,8 @@ doswitch:
 			 */
 			n = *fmt;
 			if (n == ']'
-			    || (__collate_load_error ? n < c :
-				__collate_range_cmp (n, c) < 0
+			    || (table->__collate_load_error ? n < c :
+				__collate_range_cmp (table, n, c) < 0
 			       )
 			   ) {
 				c = '-';
@@ -867,14 +886,14 @@ doswitch:
 			}
 			fmt++;
 			/* fill in the range */
-			if (__collate_load_error) {
+			if (table->__collate_load_error) {
 				do {
 					tab[++c] = v;
 				} while (c < n);
 			} else {
 				for (i = 0; i < 256; i ++)
-					if (   __collate_range_cmp (c, i) < 0
-					    && __collate_range_cmp (i, n) <= 0
+					if (   __collate_range_cmp (table, c, i) < 0
+					    && __collate_range_cmp (table, i, n) <= 0
 					   )
 						tab[i] = v;
 			}
@@ -908,7 +927,7 @@ doswitch:
 
 #ifndef NO_FLOATING_POINT
 static int
-parsefloat(FILE *fp, char *buf, char *end)
+parsefloat(FILE *fp, char *buf, char *end, locale_t locale)
 {
 	char *commit, *p;
 	int infnanpos = 0, decptpos = 0;
@@ -917,7 +936,7 @@ parsefloat(FILE *fp, char *buf, char *end)
 		S_DIGITS, S_DECPT, S_FRAC, S_EXP, S_EXPDIGITS
 	} state = S_START;
 	unsigned char c;
-	const char *decpt = localeconv()->decimal_point;
+	const char *decpt = localeconv_l(locale)->decimal_point;
 	_Bool gotmantdig = 0, ishex = 0;
 
 	/*
diff --git a/lib/libc/stdio/vfwprintf.c b/lib/libc/stdio/vfwprintf.c
index d34f559..73e37c9 100644
--- a/lib/libc/stdio/vfwprintf.c
+++ b/lib/libc/stdio/vfwprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -65,10 +70,11 @@ __FBSDID("$FreeBSD$");
 #include "local.h"
 #include "fvwrite.h"
 #include "printflocal.h"
+#include "xlocale_private.h"
 
-static int	__sprint(FILE *, struct __suio *);
-static int	__sbprintf(FILE *, const wchar_t *, va_list) __noinline;
-static wint_t	__xfputwc(wchar_t, FILE *);
+static int	__sprint(FILE *, struct __suio *, locale_t);
+static int	__sbprintf(FILE *, locale_t, const wchar_t *, va_list) __noinline;
+static wint_t	__xfputwc(wchar_t, FILE *, locale_t);
 static wchar_t	*__mbsconv(char *, int);
 
 #define	CHAR	wchar_t
@@ -85,28 +91,28 @@ struct grouping_state {
 static const mbstate_t initial_mbs;
 
 static inline wchar_t
-get_decpt(void)
+get_decpt(locale_t locale)
 {
 	mbstate_t mbs;
 	wchar_t decpt;
 	int nconv;
 
 	mbs = initial_mbs;
-	nconv = mbrtowc(&decpt, localeconv()->decimal_point, MB_CUR_MAX, &mbs);
+	nconv = mbrtowc(&decpt, localeconv_l(locale)->decimal_point, MB_CUR_MAX, &mbs);
 	if (nconv == (size_t)-1 || nconv == (size_t)-2)
 		decpt = '.';    /* failsafe */
 	return (decpt);
 }
 
 static inline wchar_t
-get_thousep(void)
+get_thousep(locale_t locale)
 {
 	mbstate_t mbs;
 	wchar_t thousep;
 	int nconv;
 
 	mbs = initial_mbs;
-	nconv = mbrtowc(&thousep, localeconv()->thousands_sep,
+	nconv = mbrtowc(&thousep, localeconv_l(locale)->thousands_sep,
 	    MB_CUR_MAX, &mbs);
 	if (nconv == (size_t)-1 || nconv == (size_t)-2)
 		thousep = '\0';    /* failsafe */
@@ -119,11 +125,11 @@ get_thousep(void)
  * of wide characters that will be printed.
  */
 static int
-grouping_init(struct grouping_state *gs, int ndigits)
+grouping_init(struct grouping_state *gs, int ndigits, locale_t locale)
 {
 
-	gs->grouping = localeconv()->grouping;
-	gs->thousands_sep = get_thousep();
+	gs->grouping = localeconv_l(locale)->grouping;
+	gs->thousands_sep = get_thousep(locale);
 
 	gs->nseps = gs->nrepeats = 0;
 	gs->lead = ndigits;
@@ -145,11 +151,11 @@ grouping_init(struct grouping_state *gs, int ndigits)
  */
 static int
 grouping_print(struct grouping_state *gs, struct io_state *iop,
-	       const CHAR *cp, const CHAR *ep)
+	       const CHAR *cp, const CHAR *ep, locale_t locale)
 {
 	const CHAR *cp0 = cp;
 
-	if (io_printandpad(iop, cp, ep, gs->lead, zeroes))
+	if (io_printandpad(iop, cp, ep, gs->lead, zeroes, locale))
 		return (-1);
 	cp += gs->lead;
 	while (gs->nseps > 0 || gs->nrepeats > 0) {
@@ -159,9 +165,9 @@ grouping_print(struct grouping_state *gs, struct io_state *iop,
 			gs->grouping--;
 			gs->nseps--;
 		}
-		if (io_print(iop, &gs->thousands_sep, 1))
+		if (io_print(iop, &gs->thousands_sep, 1, locale))
 			return (-1);
-		if (io_printandpad(iop, cp, ep, *gs->grouping, zeroes))
+		if (io_printandpad(iop, cp, ep, *gs->grouping, zeroes, locale))
 			return (-1);
 		cp += *gs->grouping;
 	}
@@ -180,7 +186,7 @@ grouping_print(struct grouping_state *gs, struct io_state *iop,
  * string eclipses the benefits of buffering.
  */
 static int
-__sprint(FILE *fp, struct __suio *uio)
+__sprint(FILE *fp, struct __suio *uio, locale_t locale)
 {
 	struct __siov *iov;
 	wchar_t *p;
@@ -191,7 +197,7 @@ __sprint(FILE *fp, struct __suio *uio)
 		p = (wchar_t *)iov->iov_base;
 		len = iov->iov_len;
 		for (i = 0; i < len; i++) {
-			if (__xfputwc(p[i], fp) == WEOF)
+			if (__xfputwc(p[i], fp, locale) == WEOF)
 				return (-1);
 		}
 	}
@@ -205,7 +211,7 @@ __sprint(FILE *fp, struct __suio *uio)
  * worries about ungetc buffers and so forth.
  */
 static int
-__sbprintf(FILE *fp, const wchar_t *fmt, va_list ap)
+__sbprintf(FILE *fp, locale_t locale, const wchar_t *fmt, va_list ap)
 {
 	int ret;
 	FILE fake;
@@ -229,7 +235,7 @@ __sbprintf(FILE *fp, const wchar_t *fmt, va_list ap)
 	fake._lbfsize = 0;	/* not actually used, but Just In Case */
 
 	/* do the work, then copy any error status */
-	ret = __vfwprintf(&fake, fmt, ap);
+	ret = __vfwprintf(&fake, locale, fmt, ap);
 	if (ret >= 0 && __fflush(&fake))
 		ret = WEOF;
 	if (fake._flags & __SERR)
@@ -242,7 +248,7 @@ __sbprintf(FILE *fp, const wchar_t *fmt, va_list ap)
  * File must already be locked.
  */
 static wint_t
-__xfputwc(wchar_t wc, FILE *fp)
+__xfputwc(wchar_t wc, FILE *fp, locale_t locale)
 {
 	mbstate_t mbs;
 	char buf[MB_LEN_MAX];
@@ -251,7 +257,7 @@ __xfputwc(wchar_t wc, FILE *fp)
 	size_t len;
 
 	if ((fp->_flags & __SSTR) == 0)
-		return (__fputwc(wc, fp));
+		return (__fputwc(wc, fp, locale));
 
 	mbs = initial_mbs;
 	if ((len = wcrtomb(buf, wc, &mbs)) == (size_t)-1) {
@@ -343,21 +349,27 @@ __mbsconv(char *mbsarg, int prec)
  * MT-safe version
  */
 int
-vfwprintf(FILE * __restrict fp, const wchar_t * __restrict fmt0, va_list ap)
+vfwprintf_l(FILE * __restrict fp, locale_t locale,
+		const wchar_t * __restrict fmt0, va_list ap)
 
 {
 	int ret;
-
+	FIX_LOCALE(locale);
 	FLOCKFILE(fp);
 	/* optimise fprintf(stderr) (and other unbuffered Unix files) */
 	if ((fp->_flags & (__SNBF|__SWR|__SRW)) == (__SNBF|__SWR) &&
 	    fp->_file >= 0)
-		ret = __sbprintf(fp, fmt0, ap);
+		ret = __sbprintf(fp, locale, fmt0, ap);
 	else
-		ret = __vfwprintf(fp, fmt0, ap);
+		ret = __vfwprintf(fp, locale, fmt0, ap);
 	FUNLOCKFILE(fp);
 	return (ret);
 }
+int
+vfwprintf(FILE * __restrict fp, const wchar_t * __restrict fmt0, va_list ap)
+{
+	return vfwprintf_l(fp, __get_locale(), fmt0, ap);
+}
 
 /*
  * The size of the buffer we use as scratch space for integer
@@ -374,7 +386,7 @@ vfwprintf(FILE * __restrict fp, const wchar_t * __restrict fmt0, va_list ap)
  * Non-MT-safe version
  */
 int
-__vfwprintf(FILE *fp, const wchar_t *fmt0, va_list ap)
+__vfwprintf(FILE *fp, locale_t locale, const wchar_t *fmt0, va_list ap)
 {
 	wchar_t *fmt;		/* format string */
 	wchar_t ch;		/* character from fmt */
@@ -437,19 +449,19 @@ __vfwprintf(FILE *fp, const wchar_t *fmt0, va_list ap)
 
 	/* BEWARE, these `goto error' on error. */
 #define	PRINT(ptr, len)	do {			\
-	if (io_print(&io, (ptr), (len)))	\
+	if (io_print(&io, (ptr), (len), locale))	\
 		goto error; \
 } while (0)
 #define	PAD(howmany, with) { \
-	if (io_pad(&io, (howmany), (with))) \
+	if (io_pad(&io, (howmany), (with), locale)) \
 		goto error; \
 }
 #define	PRINTANDPAD(p, ep, len, with) {	\
-	if (io_printandpad(&io, (p), (ep), (len), (with))) \
+	if (io_printandpad(&io, (p), (ep), (len), (with), locale)) \
 		goto error; \
 }
 #define	FLUSH() { \
-	if (io_flush(&io)) \
+	if (io_flush(&io, locale)) \
 		goto error; \
 }
 
@@ -529,7 +541,7 @@ __vfwprintf(FILE *fp, const wchar_t *fmt0, va_list ap)
 	io_init(&io, fp);
 	ret = 0;
 #ifndef NO_FLOATING_POINT
-	decimal_point = get_decpt();
+	decimal_point = get_decpt(locale);
 #endif
 
 	/*
@@ -816,7 +828,7 @@ fp_common:
 				if (prec || flags & ALT)
 					size += prec + 1;
 				if ((flags & GROUPING) && expt > 0)
-					size += grouping_init(&gs, expt);
+					size += grouping_init(&gs, expt, locale);
 			}
 			break;
 #endif /* !NO_FLOATING_POINT */
@@ -955,7 +967,7 @@ number:			if ((dprec = prec) >= 0)
 			if (size > BUF)	/* should never happen */
 				abort();
 			if ((flags & GROUPING) && size != 0)
-				size += grouping_init(&gs, size);
+				size += grouping_init(&gs, size, locale);
 			break;
 		default:	/* "%?" prints ?, unless ? is NUL */
 			if (ch == '\0')
@@ -1018,7 +1030,7 @@ number:			if ((dprec = prec) >= 0)
 			/* leading zeroes from decimal precision */
 			PAD(dprec - size, zeroes);
 			if (gs.grouping) {
-				if (grouping_print(&gs, &io, cp, buf+BUF) < 0)
+				if (grouping_print(&gs, &io, cp, buf+BUF, locale) < 0)
 					goto error;
 			} else {
 				PRINT(cp, size);
@@ -1036,7 +1048,7 @@ number:			if ((dprec = prec) >= 0)
 				} else {
 					if (gs.grouping) {
 						n = grouping_print(&gs, &io,
-						    cp, convbuf + ndig);
+						    cp, convbuf + ndig, locale);
 						if (n < 0)
 							goto error;
 						cp += n;
diff --git a/lib/libc/stdio/vfwscanf.c b/lib/libc/stdio/vfwscanf.c
index 60c7c71..22b0827 100644
--- a/lib/libc/stdio/vfwscanf.c
+++ b/lib/libc/stdio/vfwscanf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -53,10 +58,7 @@ __FBSDID("$FreeBSD$");
 
 #include "libc_private.h"
 #include "local.h"
-
-#ifndef NO_FLOATING_POINT
-#include <locale.h>
-#endif
+#include "xlocale_private.h"
 
 #define	BUF		513	/* Maximum length of numeric string. */
 
@@ -96,7 +98,7 @@ __FBSDID("$FreeBSD$");
 #define	CT_FLOAT	4	/* %[efgEFG] conversion */
 
 #ifndef NO_FLOATING_POINT
-static int parsefloat(FILE *, wchar_t *, wchar_t *);
+static int parsefloat(FILE *, wchar_t *, wchar_t *, locale_t);
 #endif
 
 #define	INCCL(_c)	\
@@ -109,22 +111,30 @@ static const mbstate_t initial_mbs;
  * MT-safe version.
  */
 int
-vfwscanf(FILE * __restrict fp, const wchar_t * __restrict fmt, va_list ap)
+vfwscanf_l(FILE * __restrict fp, locale_t locale,
+		const wchar_t * __restrict fmt, va_list ap)
 {
 	int ret;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(fp);
 	ORIENT(fp, 1);
-	ret = __vfwscanf(fp, fmt, ap);
+	ret = __vfwscanf(fp, locale, fmt, ap);
 	FUNLOCKFILE(fp);
 	return (ret);
 }
+int
+vfwscanf(FILE * __restrict fp, const wchar_t * __restrict fmt, va_list ap)
+{
+	return vfwscanf_l(fp, __get_locale(), fmt, ap);
+}
 
 /*
  * Non-MT-safe version.
  */
 int
-__vfwscanf(FILE * __restrict fp, const wchar_t * __restrict fmt, va_list ap)
+__vfwscanf(FILE * __restrict fp, locale_t locale,
+		const wchar_t * __restrict fmt, va_list ap)
 {
 	wint_t c;		/* character from format, or conversion */
 	size_t width;		/* field width, or 0 */
@@ -159,11 +169,11 @@ __vfwscanf(FILE * __restrict fp, const wchar_t * __restrict fmt, va_list ap)
 		if (c == 0)
 			return (nassigned);
 		if (iswspace(c)) {
-			while ((c = __fgetwc(fp)) != WEOF &&
-			    iswspace(c))
+			while ((c = __fgetwc(fp, locale)) != WEOF &&
+			    iswspace_l(c, locale))
 				;
 			if (c != WEOF)
-				__ungetwc(c, fp);
+				__ungetwc(c, fp, locale);
 			continue;
 		}
 		if (c != '%')
@@ -178,10 +188,10 @@ again:		c = *fmt++;
 		switch (c) {
 		case '%':
 literal:
-			if ((wi = __fgetwc(fp)) == WEOF)
+			if ((wi = __fgetwc(fp, locale)) == WEOF)
 				goto input_failure;
 			if (wi != c) {
-				__ungetwc(wi, fp);
+				__ungetwc(wi, fp, locale);
 				goto input_failure;
 			}
 			nread++;
@@ -341,11 +351,11 @@ literal:
 		 * that suppress this.
 		 */
 		if ((flags & NOSKIP) == 0) {
-			while ((wi = __fgetwc(fp)) != WEOF && iswspace(wi))
+			while ((wi = __fgetwc(fp, locale)) != WEOF && iswspace(wi))
 				nread++;
 			if (wi == WEOF)
 				goto input_failure;
-			__ungetwc(wi, fp);
+			__ungetwc(wi, fp, locale);
 		}
 
 		/*
@@ -362,7 +372,7 @@ literal:
 					p = va_arg(ap, wchar_t *);
 				n = 0;
 				while (width-- != 0 &&
-				    (wi = __fgetwc(fp)) != WEOF) {
+				    (wi = __fgetwc(fp, locale)) != WEOF) {
 					if (!(flags & SUPPRESS))
 						*p++ = (wchar_t)wi;
 					n++;
@@ -378,7 +388,7 @@ literal:
 				n = 0;
 				mbs = initial_mbs;
 				while (width != 0 &&
-				    (wi = __fgetwc(fp)) != WEOF) {
+				    (wi = __fgetwc(fp, locale)) != WEOF) {
 					if (width >= MB_CUR_MAX &&
 					    !(flags & SUPPRESS)) {
 						nconv = wcrtomb(mbp, wi, &mbs);
@@ -390,7 +400,7 @@ literal:
 						if (nconv == (size_t)-1)
 							goto input_failure;
 						if (nconv > width) {
-							__ungetwc(wi, fp);
+							__ungetwc(wi, fp, locale);
 							break;
 						}
 						if (!(flags & SUPPRESS))
@@ -418,20 +428,20 @@ literal:
 			/* take only those things in the class */
 			if ((flags & SUPPRESS) && (flags & LONG)) {
 				n = 0;
-				while ((wi = __fgetwc(fp)) != WEOF &&
+				while ((wi = __fgetwc(fp, locale)) != WEOF &&
 				    width-- != 0 && INCCL(wi))
 					n++;
 				if (wi != WEOF)
-					__ungetwc(wi, fp);
+					__ungetwc(wi, fp, locale);
 				if (n == 0)
 					goto match_failure;
 			} else if (flags & LONG) {
 				p0 = p = va_arg(ap, wchar_t *);
-				while ((wi = __fgetwc(fp)) != WEOF &&
+				while ((wi = __fgetwc(fp, locale)) != WEOF &&
 				    width-- != 0 && INCCL(wi))
 					*p++ = (wchar_t)wi;
 				if (wi != WEOF)
-					__ungetwc(wi, fp);
+					__ungetwc(wi, fp, locale);
 				n = p - p0;
 				if (n == 0)
 					goto match_failure;
@@ -442,7 +452,7 @@ literal:
 					mbp = va_arg(ap, char *);
 				n = 0;
 				mbs = initial_mbs;
-				while ((wi = __fgetwc(fp)) != WEOF &&
+				while ((wi = __fgetwc(fp, locale)) != WEOF &&
 				    width != 0 && INCCL(wi)) {
 					if (width >= MB_CUR_MAX &&
 					   !(flags & SUPPRESS)) {
@@ -466,7 +476,7 @@ literal:
 					n++;
 				}
 				if (wi != WEOF)
-					__ungetwc(wi, fp);
+					__ungetwc(wi, fp, locale);
 				if (!(flags & SUPPRESS)) {
 					*mbp = 0;
 					nassigned++;
@@ -481,29 +491,29 @@ literal:
 			if (width == 0)
 				width = (size_t)~0;
 			if ((flags & SUPPRESS) && (flags & LONG)) {
-				while ((wi = __fgetwc(fp)) != WEOF &&
+				while ((wi = __fgetwc(fp, locale)) != WEOF &&
 				    width-- != 0 &&
 				    !iswspace(wi))
 					nread++;
 				if (wi != WEOF)
-					__ungetwc(wi, fp);
+					__ungetwc(wi, fp, locale);
 			} else if (flags & LONG) {
 				p0 = p = va_arg(ap, wchar_t *);
-				while ((wi = __fgetwc(fp)) != WEOF &&
+				while ((wi = __fgetwc(fp, locale)) != WEOF &&
 				    width-- != 0 &&
 				    !iswspace(wi)) {
 					*p++ = (wchar_t)wi;
 					nread++;
 				}
 				if (wi != WEOF)
-					__ungetwc(wi, fp);
+					__ungetwc(wi, fp, locale);
 				*p = '\0';
 				nassigned++;
 			} else {
 				if (!(flags & SUPPRESS))
 					mbp = va_arg(ap, char *);
 				mbs = initial_mbs;
-				while ((wi = __fgetwc(fp)) != WEOF &&
+				while ((wi = __fgetwc(fp, locale)) != WEOF &&
 				    width != 0 &&
 				    !iswspace(wi)) {
 					if (width >= MB_CUR_MAX &&
@@ -528,7 +538,7 @@ literal:
 					nread++;
 				}
 				if (wi != WEOF)
-					__ungetwc(wi, fp);
+					__ungetwc(wi, fp, locale);
 				if (!(flags & SUPPRESS)) {
 					*mbp = 0;
 					nassigned++;
@@ -544,7 +554,7 @@ literal:
 				width = sizeof(buf) / sizeof(*buf) - 1;
 			flags |= SIGNOK | NDIGITS | NZDIGITS;
 			for (p = buf; width; width--) {
-				c = __fgetwc(fp);
+				c = __fgetwc(fp, locale);
 				/*
 				 * Switch on the character; `goto ok'
 				 * if we accept it as a part of number.
@@ -628,7 +638,7 @@ literal:
 				 * for a number.  Stop accumulating digits.
 				 */
 				if (c != WEOF)
-					__ungetwc(c, fp);
+					__ungetwc(c, fp, locale);
 				break;
 		ok:
 				/*
@@ -644,13 +654,13 @@ literal:
 			 */
 			if (flags & NDIGITS) {
 				if (p > buf)
-					__ungetwc(*--p, fp);
+					__ungetwc(*--p, fp, locale);
 				goto match_failure;
 			}
 			c = p[-1];
 			if (c == 'x' || c == 'X') {
 				--p;
-				__ungetwc(c, fp);
+				__ungetwc(c, fp, locale);
 			}
 			if ((flags & SUPPRESS) == 0) {
 				uintmax_t res;
@@ -691,7 +701,7 @@ literal:
 			if (width == 0 || width > sizeof(buf) /
 			    sizeof(*buf) - 1)
 				width = sizeof(buf) / sizeof(*buf) - 1;
-			if ((width = parsefloat(fp, buf, buf + width)) == 0)
+			if ((width = parsefloat(fp, buf, buf + width, locale)) == 0)
 				goto match_failure;
 			if ((flags & SUPPRESS) == 0) {
 				if (flags & LONGDBL) {
@@ -720,7 +730,7 @@ match_failure:
 
 #ifndef NO_FLOATING_POINT
 static int
-parsefloat(FILE *fp, wchar_t *buf, wchar_t *end)
+parsefloat(FILE *fp, wchar_t *buf, wchar_t *end, locale_t locale)
 {
 	mbstate_t mbs;
 	size_t nconv;
@@ -751,7 +761,7 @@ parsefloat(FILE *fp, wchar_t *buf, wchar_t *end)
 	commit = buf - 1;
 	c = WEOF;
 	for (p = buf; p < end; ) {
-		if ((c = __fgetwc(fp)) == WEOF)
+		if ((c = __fgetwc(fp, locale)) == WEOF)
 			break;
 reswitch:
 		switch (state) {
@@ -871,9 +881,9 @@ reswitch:
 
 parsedone:
 	if (c != WEOF)
-		__ungetwc(c, fp);
+		__ungetwc(c, fp, locale);
 	while (commit < --p)
-		__ungetwc(*p, fp);
+		__ungetwc(*p, fp, locale);
 	*++commit = '\0';
 	return (commit - buf);
 }
diff --git a/lib/libc/stdio/vprintf.c b/lib/libc/stdio/vprintf.c
index a610b7d..c15ef4d 100644
--- a/lib/libc/stdio/vprintf.c
+++ b/lib/libc/stdio/vprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,10 +42,15 @@ static char sccsid[] = "@(#)vprintf.c	8.1 (Berkeley) 6/4/93";
 __FBSDID("$FreeBSD$");
 
 #include <stdio.h>
+#include <xlocale.h>
 
 int
 vprintf(const char * __restrict fmt, __va_list ap)
 {
-
 	return (vfprintf(stdout, fmt, ap));
 }
+int
+vprintf_l(locale_t locale, const char * __restrict fmt, __va_list ap)
+{
+	return (vfprintf_l(stdout, locale, fmt, ap));
+}
diff --git a/lib/libc/stdio/vscanf.c b/lib/libc/stdio/vscanf.c
index 25bce72..d56bb3b 100644
--- a/lib/libc/stdio/vscanf.c
+++ b/lib/libc/stdio/vscanf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Donn Seeley at UUNET Technologies, Inc.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,16 +46,26 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 #include "libc_private.h"
 #include "local.h"
+#include "xlocale_private.h"
 
 int
-vscanf(fmt, ap)
+vscanf_l(locale, fmt, ap)
+	locale_t locale;
 	const char * __restrict fmt;
 	__va_list ap;
 {
 	int retval;
+	FIX_LOCALE(locale);
 
 	FLOCKFILE(stdin);
-	retval = __svfscanf(stdin, fmt, ap);
+	retval = __svfscanf(stdin, locale, fmt, ap);
 	FUNLOCKFILE(stdin);
 	return (retval);
 }
+int
+vscanf(fmt, ap)
+	const char * __restrict fmt;
+	__va_list ap;
+{
+	return vscanf_l(__get_locale(), fmt, ap);
+}
diff --git a/lib/libc/stdio/vsnprintf.c b/lib/libc/stdio/vsnprintf.c
index 70e4c53..d2bad0f 100644
--- a/lib/libc/stdio/vsnprintf.c
+++ b/lib/libc/stdio/vsnprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,15 +44,17 @@ __FBSDID("$FreeBSD$");
 #include <limits.h>
 #include <stdio.h>
 #include "local.h"
+#include "xlocale_private.h"
 
 int
-vsnprintf(char * __restrict str, size_t n, const char * __restrict fmt,
-    __va_list ap)
+vsnprintf_l(char * __restrict str, size_t n, locale_t locale, 
+		const char * __restrict fmt, __va_list ap)
 {
 	size_t on;
 	int ret;
 	char dummy[2];
 	FILE f = FAKE_FILE;
+	FIX_LOCALE(locale);
 
 	on = n;
 	if (n != 0)
@@ -64,8 +71,14 @@ vsnprintf(char * __restrict str, size_t n, const char * __restrict fmt,
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;
 	f._bf._size = f._w = n;
-	ret = __vfprintf(&f, fmt, ap);
+	ret = __vfprintf(&f, locale, fmt, ap);
 	if (on > 0)
 		*f._p = '\0';
 	return (ret);
 }
+int
+vsnprintf(char * __restrict str, size_t n, const char * __restrict fmt,
+    __va_list ap)
+{
+	return vsnprintf_l(str, n, __get_locale(), fmt, ap);
+}
diff --git a/lib/libc/stdio/vsprintf.c b/lib/libc/stdio/vsprintf.c
index 3890af7..04f2df3 100644
--- a/lib/libc/stdio/vsprintf.c
+++ b/lib/libc/stdio/vsprintf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,17 +44,25 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <limits.h>
 #include "local.h"
+#include "xlocale_private.h"
 
 int
-vsprintf(char * __restrict str, const char * __restrict fmt, __va_list ap)
+vsprintf_l(char * __restrict str, locale_t locale,
+		const char * __restrict fmt, __va_list ap)
 {
 	int ret;
 	FILE f = FAKE_FILE;
+	FIX_LOCALE(locale);
 
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;
 	f._bf._size = f._w = INT_MAX;
-	ret = __vfprintf(&f, fmt, ap);
+	ret = __vfprintf(&f, locale, fmt, ap);
 	*f._p = 0;
 	return (ret);
 }
+int
+vsprintf(char * __restrict str, const char * __restrict fmt, __va_list ap)
+{
+	return vsprintf_l(str, __get_locale(), fmt, ap);
+}
diff --git a/lib/libc/stdio/vsscanf.c b/lib/libc/stdio/vsscanf.c
index 82429c6..5668ab5 100644
--- a/lib/libc/stdio/vsscanf.c
+++ b/lib/libc/stdio/vsscanf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * This code is derived from software contributed to Berkeley by
  * Donn Seeley at UUNET Technologies, Inc.
  *
@@ -39,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <string.h>
 #include "local.h"
+#include "xlocale_private.h"
 
 static int
 eofread(void *, char *, int);
@@ -52,14 +58,21 @@ eofread(void *cookie, char *buf, int len)
 }
 
 int
-vsscanf(const char * __restrict str, const char * __restrict fmt,
-	__va_list ap)
+vsscanf_l(const char * __restrict str, locale_t locale,
+		const char * __restrict fmt, __va_list ap)
 {
 	FILE f = FAKE_FILE;
+	FIX_LOCALE(locale);
 
 	f._flags = __SRD;
 	f._bf._base = f._p = (unsigned char *)str;
 	f._bf._size = f._r = strlen(str);
 	f._read = eofread;
-	return (__svfscanf(&f, fmt, ap));
+	return (__svfscanf(&f, locale, fmt, ap));
+}
+int
+vsscanf(const char * __restrict str, const char * __restrict fmt,
+	__va_list ap)
+{
+	return vsscanf_l(str, __get_locale(), fmt, ap);
 }
diff --git a/lib/libc/stdio/vswprintf.c b/lib/libc/stdio/vswprintf.c
index 2cfe724..023c537 100644
--- a/lib/libc/stdio/vswprintf.c
+++ b/lib/libc/stdio/vswprintf.c
@@ -4,6 +4,11 @@
  * Copyright (c) 1997 Todd C. Miller <Todd.Miller@courtesan.com>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,10 +43,11 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <wchar.h>
 #include "local.h"
+#include "xlocale_private.h"
 
 int
-vswprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt,
-    __va_list ap)
+vswprintf_l(wchar_t * __restrict s, size_t n, locale_t locale,
+		const wchar_t * __restrict fmt, __va_list ap)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
@@ -49,6 +55,7 @@ vswprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt,
 	char *mbp;
 	int ret, sverrno;
 	size_t nwc;
+	FIX_LOCALE(locale);
 
 	if (n == 0) {
 		errno = EINVAL;
@@ -62,7 +69,7 @@ vswprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt,
 		return (-1);
 	}
 	f._bf._size = f._w = 127;		/* Leave room for the NUL */
-	ret = __vfwprintf(&f, fmt, ap);
+	ret = __vfwprintf(&f, locale, fmt, ap);
 	if (ret < 0) {
 		sverrno = errno;
 		free(f._bf._base);
@@ -76,7 +83,7 @@ vswprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt,
 	 * fputwc() did in __vfwprintf().
 	 */
 	mbs = initial;
-	nwc = mbsrtowcs(s, (const char **)&mbp, n, &mbs);
+	nwc = mbsrtowcs_l(s, (const char **)&mbp, n, &mbs, locale);
 	free(f._bf._base);
 	if (nwc == (size_t)-1) {
 		errno = EILSEQ;
@@ -90,3 +97,9 @@ vswprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt,
 
 	return (ret);
 }
+int
+vswprintf(wchar_t * __restrict s, size_t n, const wchar_t * __restrict fmt,
+    __va_list ap)
+{
+	return vswprintf_l(s, n, __get_locale(), fmt, ap);
+}
diff --git a/lib/libc/stdio/vswscanf.c b/lib/libc/stdio/vswscanf.c
index f06fc02..f646e85 100644
--- a/lib/libc/stdio/vswscanf.c
+++ b/lib/libc/stdio/vswscanf.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Donn Seeley at UUNET Technologies, Inc.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -46,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 #include "local.h"
+#include "xlocale_private.h"
 
 static int	eofread(void *, char *, int);
 
@@ -57,8 +63,8 @@ eofread(void *cookie, char *buf, int len)
 }
 
 int
-vswscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt,
-    va_list ap)
+vswscanf_l(const wchar_t * __restrict str, locale_t locale,
+		const wchar_t * __restrict fmt, va_list ap)
 {
 	static const mbstate_t initial;
 	mbstate_t mbs;
@@ -67,6 +73,7 @@ vswscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt,
 	size_t mlen;
 	int r;
 	const wchar_t *strp;
+	FIX_LOCALE(locale);
 
 	/*
 	 * XXX Convert the wide character string to multibyte, which
@@ -76,7 +83,7 @@ vswscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt,
 		return (EOF);
 	mbs = initial;
 	strp = str;
-	if ((mlen = wcsrtombs(mbstr, &strp, SIZE_T_MAX, &mbs)) == (size_t)-1) {
+	if ((mlen = wcsrtombs_l(mbstr, &strp, SIZE_T_MAX, &mbs, locale)) == (size_t)-1) {
 		free(mbstr);
 		return (EOF);
 	}
@@ -84,8 +91,14 @@ vswscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt,
 	f._bf._base = f._p = (unsigned char *)mbstr;
 	f._bf._size = f._r = mlen;
 	f._read = eofread;
-	r = __vfwscanf(&f, fmt, ap);
+	r = __vfwscanf(&f, locale, fmt, ap);
 	free(mbstr);
 
 	return (r);
 }
+int
+vswscanf(const wchar_t * __restrict str, const wchar_t * __restrict fmt,
+    va_list ap)
+{
+	return vswscanf_l(str, __get_locale(), fmt, ap);
+}
diff --git a/lib/libc/stdio/vwprintf.c b/lib/libc/stdio/vwprintf.c
index 91212a8..e09a30e 100644
--- a/lib/libc/stdio/vwprintf.c
+++ b/lib/libc/stdio/vwprintf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,10 +35,15 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 vwprintf(const wchar_t * __restrict fmt, va_list ap)
 {
-
 	return (vfwprintf(stdout, fmt, ap));
 }
+int
+vwprintf_l(locale_t locale, const wchar_t * __restrict fmt, va_list ap)
+{
+	return (vfwprintf_l(stdout, locale, fmt, ap));
+}
diff --git a/lib/libc/stdio/vwscanf.c b/lib/libc/stdio/vwscanf.c
index 4a21af2..730beee 100644
--- a/lib/libc/stdio/vwscanf.c
+++ b/lib/libc/stdio/vwscanf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,10 +35,15 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 vwscanf(const wchar_t * __restrict fmt, va_list ap)
 {
-
 	return (vfwscanf(stdin, fmt, ap));
 }
+int
+vwscanf_l(locale_t locale, const wchar_t * __restrict fmt, va_list ap)
+{
+	return (vfwscanf_l(stdin, locale, fmt, ap));
+}
diff --git a/lib/libc/stdio/wprintf.c b/lib/libc/stdio/wprintf.c
index 92426f6..fdffa86 100644
--- a/lib/libc/stdio/wprintf.c
+++ b/lib/libc/stdio/wprintf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 wprintf(const wchar_t * __restrict fmt, ...)
@@ -43,3 +49,15 @@ wprintf(const wchar_t * __restrict fmt, ...)
 
 	return (ret);
 }
+int
+wprintf_l(locale_t locale, const wchar_t * __restrict fmt, ...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vfwprintf_l(stdout, locale, fmt, ap);
+	va_end(ap);
+
+	return (ret);
+}
diff --git a/lib/libc/stdio/wscanf.c b/lib/libc/stdio/wscanf.c
index 5e27b40..22ce9bd 100644
--- a/lib/libc/stdio/wscanf.c
+++ b/lib/libc/stdio/wscanf.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -30,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include <stdio.h>
 #include <wchar.h>
+#include <xlocale.h>
 
 int
 wscanf(const wchar_t * __restrict fmt, ...)
@@ -43,3 +49,15 @@ wscanf(const wchar_t * __restrict fmt, ...)
 
 	return (r);
 }
+int
+wscanf_l(locale_t locale, const wchar_t * __restrict fmt, ...)
+{
+	va_list ap;
+	int r;
+
+	va_start(ap, fmt);
+	r = vfwscanf_l(stdin, locale, fmt, ap);
+	va_end(ap);
+
+	return (r);
+}
diff --git a/lib/libc/stdlib/Symbol.map b/lib/libc/stdlib/Symbol.map
index d159bed..c268ab9 100644
--- a/lib/libc/stdlib/Symbol.map
+++ b/lib/libc/stdlib/Symbol.map
@@ -92,6 +92,24 @@ FBSD_1.0 {
 	twalk;
 };
 
+FBSD_1.3 {
+	atof_l;
+	atoi_l;
+	atol_l;
+	atoll_l;
+	strtod_l;
+	strtol_l;
+	strtoll_l;
+	strtof_l;
+	strtoimax_l;
+	strtold_l;
+	strtoq_l;
+	strtoul_l;
+	strtoull_l;
+	strtoumax_l;
+	strtouq_l;
+};
+
 FBSDprivate_1.0 {
 	_malloc_thread_cleanup;
 	_malloc_prefork;
diff --git a/lib/libc/stdlib/atof.c b/lib/libc/stdlib/atof.c
index 51e482e..746ceac 100644
--- a/lib/libc/stdlib/atof.c
+++ b/lib/libc/stdlib/atof.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1988, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,6 +39,7 @@ static char sccsid[] = "@(#)atof.c	8.1 (Berkeley) 6/4/93";
 __FBSDID("$FreeBSD$");
 
 #include <stdlib.h>
+#include <xlocale.h>
 
 double
 atof(ascii)
@@ -41,3 +47,11 @@ atof(ascii)
 {
 	return strtod(ascii, (char **)NULL);
 }
+
+double
+atof_l(ascii, locale)
+	const char *ascii;
+	locale_t locale;
+{
+	return strtod_l(ascii, (char **)NULL, locale);
+}
diff --git a/lib/libc/stdlib/atoi.c b/lib/libc/stdlib/atoi.c
index 0ffcd03..31a8676 100644
--- a/lib/libc/stdlib/atoi.c
+++ b/lib/libc/stdlib/atoi.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1988, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,6 +39,7 @@ static char sccsid[] = "@(#)atoi.c	8.1 (Berkeley) 6/4/93";
 __FBSDID("$FreeBSD$");
 
 #include <stdlib.h>
+#include <xlocale.h>
 
 int
 atoi(str)
@@ -41,3 +47,11 @@ atoi(str)
 {
 	return (int)strtol(str, (char **)NULL, 10);
 }
+
+int
+atoi_l(str, locale)
+	const char *str;
+	locale_t locale;
+{
+	return (int)strtol_l(str, (char **)NULL, 10, locale);
+}
diff --git a/lib/libc/stdlib/atol.c b/lib/libc/stdlib/atol.c
index c9cf2e8..7ebbe01 100644
--- a/lib/libc/stdlib/atol.c
+++ b/lib/libc/stdlib/atol.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1988, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -34,6 +39,7 @@ static char sccsid[] = "@(#)atol.c	8.1 (Berkeley) 6/4/93";
 __FBSDID("$FreeBSD$");
 
 #include <stdlib.h>
+#include <xlocale.h>
 
 long
 atol(str)
@@ -41,3 +47,11 @@ atol(str)
 {
 	return strtol(str, (char **)NULL, 10);
 }
+
+long
+atol_l(str, locale)
+	const char *str;
+	locale_t locale;
+{
+	return strtol_l(str, (char **)NULL, 10, locale);
+}
diff --git a/lib/libc/stdlib/atoll.c b/lib/libc/stdlib/atoll.c
index 203c174..cbb5459 100644
--- a/lib/libc/stdlib/atoll.c
+++ b/lib/libc/stdlib/atoll.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1988, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -31,6 +36,7 @@
 __FBSDID("$FreeBSD$");
 
 #include <stdlib.h>
+#include <xlocale.h>
 
 long long
 atoll(str)
@@ -38,3 +44,11 @@ atoll(str)
 {
 	return strtoll(str, (char **)NULL, 10);
 }
+
+long long
+atoll_l(str, locale)
+	const char *str;
+	locale_t locale;
+{
+	return strtoll_l(str, (char **)NULL, 10, locale);
+}
diff --git a/lib/libc/stdlib/realpath.c b/lib/libc/stdlib/realpath.c
index 66bb8da..2c9562e 100644
--- a/lib/libc/stdlib/realpath.c
+++ b/lib/libc/stdlib/realpath.c
@@ -212,7 +212,8 @@ realpath(const char * __restrict path, char * __restrict resolved)
 					symlink[slen] = '/';
 					symlink[slen + 1] = 0;
 				}
-				left_len = strlcat(symlink, left, sizeof(left));
+				left_len = strlcat(symlink, left,
+				    sizeof(symlink));
 				if (left_len >= sizeof(left)) {
 					if (m)
 						free(resolved);
diff --git a/lib/libc/stdlib/strfmon.c b/lib/libc/stdlib/strfmon.c
index 6c28254..b82797d 100644
--- a/lib/libc/stdlib/strfmon.c
+++ b/lib/libc/stdlib/strfmon.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2001 Alexey Zelkin <phantom@FreeBSD.org>
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -38,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include "xlocale_private.h"
 
 /* internal flags */
 #define	NEED_GROUPING		0x01	/* print digits grouped (default) */
@@ -92,11 +98,10 @@ static void __setup_vars(int, char *, char *, char *, char **);
 static int __calc_left_pad(int, char *);
 static char *__format_grouped_double(double, int *, int, int, int);
 
-ssize_t
-strfmon(char * __restrict s, size_t maxsize, const char * __restrict format,
-    ...)
+static ssize_t
+vstrfmon_l(char * __restrict s, size_t maxsize, locale_t loc,
+		const char * __restrict format, va_list ap)
 {
-	va_list		ap;
 	char 		*dst;		/* output destination pointer */
 	const char 	*fmt;		/* current format poistion pointer */
 	struct lconv 	*lc;		/* pointer to lconv structure */
@@ -119,10 +124,10 @@ strfmon(char * __restrict s, size_t maxsize, const char * __restrict format,
 
 	char		*tmpptr;	/* temporary vars */
 	int		sverrno;
+	FIX_LOCALE(loc);
 
-        va_start(ap, format);
 
-	lc = localeconv();
+	lc = localeconv_l(loc);
 	dst = s;
 	fmt = format;
 	asciivalue = NULL;
@@ -380,7 +385,6 @@ strfmon(char * __restrict s, size_t maxsize, const char * __restrict format,
 	}
 
 	PRINT('\0');
-	va_end(ap);
 	free(asciivalue);
 	free(currency_symbol);
 	return (dst - s - 1);	/* return size of put data except trailing '\0' */
@@ -399,9 +403,32 @@ end_error:
 	if (currency_symbol != NULL)
 		free(currency_symbol);
 	errno = sverrno;
-	va_end(ap);
 	return (-1);
 }
+ssize_t
+strfmon_l(char * __restrict s, size_t maxsize, locale_t loc, const char * __restrict format,
+    ...)
+{
+	size_t ret;
+	va_list ap;
+	va_start(ap, format);
+	ret = vstrfmon_l(s, maxsize, loc, format, ap);
+	va_end(ap);
+	return ret;
+}
+
+ssize_t
+strfmon(char * __restrict s, size_t maxsize, const char * __restrict format,
+    ...)
+{
+	size_t ret;
+	va_list ap;
+	va_start(ap, format);
+	ret = vstrfmon_l(s, maxsize, __get_locale(), format, ap);
+	va_end(ap);
+	return ret;
+}
+
 
 static void
 __setup_vars(int flags, char *cs_precedes, char *sep_by_space,
diff --git a/lib/libc/stdlib/strtoimax.c b/lib/libc/stdlib/strtoimax.c
index 4f0063c..9be773b 100644
--- a/lib/libc/stdlib/strtoimax.c
+++ b/lib/libc/stdlib/strtoimax.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <errno.h>
 #include <stdlib.h>
 #include <inttypes.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to an intmax_t integer.
@@ -45,13 +51,15 @@ __FBSDID("$FreeBSD$");
  * alphabets and digits are each contiguous.
  */
 intmax_t
-strtoimax(const char * __restrict nptr, char ** __restrict endptr, int base)
+strtoimax_l(const char * __restrict nptr, char ** __restrict endptr, int base,
+		locale_t locale)
 {
 	const char *s;
 	uintmax_t acc;
 	char c;
 	uintmax_t cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * Skip white space and pick up leading +/- sign if any.
@@ -61,7 +69,7 @@ strtoimax(const char * __restrict nptr, char ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (isspace((unsigned char)c));
+	} while (isspace_l((unsigned char)c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -138,3 +146,8 @@ noconv:
 		*endptr = (char *)(any ? s - 1 : nptr);
 	return (acc);
 }
+intmax_t
+strtoimax(const char * __restrict nptr, char ** __restrict endptr, int base)
+{
+	return strtoimax_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/stdlib/strtol.c b/lib/libc/stdlib/strtol.c
index 0d6f736..f80ecb4 100644
--- a/lib/libc/stdlib/strtol.c
+++ b/lib/libc/stdlib/strtol.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <errno.h>
 #include <stdlib.h>
+#include "xlocale_private.h"
 
 
 /*
@@ -46,13 +52,15 @@ __FBSDID("$FreeBSD$");
  * alphabets and digits are each contiguous.
  */
 long
-strtol(const char * __restrict nptr, char ** __restrict endptr, int base)
+strtol_l(const char * __restrict nptr, char ** __restrict endptr, int base,
+		locale_t locale)
 {
 	const char *s;
 	unsigned long acc;
 	char c;
 	unsigned long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * Skip white space and pick up leading +/- sign if any.
@@ -62,7 +70,7 @@ strtol(const char * __restrict nptr, char ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (isspace((unsigned char)c));
+	} while (isspace_l((unsigned char)c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -138,3 +146,13 @@ noconv:
 		*endptr = (char *)(any ? s - 1 : nptr);
 	return (acc);
 }
+long
+strtol(const char * __restrict nptr, char ** __restrict endptr, int base)
+{
+	return strtol_l(nptr, endptr, base, __get_locale());
+}
+long double
+strtold(const char * __restrict nptr, char ** __restrict endptr)
+{
+	return strtold_l(nptr, endptr, __get_locale());
+}
diff --git a/lib/libc/stdlib/strtoll.c b/lib/libc/stdlib/strtoll.c
index 2e5547d..6783327 100644
--- a/lib/libc/stdlib/strtoll.c
+++ b/lib/libc/stdlib/strtoll.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <errno.h>
 #include <ctype.h>
 #include <stdlib.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to a long long integer.
@@ -45,13 +51,15 @@ __FBSDID("$FreeBSD$");
  * alphabets and digits are each contiguous.
  */
 long long
-strtoll(const char * __restrict nptr, char ** __restrict endptr, int base)
+strtoll_l(const char * __restrict nptr, char ** __restrict endptr, int base,
+		locale_t locale)
 {
 	const char *s;
 	unsigned long long acc;
 	char c;
 	unsigned long long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * Skip white space and pick up leading +/- sign if any.
@@ -61,7 +69,7 @@ strtoll(const char * __restrict nptr, char ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (isspace((unsigned char)c));
+	} while (isspace_l((unsigned char)c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -138,3 +146,8 @@ noconv:
 		*endptr = (char *)(any ? s - 1 : nptr);
 	return (acc);
 }
+long long
+strtoll(const char * __restrict nptr, char ** __restrict endptr, int base)
+{
+	return strtoll_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/stdlib/strtoul.c b/lib/libc/stdlib/strtoul.c
index e8d1e11..0ae0661 100644
--- a/lib/libc/stdlib/strtoul.c
+++ b/lib/libc/stdlib/strtoul.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <ctype.h>
 #include <errno.h>
 #include <stdlib.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to an unsigned long integer.
@@ -45,13 +51,14 @@ __FBSDID("$FreeBSD$");
  * alphabets and digits are each contiguous.
  */
 unsigned long
-strtoul(const char * __restrict nptr, char ** __restrict endptr, int base)
+strtoul_l(const char * __restrict nptr, char ** __restrict endptr, int base, locale_t locale)
 {
 	const char *s;
 	unsigned long acc;
 	char c;
 	unsigned long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtol for comments as to the logic used.
@@ -59,7 +66,7 @@ strtoul(const char * __restrict nptr, char ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (isspace((unsigned char)c));
+	} while (isspace_l((unsigned char)c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -116,3 +123,8 @@ noconv:
 		*endptr = (char *)(any ? s - 1 : nptr);
 	return (acc);
 }
+unsigned long
+strtoul(const char * __restrict nptr, char ** __restrict endptr, int base)
+{
+	return strtoul_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/stdlib/strtoull.c b/lib/libc/stdlib/strtoull.c
index f11910c..4b7f1c9 100644
--- a/lib/libc/stdlib/strtoull.c
+++ b/lib/libc/stdlib/strtoull.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <errno.h>
 #include <ctype.h>
 #include <stdlib.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to an unsigned long long integer.
@@ -45,13 +51,15 @@ __FBSDID("$FreeBSD$");
  * alphabets and digits are each contiguous.
  */
 unsigned long long
-strtoull(const char * __restrict nptr, char ** __restrict endptr, int base)
+strtoull_l(const char * __restrict nptr, char ** __restrict endptr, int base,
+		locale_t locale)
 {
 	const char *s;
 	unsigned long long acc;
 	char c;
 	unsigned long long cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtoq for comments as to the logic used.
@@ -59,7 +67,7 @@ strtoull(const char * __restrict nptr, char ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (isspace((unsigned char)c));
+	} while (isspace_l((unsigned char)c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -116,3 +124,8 @@ noconv:
 		*endptr = (char *)(any ? s - 1 : nptr);
 	return (acc);
 }
+unsigned long long
+strtoull(const char * __restrict nptr, char ** __restrict endptr, int base)
+{
+	return strtoull_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/stdlib/strtoumax.c b/lib/libc/stdlib/strtoumax.c
index 23050c2..d4362bb 100644
--- a/lib/libc/stdlib/strtoumax.c
+++ b/lib/libc/stdlib/strtoumax.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -37,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <errno.h>
 #include <stdlib.h>
 #include <inttypes.h>
+#include "xlocale_private.h"
 
 /*
  * Convert a string to a uintmax_t integer.
@@ -45,13 +51,15 @@ __FBSDID("$FreeBSD$");
  * alphabets and digits are each contiguous.
  */
 uintmax_t
-strtoumax(const char * __restrict nptr, char ** __restrict endptr, int base)
+strtoumax_l(const char * __restrict nptr, char ** __restrict endptr, int base,
+		locale_t locale)
 {
 	const char *s;
 	uintmax_t acc;
 	char c;
 	uintmax_t cutoff;
 	int neg, any, cutlim;
+	FIX_LOCALE(locale);
 
 	/*
 	 * See strtoimax for comments as to the logic used.
@@ -59,7 +67,7 @@ strtoumax(const char * __restrict nptr, char ** __restrict endptr, int base)
 	s = nptr;
 	do {
 		c = *s++;
-	} while (isspace((unsigned char)c));
+	} while (isspace_l((unsigned char)c, locale));
 	if (c == '-') {
 		neg = 1;
 		c = *s++;
@@ -116,3 +124,8 @@ noconv:
 		*endptr = (char *)(any ? s - 1 : nptr);
 	return (acc);
 }
+uintmax_t
+strtoumax(const char * __restrict nptr, char ** __restrict endptr, int base)
+{
+	return strtoumax_l(nptr, endptr, base, __get_locale());
+}
diff --git a/lib/libc/stdtime/strftime.c b/lib/libc/stdtime/strftime.c
index dac910b..d3571ad 100644
--- a/lib/libc/stdtime/strftime.c
+++ b/lib/libc/stdtime/strftime.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1989 The Regents of the University of California.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms are permitted
  * provided that the above copyright notice and this paragraph are
  * duplicated in all such forms and that any documentation,
@@ -43,7 +48,7 @@ __FBSDID("$FreeBSD$");
 static char *	_add(const char *, char *, const char *);
 static char *	_conv(int, const char *, char *, const char *);
 static char *	_fmt(const char *, const struct tm *, char *, const char *,
-			int *);
+			int *, locale_t);
 static char *	_yconv(int, int, int, int, char *, const char *);
 
 extern char *	tzname[];
@@ -82,29 +87,30 @@ static const char* fmt_padding[][4] = {
 };
 
 size_t
-strftime(char * __restrict s, size_t maxsize, const char * __restrict format,
-    const struct tm * __restrict t)
+strftime_l(char * __restrict s, size_t maxsize, const char * __restrict format,
+    const struct tm * __restrict t, locale_t loc)
 {
 	char *	p;
 	int	warn;
+	FIX_LOCALE(loc);
 
 	tzset();
 	warn = IN_NONE;
-	p = _fmt(((format == NULL) ? "%c" : format), t, s, s + maxsize, &warn);
+	p = _fmt(((format == NULL) ? "%c" : format), t, s, s + maxsize, &warn, loc);
 #ifndef NO_RUN_TIME_WARNINGS_ABOUT_YEAR_2000_PROBLEMS_THANK_YOU
 	if (warn != IN_NONE && getenv(YEAR_2000_NAME) != NULL) {
-		(void) fprintf(stderr, "\n");
+		(void) fprintf_l(stderr, loc, "\n");
 		if (format == NULL)
-			(void) fprintf(stderr, "NULL strftime format ");
-		else	(void) fprintf(stderr, "strftime format \"%s\" ",
+			(void) fprintf_l(stderr, loc, "NULL strftime format ");
+		else	(void) fprintf_l(stderr, loc, "strftime format \"%s\" ",
 				format);
-		(void) fprintf(stderr, "yields only two digits of years in ");
+		(void) fprintf_l(stderr, loc, "yields only two digits of years in ");
 		if (warn == IN_SOME)
-			(void) fprintf(stderr, "some locales");
+			(void) fprintf_l(stderr, loc, "some locales");
 		else if (warn == IN_THIS)
-			(void) fprintf(stderr, "the current locale");
-		else	(void) fprintf(stderr, "all locales");
-		(void) fprintf(stderr, "\n");
+			(void) fprintf_l(stderr, loc, "the current locale");
+		else	(void) fprintf_l(stderr, loc, "all locales");
+		(void) fprintf_l(stderr, loc, "\n");
 	}
 #endif /* !defined NO_RUN_TIME_WARNINGS_ABOUT_YEAR_2000_PROBLEMS_THANK_YOU */
 	if (p == s + maxsize)
@@ -113,16 +119,24 @@ strftime(char * __restrict s, size_t maxsize, const char * __restrict format,
 	return p - s;
 }
 
+size_t
+strftime(char * __restrict s, size_t maxsize, const char * __restrict format,
+    const struct tm * __restrict t)
+{
+	return strftime_l(s, maxsize, format, t, __get_locale());
+}
+
 static char *
-_fmt(format, t, pt, ptlim, warnp)
+_fmt(format, t, pt, ptlim, warnp, loc)
 const char *		format;
 const struct tm * const	t;
 char *			pt;
 const char * const	ptlim;
 int *			warnp;
+locale_t	loc;
 {
 	int Ealternative, Oalternative, PadIndex;
-	struct lc_time_T *tptr = __get_current_time_locale();
+	struct lc_time_T *tptr = __get_current_time_locale(loc);
 
 	for ( ; *format; ++format) {
 		if (*format == '%') {
@@ -175,7 +189,7 @@ label:
 				{
 				int warn2 = IN_SOME;
 
-				pt = _fmt(tptr->c_fmt, t, pt, ptlim, &warn2);
+				pt = _fmt(tptr->c_fmt, t, pt, ptlim, &warn2, loc);
 				if (warn2 == IN_ALL)
 					warn2 = IN_THIS;
 				if (warn2 > *warnp)
@@ -183,7 +197,7 @@ label:
 				}
 				continue;
 			case 'D':
-				pt = _fmt("%m/%d/%y", t, pt, ptlim, warnp);
+				pt = _fmt("%m/%d/%y", t, pt, ptlim, warnp, loc);
 				continue;
 			case 'd':
 				pt = _conv(t->tm_mday, fmt_padding[PAD_FMT_DAYOFMONTH][PadIndex],
@@ -216,7 +230,7 @@ label:
 					fmt_padding[PAD_FMT_SDAYOFMONTH][PadIndex], pt, ptlim);
 				continue;
 			case 'F':
-				pt = _fmt("%Y-%m-%d", t, pt, ptlim, warnp);
+				pt = _fmt("%Y-%m-%d", t, pt, ptlim, warnp, loc);
 				continue;
 			case 'H':
 				pt = _conv(t->tm_hour, fmt_padding[PAD_FMT_HMS][PadIndex],
@@ -285,11 +299,11 @@ label:
 					pt, ptlim);
 				continue;
 			case 'R':
-				pt = _fmt("%H:%M", t, pt, ptlim, warnp);
+				pt = _fmt("%H:%M", t, pt, ptlim, warnp, loc);
 				continue;
 			case 'r':
 				pt = _fmt(tptr->ampm_fmt, t, pt, ptlim,
-					warnp);
+					warnp, loc);
 				continue;
 			case 'S':
 				pt = _conv(t->tm_sec, fmt_padding[PAD_FMT_HMS][PadIndex],
@@ -313,7 +327,7 @@ label:
 				}
 				continue;
 			case 'T':
-				pt = _fmt("%H:%M:%S", t, pt, ptlim, warnp);
+				pt = _fmt("%H:%M:%S", t, pt, ptlim, warnp, loc);
 				continue;
 			case 't':
 				pt = _add("\t", pt, ptlim);
@@ -428,7 +442,7 @@ label:
 				** "date as dd-bbb-YYYY"
 				** (ado, 1993-05-24)
 				*/
-				pt = _fmt("%e-%b-%Y", t, pt, ptlim, warnp);
+				pt = _fmt("%e-%b-%Y", t, pt, ptlim, warnp, loc);
 				continue;
 			case 'W':
 				pt = _conv((t->tm_yday + DAYSPERWEEK -
@@ -441,13 +455,13 @@ label:
 				pt = _conv(t->tm_wday, "%d", pt, ptlim);
 				continue;
 			case 'X':
-				pt = _fmt(tptr->X_fmt, t, pt, ptlim, warnp);
+				pt = _fmt(tptr->X_fmt, t, pt, ptlim, warnp, loc);
 				continue;
 			case 'x':
 				{
 				int	warn2 = IN_SOME;
 
-				pt = _fmt(tptr->x_fmt, t, pt, ptlim, &warn2);
+				pt = _fmt(tptr->x_fmt, t, pt, ptlim, &warn2, loc);
 				if (warn2 == IN_ALL)
 					warn2 = IN_THIS;
 				if (warn2 > *warnp)
@@ -534,7 +548,7 @@ label:
 				continue;
 			case '+':
 				pt = _fmt(tptr->date_fmt, t, pt, ptlim,
-					warnp);
+					warnp, loc);
 				continue;
 			case '-':
 				if (PadIndex != PAD_DEFAULT)
diff --git a/lib/libc/stdtime/strptime.c b/lib/libc/stdtime/strptime.c
index 401350e..fb94dcd 100644
--- a/lib/libc/stdtime/strptime.c
+++ b/lib/libc/stdtime/strptime.c
@@ -22,6 +22,11 @@
 /*
  * Copyright (c) 1994 Powerdog Industries.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -72,19 +77,20 @@ __FBSDID("$FreeBSD$");
 #include "libc_private.h"
 #include "timelocal.h"
 
-static char * _strptime(const char *, const char *, struct tm *, int *);
+static char * _strptime(const char *, const char *, struct tm *, int *, locale_t);
 
 #define asizeof(a)	(sizeof (a) / sizeof ((a)[0]))
 
 static char *
-_strptime(const char *buf, const char *fmt, struct tm *tm, int *GMTp)
+_strptime(const char *buf, const char *fmt, struct tm *tm, int *GMTp,
+		locale_t locale)
 {
 	char	c;
 	const char *ptr;
 	int	i,
 		len;
 	int Ealternative, Oalternative;
-	struct lc_time_T *tptr = __get_current_time_locale();
+	struct lc_time_T *tptr = __get_current_time_locale(locale);
 
 	ptr = fmt;
 	while (*ptr != 0) {
@@ -94,8 +100,9 @@ _strptime(const char *buf, const char *fmt, struct tm *tm, int *GMTp)
 		c = *ptr++;
 
 		if (c != '%') {
-			if (isspace((unsigned char)c))
-				while (*buf != 0 && isspace((unsigned char)*buf))
+			if (isspace_l((unsigned char)c, locale))
+				while (*buf != 0 && 
+				       isspace_l((unsigned char)*buf, locale))
 					buf++;
 			else if (c != *buf++)
 				return 0;
@@ -114,18 +121,19 @@ label:
 			break;
 
 		case '+':
-			buf = _strptime(buf, tptr->date_fmt, tm, GMTp);
+			buf = _strptime(buf, tptr->date_fmt, tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'C':
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			/* XXX This will break for 3-digit centuries. */
 			len = 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++) {
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -137,13 +145,13 @@ label:
 			break;
 
 		case 'c':
-			buf = _strptime(buf, tptr->c_fmt, tm, GMTp);
+			buf = _strptime(buf, tptr->c_fmt, tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'D':
-			buf = _strptime(buf, "%m/%d/%y", tm, GMTp);
+			buf = _strptime(buf, "%m/%d/%y", tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
@@ -161,47 +169,48 @@ label:
 			goto label;
 
 		case 'F':
-			buf = _strptime(buf, "%Y-%m-%d", tm, GMTp);
+			buf = _strptime(buf, "%Y-%m-%d", tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'R':
-			buf = _strptime(buf, "%H:%M", tm, GMTp);
+			buf = _strptime(buf, "%H:%M", tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'r':
-			buf = _strptime(buf, tptr->ampm_fmt, tm, GMTp);
+			buf = _strptime(buf, tptr->ampm_fmt, tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'T':
-			buf = _strptime(buf, "%H:%M:%S", tm, GMTp);
+			buf = _strptime(buf, "%H:%M:%S", tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'X':
-			buf = _strptime(buf, tptr->X_fmt, tm, GMTp);
+			buf = _strptime(buf, tptr->X_fmt, tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'x':
-			buf = _strptime(buf, tptr->x_fmt, tm, GMTp);
+			buf = _strptime(buf, tptr->x_fmt, tm, GMTp, locale);
 			if (buf == 0)
 				return 0;
 			break;
 
 		case 'j':
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = 3;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++){
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -214,14 +223,16 @@ label:
 
 		case 'M':
 		case 'S':
-			if (*buf == 0 || isspace((unsigned char)*buf))
+			if (*buf == 0 ||
+				isspace_l((unsigned char)*buf, locale))
 				break;
 
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+				isdigit_l((unsigned char)*buf, locale); buf++){
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -237,8 +248,10 @@ label:
 				tm->tm_sec = i;
 			}
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+				isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
@@ -254,11 +267,12 @@ label:
 			 * XXX The %l specifier may gobble one too many
 			 * digits if used incorrectly.
 			 */
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++) {
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -271,8 +285,10 @@ label:
 
 			tm->tm_hour = i;
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+			    isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
@@ -282,7 +298,7 @@ label:
 			 * specifiers.
 			 */
 			len = strlen(tptr->am);
-			if (strncasecmp(buf, tptr->am, len) == 0) {
+			if (strncasecmp_l(buf, tptr->am, len, locale) == 0) {
 				if (tm->tm_hour > 12)
 					return 0;
 				if (tm->tm_hour == 12)
@@ -292,7 +308,7 @@ label:
 			}
 
 			len = strlen(tptr->pm);
-			if (strncasecmp(buf, tptr->pm, len) == 0) {
+			if (strncasecmp_l(buf, tptr->pm, len, locale) == 0) {
 				if (tm->tm_hour > 12)
 					return 0;
 				if (tm->tm_hour != 12)
@@ -307,12 +323,12 @@ label:
 		case 'a':
 			for (i = 0; i < asizeof(tptr->weekday); i++) {
 				len = strlen(tptr->weekday[i]);
-				if (strncasecmp(buf, tptr->weekday[i],
-						len) == 0)
+				if (strncasecmp_l(buf, tptr->weekday[i],
+						len, locale) == 0)
 					break;
 				len = strlen(tptr->wday[i]);
-				if (strncasecmp(buf, tptr->wday[i],
-						len) == 0)
+				if (strncasecmp_l(buf, tptr->wday[i],
+						len, locale) == 0)
 					break;
 			}
 			if (i == asizeof(tptr->weekday))
@@ -330,11 +346,12 @@ label:
 			 * point to calculate a real value, so just check the
 			 * range for now.
 			 */
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++) {
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -342,13 +359,15 @@ label:
 			if (i > 53)
 				return 0;
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+			    isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
 		case 'w':
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			i = *buf - '0';
@@ -357,8 +376,10 @@ label:
 
 			tm->tm_wday = i;
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+			    isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
@@ -372,11 +393,12 @@ label:
 			 * XXX The %e specifier may gobble one too many
 			 * digits if used incorrectly.
 			 */
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++) {
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -386,8 +408,10 @@ label:
 
 			tm->tm_mday = i;
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+			    isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
@@ -398,15 +422,15 @@ label:
 				if (Oalternative) {
 					if (c == 'B') {
 						len = strlen(tptr->alt_month[i]);
-						if (strncasecmp(buf,
+						if (strncasecmp_l(buf,
 								tptr->alt_month[i],
-								len) == 0)
+								len, locale) == 0)
 							break;
 					}
 				} else {
 					len = strlen(tptr->month[i]);
-					if (strncasecmp(buf, tptr->month[i],
-							len) == 0)
+					if (strncasecmp_l(buf, tptr->month[i],
+							len, locale) == 0)
 						break;
 				}
 			}
@@ -417,8 +441,8 @@ label:
 			if (i == asizeof(tptr->month) && !Oalternative) {
 				for (i = 0; i < asizeof(tptr->month); i++) {
 					len = strlen(tptr->mon[i]);
-					if (strncasecmp(buf, tptr->mon[i],
-							len) == 0)
+					if (strncasecmp_l(buf, tptr->mon[i],
+							len, locale) == 0)
 						break;
 				}
 			}
@@ -430,11 +454,12 @@ label:
 			break;
 
 		case 'm':
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++) {
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -444,8 +469,10 @@ label:
 
 			tm->tm_mon = i - 1;
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+			    isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
@@ -458,7 +485,7 @@ label:
 
 			sverrno = errno;
 			errno = 0;
-			n = strtol(buf, &cp, 10);
+			n = strtol_l(buf, &cp, 10, locale);
 			if (errno == ERANGE || (long)(t = n) != n) {
 				errno = sverrno;
 				return 0;
@@ -472,14 +499,16 @@ label:
 
 		case 'Y':
 		case 'y':
-			if (*buf == 0 || isspace((unsigned char)*buf))
+			if (*buf == 0 ||
+			    isspace_l((unsigned char)*buf, locale))
 				break;
 
-			if (!isdigit((unsigned char)*buf))
+			if (!isdigit_l((unsigned char)*buf, locale))
 				return 0;
 
 			len = (c == 'Y') ? 4 : 2;
-			for (i = 0; len && *buf != 0 && isdigit((unsigned char)*buf); buf++) {
+			for (i = 0; len && *buf != 0 &&
+			     isdigit_l((unsigned char)*buf, locale); buf++) {
 				i *= 10;
 				i += *buf - '0';
 				len--;
@@ -493,8 +522,10 @@ label:
 
 			tm->tm_year = i;
 
-			if (*buf != 0 && isspace((unsigned char)*buf))
-				while (*ptr != 0 && !isspace((unsigned char)*ptr))
+			if (*buf != 0 &&
+			    isspace_l((unsigned char)*buf, locale))
+				while (*ptr != 0 &&
+				       !isspace_l((unsigned char)*ptr, locale))
 					ptr++;
 			break;
 
@@ -503,7 +534,9 @@ label:
 			const char *cp;
 			char *zonestr;
 
-			for (cp = buf; *cp && isupper((unsigned char)*cp); ++cp) {/*empty*/}
+			for (cp = buf; *cp &&
+			     isupper_l((unsigned char)*cp, locale); ++cp) {
+				/*empty*/}
 			if (cp - buf) {
 				zonestr = alloca(cp - buf + 1);
 				strncpy(zonestr, buf, cp - buf);
@@ -537,7 +570,7 @@ label:
 			buf++;
 			i = 0;
 			for (len = 4; len > 0; len--) {
-				if (isdigit((unsigned char)*buf)) {
+				if (isdigit_l((unsigned char)*buf, locale)) {
 					i *= 10;
 					i += *buf - '0';
 					buf++;
@@ -557,14 +590,15 @@ label:
 
 
 char *
-strptime(const char * __restrict buf, const char * __restrict fmt,
-    struct tm * __restrict tm)
+strptime_l(const char * __restrict buf, const char * __restrict fmt,
+    struct tm * __restrict tm, locale_t loc)
 {
 	char *ret;
 	int gmt;
+	FIX_LOCALE(loc);
 
 	gmt = 0;
-	ret = _strptime(buf, fmt, tm, &gmt);
+	ret = _strptime(buf, fmt, tm, &gmt, loc);
 	if (ret && gmt) {
 		time_t t = timegm(tm);
 		localtime_r(&t, tm);
@@ -572,3 +606,9 @@ strptime(const char * __restrict buf, const char * __restrict fmt,
 
 	return (ret);
 }
+char *
+strptime(const char * __restrict buf, const char * __restrict fmt,
+    struct tm * __restrict tm)
+{
+	return strptime_l(buf, fmt, tm, __get_locale());
+}
diff --git a/lib/libc/stdtime/timelocal.c b/lib/libc/stdtime/timelocal.c
index 6917e6b..3d9d096 100644
--- a/lib/libc/stdtime/timelocal.c
+++ b/lib/libc/stdtime/timelocal.c
@@ -3,6 +3,11 @@
  * Copyright (c) 1997 FreeBSD Inc.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,9 +38,13 @@ __FBSDID("$FreeBSD$");
 #include "ldpart.h"
 #include "timelocal.h"
 
-static struct lc_time_T _time_locale;
-static int _time_using_locale;
-static char *time_locale_buf;
+struct xlocale_time {
+	struct xlocale_component header;
+	char *buffer;
+	struct lc_time_T locale;
+};
+
+struct xlocale_time __xlocale_global_time;
 
 #define LCTIME_SIZE (sizeof(struct lc_time_T) / sizeof(char *))
 
@@ -99,19 +108,47 @@ static const struct lc_time_T	_C_time_locale = {
 	"%I:%M:%S %p"
 };
 
+static void destruct_time(void *v)
+{
+	struct xlocale_time *l = v;
+	if (l->buffer)
+		free(l->buffer);
+	free(l);
+}
+
+#include <stdio.h>
 struct lc_time_T *
-__get_current_time_locale(void)
+__get_current_time_locale(locale_t loc)
 {
-	return (_time_using_locale
-		? &_time_locale
+	return (loc->using_time_locale
+		? &((struct xlocale_time *)loc->components[XLC_TIME])->locale
 		: (struct lc_time_T *)&_C_time_locale);
 }
 
+static int
+time_load_locale(struct xlocale_time *l, int *using_locale, const char *name)
+{
+	struct lc_time_T *time_locale = &l->locale;
+	return (__part_load_locale(name, using_locale,
+			&l->buffer, "LC_TIME",
+			LCTIME_SIZE, LCTIME_SIZE,
+			(const char **)time_locale));
+}
 int
 __time_load_locale(const char *name)
 {
-	return (__part_load_locale(name, &_time_using_locale,
-			&time_locale_buf, "LC_TIME",
-			LCTIME_SIZE, LCTIME_SIZE,
-			(const char **)&_time_locale));
+	return time_load_locale(&__xlocale_global_time,
+			&__xlocale_global_locale.using_time_locale, name);
 }
+void* __time_load(const char* name, locale_t loc)
+{
+	struct xlocale_time *new = calloc(sizeof(struct xlocale_time), 1);
+	new->header.header.destructor = destruct_time;
+	if (time_load_locale(new, &loc->using_time_locale, name) == _LDP_ERROR)
+	{
+		xlocale_release(new);
+		return NULL;
+	}
+	return new;
+}
+
diff --git a/lib/libc/stdtime/timelocal.h b/lib/libc/stdtime/timelocal.h
index 5d26bf9..2e44415 100644
--- a/lib/libc/stdtime/timelocal.h
+++ b/lib/libc/stdtime/timelocal.h
@@ -2,6 +2,11 @@
  * Copyright (c) 1997-2002 FreeBSD Project.
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -28,6 +33,7 @@
 
 #ifndef _TIMELOCAL_H_
 #define	_TIMELOCAL_H_
+#include "xlocale_private.h"
 
 /*
  * Private header file for the strftime and strptime localization
@@ -49,7 +55,7 @@ struct lc_time_T {
 	const char	*ampm_fmt;
 };
 
-struct lc_time_T *__get_current_time_locale(void);
+struct lc_time_T *__get_current_time_locale(locale_t);
 int	__time_load_locale(const char *);
 
 #endif /* !_TIMELOCAL_H_ */
diff --git a/lib/libc/string/Symbol.map b/lib/libc/string/Symbol.map
index 19f3382..ef23465 100644
--- a/lib/libc/string/Symbol.map
+++ b/lib/libc/string/Symbol.map
@@ -91,6 +91,14 @@ FBSD_1.1 {
 	wcsnlen;
 };
 
+FBSD_1.3 {
+	strcasecmp_l;
+	strcasestr_l;
+	strncasecmp_l;
+	wcswidth_l;
+	wcwidth_l;
+};
+
 FBSDprivate_1.0 {
 	__strtok_r;
 };
diff --git a/lib/libc/string/strcasecmp.c b/lib/libc/string/strcasecmp.c
index 4a474fe..e173f6d 100644
--- a/lib/libc/string/strcasecmp.c
+++ b/lib/libc/string/strcasecmp.c
@@ -2,6 +2,11 @@
  * Copyright (c) 1987, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,36 +40,50 @@ __FBSDID("$FreeBSD$");
 
 #include <strings.h>
 #include <ctype.h>
+#include "xlocale_private.h"
 
 typedef unsigned char u_char;
 
 int
-strcasecmp(const char *s1, const char *s2)
+strcasecmp_l(const char *s1, const char *s2, locale_t locale)
 {
 	const u_char
 			*us1 = (const u_char *)s1,
 			*us2 = (const u_char *)s2;
+	FIX_LOCALE(locale);
 
-	while (tolower(*us1) == tolower(*us2++))
+	while (tolower_l(*us1, locale) == tolower_l(*us2++, locale))
 		if (*us1++ == '\0')
 			return (0);
-	return (tolower(*us1) - tolower(*--us2));
+	return (tolower_l(*us1, locale) - tolower_l(*--us2, locale));
+}
+int
+strcasecmp(const char *s1, const char *s2)
+{
+	return strcasecmp_l(s1, s2, __get_locale());
 }
 
 int
-strncasecmp(const char *s1, const char *s2, size_t n)
+strncasecmp_l(const char *s1, const char *s2, size_t n, locale_t locale)
 {
+	FIX_LOCALE(locale);
 	if (n != 0) {
 		const u_char
 				*us1 = (const u_char *)s1,
 				*us2 = (const u_char *)s2;
 
 		do {
-			if (tolower(*us1) != tolower(*us2++))
-				return (tolower(*us1) - tolower(*--us2));
+			if (tolower_l(*us1, locale) != tolower_l(*us2++, locale))
+				return (tolower_l(*us1, locale) - tolower_l(*--us2, locale));
 			if (*us1++ == '\0')
 				break;
 		} while (--n != 0);
 	}
 	return (0);
 }
+
+int
+strncasecmp(const char *s1, const char *s2, size_t n)
+{
+	return strncasecmp_l(s1, s2, n, __get_locale());
+}
diff --git a/lib/libc/string/strcasestr.c b/lib/libc/string/strcasestr.c
index 9b28bf5..5f39648 100644
--- a/lib/libc/string/strcasestr.c
+++ b/lib/libc/string/strcasestr.c
@@ -5,6 +5,11 @@
  * This code is derived from software contributed to Berkeley by
  * Chris Torek.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,26 +40,33 @@ __FBSDID("$FreeBSD$");
 
 #include <ctype.h>
 #include <string.h>
+#include "xlocale_private.h"
 
 /*
  * Find the first occurrence of find in s, ignore case.
  */
 char *
-strcasestr(const char *s, const char *find)
+strcasestr_l(const char *s, const char *find, locale_t locale)
 {
 	char c, sc;
 	size_t len;
+	FIX_LOCALE(locale);
 
 	if ((c = *find++) != 0) {
-		c = tolower((unsigned char)c);
+		c = tolower_l((unsigned char)c, locale);
 		len = strlen(find);
 		do {
 			do {
 				if ((sc = *s++) == 0)
 					return (NULL);
-			} while ((char)tolower((unsigned char)sc) != c);
-		} while (strncasecmp(s, find, len) != 0);
+			} while ((char)tolower_l((unsigned char)sc, locale) != c);
+		} while (strncasecmp_l(s, find, len, locale) != 0);
 		s--;
 	}
 	return ((char *)s);
 }
+char *
+strcasestr(const char *s, const char *find)
+{
+	return strcasestr_l(s, find, __get_locale());
+}
diff --git a/lib/libc/string/strcoll.c b/lib/libc/string/strcoll.c
index 0da5c57..a918fca 100644
--- a/lib/libc/string/strcoll.c
+++ b/lib/libc/string/strcoll.c
@@ -3,6 +3,11 @@
  *		at Electronni Visti IA, Kiev, Ukraine.
  *			All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -32,21 +37,26 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include "collate.h"
 
+#include <stdio.h>
+
 int
-strcoll(const char *s, const char *s2)
+strcoll_l(const char *s, const char *s2, locale_t locale)
 {
 	int len, len2, prim, prim2, sec, sec2, ret, ret2;
 	const char *t, *t2;
 	char *tt, *tt2;
+	FIX_LOCALE(locale);
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)locale->components[XLC_COLLATE];
 
-	if (__collate_load_error)
+	if (table->__collate_load_error)
 		return strcmp(s, s2);
 
 	len = len2 = 1;
 	ret = ret2 = 0;
-	if (__collate_substitute_nontrivial) {
-		t = tt = __collate_substitute(s);
-		t2 = tt2 = __collate_substitute(s2);
+	if (table->__collate_substitute_nontrivial) {
+		t = tt = __collate_substitute(table, s);
+		t2 = tt2 = __collate_substitute(table, s2);
 	} else {
 		tt = tt2 = NULL;
 		t = s;
@@ -55,11 +65,11 @@ strcoll(const char *s, const char *s2)
 	while(*t && *t2) {
 		prim = prim2 = 0;
 		while(*t && !prim) {
-			__collate_lookup(t, &len, &prim, &sec);
+			__collate_lookup(table, t, &len, &prim, &sec);
 			t += len;
 		}
 		while(*t2 && !prim2) {
-			__collate_lookup(t2, &len2, &prim2, &sec2);
+			__collate_lookup(table, t2, &len2, &prim2, &sec2);
 			t2 += len2;
 		}
 		if(!prim || !prim2)
@@ -83,3 +93,10 @@ strcoll(const char *s, const char *s2)
 
 	return ret;
 }
+
+int
+strcoll(const char *s, const char *s2)
+{
+	return strcoll_l(s, s2, __get_locale());
+}
+
diff --git a/lib/libc/string/strxfrm.c b/lib/libc/string/strxfrm.c
index a4c8019..b758b0c 100644
--- a/lib/libc/string/strxfrm.c
+++ b/lib/libc/string/strxfrm.c
@@ -3,6 +3,11 @@
  *		at Electronni Visti IA, Kiev, Ukraine.
  *			All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -33,11 +38,22 @@ __FBSDID("$FreeBSD$");
 #include "collate.h"
 
 size_t
+strxfrm_l(char * __restrict dest, const char * __restrict src, size_t len, locale_t loc);
+size_t
 strxfrm(char * __restrict dest, const char * __restrict src, size_t len)
 {
+	return strxfrm_l(dest, src, len, __get_locale());
+}
+
+size_t
+strxfrm_l(char * __restrict dest, const char * __restrict src, size_t len, locale_t locale)
+{
 	int prim, sec, l;
 	size_t slen;
 	char *s, *ss;
+	FIX_LOCALE(locale);
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)locale->components[XLC_COLLATE];
 
 	if (!*src) {
 		if (len > 0)
@@ -45,15 +61,15 @@ strxfrm(char * __restrict dest, const char * __restrict src, size_t len)
 		return 0;
 	}
 
-	if (__collate_load_error)
+	if (table->__collate_load_error)
 		return strlcpy(dest, src, len);
 
 	slen = 0;
 	prim = sec = 0;
-	ss = s = __collate_substitute(src);
+	ss = s = __collate_substitute(table, src);
 	while (*s) {
 		while (*s && !prim) {
-			__collate_lookup(s, &l, &prim, &sec);
+			__collate_lookup(table, s, &l, &prim, &sec);
 			s += l;
 		}
 		if (prim) {
diff --git a/lib/libc/string/wcscoll.c b/lib/libc/string/wcscoll.c
index dbfbcfa..3c51015 100644
--- a/lib/libc/string/wcscoll.c
+++ b/lib/libc/string/wcscoll.c
@@ -2,6 +2,11 @@
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -41,12 +46,15 @@ static char *__mbsdup(const wchar_t *);
  * with extended character sets.
  */
 int
-wcscoll(const wchar_t *ws1, const wchar_t *ws2)
+wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t locale)
 {
 	char *mbs1, *mbs2;
 	int diff, sverrno;
+	FIX_LOCALE(locale);
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)locale->components[XLC_COLLATE];
 
-	if (__collate_load_error || MB_CUR_MAX > 1)
+	if (table->__collate_load_error || MB_CUR_MAX > 1)
 		/*
 		 * Locale has no special collating order, could not be
 		 * loaded, or has an extended character set; do a fast binary
@@ -67,7 +75,7 @@ wcscoll(const wchar_t *ws1, const wchar_t *ws2)
 		return (wcscmp(ws1, ws2));
 	}
 
-	diff = strcoll(mbs1, mbs2);
+	diff = strcoll_l(mbs1, mbs2, locale);
 	sverrno = errno;
 	free(mbs1);
 	free(mbs2);
@@ -76,6 +84,12 @@ wcscoll(const wchar_t *ws1, const wchar_t *ws2)
 	return (diff);
 }
 
+int
+wcscoll(const wchar_t *ws1, const wchar_t *ws2)
+{
+	return wcscoll_l(ws1, ws2, __get_locale());
+}
+
 static char *
 __mbsdup(const wchar_t *ws)
 {
diff --git a/lib/libc/string/wcswidth.c b/lib/libc/string/wcswidth.c
index b142074..4095c8d 100644
--- a/lib/libc/string/wcswidth.c
+++ b/lib/libc/string/wcswidth.c
@@ -10,6 +10,11 @@
  * This code is derived from software contributed to Berkeley by
  * Paul Borman at Krystal Technologies.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,19 +44,26 @@
 __FBSDID("$FreeBSD$");
 
 #include <wchar.h>
+#include "xlocale_private.h"
 
 int
-wcswidth(const wchar_t *pwcs, size_t n)
+wcswidth_l(const wchar_t *pwcs, size_t n, locale_t locale)
 {
 	wchar_t wc;
 	int len, l;
+	FIX_LOCALE(locale);
 
 	len = 0;
 	while (n-- > 0 && (wc = *pwcs++) != L'\0') {
-		if ((l = wcwidth(wc)) < 0)
+		if ((l = wcwidth_l(wc, locale)) < 0)
 			return (-1);
 		len += l;
 	}
 	return (len);
 }
 
+int
+wcswidth(const wchar_t *pwcs, size_t n)
+{
+	return wcswidth_l(pwcs, n, __get_locale());
+}
diff --git a/lib/libc/string/wcsxfrm.c b/lib/libc/string/wcsxfrm.c
index 5e47ad9..cea667e 100644
--- a/lib/libc/string/wcsxfrm.c
+++ b/lib/libc/string/wcsxfrm.c
@@ -3,6 +3,11 @@
  *		at Electronni Visti IA, Kiev, Ukraine.
  *			All rights reserved.
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ * Portions of this software were developed by David Chisnall
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -43,11 +48,14 @@ static char *__mbsdup(const wchar_t *);
  * the logic used.
  */
 size_t
-wcsxfrm(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len)
+wcsxfrm_l(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len, locale_t locale)
 {
 	int prim, sec, l;
 	size_t slen;
 	char *mbsrc, *s, *ss;
+	FIX_LOCALE(locale);
+	struct xlocale_collate *table =
+		(struct xlocale_collate*)locale->components[XLC_COLLATE];
 
 	if (*src == L'\0') {
 		if (len != 0)
@@ -55,7 +63,7 @@ wcsxfrm(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len)
 		return (0);
 	}
 
-	if (__collate_load_error || MB_CUR_MAX > 1) {
+	if (table->__collate_load_error || MB_CUR_MAX > 1) {
 		slen = wcslen(src);
 		if (len > 0) {
 			if (slen < len)
@@ -71,10 +79,10 @@ wcsxfrm(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len)
 	mbsrc = __mbsdup(src);
 	slen = 0;
 	prim = sec = 0;
-	ss = s = __collate_substitute(mbsrc);
+	ss = s = __collate_substitute(table, mbsrc);
 	while (*s != '\0') {
 		while (*s != '\0' && prim == 0) {
-			__collate_lookup(s, &l, &prim, &sec);
+			__collate_lookup(table, s, &l, &prim, &sec);
 			s += l;
 		}
 		if (prim != 0) {
@@ -93,6 +101,11 @@ wcsxfrm(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len)
 
 	return (slen);
 }
+size_t
+wcsxfrm(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len)
+{
+	return wcsxfrm_l(dest, src, len, __get_locale());
+}
 
 static char *
 __mbsdup(const wchar_t *ws)
diff --git a/lib/libc/sys/Makefile.inc b/lib/libc/sys/Makefile.inc
index fe5061d..ea38b51 100644
--- a/lib/libc/sys/Makefile.inc
+++ b/lib/libc/sys/Makefile.inc
@@ -80,7 +80,7 @@ MAN+=	abort2.2 accept.2 access.2 acct.2 adjtime.2 \
 	bind.2 brk.2 cap_enter.2 cap_new.2 chdir.2 chflags.2 \
 	chmod.2 chown.2 chroot.2 clock_gettime.2 close.2 closefrom.2 \
 	connect.2 cpuset.2 cpuset_getaffinity.2 dup.2 execve.2 _exit.2 \
-	extattr_get_file.2 fcntl.2 fhopen.2 flock.2 fork.2 fsync.2 \
+	extattr_get_file.2 fcntl.2 ffclock.2 fhopen.2 flock.2 fork.2 fsync.2 \
 	getdirentries.2 getdtablesize.2 \
 	getfh.2 getfsstat.2 getgid.2 getgroups.2 getitimer.2 getlogin.2 \
 	getloginclass.2 getpeername.2 getpgrp.2 getpid.2 getpriority.2 \
@@ -96,7 +96,8 @@ MAN+=	abort2.2 accept.2 access.2 acct.2 adjtime.2 \
 	mq_setattr.2 \
 	msgctl.2 msgget.2 msgrcv.2 msgsnd.2 \
 	msync.2 munmap.2 nanosleep.2 nfssvc.2 ntp_adjtime.2 open.2 \
-	pathconf.2 pdfork.2 pipe.2 poll.2 posix_fallocate.2 posix_openpt.2 profil.2 \
+	pathconf.2 pdfork.2 pipe.2 poll.2 posix_fadvise.2 posix_fallocate.2 \
+	posix_openpt.2 profil.2 \
 	pselect.2 ptrace.2 quotactl.2 \
 	read.2 readlink.2 reboot.2 recv.2 rename.2 revoke.2 rfork.2 rmdir.2 \
 	rtprio.2
@@ -141,6 +142,8 @@ MLINKS+=extattr_get_file.2 extattr.2 \
 	extattr_get_file.2 extattr_set_fd.2 \
 	extattr_get_file.2 extattr_set_file.2 \
 	extattr_get_file.2 extattr_set_link.2
+MLINKS+=ffclock.2 ffclock_getcounter.2 ffclock.2 ffclock_getestimate.2 \
+	ffclock.2 ffclock_setestimate.2
 MLINKS+=fhopen.2 fhstat.2 fhopen.2 fhstatfs.2
 MLINKS+=getdirentries.2 getdents.2
 MLINKS+=getfh.2 lgetfh.2
diff --git a/lib/libc/sys/Symbol.map b/lib/libc/sys/Symbol.map
index 095751a..f1c1567 100644
--- a/lib/libc/sys/Symbol.map
+++ b/lib/libc/sys/Symbol.map
@@ -365,6 +365,9 @@ FBSD_1.2 {
 	cap_getmode;
 	cap_new;
 	cap_getrights;
+	ffclock_getcounter;
+	ffclock_getestimate;
+	ffclock_setestimate;
 	getloginclass;
 	pdfork;
 	pdgetpid;
@@ -378,6 +381,10 @@ FBSD_1.2 {
 	setloginclass;
 };
 
+FBSD_1.3 {
+	posix_fadvise;
+};
+
 FBSDprivate_1.0 {
 	___acl_aclcheck_fd;
 	__sys___acl_aclcheck_fd;
diff --git a/lib/libc/sys/ffclock.2 b/lib/libc/sys/ffclock.2
new file mode 100644
index 0000000..0e8f09b
--- /dev/null
+++ b/lib/libc/sys/ffclock.2
@@ -0,0 +1,177 @@
+.\" Copyright (c) 2011 The University of Melbourne
+.\" All rights reserved.
+.\"
+.\" This documentation was written by Julien Ridoux at the University of
+.\" Melbourne under sponsorship from the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd November 21, 2011
+.Dt FFCLOCK 2
+.Os
+.Sh NAME
+.Nm ffclock_getcounter ,
+.Nm ffclock_getestimate ,
+.Nm ffclock_setestimate
+.Nd Retrieve feed-forward counter, get and set feed-forward clock estimates.
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In sys/timeffc.h
+.Ft int
+.Fn ffclock_getcounter "ffcounter *ffcount"
+.Ft int
+.Fn ffclock_getestimate "struct ffclock_estimate *cest"
+.Ft int
+.Fn ffclock_setestimate "struct ffclock_estimate *cest"
+.Sh DESCRIPTION
+The ffclock is an alternative method to synchronise the system clock.
+The ffclock implements a feed-forward paradigm and decouples the timestamping
+and timekeeping kernel functions.
+This ensures that past clock errors do not affect current timekeeping, an
+approach radically different from the feedback alternative implemented by the
+ntpd daemon when adjusting the system clock.
+The feed-forward approach has demonstrated better performance and higher
+robustness than a feedback approach when synchronising over the network.
+.Pp
+In the feed-forward context, a
+.Em timestamp
+is a cumulative value of the ticks of the timecounter, which can be converted
+into seconds by using the feed-forward
+.Em clock estimates.
+.Pp
+The
+.Fn ffclock_getcounter
+system call allows the calling process to retrieve the current value of the
+feed-forward counter maintained by the kernel.
+.Pp
+The
+.Fn ffclock_getestimate
+and
+.Fn ffclock_setestimate
+system calls allow the caller to get and set the kernel's feed-forward clock
+parameter estimates respectively.
+The
+.Fn ffclock_setestimate
+system call should be invoked by a single instance of a feed-forward
+synchronisation daemon.
+The
+.Fn ffclock_getestimate
+system call can be called by any process to retrieve the feed-forward clock
+estimates.
+.Pp
+The feed-forward approach does not require that the clock estimates be retrieved
+every time a timestamp is to be converted into seconds.
+The number of system calls can therefore be greatly reduced if the calling
+process retrieves the clock estimates from the clock synchronisation daemon
+instead.
+The
+.Fn ffclock_getestimate
+must be used when the feed-forward synchronisation daemon is not running
+.Po see
+.Sx USAGE
+below
+.Pc .
+.Pp
+The clock parameter estimates structure pointed to by
+.Fa cest
+is defined in
+.In sys/timeffc.h
+as:
+.Bd -literal
+struct ffclock_estimate {
+	struct bintime update_time;    /* Time of last estimates update. */
+	ffcounter      update_ffcount; /* Counter value at last update. */
+	ffcounter      leapsec_next;   /* Counter value of next leap second. */
+	uint64_t       period;         /* Estimate of counter period. */
+	uint32_t       errb_abs;       /* Bound on absolute clock error [ns]. */
+	uint32_t       errb_rate;      /* Bound on counter rate error [ps/s]. */
+	uint32_t       status;         /* Clock status. */
+	int16_t        leapsec_total;  /* All leap seconds seen so far. */
+	int8_t         leapsec;        /* Next leap second (in {-1,0,1}). */
+};
+.Ed
+.Pp
+Only the super-user may set the feed-forward clock estimates.
+.Sh RETURN VALUES
+.Rv -std
+.Sh ERRORS
+The following error codes may be set in
+.Va errno :
+.Bl -tag -width Er
+.It Bq Er EFAULT
+The
+.Fa ffcount
+or
+.Fa cest
+pointer referenced invalid memory.
+.It Bq Er EPERM
+A user other than the super-user attempted to set the feed-forward clock
+parameter estimates.
+.El
+.Sh USAGE
+The feed-forward paradigm enables the definition of specialised clock functions.
+.Pp
+In its simplest form,
+.Fn ffclock_getcounter
+can be used to establish strict order between events or to measure small time
+intervals very accurately with a minimum performance cost.
+.Pp
+Different methods exist to access absolute time
+.Po or
+.Qq wall-clock time
+.Pc tracked by the ffclock.
+The simplest method uses the ffclock sysctl interface
+.Va kern.ffclock
+to make the system clock return the ffclock time.
+The
+.Xr clock_gettime 2
+system call can then be used to retrieve the current time seen by the
+feed-forward clock.
+Note that this setting affects the entire system and that a feed-forward
+synchronisation daemon should be running.
+.Pp
+A less automated method consists of retrieving the feed-forward counter
+timestamp from the kernel and using the feed-forward clock parameter estimates
+to convert the timestamp into seconds.
+The feed-forward clock parameter estimates can be retrieved from the kernel or
+from the synchronisation daemon directly (preferred).
+This method allows converting timestamps using different clock models as needed
+by the application, while collecting meaningful upper bounds on current clock
+error.
+.Sh SEE ALSO
+.Xr date 1 ,
+.Xr adjtime 2 ,
+.Xr clock_gettime 2 ,
+.Xr ctime 3
+.Sh HISTORY
+Feed-forward clock support first appeared in
+.Fx 10.0 .
+.Sh AUTHORS
+.An -nosplit
+The feed-forward clock support was written by
+.An Julien Ridoux Aq jridoux@unimelb.edu.au
+in collaboration with
+.An Darryl Veitch Aq dveitch@unimelb.edu.au
+at the University of Melbourne under sponsorship from the FreeBSD Foundation.
diff --git a/lib/libc/sys/flock.2 b/lib/libc/sys/flock.2
index 93f1f66..6106bf4 100644
--- a/lib/libc/sys/flock.2
+++ b/lib/libc/sys/flock.2
@@ -28,7 +28,7 @@
 .\"     @(#)flock.2	8.2 (Berkeley) 12/11/93
 .\" $FreeBSD$
 .\"
-.Dd January 22, 2008
+.Dd November 9, 2011
 .Dt FLOCK 2
 .Os
 .Sh NAME
@@ -154,6 +154,8 @@ refers to an object other than a file.
 The argument
 .Fa fd
 refers to an object that does not support file locking.
+.It Bq Er ENOLCK
+A lock was requested, but no locks are available.
 .El
 .Sh SEE ALSO
 .Xr close 2 ,
diff --git a/lib/libc/sys/getsockopt.2 b/lib/libc/sys/getsockopt.2
index 7370b6b..edd92c3 100644
--- a/lib/libc/sys/getsockopt.2
+++ b/lib/libc/sys/getsockopt.2
@@ -28,7 +28,7 @@
 .\"     @(#)getsockopt.2	8.4 (Berkeley) 5/2/95
 .\" $FreeBSD$
 .\"
-.Dd June 13, 2008
+.Dd November 21, 2011
 .Dt GETSOCKOPT 2
 .Os
 .Sh NAME
@@ -533,7 +533,9 @@ on a non-listening socket was attempted.
 .Sh HISTORY
 The
 .Fn getsockopt
-system call appeared in
+and
+.Fn setsockopt
+system calls appeared in
 .Bx 4.2 .
 .Sh BUGS
 Several of the socket options should be handled at lower levels of the system.
diff --git a/lib/libc/sys/madvise.2 b/lib/libc/sys/madvise.2
index 48f0e5a..b5ea6b2 100644
--- a/lib/libc/sys/madvise.2
+++ b/lib/libc/sys/madvise.2
@@ -169,7 +169,8 @@ was specified and the process does not have superuser privileges.
 .Xr mincore 2 ,
 .Xr mprotect 2 ,
 .Xr msync 2 ,
-.Xr munmap 2
+.Xr munmap 2 ,
+.Xr posix_fadvise 2
 .Sh STANDARDS
 The
 .Fn posix_madvise
diff --git a/lib/libc/sys/posix_fadvise.2 b/lib/libc/sys/posix_fadvise.2
new file mode 100644
index 0000000..bdf321f
--- /dev/null
+++ b/lib/libc/sys/posix_fadvise.2
@@ -0,0 +1,139 @@
+.\" Copyright (c) 1991, 1993
+.\"	The Regents of the University of California.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 4. Neither the name of the University nor the names of its contributors
+.\"    may be used to endorse or promote products derived from this software
+.\"    without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"	@(#)madvise.2	8.1 (Berkeley) 6/9/93
+.\" $FreeBSD$
+.\"
+.Dd October 26, 2011
+.Dt POSIX_FADVISE 2
+.Os
+.Sh NAME
+.Nm posix_fadvise
+.Nd give advice about use of file data
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In fcntl.h
+.Ft int
+.Fn posix_fadvise "int fd" "off_t offset" "off_t len" "int advice"
+.Sh DESCRIPTION
+The
+.Fn posix_fadvise
+system call
+allows a process to describe to the system its data access behavior for an
+open file descriptor
+.Fa fd .
+The advice covers the data starting at offset
+.Fa offset
+and continuing for
+.Fa len
+bytes.
+If
+.Fa len
+is zero,
+all data from
+.Fa offset
+to the end of the file is covered.
+.Pp
+The behavior is specified by the
+.Fa advice
+parameter and may be one of:
+.Bl -tag -width POSIX_FADV_SEQUENTIAL
+.It Dv POSIX_FADV_NORMAL
+Tells the system to revert to the default data access behavior.
+.It Dv POSIX_FADV_RANDOM
+Is a hint that file data will be accessed randomly,
+and prefetching is likely not advantageous.
+.It Dv POSIX_FADV_SEQUENTIAL
+Tells the system that file data will be accessed sequentially.
+This currently does nothing as the default behavior uses heuristics to
+detect sequential behavior.
+.It Dv POSIX_FADV_WILLNEED
+Tells the system that the specified data will be accessed in the near future.
+The system may initiate an asychronous read of the data if it is not already
+present in memory.
+.It Dv POSIX_FADV_DONTNEED
+Tells the system that the specified data will not be accessed in the near
+future.
+The system may decrease the in-memory priority of clean data within the
+specified range and future access to this data may require a read operation.
+.It Dv POSIX_FADV_NOREUSE
+Tells the system that the specified data will only be accessed once and
+then not reused.
+Accesses to data within the specified range are treated as if the file
+descriptor has the
+.Dv O_DIRECT
+flag enabled.
+.El
+.Pp
+.Sh RETURN VALUES
+.Rv -std posix_fadvise
+.Sh ERRORS
+The
+.Fn posix_fadvise
+system call will fail if:
+.Bl -tag -width Er
+.It Bq Er EBADF
+The
+.Fa fd
+argument is not a valid file descriptor.
+.It Bq Er EINVAL
+The
+.Fa advice
+argument is not valid.
+.It Bq Er EINVAL
+The
+.Fa offset
+or
+.Fa len
+arguments are negative,
+or
+.Fa offset
++
+.Fa len
+is greater than the maximum file size.
+.It Bq Er ENODEV
+The
+.Fa fd
+argument does not refer to a regular file.
+.It Bq Er ESPIPE
+The
+.Fa fd
+argument is associated with a pipe or FIFO.
+.El
+.Sh SEE ALSO
+.Xr madvise 2
+.Sh STANDARDS
+The
+.Fn posix_fadvise
+interface conforms to
+.St -p1003.1-2001 .
+.Sh HISTORY
+The
+.Fn posix_fadvise
+system call first appeared in
+.Fx 10.0 .
diff --git a/lib/libc/sys/stat.2 b/lib/libc/sys/stat.2
index 7f6d533..b6c03ab 100644
--- a/lib/libc/sys/stat.2
+++ b/lib/libc/sys/stat.2
@@ -28,7 +28,7 @@
 .\"     @(#)stat.2	8.4 (Berkeley) 5/1/95
 .\" $FreeBSD$
 .\"
-.Dd March 28, 2010
+.Dd November 17, 2011
 .Dt STAT 2
 .Os
 .Sh NAME
@@ -411,7 +411,7 @@ and
 system calls are expected to conform to
 .St -p1003.1-90 .
 The
-.Fn fchownat
+.Fn fstatat
 system call follows The Open Group Extended API Set 2 specification.
 .Sh HISTORY
 The
diff --git a/lib/libkiconv/xlat16_iconv.c b/lib/libkiconv/xlat16_iconv.c
index ce06324..dc9dc86 100644
--- a/lib/libkiconv/xlat16_iconv.c
+++ b/lib/libkiconv/xlat16_iconv.c
@@ -74,6 +74,18 @@ kiconv_add_xlat16_cspair(const char *tocode, const char *fromcode, int flag)
 	struct xlat16_table xt;
 	void *data;
 	char *p;
+	const char unicode[] = ENCODING_UNICODE;
+
+	if ((flag & KICONV_WCTYPE) == 0 &&
+	    strcmp(unicode, tocode) != 0 &&
+	    strcmp(unicode, fromcode) != 0 &&
+	    kiconv_lookupconv(unicode) == 0) {
+		error = kiconv_add_xlat16_cspair(unicode, fromcode, flag);
+		if (error)
+			return (-1);
+		error = kiconv_add_xlat16_cspair(tocode, unicode, flag);
+		return (error);
+	}
 
 	if (kiconv_lookupcs(tocode, fromcode) == 0)
 		return (0);
diff --git a/lib/libpam/Makefile.inc b/lib/libpam/Makefile.inc
index 1c1e513..923d7bd 100644
--- a/lib/libpam/Makefile.inc
+++ b/lib/libpam/Makefile.inc
@@ -31,4 +31,6 @@ DEBUG_FLAGS+=	-DDEBUG
 SHLIB_MAJOR=	5
 PAM_MOD_DIR=	${LIBDIR}
 
+STATIC_CFLAGS+=	-DOPENPAM_STATIC_MODULES
+
 .include "../Makefile.inc"
diff --git a/lib/libpam/modules/Makefile.inc b/lib/libpam/modules/Makefile.inc
index feb5da0..00bfbf9 100644
--- a/lib/libpam/modules/Makefile.inc
+++ b/lib/libpam/modules/Makefile.inc
@@ -5,20 +5,8 @@ PAMDIR=		${.CURDIR}/../../../../contrib/openpam
 NO_INSTALLLIB=
 NO_PROFILE=
 
-CFLAGS+= -I${PAMDIR}/include -I${.CURDIR}/../../libpam
+CFLAGS+=	-I${PAMDIR}/include -I${.CURDIR}/../../libpam
 
-# This is nasty.
-# For the static case, libpam.a depends on the modules.
-# For the dynamic case, the modules depend on libpam.so.N
-.if defined(_NO_LIBPAM_SO_YET)
-NO_PIC=
-.else
 SHLIB_NAME?=	${LIB}.so.${SHLIB_MAJOR}
-DPADD+=		${LIBPAM}
-LDADD+=		-lpam
-.endif
-
-.c.o:
-	${CC} ${CFLAGS} -DOPENPAM_STATIC_MODULES -c ${.IMPSRC}
 
 .include "../Makefile.inc"
diff --git a/lib/libpam/modules/pam_lastlog/pam_lastlog.c b/lib/libpam/modules/pam_lastlog/pam_lastlog.c
index 14e47b80..72bb942 100644
--- a/lib/libpam/modules/pam_lastlog/pam_lastlog.c
+++ b/lib/libpam/modules/pam_lastlog/pam_lastlog.c
@@ -47,6 +47,8 @@ __FBSDID("$FreeBSD$");
 #define _BSD_SOURCE
 
 #include <sys/time.h>
+
+#include <paths.h>
 #include <pwd.h>
 #include <stdlib.h>
 #include <string.h>
@@ -96,6 +98,9 @@ pam_sm_open_session(pam_handle_t *pamh, int flags,
 		pam_err = PAM_SERVICE_ERR;
 		goto err;
 	}
+	/* Strip /dev/ component. */
+	if (strncmp(tty, _PATH_DEV, sizeof(_PATH_DEV) - 1) == 0)
+		tty = (const char *)tty + sizeof(_PATH_DEV) - 1;
 
 	if ((flags & PAM_SILENT) == 0) {
 		if (setutxdb(UTXDB_LASTLOGIN, NULL) != 0) {
diff --git a/lib/libpam/modules/pam_ssh/pam_ssh.c b/lib/libpam/modules/pam_ssh/pam_ssh.c
index bc5f522..ab4990b 100644
--- a/lib/libpam/modules/pam_ssh/pam_ssh.c
+++ b/lib/libpam/modules/pam_ssh/pam_ssh.c
@@ -93,7 +93,8 @@ static char *const pam_ssh_agent_envp[] = { NULL };
  * struct pam_ssh_key containing the key and its comment.
  */
 static struct pam_ssh_key *
-pam_ssh_load_key(const char *dir, const char *kfn, const char *passphrase)
+pam_ssh_load_key(const char *dir, const char *kfn, const char *passphrase,
+    int nullok)
 {
 	struct pam_ssh_key *psk;
 	char fn[PATH_MAX];
@@ -103,7 +104,21 @@ pam_ssh_load_key(const char *dir, const char *kfn, const char *passphrase)
 	if (snprintf(fn, sizeof(fn), "%s/%s", dir, kfn) > (int)sizeof(fn))
 		return (NULL);
 	comment = NULL;
-	key = key_load_private(fn, passphrase, &comment);
+	/*
+	 * If the key is unencrypted, OpenSSL ignores the passphrase, so
+	 * it will seem like the user typed in the right one.  This allows
+	 * a user to circumvent nullok by providing a dummy passphrase.
+	 * Verify that the key really *is* encrypted by trying to load it
+	 * with an empty passphrase, and if the key is not encrypted,
+	 * accept only an empty passphrase.
+	 */
+	key = key_load_private(fn, NULL, &comment);
+	if (key != NULL && !(*passphrase == '\0' && nullok)) {
+		key_free(key);
+		return (NULL);
+	}
+	if (key == NULL)
+		key = key_load_private(fn, passphrase, &comment);
 	if (key == NULL) {
 		openpam_log(PAM_LOG_DEBUG, "failed to load key from %s", fn);
 		return (NULL);
@@ -170,9 +185,6 @@ pam_sm_authenticate(pam_handle_t *pamh, int flags __unused,
 	if (pam_err != PAM_SUCCESS)
 		return (pam_err);
 
-	if (*passphrase == '\0' && !nullok)
-		goto skip_keys;
-
 	/* switch to user credentials */
 	pam_err = openpam_borrow_cred(pamh, pwd);
 	if (pam_err != PAM_SUCCESS)
@@ -180,7 +192,7 @@ pam_sm_authenticate(pam_handle_t *pamh, int flags __unused,
 
 	/* try to load keys from all keyfiles we know of */
 	for (kfn = pam_ssh_keyfiles; *kfn != NULL; ++kfn) {
-		psk = pam_ssh_load_key(pwd->pw_dir, *kfn, passphrase);
+		psk = pam_ssh_load_key(pwd->pw_dir, *kfn, passphrase, nullok);
 		if (psk != NULL) {
 			pam_set_data(pamh, *kfn, psk, pam_ssh_free_key);
 			++nkeys;
@@ -190,7 +202,6 @@ pam_sm_authenticate(pam_handle_t *pamh, int flags __unused,
 	/* switch back to arbitrator credentials */
 	openpam_restore_cred(pamh);
 
- skip_keys:
 	/*
 	 * If we tried an old token and didn't get anything, and
 	 * try_first_pass was specified, try again after prompting the
diff --git a/lib/libpam/modules/pam_unix/pam_unix.8 b/lib/libpam/modules/pam_unix/pam_unix.8
index 82354d5..1345e82 100644
--- a/lib/libpam/modules/pam_unix/pam_unix.8
+++ b/lib/libpam/modules/pam_unix/pam_unix.8
@@ -199,3 +199,9 @@ password database.
 .Xr pam 8 ,
 .Xr pw 8 ,
 .Xr yp 8
+.Sh BUGS
+The
+.Nm
+module ignores the
+.Dv PAM_CHANGE_EXPIRED_AUTHTOK
+flag.
diff --git a/lib/libpam/modules/pam_unix/pam_unix.c b/lib/libpam/modules/pam_unix/pam_unix.c
index 5c6c8f7..415004a 100644
--- a/lib/libpam/modules/pam_unix/pam_unix.c
+++ b/lib/libpam/modules/pam_unix/pam_unix.c
@@ -81,8 +81,6 @@ static char password_hash[] =		PASSWORD_HASH;
 #define PAM_OPT_LOCAL_PASS	"local_pass"
 #define PAM_OPT_NIS_PASS	"nis_pass"
 
-char *tempname = NULL;
-
 /*
  * authentication management
  */
diff --git a/lib/librt/timer.c b/lib/librt/timer.c
index f02e761..90269c2 100644
--- a/lib/librt/timer.c
+++ b/lib/librt/timer.c
@@ -102,12 +102,14 @@ __timer_create(clockid_t clockid, struct sigevent *evp, timer_t *timerid)
 	}
 
 	if (__sigev_check_init()) {
+		free(timer);
 		errno = EINVAL;
 		return (-1);
 	}
 
 	sn = __sigev_alloc(SI_TIMER, evp, NULL, 0);
 	if (sn == NULL) {
+		free(timer);
 		errno = EAGAIN;
 		return (-1);
 	}
diff --git a/lib/libthr/thread/thr_umtx.c b/lib/libthr/thread/thr_umtx.c
index 33c3637..e962378 100644
--- a/lib/libthr/thread/thr_umtx.c
+++ b/lib/libthr/thread/thr_umtx.c
@@ -231,7 +231,7 @@ _thr_ucond_init(struct ucond *cv)
 
 int
 _thr_ucond_wait(struct ucond *cv, struct umutex *m,
-	const struct timespec *timeout, int check_unparking)
+	const struct timespec *timeout, int flags)
 {
 	if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 &&
 	    timeout->tv_nsec <= 0))) {
@@ -239,8 +239,7 @@ _thr_ucond_wait(struct ucond *cv, struct umutex *m,
 		_thr_umutex_unlock(m, TID(curthread));
                 return (ETIMEDOUT);
 	}
-	return _umtx_op_err(cv, UMTX_OP_CV_WAIT,
-		     check_unparking ? UMTX_CHECK_UNPARKING : 0, 
+	return _umtx_op_err(cv, UMTX_OP_CV_WAIT, flags,
 		     m, __DECONST(void*, timeout));
 }
  
diff --git a/lib/libusb/Makefile b/lib/libusb/Makefile
index b76618a..8d78619 100644
--- a/lib/libusb/Makefile
+++ b/lib/libusb/Makefile
@@ -74,6 +74,10 @@ MLINKS += libusb.3 libusb_get_config_descriptor.3
 MLINKS += libusb.3 libusb_get_config_descriptor_by_value.3
 MLINKS += libusb.3 libusb_free_config_descriptor.3
 MLINKS += libusb.3 libusb_get_string_descriptor_ascii.3
+MLINKS += libusb.3 libusb_parse_ss_endpoint_comp.3
+MLINKS += libusb.3 libusb_free_ss_endpoint_comp.3
+MLINKS += libusb.3 libusb_parse_bos_descriptor.3
+MLINKS += libusb.3 libusb_free_bos_descriptor.3
 MLINKS += libusb.3 libusb_alloc_transfer.3
 MLINKS += libusb.3 libusb_free_transfer.3
 MLINKS += libusb.3 libusb_submit_transfer.3
diff --git a/lib/libusb/libusb.3 b/lib/libusb/libusb.3
index 7255f2b..4e4dd10 100644
--- a/lib/libusb/libusb.3
+++ b/lib/libusb/libusb.3
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 16, 2011
+.Dd November 9, 2011
 .Dt LIBUSB 3
 .Os
 .Sh NAME
@@ -316,6 +316,40 @@ Retrieve a string descriptor in C style ASCII.
 Returns the positive number of bytes in the resulting ASCII string
 on success and a LIBUSB_ERROR code on failure.
 .Pp
+.Ft int
+.Fn libusb_parse_ss_endpoint_comp "const void *buf" "int len" "libusb_ss_endpoint_companion_descriptor **ep_comp"
+This function parses the USB 3.0 endpoint companion descriptor in host endian format pointed to by
+.Fa buf
+and having a length of
+.Fa len.
+Typically these arguments are the extra and extra_length fields of the
+endpoint descriptor.
+On success the pointer to resulting descriptor is stored at the location given by
+.Fa ep_comp.
+Returns zero on success and a LIBUSB_ERROR code on failure.
+On success the parsed USB 3.0 endpoint companion descriptor must be
+freed using the libusb_free_ss_endpoint_comp function.
+.Pp
+.Ft void
+.Fn libusb_free_ss_endpoint_comp "libusb_ss_endpoint_companion_descriptor *ep_comp"
+This function is NULL safe and frees a parsed USB 3.0 endpoint companion descriptor.
+.Pp
+.Ft int
+.Fn libusb_parse_bos_descriptor "const void *buf" "int len" "libusb_bos_descriptor **bos"
+This function parses a Binary Object Store, BOS, descriptor into host endian format pointed to by
+.Fa buf
+and having a length of
+.Fa len.
+On success the pointer to resulting descriptor is stored at the location given by
+.Fa bos.
+Returns zero on success and a LIBUSB_ERROR code on failure.
+On success the parsed BOS descriptor must be freed using the
+libusb_free_bos_descriptor function.
+.Pp
+.Ft void
+.Fn libusb_free_bos_descriptor "libusb_bos_descriptor *bos"
+This function is NULL safe and frees a parsed BOS descriptor.
+.Pp
 .Sh USB ASYNCHRONOUS I/O
 .Pp
 .Ft struct libusb_transfer *
diff --git a/lib/libusb/libusb.h b/lib/libusb/libusb.h
index 63e3ebc..5a4d2df 100644
--- a/lib/libusb/libusb.h
+++ b/lib/libusb/libusb.h
@@ -63,6 +63,16 @@ enum libusb_descriptor_type {
 	LIBUSB_DT_REPORT = 0x22,
 	LIBUSB_DT_PHYSICAL = 0x23,
 	LIBUSB_DT_HUB = 0x29,
+	LIBUSB_DT_BOS = 0x0f,
+	LIBUSB_DT_DEVICE_CAPABILITY = 0x10,
+	LIBUSB_DT_SS_ENDPOINT_COMPANION = 0x30,
+};
+
+enum libusb_device_capability_type {
+	LIBUSB_WIRELESS_USB_DEVICE_CAPABILITY = 0x1,
+	LIBUSB_USB_2_0_EXTENSION_DEVICE_CAPABILITY = 0x2,
+	LIBUSB_SS_USB_DEVICE_CAPABILITY = 0x3,
+	LIBUSB_CONTAINER_ID_DEVICE_CAPABILITY = 0x4,
 };
 
 #define	LIBUSB_DT_DEVICE_SIZE		18
@@ -71,6 +81,10 @@ enum libusb_descriptor_type {
 #define	LIBUSB_DT_ENDPOINT_SIZE		7
 #define	LIBUSB_DT_ENDPOINT_AUDIO_SIZE	9
 #define	LIBUSB_DT_HUB_NONVAR_SIZE	7
+#define	LIBUSB_DT_SS_ENDPOINT_COMPANION_SIZE	6
+#define	LIBUSB_DT_BOS_SIZE		5
+#define	LIBUSB_USB_2_0_EXTENSION_DEVICE_CAPABILITY_SIZE	7
+#define	LIBUSB_SS_USB_DEVICE_CAPABILITY_SIZE	10
 
 #define	LIBUSB_ENDPOINT_ADDRESS_MASK	0x0f
 #define	LIBUSB_ENDPOINT_DIR_MASK	0x80
@@ -230,6 +244,14 @@ typedef struct libusb_endpoint_descriptor {
 	int	extra_length;
 }	libusb_endpoint_descriptor __aligned(sizeof(void *));
 
+typedef struct libusb_ss_endpoint_companion_descriptor {
+	uint8_t bLength;
+	uint8_t bDescriptorType;
+	uint8_t bMaxBurst;
+	uint8_t bmAttributes;
+	uint16_t wBytesPerInterval;
+}	libusb_ss_endpoint_companion_descriptor __aligned(sizeof(void *));
+
 typedef struct libusb_interface_descriptor {
 	uint8_t	bLength;
 	uint8_t	bDescriptorType;
@@ -264,6 +286,39 @@ typedef struct libusb_config_descriptor {
 	int	extra_length;
 }	libusb_config_descriptor __aligned(sizeof(void *));
 
+typedef struct libusb_usb_2_0_device_capability_descriptor {
+	uint8_t bLength;
+	uint8_t bDescriptorType;
+	uint8_t bDevCapabilityType;
+	uint32_t bmAttributes;
+#define LIBUSB_USB_2_0_CAPABILITY_LPM_SUPPORT  (1 << 1)
+}	libusb_usb_2_0_device_capability_descriptor __aligned(sizeof(void *));
+
+typedef struct libusb_ss_usb_device_capability_descriptor {
+	uint8_t bLength;
+	uint8_t bDescriptorType;
+	uint8_t bDevCapabilityType;
+	uint8_t bmAttributes;
+#define LIBUSB_SS_USB_CAPABILITY_LPM_SUPPORT   (1 << 1)
+	uint16_t wSpeedSupported;
+#define LIBUSB_CAPABILITY_LOW_SPEED_OPERATION  (1)
+#define LIBUSB_CAPABILITY_FULL_SPEED_OPERATION (1 << 1)
+#define LIBUSB_CAPABILITY_HIGH_SPEED_OPERATION (1 << 2)
+#define LIBUSB_CAPABILITY_5GBPS_OPERATION      (1 << 3)
+	uint8_t bFunctionalitySupport;
+	uint8_t bU1DevExitLat;
+	uint16_t wU2DevExitLat;
+}	libusb_ss_usb_device_capability_descriptor __aligned(sizeof(void *));
+
+typedef struct libusb_bos_descriptor {
+	uint8_t bLength;
+	uint8_t bDescriptorType;
+	uint16_t wTotalLength;
+	uint8_t bNumDeviceCapabilities;
+	struct libusb_usb_2_0_device_capability_descriptor *usb_2_0_ext_cap;
+	struct libusb_ss_usb_device_capability_descriptor *ss_usb_cap;
+}	libusb_bos_descriptor __aligned(sizeof(void *));
+
 typedef struct libusb_control_setup {
 	uint8_t	bmRequestType;
 	uint8_t	bRequest;
@@ -345,6 +400,10 @@ int	libusb_get_config_descriptor_by_value(libusb_device * dev, uint8_t bConfigur
 void	libusb_free_config_descriptor(struct libusb_config_descriptor *config);
 int	libusb_get_string_descriptor_ascii(libusb_device_handle * devh, uint8_t desc_index, uint8_t *data, int length);
 int	libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type, uint8_t desc_index, uint8_t *data, int length);
+int	libusb_parse_ss_endpoint_comp(const void *buf, int len, struct libusb_ss_endpoint_companion_descriptor **ep_comp);
+void	libusb_free_ss_endpoint_comp(struct libusb_ss_endpoint_companion_descriptor *ep_comp);
+int	libusb_parse_bos_descriptor(const void *buf, int len, struct libusb_bos_descriptor **bos);
+void	libusb_free_bos_descriptor(struct libusb_bos_descriptor *bos);
 
 /* Asynchronous device I/O */
 
diff --git a/lib/libusb/libusb10_desc.c b/lib/libusb/libusb10_desc.c
index 3448679..6d5822e 100644
--- a/lib/libusb/libusb10_desc.c
+++ b/lib/libusb/libusb10_desc.c
@@ -298,7 +298,7 @@ libusb_get_string_descriptor_ascii(libusb_device_handle *pdev,
     uint8_t desc_index, unsigned char *data, int length)
 {
 	if (pdev == NULL || data == NULL || length < 1)
-		return (LIBUSB20_ERROR_INVALID_PARAM);
+		return (LIBUSB_ERROR_INVALID_PARAM);
 
 	if (length > 65535)
 		length = 65535;
@@ -318,7 +318,7 @@ libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type,
     uint8_t desc_index, uint8_t *data, int length)
 {
 	if (devh == NULL || data == NULL || length < 1)
-		return (LIBUSB20_ERROR_INVALID_PARAM);
+		return (LIBUSB_ERROR_INVALID_PARAM);
 
 	if (length > 65535)
 		length = 65535;
@@ -327,3 +327,172 @@ libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type,
 	    LIBUSB_REQUEST_GET_DESCRIPTOR, (desc_type << 8) | desc_index, 0, data,
 	    length, 1000));
 }
+
+int
+libusb_parse_ss_endpoint_comp(const void *buf, int len,
+    struct libusb_ss_endpoint_companion_descriptor **ep_comp)
+{
+	if (buf == NULL || ep_comp == NULL || len < 1)
+		return (LIBUSB_ERROR_INVALID_PARAM);
+
+	if (len > 65535)
+		len = 65535;
+
+	*ep_comp = NULL;
+
+	while (len != 0) {
+		uint8_t dlen;
+		uint8_t dtype;
+
+		dlen = ((const uint8_t *)buf)[0];
+		dtype = ((const uint8_t *)buf)[1];
+
+		if (dlen < 2 || dlen > len)
+			break;
+
+		if (dlen >= LIBUSB_DT_SS_ENDPOINT_COMPANION_SIZE &&
+		    dtype == LIBUSB_DT_SS_ENDPOINT_COMPANION) {
+			struct libusb_ss_endpoint_companion_descriptor *ptr;
+
+			ptr = malloc(sizeof(*ptr));
+			if (ptr == NULL)
+				return (LIBUSB_ERROR_NO_MEM);
+
+			ptr->bLength = LIBUSB_DT_SS_ENDPOINT_COMPANION_SIZE;
+			ptr->bDescriptorType = dtype;
+			ptr->bMaxBurst = ((const uint8_t *)buf)[2];
+			ptr->bmAttributes = ((const uint8_t *)buf)[3];
+			ptr->wBytesPerInterval = ((const uint8_t *)buf)[4] |
+			    (((const uint8_t *)buf)[5] << 8);
+
+			*ep_comp = ptr;
+
+			return (0);	/* success */
+		}
+
+		buf = ((const uint8_t *)buf) + dlen;
+		len -= dlen;
+	}
+	return (LIBUSB_ERROR_IO);
+}
+
+void
+libusb_free_ss_endpoint_comp(struct libusb_ss_endpoint_companion_descriptor *ep_comp)
+{
+	if (ep_comp == NULL)
+		return;
+
+	free(ep_comp);
+}
+
+int
+libusb_parse_bos_descriptor(const void *buf, int len,
+    struct libusb_bos_descriptor **bos)
+{
+	struct libusb_bos_descriptor *ptr;
+	struct libusb_usb_2_0_device_capability_descriptor *dcap_20;
+	struct libusb_ss_usb_device_capability_descriptor *ss_cap;
+
+	if (buf == NULL || bos == NULL || len < 1)
+		return (LIBUSB_ERROR_INVALID_PARAM);
+
+	if (len > 65535)
+		len = 65535;
+
+	*bos = ptr = NULL;
+
+	while (len != 0) {
+		uint8_t dlen;
+		uint8_t dtype;
+
+		dlen = ((const uint8_t *)buf)[0];
+		dtype = ((const uint8_t *)buf)[1];
+
+		if (dlen < 2 || dlen > len)
+			break;
+
+		if (dlen >= LIBUSB_DT_BOS_SIZE &&
+		    dtype == LIBUSB_DT_BOS) {
+
+			ptr = malloc(sizeof(*ptr) + sizeof(*dcap_20) +
+			    sizeof(*ss_cap));
+
+			if (ptr == NULL)
+				return (LIBUSB_ERROR_NO_MEM);
+
+			*bos = ptr;
+
+			ptr->bLength = LIBUSB_DT_BOS_SIZE;
+			ptr->bDescriptorType = dtype;
+			ptr->wTotalLength = ((const uint8_t *)buf)[2] |
+			    (((const uint8_t *)buf)[3] << 8);
+			ptr->bNumDeviceCapabilities = ((const uint8_t *)buf)[4];
+			ptr->usb_2_0_ext_cap = NULL;
+			ptr->ss_usb_cap = NULL;
+
+			dcap_20 = (void *)(ptr + 1);
+			ss_cap = (void *)(dcap_20 + 1);
+		}
+		if (dlen >= 3 &&
+		    ptr != NULL &&
+		    dtype == LIBUSB_DT_DEVICE_CAPABILITY) {
+			switch (((const uint8_t *)buf)[2]) {
+			case LIBUSB_USB_2_0_EXTENSION_DEVICE_CAPABILITY:
+				if (ptr->usb_2_0_ext_cap != NULL)
+					break;
+				if (dlen < LIBUSB_USB_2_0_EXTENSION_DEVICE_CAPABILITY_SIZE)
+					break;
+
+				ptr->usb_2_0_ext_cap = dcap_20;
+
+				dcap_20->bLength = LIBUSB_USB_2_0_EXTENSION_DEVICE_CAPABILITY_SIZE;
+				dcap_20->bDescriptorType = dtype;
+				dcap_20->bDevCapabilityType = ((const uint8_t *)buf)[2];
+				dcap_20->bmAttributes = ((const uint8_t *)buf)[3] |
+				    (((const uint8_t *)buf)[4] << 8) |
+				    (((const uint8_t *)buf)[5] << 16) |
+				    (((const uint8_t *)buf)[6] << 24);
+				break;
+
+			case LIBUSB_SS_USB_DEVICE_CAPABILITY:
+				if (ptr->ss_usb_cap != NULL)
+					break;
+				if (dlen < LIBUSB_SS_USB_DEVICE_CAPABILITY_SIZE)
+					break;
+
+				ptr->ss_usb_cap = ss_cap;
+
+				ss_cap->bLength = LIBUSB_SS_USB_DEVICE_CAPABILITY_SIZE;
+				ss_cap->bDescriptorType = dtype;
+				ss_cap->bDevCapabilityType = ((const uint8_t *)buf)[2];
+				ss_cap->bmAttributes = ((const uint8_t *)buf)[3];
+				ss_cap->wSpeedSupported = ((const uint8_t *)buf)[4] |
+				    (((const uint8_t *)buf)[5] << 8);
+				ss_cap->bFunctionalitySupport = ((const uint8_t *)buf)[6];
+				ss_cap->bU1DevExitLat = ((const uint8_t *)buf)[7];
+				ss_cap->wU2DevExitLat = ((const uint8_t *)buf)[8] |
+				    (((const uint8_t *)buf)[9] << 8);
+				break;
+
+			default:
+				break;
+			}
+		}
+
+		buf = ((const uint8_t *)buf) + dlen;
+		len -= dlen;
+	}
+	if (ptr != NULL)
+		return (0);		/* success */
+
+	return (LIBUSB_ERROR_IO);
+}
+
+void
+libusb_free_bos_descriptor(struct libusb_bos_descriptor *bos)
+{
+	if (bos == NULL)
+		return;
+
+	free(bos);
+}
diff --git a/lib/libusb/libusb20_desc.c b/lib/libusb/libusb20_desc.c
index 5ee61b2..0781067 100644
--- a/lib/libusb/libusb20_desc.c
+++ b/lib/libusb/libusb20_desc.c
@@ -41,6 +41,10 @@ LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_ENDPOINT_DESC);
 LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_INTERFACE_DESC);
 LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_CONFIG_DESC);
 LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_CONTROL_SETUP);
+LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_SS_ENDPT_COMP_DESC);
+LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_USB_20_DEVCAP_DESC);
+LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_SS_USB_DEVCAP_DESC);
+LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_BOS_DESCRIPTOR);
 
 /*------------------------------------------------------------------------*
  *	libusb20_parse_config_desc
diff --git a/lib/libusb/libusb20_desc.h b/lib/libusb/libusb20_desc.h
index e5e7c94..a069ee9 100644
--- a/lib/libusb/libusb20_desc.h
+++ b/lib/libusb/libusb20_desc.h
@@ -264,6 +264,43 @@ LIBUSB20_MAKE_STRUCT(LIBUSB20_CONFIG_DESC);
 
 LIBUSB20_MAKE_STRUCT(LIBUSB20_CONTROL_SETUP);
 
+#define	LIBUSB20_SS_ENDPT_COMP_DESC(m,n) \
+  m(n, UINT8_T,  bLength, ) \
+  m(n, UINT8_T,  bDescriptorType, ) \
+  m(n, UINT8_T,  bMaxBurst, ) \
+  m(n, UINT8_T,  bmAttributes, ) \
+  m(n, UINT16_T, wBytesPerInterval, ) \
+
+LIBUSB20_MAKE_STRUCT(LIBUSB20_SS_ENDPT_COMP_DESC);
+
+#define	LIBUSB20_USB_20_DEVCAP_DESC(m,n) \
+  m(n, UINT8_T,  bLength, ) \
+  m(n, UINT8_T,  bDescriptorType, ) \
+  m(n, UINT8_T,  bDevCapabilityType, ) \
+  m(n, UINT32_T, bmAttributes, ) \
+
+LIBUSB20_MAKE_STRUCT(LIBUSB20_USB_20_DEVCAP_DESC);
+
+#define	LIBUSB20_SS_USB_DEVCAP_DESC(m,n) \
+  m(n, UINT8_T,  bLength, ) \
+  m(n, UINT8_T,  bDescriptorType, ) \
+  m(n, UINT8_T,  bDevCapabilityType, ) \
+  m(n, UINT8_T,  bmAttributes, ) \
+  m(n, UINT16_T, wSpeedSupported, ) \
+  m(n, UINT8_T,  bFunctionalitySupport, ) \
+  m(n, UINT8_T,  bU1DevExitLat, ) \
+  m(n, UINT16_T, wU2DevExitLat, ) \
+
+LIBUSB20_MAKE_STRUCT(LIBUSB20_SS_USB_DEVCAP_DESC);
+
+#define	LIBUSB20_BOS_DESCRIPTOR(m,n) \
+  m(n, UINT8_T,  bLength, ) \
+  m(n, UINT8_T,  bDescriptorType, ) \
+  m(n, UINT16_T, wTotalLength, ) \
+  m(n, UINT8_T,  bNumDeviceCapabilities, ) \
+
+LIBUSB20_MAKE_STRUCT(LIBUSB20_BOS_DESCRIPTOR);
+
 /* standard USB stuff */
 
 /** \ingroup desc
@@ -333,6 +370,24 @@ enum libusb20_descriptor_type {
 
 	/** Hub descriptor */
 	LIBUSB20_DT_HUB = 0x29,
+
+	/** Binary Object Store, BOS */
+	LIBUSB20_DT_BOS = 0x0f,
+
+	/** Device Capability */
+	LIBUSB20_DT_DEVICE_CAPABILITY = 0x10,
+
+	/** SuperSpeed endpoint companion */
+	LIBUSB20_DT_SS_ENDPOINT_COMPANION = 0x30,
+};
+
+/** \ingroup desc
+ * Device capability types as defined by the USB specification. */
+enum libusb20_device_capability_type {
+	LIBUSB20_WIRELESS_USB_DEVICE_CAPABILITY = 0x1,
+	LIBUSB20_USB_2_0_EXTENSION_DEVICE_CAPABILITY = 0x2,
+	LIBUSB20_SS_USB_DEVICE_CAPABILITY = 0x3,
+	LIBUSB20_CONTAINER_ID_DEVICE_CAPABILITY = 0x4,
 };
 
 /* Descriptor sizes per descriptor type */
@@ -342,6 +397,10 @@ enum libusb20_descriptor_type {
 #define	LIBUSB20_DT_ENDPOINT_SIZE		7
 #define	LIBUSB20_DT_ENDPOINT_AUDIO_SIZE		9	/* Audio extension */
 #define	LIBUSB20_DT_HUB_NONVAR_SIZE		7
+#define	LIBUSB20_DT_SS_ENDPOINT_COMPANION_SIZE	6
+#define	LIBUSB20_DT_BOS_SIZE		5
+#define	LIBUSB20_USB_2_0_EXTENSION_DEVICE_CAPABILITY_SIZE	7
+#define	LIBUSB20_SS_USB_DEVICE_CAPABILITY_SIZE	10
 
 #define	LIBUSB20_ENDPOINT_ADDRESS_MASK	0x0f	/* in bEndpointAddress */
 #define	LIBUSB20_ENDPOINT_DIR_MASK	0x80
diff --git a/lib/msun/src/math.h b/lib/msun/src/math.h
index 69b138e..9389dda 100644
--- a/lib/msun/src/math.h
+++ b/lib/msun/src/math.h
@@ -395,32 +395,32 @@ float	significandf(float);
  * long double versions of ISO/POSIX math functions
  */
 #if __ISO_C_VISIBLE >= 1999
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	acoshl(long double);
 #endif
 long double	acosl(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	asinhl(long double);
 #endif
 long double	asinl(long double);
 long double	atan2l(long double, long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	atanhl(long double);
 #endif
 long double	atanl(long double);
 long double	cbrtl(long double);
 long double	ceill(long double);
 long double	copysignl(long double, long double) __pure2;
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	coshl(long double);
 #endif
 long double	cosl(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	erfcl(long double);
 long double	erfl(long double);
 #endif
 long double	exp2l(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	expl(long double);
 long double	expm1l(long double);
 #endif
@@ -435,18 +435,18 @@ long double	frexpl(long double value, int *); /* fundamentally !__pure2 */
 long double	hypotl(long double, long double);
 int		ilogbl(long double) __pure2;
 long double	ldexpl(long double, int);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	lgammal(long double);
 #endif
 long long	llrintl(long double);
 long long	llroundl(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	log10l(long double);
 long double	log1pl(long double);
 long double	log2l(long double);
 #endif
 long double	logbl(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	logl(long double);
 #endif
 long		lrintl(long double);
@@ -458,7 +458,7 @@ long double	nextafterl(long double, long double);
 double		nexttoward(double, long double);
 float		nexttowardf(float, long double);
 long double	nexttowardl(long double, long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	powl(long double, long double);
 #endif
 long double	remainderl(long double, long double);
@@ -467,16 +467,16 @@ long double	rintl(long double);
 long double	roundl(long double);
 long double	scalblnl(long double, long);
 long double	scalbnl(long double, int);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	sinhl(long double);
 #endif
 long double	sinl(long double);
 long double	sqrtl(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	tanhl(long double);
 #endif
 long double	tanl(long double);
-#if 0
+#if _DECLARE_C99_LDBL_MATH
 long double	tgammal(long double);
 #endif
 long double	truncl(long double);
diff --git a/libexec/rshd/rshd.c b/libexec/rshd/rshd.c
index ec17251..82af6a4 100644
--- a/libexec/rshd/rshd.c
+++ b/libexec/rshd/rshd.c
@@ -317,7 +317,7 @@ doit(struct sockaddr *fromp)
 	}
 
 	if ((pam_err = pam_set_item(pamh, PAM_RUSER, ruser)) != PAM_SUCCESS ||
-	    (pam_err = pam_set_item(pamh, PAM_RHOST, rhost) != PAM_SUCCESS)) {
+	    (pam_err = pam_set_item(pamh, PAM_RHOST, rhost)) != PAM_SUCCESS) {
 		syslog(LOG_ERR|LOG_AUTH, "pam_set_item(): %s",
 		    pam_strerror(pamh, pam_err));
 		rshd_errx(1, "Login incorrect.");
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 36102ba..0d4e972 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -1657,6 +1657,7 @@ load_object(const char *name, const Obj_Entry *refobj, int flags)
     }
     if (flags & RTLD_LO_NOLOAD) {
 	free(path);
+	close(fd);
 	return (NULL);
     }
 
diff --git a/release/doc/en_US.ISO8859-1/hardware/article.sgml b/release/doc/en_US.ISO8859-1/hardware/article.sgml
index 65e17ed..6de0f9f 100644
--- a/release/doc/en_US.ISO8859-1/hardware/article.sgml
+++ b/release/doc/en_US.ISO8859-1/hardware/article.sgml
@@ -30,6 +30,8 @@
       <year>2007</year>
       <year>2008</year>
       <year>2009</year>
+      <year>2010</year>
+      <year>2011</year>
       <holder role="mailto:doc@FreeBSD.org">The &os; Documentation Project</holder>
     </copyright>
 
@@ -798,8 +800,12 @@
 
       &hwlist.mly;
 
+      &hwlist.mps;
+
       &hwlist.mpt;
 
+      &hwlist.mvs;
+
       &hwlist.ncr;
 
       &hwlist.ncv;
@@ -948,6 +954,8 @@
 
       &hwlist.pcn;
 
+      &hwlist.qlxgb;
+
       &hwlist.re;
 
       &hwlist.rl;
@@ -956,6 +964,8 @@
 
       &hwlist.sf;
 
+      &hwlist.sfxge;
+
       &hwlist.sge;
 
       &hwlist.sis;
diff --git a/release/doc/share/misc/dev.archlist.txt b/release/doc/share/misc/dev.archlist.txt
index 5ca6feb..10b7853 100644
--- a/release/doc/share/misc/dev.archlist.txt
+++ b/release/doc/share/misc/dev.archlist.txt
@@ -100,6 +100,7 @@ ohci	i386,pc98,ia64,amd64,powerpc
 oltr	i386
 pcn	i386,pc98,ia64,amd64
 pst	i386
+qlxgb	amd64
 rc	i386
 ral	i386,amd64
 rue	i386,pc98,amd64
@@ -107,6 +108,7 @@ rum	i386,amd64
 run	i386,amd64
 safe	i386,pc98,amd64
 sbp	i386,sparc64,ia64,amd64
+sfgxe	amd64
 sn	i386,amd64
 snc	pc98
 snd_ad1816	i386,amd64
diff --git a/release/generate-release.sh b/release/generate-release.sh
index cdba026..6a95e36 100755
--- a/release/generate-release.sh
+++ b/release/generate-release.sh
@@ -14,6 +14,7 @@
 #  SVNROOT:    SVN URL to FreeBSD source repository (by default, 
 #   svn://svn.freebsd.org/base)
 #  MAKE_FLAGS: optional flags to pass to make (e.g. -j)
+#  RELSTRING:  optional base name for media images (e.g. FreeBSD-9.0-RC2-amd64)
 # 
 #  Note: Since this requires a chroot, release cross-builds will not work!
 #
@@ -65,3 +66,12 @@ chroot $2 make -C /usr/src $MAKE_FLAGS buildworld buildkernel
 chroot $2 make -C /usr/src/release release
 chroot $2 make -C /usr/src/release install DESTDIR=/R
 
+: ${RELSTRING=`chroot $2 uname -s`-`chroot $2 uname -r`-`chroot $2 uname -p`}
+
+cd $2/R
+for i in release.iso bootonly.iso memstick; do
+	mv $i $RELSTRING-$i
+done
+sha256 $RELSTRING-* > CHECKSUM.SHA256
+md5 $RELSTRING-* > CHECKSUM.MD5
+
diff --git a/release/ia64/mkisoimages.sh b/release/ia64/mkisoimages.sh
index f17185b..59fe025 100644
--- a/release/ia64/mkisoimages.sh
+++ b/release/ia64/mkisoimages.sh
@@ -64,6 +64,8 @@ if [ $bootable = yes ]; then
 	cp $BASE/boot/mfsroot.gz $MNT/boot
     fi
     cp $BASE/boot/support.4th $MNT/boot
+    cp $BASE/boot/check-password.4th $MNT/boot
+    cp $BASE/boot/screen.4th $MNT/boot
     mv $MNT/boot/loader.efi $MNT/efi/boot/bootia64.efi
     umount $MNT
     mdconfig -d -u $md
diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile
index 81d230d..9eea239 100644
--- a/rescue/rescue/Makefile
+++ b/rescue/rescue/Makefile
@@ -181,7 +181,7 @@ CRUNCH_BUILDOPTS_dhclient= -DRELEASE_CRUNCH -Dlint
 #
 CRUNCH_SRCDIRS+= usr.bin
 
-CRUNCH_PROGS_usr.bin= head mt sed tail tee
+CRUNCH_PROGS_usr.bin= head mt nc sed tail tee
 
 CRUNCH_PROGS_usr.bin+= gzip
 CRUNCH_ALIAS_gzip= gunzip gzcat zcat
diff --git a/sbin/atm/atmconfig/atmconfig_device.c b/sbin/atm/atmconfig/atmconfig_device.c
index 082494d..f2f0838 100644
--- a/sbin/atm/atmconfig/atmconfig_device.c
+++ b/sbin/atm/atmconfig/atmconfig_device.c
@@ -103,7 +103,7 @@ static struct atmhw_list atmhw_list;
 /*
  * Read ATM hardware table
  */
-const struct snmp_table atmhw_table = {
+static const struct snmp_table atmhw_table = {
 	OIDX_begemotAtmHWTable,
 	OIDX_begemotAtmIfTableLastChange, 2,
 	sizeof(struct atmhw),
diff --git a/sbin/atm/atmconfig/diag.c b/sbin/atm/atmconfig/diag.c
index 699d5cc..0211318 100644
--- a/sbin/atm/atmconfig/diag.c
+++ b/sbin/atm/atmconfig/diag.c
@@ -57,7 +57,7 @@ static void diag_phy_print(int, char *[]);
 static void diag_phy_stats(int, char *[]);
 static void diag_stats(int, char *[]);
 
-const struct cmdtab diag_phy_tab[] = {
+static const struct cmdtab diag_phy_tab[] = {
 	{ "show",	NULL, 		diag_phy_show },
 	{ "set",	NULL, 		diag_phy_set },
 	{ "stats",	NULL,		diag_phy_stats },
diff --git a/sbin/atm/atmconfig/main.c b/sbin/atm/atmconfig/main.c
index d2543f7..73e1fac 100644
--- a/sbin/atm/atmconfig/main.c
+++ b/sbin/atm/atmconfig/main.c
@@ -48,7 +48,7 @@ __FBSDID("$FreeBSD$");
 #include "private.h"
 
 /* verbosity level */
-int verbose;
+static int verbose;
 
 /* notitle option */
 static int notitle;
diff --git a/sbin/badsect/badsect.c b/sbin/badsect/badsect.c
index 9389171..f180410 100644
--- a/sbin/badsect/badsect.c
+++ b/sbin/badsect/badsect.c
@@ -71,9 +71,9 @@ __FBSDID("$FreeBSD$");
 
 #define sblock	disk.d_fs
 #define	acg	disk.d_cg
-struct	uufsd disk;
-struct	fs *fs = &sblock;
-int	errs;
+static struct	uufsd disk;
+static struct	fs *fs = &sblock;
+static int	errs;
 
 int	chkuse(daddr_t, int);
 
diff --git a/sbin/bsdlabel/bsdlabel.c b/sbin/bsdlabel/bsdlabel.c
index fbd978a..ac83a41 100644
--- a/sbin/bsdlabel/bsdlabel.c
+++ b/sbin/bsdlabel/bsdlabel.c
@@ -63,7 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/disk.h>
 #define DKTYPENAMES
 #define FSTYPENAMES
-#define MAXPARTITIONS	26
+#define MAXPARTITIONS	20
 #include <sys/disklabel.h>
 
 #include <unistd.h>
@@ -80,7 +80,7 @@ __FBSDID("$FreeBSD$");
 #include "pathnames.h"
 
 static void	makelabel(const char *, struct disklabel *);
-static int	geom_bsd_available(void);
+static int	geom_class_available(const char *);
 static int	writelabel(void);
 static int	readlabel(int flag);
 static void	display(FILE *, const struct disklabel *);
@@ -355,7 +355,7 @@ readboot(void)
 }
 
 static int
-geom_bsd_available(void)
+geom_class_available(const char *name)
 {
 	struct gclass *class;
 	struct gmesh mesh;
@@ -366,7 +366,7 @@ geom_bsd_available(void)
 		errc(1, error, "Cannot get GEOM tree");
 
 	LIST_FOREACH(class, &mesh.lg_class, lg_class) {
-		if (strcmp(class->lg_name, "BSD") == 0) {
+		if (strcmp(class->lg_name, name) == 0) {
 			geom_deletetree(&mesh);
 			return (1);
 		}
@@ -411,8 +411,20 @@ writelabel(void)
 		} else
 			serrno = errno;
 
+		if (geom_class_available("PART") != 0) {
+			/*
+			 * Since we weren't able open provider for
+			 * writing, then recommend user to use gpart(8).
+			 */
+			warnc(serrno,
+			    "cannot open provider %s for writing label",
+			    specname);
+			warnx("Try to use gpart(8).");
+			return (1);
+		}
+
 		/* Give up if GEOM_BSD is not available. */
-		if (geom_bsd_available() == 0) {
+		if (geom_class_available("BSD") == 0) {
 			warnc(serrno, "%s", specname);
 			return (1);
 		}
@@ -831,11 +843,16 @@ getasciilabel(FILE *f, struct disklabel *lp)
 			continue;
 		}
 		if (sscanf(cp, "%lu partitions", &v) == 1) {
-			if (v == 0 || v > MAXPARTITIONS) {
+			if (v > MAXPARTITIONS) {
 				fprintf(stderr,
 				    "line %d: bad # of partitions\n", lineno);
 				lp->d_npartitions = MAXPARTITIONS;
 				errors++;
+			} else if (v < DEFPARTITIONS) {
+				fprintf(stderr,
+				    "line %d: bad # of partitions\n", lineno);
+				lp->d_npartitions = DEFPARTITIONS;
+				errors++;
 			} else
 				lp->d_npartitions = v;
 			continue;
@@ -1149,23 +1166,41 @@ checklabel(struct disklabel *lp)
 			errors++;
 		} else if (lp->d_bbsize % lp->d_secsize)
 			warnx("boot block size %% sector-size != 0");
-		if (lp->d_npartitions > MAXPARTITIONS)
+		if (lp->d_npartitions > MAXPARTITIONS) {
 			warnx("number of partitions (%lu) > MAXPARTITIONS (%d)",
 			    (u_long)lp->d_npartitions, MAXPARTITIONS);
+			errors++;
+		}
+		if (lp->d_npartitions < DEFPARTITIONS) {
+			warnx("number of partitions (%lu) < DEFPARTITIONS (%d)",
+			    (u_long)lp->d_npartitions, DEFPARTITIONS);
+			errors++;
+		}
 	} else {
 		struct disklabel *vl;
 
 		vl = getvirginlabel();
-		lp->d_secsize = vl->d_secsize;
-		lp->d_nsectors = vl->d_nsectors;
-		lp->d_ntracks = vl->d_ntracks;
-		lp->d_ncylinders = vl->d_ncylinders;
-		lp->d_rpm = vl->d_rpm;
-		lp->d_interleave = vl->d_interleave;
-		lp->d_secpercyl = vl->d_secpercyl;
-		lp->d_secperunit = vl->d_secperunit;
-		lp->d_bbsize = vl->d_bbsize;
-		lp->d_npartitions = vl->d_npartitions;
+		if (lp->d_secsize == 0)
+			lp->d_secsize = vl->d_secsize;
+		if (lp->d_nsectors == 0)
+			lp->d_nsectors = vl->d_nsectors;
+		if (lp->d_ntracks == 0)
+			lp->d_ntracks = vl->d_ntracks;
+		if (lp->d_ncylinders == 0)
+			lp->d_ncylinders = vl->d_ncylinders;
+		if (lp->d_rpm == 0)
+			lp->d_rpm = vl->d_rpm;
+		if (lp->d_interleave == 0)
+			lp->d_interleave = vl->d_interleave;
+		if (lp->d_secpercyl == 0)
+			lp->d_secpercyl = vl->d_secpercyl;
+		if (lp->d_secperunit == 0)
+			lp->d_secperunit = vl->d_secperunit;
+		if (lp->d_bbsize == 0)
+			lp->d_bbsize = vl->d_bbsize;
+		if (lp->d_npartitions < DEFPARTITIONS ||
+		    lp->d_npartitions > MAXPARTITIONS)
+			lp->d_npartitions = vl->d_npartitions;
 	}
 
 
diff --git a/sbin/camcontrol/camcontrol.c b/sbin/camcontrol/camcontrol.c
index 9ec83f5..7612f5c 100644
--- a/sbin/camcontrol/camcontrol.c
+++ b/sbin/camcontrol/camcontrol.c
@@ -140,7 +140,7 @@ static const char smppc_opts[] = "a:A:d:lm:M:o:p:s:S:T:";
 static const char smpphylist_opts[] = "lq";
 #endif
 
-struct camcontrol_opts option_table[] = {
+static struct camcontrol_opts option_table[] = {
 #ifndef MINIMALISTIC
 	{"tur", CAM_CMD_TUR, CAM_ARG_NONE, NULL},
 	{"inquiry", CAM_CMD_INQUIRY, CAM_ARG_NONE, "DSR"},
@@ -207,8 +207,8 @@ struct cam_devlist {
 	path_id_t path_id;
 };
 
-cam_cmdmask cmdlist;
-cam_argmask arglist;
+static cam_cmdmask cmdlist;
+static cam_argmask arglist;
 
 camcontrol_optret getoption(struct camcontrol_opts *table, char *arg,
 			    uint32_t *cmdnum, cam_argmask *argnum,
@@ -4646,7 +4646,7 @@ bailout:
 	return (error);
 }
 
-struct camcontrol_opts phy_ops[] = {
+static struct camcontrol_opts phy_ops[] = {
 	{"nop", SMP_PC_PHY_OP_NOP, CAM_ARG_NONE, NULL},
 	{"linkreset", SMP_PC_PHY_OP_LINK_RESET, CAM_ARG_NONE, NULL},
 	{"hardreset", SMP_PC_PHY_OP_HARD_RESET, CAM_ARG_NONE, NULL},
diff --git a/sbin/ccdconfig/ccdconfig.c b/sbin/ccdconfig/ccdconfig.c
index 5630c45..6324150 100644
--- a/sbin/ccdconfig/ccdconfig.c
+++ b/sbin/ccdconfig/ccdconfig.c
@@ -59,7 +59,7 @@ static	int lineno = 0;
 static	int verbose = 0;
 static	const char *ccdconf = _PATH_CCDCONF;
 
-struct	flagval {
+static struct flagval {
 	const char	*fv_flag;
 	int		fv_val;
 } flagvaltab[] = {
diff --git a/sbin/devfs/rule.c b/sbin/devfs/rule.c
index 25a7d28..e4726d7 100644
--- a/sbin/devfs/rule.c
+++ b/sbin/devfs/rule.c
@@ -76,7 +76,7 @@ static struct intstr ist_type[] = {
 	{ NULL,			-1 }
 };
 
-devfs_rsnum in_rsnum;
+static devfs_rsnum in_rsnum;
 
 int
 rule_main(int ac, char **av)
diff --git a/sbin/dmesg/dmesg.c b/sbin/dmesg/dmesg.c
index f2e513f..002732c 100644
--- a/sbin/dmesg/dmesg.c
+++ b/sbin/dmesg/dmesg.c
@@ -60,11 +60,9 @@ __FBSDID("$FreeBSD$");
 #include <vis.h>
 #include <sys/syslog.h>
 
-char s_msgbufp[] = "_msgbufp";
-
-struct nlist nl[] = {
+static struct nlist nl[] = {
 #define	X_MSGBUF	0
-	{ s_msgbufp, 0, 0, 0, 0 },
+	{ "_msgbufp", 0, 0, 0, 0 },
 	{ NULL, 0, 0, 0, 0 },
 };
 
diff --git a/sbin/dumpfs/dumpfs.c b/sbin/dumpfs/dumpfs.c
index c54fa2b..a21f2f2 100644
--- a/sbin/dumpfs/dumpfs.c
+++ b/sbin/dumpfs/dumpfs.c
@@ -77,7 +77,7 @@ static const char rcsid[] =
 #define	afs	disk.d_fs
 #define	acg	disk.d_cg
 
-struct uufsd disk;
+static struct uufsd disk;
 
 int	dumpfs(const char *);
 int	dumpfsid(void);
diff --git a/sbin/fdisk/fdisk.8 b/sbin/fdisk/fdisk.8
index 1d04239..1764b64 100644
--- a/sbin/fdisk/fdisk.8
+++ b/sbin/fdisk/fdisk.8
@@ -476,6 +476,7 @@ The default boot code.
 .Sh SEE ALSO
 .Xr boot0cfg 8 ,
 .Xr bsdlabel 8 ,
+.Xr gpart 8 ,
 .Xr newfs 8
 .Sh BUGS
 The default boot code will not necessarily handle all slice types
diff --git a/sbin/fdisk/fdisk.c b/sbin/fdisk/fdisk.c
index ba1ee00..c784b86 100644
--- a/sbin/fdisk/fdisk.c
+++ b/sbin/fdisk/fdisk.c
@@ -47,7 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-int iotest;
+static int iotest;
 
 #define NO_DISK_SECTORS ((u_int32_t)-1)
 #define NO_TRACK_CYLINDERS 1023
@@ -232,6 +232,7 @@ get_type(int t)
 }
 
 
+static int geom_class_available(const char *);
 static void print_s0(void);
 static void print_part(const struct dos_partition *);
 static void init_sector0(unsigned long start);
@@ -487,7 +488,7 @@ print_part(const struct dos_partition *partp)
 	    get_type(partp->dp_typ));
 	printf("    start %lu, size %lu (%ju Meg), flag %x%s\n",
 		(u_long)partp->dp_start,
-		(u_long)partp->dp_size, 
+		(u_long)partp->dp_size,
 		(uintmax_t)part_mb,
 		partp->dp_flag,
 		partp->dp_flag == ACTIVE ? " (active)" : "");
@@ -513,6 +514,8 @@ init_boot(void)
 	if ((fdesc = open(fname, O_RDONLY)) == -1 ||
 	    fstat(fdesc, &sb) == -1)
 		err(1, "%s", fname);
+	if (sb.st_size == 0)
+		errx(1, "%s is empty, must not be.", fname);
 	if ((mboot.bootinst_size = sb.st_size) % secsize != 0)
 		errx(1, "%s: length must be a multiple of sector size", fname);
 	if (mboot.bootinst != NULL)
@@ -767,49 +770,76 @@ read_disk(off_t sector, void *buf)
 }
 
 static int
-write_disk(off_t sector, void *buf)
+geom_class_available(const char *name)
 {
+	struct gclass *class;
+	struct gmesh mesh;
 	int error;
+
+	error = geom_gettree(&mesh);
+	if (error != 0)
+		errc(1, error, "Cannot get GEOM tree");
+
+	LIST_FOREACH(class, &mesh.lg_class, lg_class) {
+		if (strcmp(class->lg_name, name) == 0) {
+			geom_deletetree(&mesh);
+			return (1);
+		}
+	}
+
+	geom_deletetree(&mesh);
+
+	return (0);
+}
+
+static int
+write_disk(off_t sector, void *buf)
+{
 	struct gctl_req *grq;
 	const char *errmsg;
-	char fbuf[BUFSIZ], *pname;
-	int i, fdw;
-
-	grq = gctl_get_handle();
-	gctl_ro_param(grq, "verb", -1, "write MBR");
-	gctl_ro_param(grq, "class", -1, "MBR");
-	pname = g_providername(fd);
-	if (pname == NULL) {
-		warn("Error getting providername for %s", disk);
-		return (-1);
-	}
-	gctl_ro_param(grq, "geom", -1, pname);
-	gctl_ro_param(grq, "data", secsize, buf);
-	errmsg = gctl_issue(grq);
-	free(pname);
-	if (errmsg == NULL) {
+	char *pname;
+	int error;
+
+	/* Check that GEOM_MBR is available */
+	if (geom_class_available("MBR") != 0) {
+		grq = gctl_get_handle();
+		gctl_ro_param(grq, "verb", -1, "write MBR");
+		gctl_ro_param(grq, "class", -1, "MBR");
+		pname = g_providername(fd);
+		if (pname == NULL) {
+			warn("Error getting providername for %s", disk);
+			return (-1);
+		}
+		gctl_ro_param(grq, "geom", -1, pname);
+		gctl_ro_param(grq, "data", secsize, buf);
+		errmsg = gctl_issue(grq);
+		free(pname);
+		if (errmsg == NULL) {
+			gctl_free(grq);
+			return(0);
+		}
+		if (!q_flag)
+			warnx("GEOM_MBR: %s", errmsg);
 		gctl_free(grq);
-		return(0);
-	}
-	if (!q_flag)	/* GEOM errors are benign, not all devices supported */
-		warnx("%s", errmsg);
-	gctl_free(grq);
-
-	error = pwrite(fd, buf, secsize, (sector * 512));
-	if (error == secsize)
-		return (0);
-
-	for (i = 1; i < 5; i++) {
-		sprintf(fbuf, "%ss%d", disk, i);
-		fdw = open(fbuf, O_RDWR, 0);
-		if (fdw < 0)
-			continue;
-		error = ioctl(fdw, DIOCSMBR, buf);
-		close(fdw);
-		if (error == 0)
+	} else {
+		/*
+		 * Try to write MBR directly. This may help when disk
+		 * is not in use.
+		 * XXX: hardcoded sectorsize
+		 */
+		error = pwrite(fd, buf, secsize, (sector * 512));
+		if (error == secsize)
 			return (0);
 	}
-	warnx("Failed to write sector zero");
+
+	/*
+	 * GEOM_MBR is not available or failed to write MBR.
+	 * Now check that we have GEOM_PART and recommend to use gpart (8).
+	 */
+	if (geom_class_available("PART") != 0)
+		warnx("Failed to write MBR. Try to use gpart(8).");
+	else
+		warnx("Failed to write sector zero");
 	return(EINVAL);
 }
 
@@ -890,7 +920,7 @@ write_s0()
 		dos_partition_enc(&mboot.bootinst[DOSPARTOFF + i * DOSPARTSIZE],
 		    &mboot.parts[i]);
 	le16enc(&mboot.bootinst[DOSMAGICOFFSET], DOSMAGIC);
-	for(sector = 0; sector < mboot.bootinst_size / secsize; sector++) 
+	for(sector = 0; sector < mboot.bootinst_size / secsize; sector++)
 		if (write_disk(sector,
 			       &mboot.bootinst[sector * secsize]) == -1) {
 			warn("can't write fdisk partition table");
@@ -920,11 +950,12 @@ ok(const char *str)
 static int
 decimal(const char *str, int *num, int deflt, uint32_t maxval)
 {
-	long long acc = 0;
+	long long acc;
 	int c;
 	char *cp;
 
 	while (1) {
+		acc = 0;
 		printf("Supply a decimal value for \"%s\" [%d] ", str, deflt);
 		fflush(stdout);
 		if (fgets(lbuf, LBUF, stdin) == NULL)
@@ -960,7 +991,6 @@ decimal(const char *str, int *num, int deflt, uint32_t maxval)
 			printf("%s is an invalid decimal number.  Try again.\n",
 				lbuf);
 	}
-
 }
 
 
@@ -1111,7 +1141,7 @@ str2sectors(const char *str)
 		return NO_DISK_SECTORS;
 	}
 
-	if (*end == 'K') 
+	if (*end == 'K')
 		val *= 1024UL / secsize;
 	else if (*end == 'M')
 		val *= 1024UL * 1024UL / secsize;
diff --git a/sbin/fdisk_pc98/fdisk.8 b/sbin/fdisk_pc98/fdisk.8
index c33e78a..aa3a6c4 100644
--- a/sbin/fdisk_pc98/fdisk.8
+++ b/sbin/fdisk_pc98/fdisk.8
@@ -446,6 +446,7 @@ Example: to make slice 1 the active slice:
 .Sh SEE ALSO
 .Xr boot98cfg 8 ,
 .Xr bsdlabel 8 ,
+.Xr gpart 8 ,
 .Xr newfs 8
 .Sh BUGS
 The default boot code will not necessarily handle all slice types
diff --git a/sbin/ffsinfo/Makefile b/sbin/ffsinfo/Makefile
index 88b267d..cb4e8f9 100644
--- a/sbin/ffsinfo/Makefile
+++ b/sbin/ffsinfo/Makefile
@@ -15,5 +15,5 @@ WARNS?=	1
 CFLAGS+=-DFS_DEBUG -I${GROWFS}
 DPADD=	${LIBUFS}
 LDADD=	-lufs
- 
-.include <bsd.prog.mk>               
+
+.include <bsd.prog.mk>
diff --git a/sbin/ffsinfo/ffsinfo.c b/sbin/ffsinfo/ffsinfo.c
index 1570da5..881fc1e 100644
--- a/sbin/ffsinfo/ffsinfo.c
+++ b/sbin/ffsinfo/ffsinfo.c
@@ -79,7 +79,7 @@ static const char rcsid[] =
 int	_dbg_lvl_ = (DL_INFO); /* DL_TRC */
 #endif /* FS_DEBUG */
 
-struct uufsd disk;
+static struct uufsd disk;
 
 #define sblock disk.d_fs
 #define acg    disk.d_cg
diff --git a/sbin/fsck/fsck.c b/sbin/fsck/fsck.c
index 5e27010..768e670 100644
--- a/sbin/fsck/fsck.c
+++ b/sbin/fsck/fsck.c
@@ -65,7 +65,7 @@ __FBSDID("$FreeBSD$");
 
 static enum { IN_LIST, NOT_IN_LIST } which = NOT_IN_LIST;
 
-TAILQ_HEAD(fstypelist, entry) opthead, selhead;
+static TAILQ_HEAD(fstypelist, entry) opthead, selhead;
 
 struct entry {
 	char *type;
diff --git a/sbin/fsck/preen.c b/sbin/fsck/preen.c
index 6204685..787bed0 100644
--- a/sbin/fsck/preen.c
+++ b/sbin/fsck/preen.c
@@ -62,7 +62,7 @@ struct partentry {
 	char		  	*p_type;	/* file system type */
 };
 
-TAILQ_HEAD(part, partentry) badh;
+static TAILQ_HEAD(part, partentry) badh;
 
 struct diskentry {
 	TAILQ_ENTRY(diskentry) 	    d_entries;
@@ -71,7 +71,7 @@ struct diskentry {
 	int			    d_pid;	/* 0 or pid of fsck proc */
 };
 
-TAILQ_HEAD(disk, diskentry) diskh;
+static TAILQ_HEAD(disk, diskentry) diskh;
 
 static int nrun = 0, ndisks = 0;
 
diff --git a/sbin/fsck_msdosfs/dir.c b/sbin/fsck_msdosfs/dir.c
index f2ecff3..1dc4456 100644
--- a/sbin/fsck_msdosfs/dir.c
+++ b/sbin/fsck_msdosfs/dir.c
@@ -153,7 +153,7 @@ freeDirTodo(struct dirTodoNode *dt)
 /*
  * The stack of unread directories
  */
-struct dirTodoNode *pendingDirectories = NULL;
+static struct dirTodoNode *pendingDirectories = NULL;
 
 /*
  * Return the full pathname for a directory entry.
diff --git a/sbin/fsirand/fsirand.c b/sbin/fsirand/fsirand.c
index 483879e..c3ae3e0 100644
--- a/sbin/fsirand/fsirand.c
+++ b/sbin/fsirand/fsirand.c
@@ -60,7 +60,7 @@ int fsirand(char *);
  */
 static int sblock_try[] = SBLOCKSEARCH;
 
-int printonly = 0, force = 0, ignorelabel = 0;
+static int printonly = 0, force = 0, ignorelabel = 0;
 
 int
 main(int argc, char *argv[])
diff --git a/sbin/geom/class/multipath/geom_multipath.c b/sbin/geom/class/multipath/geom_multipath.c
index 6364bc3..ecbaa37 100644
--- a/sbin/geom/class/multipath/geom_multipath.c
+++ b/sbin/geom/class/multipath/geom_multipath.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <strings.h>
 #include <assert.h>
 #include <libgeom.h>
+#include <unistd.h>
 #include <uuid.h>
 #include <geom/multipath/g_multipath.h>
 
@@ -48,31 +49,58 @@ uint32_t version = G_MULTIPATH_VERSION;
 static void mp_main(struct gctl_req *, unsigned int);
 static void mp_label(struct gctl_req *);
 static void mp_clear(struct gctl_req *);
-static void mp_add(struct gctl_req *);
 
 struct g_command class_commands[] = {
 	{
-		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
-		"[-v] name prov ..."
+		"create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, NULL,
+		{
+			{ 'A', "active_active", NULL, G_TYPE_BOOL },
+			G_OPT_SENTINEL
+		},
+		"[-vA] name prov ..."
 	},
 	{
-		"add", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
-		"[-v] name prov ..."
+		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main,
+		{
+			{ 'A', "active_active", NULL, G_TYPE_BOOL },
+			G_OPT_SENTINEL
+		},
+		"[-vA] name prov ..."
 	},
 	{
-		"destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
-		"[-v] prov ..."
+		"add", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
 	},
 	{
-		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
-		"[-v] prov ..."
+		"remove", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
+	},
+	{
+		"fail", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
+	},
+	{
+		"restore", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
 	},
 	{
 		"rotate", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
-		"[-v] prov ..."
+		"[-v] name"
 	},
 	{
 		"getactive", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"stop", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
 		"[-v] prov ..."
 	},
 	G_CMD_SENTINEL
@@ -90,8 +118,6 @@ mp_main(struct gctl_req *req, unsigned int flags __unused)
 	}
 	if (strcmp(name, "label") == 0) {
 		mp_label(req);
-	} else if (strcmp(name, "add") == 0) {
-		mp_add(req);
 	} else if (strcmp(name, "clear") == 0) {
 		mp_clear(req);
 	} else {
@@ -103,13 +129,14 @@ static void
 mp_label(struct gctl_req *req)
 {
 	struct g_multipath_metadata md;
-	off_t disksiz = 0, msize;
-	uint8_t *sector;
+	off_t disksize = 0, msize;
+	uint8_t *sector, *rsector;
 	char *ptr;
 	uuid_t uuid;
-	uint32_t secsize = 0, ssize, status;
-	const char *name, *mpname;
-	int error, i, nargs;
+	ssize_t secsize = 0, ssize;
+	uint32_t status;
+	const char *name, *name2, *mpname;
+	int error, i, nargs, fd;
 
 	nargs = gctl_get_int(req, "nargs");
 	if (nargs < 2) {
@@ -132,14 +159,14 @@ mp_label(struct gctl_req *req)
 		}
 		if (i == 1) {
 			secsize = ssize;
-			disksiz	= msize;
+			disksize = msize;
 		} else {
 			if (secsize != ssize) {
-				gctl_error(req, "%s sector size %u different.",
-				    name, ssize);
+				gctl_error(req, "%s sector size %ju different.",
+				    name, (intmax_t)ssize);
 				return;
 			}
-			if (disksiz != msize) {
+			if (disksize != msize) {
 				gctl_error(req, "%s media size %ju different.",
 				    name, (intmax_t)msize);
 				return;
@@ -155,7 +182,7 @@ mp_label(struct gctl_req *req)
 	md.md_version = G_MULTIPATH_VERSION;
 	mpname = gctl_get_ascii(req, "arg0");
 	strlcpy(md.md_name, mpname, sizeof(md.md_name));
-	md.md_size = disksiz;
+	md.md_size = disksize;
 	md.md_sectorsize = secsize;
 	uuid_create(&uuid, &status);
 	if (status != uuid_s_ok) {
@@ -168,19 +195,10 @@ mp_label(struct gctl_req *req)
 		return;
 	}
 	strlcpy(md.md_uuid, ptr, sizeof (md.md_uuid));
+	md.md_active_active = gctl_get_int(req, "active_active");
 	free(ptr);
 
 	/*
-	 * Clear metadata on initial provider first.
-	 */
-	name = gctl_get_ascii(req, "arg1");
-	error = g_metadata_clear(name, NULL);
-	if (error != 0) {
-		gctl_error(req, "cannot clear metadata on %s: %s.", name, strerror(error));
-		return;
-	}
-
-	/*
 	 * Allocate a sector to write as metadata.
 	 */
 	sector = malloc(secsize);
@@ -189,6 +207,12 @@ mp_label(struct gctl_req *req)
 		return;
 	}
 	memset(sector, 0, secsize);
+	rsector = malloc(secsize);
+	if (rsector == NULL) {
+		free(sector);
+		gctl_error(req, "unable to allocate metadata buffer");
+		return;
+	}
 
 	/*
 	 * encode the metadata
@@ -198,6 +222,7 @@ mp_label(struct gctl_req *req)
 	/*
 	 * Store metadata on the initial provider.
 	 */
+	name = gctl_get_ascii(req, "arg1");
 	error = g_metadata_store(name, sector, secsize);
 	if (error != 0) {
 		gctl_error(req, "cannot store metadata on %s: %s.", name, strerror(error));
@@ -205,20 +230,29 @@ mp_label(struct gctl_req *req)
 	}
 
 	/*
-	 * Now add the rest of the providers.
+	 * Now touch the rest of the providers to hint retaste.
 	 */
-	error = gctl_change_param(req, "verb", -1, "add");
-	if (error) {
-		gctl_error(req, "unable to change verb to \"add\": %s.", strerror(error));
-		return;
-	}
 	for (i = 2; i < nargs; i++) {
-		error = gctl_change_param(req, "arg1", -1, gctl_get_ascii(req, "arg%d", i));
-		if (error) {
-			gctl_error(req, "unable to add %s to %s: %s.", gctl_get_ascii(req, "arg%d", i), mpname, strerror(error));
+		name2 = gctl_get_ascii(req, "arg%d", i);
+		fd = g_open(name2, 1);
+		if (fd < 0) {
+			fprintf(stderr, "Unable to open %s: %s.\n",
+			    name2, strerror(errno));
+			continue;
+		}
+		if (pread(fd, rsector, secsize, disksize - secsize) !=
+		    (ssize_t)secsize) {
+			fprintf(stderr, "Unable to read metadata from %s: %s.\n",
+			    name2, strerror(errno));
+			g_close(fd);
 			continue;
 		}
-		mp_add(req);
+		g_close(fd);
+		if (memcmp(sector, rsector, secsize)) {
+			fprintf(stderr, "No metadata found on %s."
+			    " It is not a path of %s.\n",
+			    name2, name);
+		}
 	}
 }
 
@@ -247,13 +281,3 @@ mp_clear(struct gctl_req *req)
 	}
 }
 
-static void
-mp_add(struct gctl_req *req)
-{
-	const char *errstr;
-
-	errstr = gctl_issue(req);
-	if (errstr != NULL && errstr[0] != '\0') {
-		gctl_error(req, "%s", errstr);
-	}
-}
diff --git a/sbin/geom/class/multipath/gmultipath.8 b/sbin/geom/class/multipath/gmultipath.8
index 8c354e4..a009b2b 100644
--- a/sbin/geom/class/multipath/gmultipath.8
+++ b/sbin/geom/class/multipath/gmultipath.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 26, 2007
+.Dd October 31, 2011
 .Dt GMULTIPATH 8
 .Os
 .Sh NAME
@@ -32,11 +32,48 @@
 .Nd "disk multipath control utility"
 .Sh SYNOPSIS
 .Nm
+.Cm create
+.Op Fl Av
+.Ar name
+.Ar prov ...
+.Nm
 .Cm label
-.Op Fl hv
+.Op Fl Av
 .Ar name
 .Ar prov ...
 .Nm
+.Cm add
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm remove
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm fail
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm restore
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm rotate
+.Op Fl v
+.Ar name
+.Nm
+.Cm getactive
+.Op Fl v
+.Ar name
+.Nm
+.Cm destroy
+.Op Fl v
+.Ar name
+.Nm
+.Cm stop
+.Op Fl v
+.Ar name
+.Nm
 .Cm clear
 .Op Fl v
 .Ar prov ...
@@ -53,27 +90,79 @@ The
 .Nm
 utility is used for device multipath configuration.
 .Pp
-Only automatic configuration is supported at the present time via the
-.Cm label
-command.
-This operation writes a label on the last sector of the underlying
-disk device with a contained name and UUID.
-The UUID guarantees uniqueness
-in a shared storage environment but is in general too cumbersome to use.
+The multipath device can be configured using two different methods:
+.Dq manual
+or
+.Dq automatic .
+When using the
+.Dq manual
+method, no metadata are stored on the devices, so the multipath
+device has to be configured by hand every time it is needed.
+Additional device paths also won't be detected automatically.
+The
+.Dq automatic
+method uses on-disk metadata to detect device and all it's paths.
+Metadata use the last sector of the underlying disk device and
+include device name and UUID.
+The UUID guarantees uniqueness in a shared storage environment
+but is in general too cumbersome to use.
 The name is what is exported via the device interface.
 .Pp
 The first argument to
 .Nm
 indicates an action to be performed:
 .Bl -tag -width ".Cm destroy"
+.It Cm create
+Create multipath device with
+.Dq manual
+method without writing any on-disk metadata.
+It is up to administrator, how to properly identify device paths.
+Kernel will only check that all given providers have same media and
+sector sizes.
+.Pp
+.Fl A
+option enables Active/Active mode, otherwise Active/Passive mode is used
+by default.
 .It Cm label
-Label the given underlying device with the specified
+Create multipath device with
+.Dq automatic
+method.
+Label the first given provider with on-disk metadata using the specified
 .Ar name .
-The kernel module
-.Pa geom_multipath.ko
-will be loaded if it is not loaded already.
+The rest of given providers will be retasted to detect these metadata.
+It reliably protects against specifying unrelated providers.
+Providers with no matching metadata detected will not be added to the device.
+.Pp
+.Fl A
+option enables Active/Active mode, otherwise Active/Passive mode is used
+by default.
+.It Cm add
+Add the given provider as a path to the given multipath device.
+Should normally be used only for devices created with
+.Dq manual
+method, unless you know what you are doing (you are sure that it is another
+device path, but tasting its metadata in regular
+.Dq automatic
+way is not possible).
+.It Cm remove
+Remove the given provider as a path from the given multipath device.
+If the last path removed, the multipath device will be destroyed.
+.It Cm fail
+Mark specified provider as a path of the specified multipath device as failed.
+If there are other paths present, new requests will be forwarded there.
+.It Cm restore
+Mark specified provider as a path of the specified multipath device as
+operational, allowing it to handle requests.
+.It Cm rotate
+Change the active provider/path in Active/Passive mode.
+.It Cm getactive
+Get the currently active provider(s)/path(s).
+.It Cm destroy
+Destroy the given multipath device clearing metadata.
+.It Cm stop
+Stop the given multipath device without clearing metadata.
 .It Cm clear
-Clear metadata on the given device.
+Clear metadata on the given provider.
 .It Cm list
 See
 .Xr geom 8 .
@@ -101,14 +190,15 @@ Debug level of the
 GEOM class.
 This can be set to 0 (default) or 1 to disable or enable various
 forms of chattiness.
+.It Va kern.geom.multipath.exclusive : No 1
+Open underlying providers exclusively, preventing individual paths access.
 .El
 .Sh EXIT STATUS
 Exit status is 0 on success, and 1 if the command fails.
 .Sh MULTIPATH ARCHITECTURE
 .Pp
-This is an active/passive
-multiple path architecture with no device knowledge or presumptions other
-than size matching built in.
+This is a multiple path architecture with no device knowledge or
+presumptions other than size matching built in.
 Therefore the user must exercise some care
 in selecting providers that do indeed represent multiple paths to the
 same underlying disk device.
@@ -133,15 +223,16 @@ of multiple pathnames refer to the same device should be left to the
 system operator who will use tools and knowledge of their own storage
 subsystem to make the correct configuration selection.
 .Pp
-As an active/passive architecture, only one path has I/O moving on it
+There are Active/Passive and Active/Active operation modes supported.
+In Active/Passive mode only one path has I/O moving on it
 at any point in time.
 This I/O continues until an I/O is returned with
 a generic I/O error or a "Nonexistent Device" error.
-When this occurs,
-the active device is kicked out of the
-.Nm MULTIPATH
-GEOM class and the next in a list is selected, the failed I/O reissued
-and the system proceeds.
+When this occurs, that path is marked FAIL, the next path
+in a list is selected as active and the failed I/O reissued.
+In Active/Active mode all paths not marked FAIL may handle I/O same time.
+Requests are distributed between paths to equalize load.
+For capable devices it allows to utilize bandwidth of all paths.
 .Pp
 When new devices are added to the system the
 .Nm MULTIPATH
@@ -149,9 +240,9 @@ GEOM class is given an opportunity to taste these new devices.
 If a new
 device has a
 .Nm MULTIPATH
-label, the device is used to either create a new
+on-disk metadata label, the device is used to either create a new
 .Nm MULTIPATH
-GEOM, or to attach to the end of the list of devices for an existing
+GEOM, or been added the list of paths for an existing
 .Nm MULTIPATH
 GEOM.
 .Pp
@@ -176,7 +267,7 @@ of an RSCN event from the Fabric Domain Controller), they can cause
 a rescan to occur and cause the attachment and configuration of any
 (now) new devices to occur, causing the taste event described above.
 .Pp
-This means that this active/passive architecture is not a one-shot path
+This means that this multipath architecture is not a one-shot path
 failover, but can be considered to be steady state as long as failed
 paths are repaired (automatically or otherwise).
 .Pp
@@ -184,7 +275,7 @@ Automatic rescanning is not a requirement.
 Nor is Fibre Channel.
 The
 same failover mechanisms work equally well for traditional "Parallel"
-SCSI but require manual intervention with
+SCSI but may require manual intervention with
 .Xr camcontrol 8
 to cause the reattachment of repaired device links.
 .Sh EXAMPLES
@@ -226,9 +317,9 @@ mount /dev/multipath/FREDa /mnt....
 .Pp
 The resultant console output looks something like:
 .Bd -literal -offset indent
-GEOM_MULTIPATH: adding da0 to Fred/b631385f-c61c-11db-b884-0011116ae789
-GEOM_MULTIPATH: da0 now active path in Fred
-GEOM_MULTIPATH: adding da2 to Fred/b631385f-c61c-11db-b884-0011116ae789
+GEOM_MULTIPATH: da0 added to FRED
+GEOM_MULTIPATH: da0 is now active path in FRED
+GEOM_MULTIPATH: da2 added to FRED
 .Ed
 .Sh SEE ALSO
 .Xr geom 4 ,
@@ -240,24 +331,6 @@ GEOM_MULTIPATH: adding da2 to Fred/b631385f-c61c-11db-b884-0011116ae789
 .Xr mount 8 ,
 .Xr newfs 8 ,
 .Xr sysctl 8
-.Sh BUGS
-The
-.Nm
-should allow for a manual method of pairing disks.
-.Pp
-There is currently no way for
-.Pa geom_multipath.ko
-to distinguish between various label instances of the same provider.
-That
-is devices such as
-.Ar da0
-and
-.Ar da0c
-can be tasted and instantiated as multiple paths for the same device.
-Technically, this is correct, but pretty useless.
-This will be fixed soon
-(I hope), but to avoid this it is a good idea to destroy any label on
-the disk object prior to labelling it with
-.Nm .
 .Sh AUTHOR
 .An Matthew Jacob Aq mjacob@FreeBSD.org
+.An Alexander Motin Aq mav@FreeBSD.org
diff --git a/sbin/geom/class/part/gpart.8 b/sbin/geom/class/part/gpart.8
index 66c30d6..b915316 100644
--- a/sbin/geom/class/part/gpart.8
+++ b/sbin/geom/class/part/gpart.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 19, 2011
+.Dd November 21, 2011
 .Dt GPART 8
 .Os
 .Sh NAME
@@ -37,52 +37,20 @@ lines in your kernel configuration file:
 .Bd -ragged -offset indent
 .Cd "options GEOM_PART_APM"
 .Cd "options GEOM_PART_BSD"
-.Cd "options GEOM_PART_EBR"
-.Cd "options GEOM_PART_EBR_COMPAT"
 .Cd "options GEOM_PART_GPT"
 .Cd "options GEOM_PART_MBR"
+.Cd "options GEOM_PART_EBR"
+.Cd "options GEOM_PART_EBR_COMPAT"
 .Cd "options GEOM_PART_PC98"
 .Cd "options GEOM_PART_VTOC8"
 .Ed
 .Pp
-The
-.Dv GEOM_PART_APM
-option adds support for the Apple Partition Map (APM)
-found on Apple Macintosh computers.
-The
-.Dv GEOM_PART_BSD
-option adds support for the traditional
-.Bx
-disklabel.
-The
-.Dv GEOM_PART_EBR
-option adds support for the Extended Boot Record (EBR),
-which is used to define a logical partition.
-The
-.Dv GEOM_PART_EBR_COMPAT
-option enables backward compatibility for partition names
-in the EBR scheme.
-Also it makes impossible any types of actions
-with such partitions.
-The
-.Dv GEOM_PART_GPT
-option adds support for the GUID Partition Table (GPT)
-found on Intel Itanium computers and Intel-based Macintosh computers.
-The
-.Dv GEOM_PART_MBR
-option adds support for the Master Boot Record (MBR)
-found on PCs and used on many removable media.
-The
-.Dv GEOM_PART_PC98
-option adds support for the MBR variant as used on
-NEC PC-98 computers.
-The
-.Dv GEOM_PART_VTOC8
-option adds support for Sun's SMI VTOC8 label as
-found on computers based on
-.Tn SPARC64
-and
-.Tn UltraSPARC.
+These options provide support for the various types of partitioning
+schemes supported by the
+.Ns Nm
+utility.  See
+.Sx "PARTITIONING SCHEMES"
+below for more details.
 .Pp
 Usage of the
 .Ns Nm
@@ -190,7 +158,7 @@ utility:
 The
 .Nm
 utility is used to partition GEOM providers, normally disks.
-The first argument of which is the action to be taken:
+The first argument is the action to be taken:
 .Bl -tag -width ".Cm bootcode"
 .\" ==== ADD ====
 .It Cm add
@@ -245,7 +213,7 @@ about its use.
 .El
 .\" ==== BACKUP ====
 .It Cm backup
-Dump a partition table to standard output in special format used by
+Dump a partition table to standard output in a special format used by the
 .Cm restore
 action.
 .\" ==== BOOTCODE ====
@@ -290,18 +258,17 @@ about its use.
 .It Cm commit
 Commit any pending changes for geom
 .Ar geom .
-All actions are being committed by default and will not result in
+All actions are committed by default and will not result in
 pending changes.
 Actions can be modified with the
 .Fl f Ar flags
-option so that they are not being committed by default.
-As such, they become pending.
+option so that they are not committed, but become pending.
 Pending changes are reflected by the geom and the
 .Nm
 utility, but they are not actually written to disk.
 The
 .Cm commit
-action will write any and all pending changes to disk.
+action will write all pending changes to disk.
 .\" ==== CREATE ====
 .It Cm create
 Create a new partitioning scheme on a provider given by
@@ -309,16 +276,16 @@ Create a new partitioning scheme on a provider given by
 The
 .Fl s Ar scheme
 option determines the scheme to use.
-The kernel needs to have support for a particular scheme before
+The kernel must have support for a particular scheme before
 that scheme can be used to partition a disk.
 .Pp
 Additional options include:
 .Bl -tag -width 10n
 .It Fl n Ar entries
 The number of entries in the partition table.
-Every partitioning scheme has a minimum and a maximum number of entries
-and this option allows tables to be created with the number of entries
-that lies anywhere between the minimum and the maximum.
+Every partitioning scheme has a minimum and maximum number of entries.
+This option allows tables to be created with a number of entries
+that is within the limits.
 Some schemes have a maximum equal to the minimum and some schemes have
 a maximum large enough to be considered unlimited.
 By default, partition tables are created with the minimum number of
@@ -392,7 +359,7 @@ about its use.
 .El
 .\" ==== RECOVER ====
 .It Cm recover
-Recover corrupt partition's scheme metadata on the geom
+Recover a corrupt partition's scheme metadata on the geom
 .Ar geom .
 See the section entitled
 .Sx RECOVERING
@@ -489,7 +456,7 @@ or all geoms if none are specified.
 Additional options include:
 .Bl -tag -width 10n
 .It Fl l
-For partition schemes that support partition labels print them
+For partitioning schemes that support partition labels, print them
 instead of partition type.
 .It Fl p
 Show provider names instead of partition indexes.
@@ -520,39 +487,95 @@ below for a discussion
 about its use.
 .El
 .El
-.\"
+.Sh PARTITIONING SCHEMES
+Several partitioning schemes are supported by the
+.Nm
+utility:
+.Bl -tag -width ".Cm VTOC8"
+.It Cm APM
+Apple Partition Map, used by PowerPC(R) Macintosh(R) computers.
+Requires the
+.Cd GEOM_PART_APM
+kernel option.
+.It Cm BSD
+Traditional BSD disklabel, usually used to subdivide MBR partitions.
+.Po
+This scheme can also be used as the sole partitioning method, without
+an MBR.
+Partition editing tools from other operating systems often do not 
+understand the bare disklabel partition layout, so this is sometimes 
+called
+.Dq dangerously dedicated .
+.Pc
+Requires the
+.Cm GEOM_PART_BSD
+kernel option.
+.It Cm GPT
+GUID Partition Table is used on Intel-based Macintosh computers and
+gradually replacing MBR on most PCs and other systems.
+Requires the
+.Cm GEOM_PART_GPT
+kernel option.
+.It Cm MBR
+Master Boot Record is used on PCs and removable media.
+Requires the
+.Cm GEOM_PART_MBR
+kernel option.
+The
+.Cm GEOM_PART_EBR
+option adds support for the Extended Boot Record (EBR),
+which is used to define a logical partition.
+The
+.Cm GEOM_PART_EBR_COMPAT
+option enables backward compatibility for partition names
+in the EBR scheme.
+It also prevents any type of actions on such partitions.
+.It Cm PC98
+An MBR variant for NEC PC-98 and compatible computers.
+Requires the
+.Cm GEOM_PART_PC98
+kernel option.
+.It Cm VTOC8
+Sun's SMI Volume Table Of Contents, used by
+.Tn SPARC64
+and
+.Tn UltraSPARC
+computers.
+Requires the
+.Cm GEOM_PART_VTOC8
+kernel option.
+.El
 .Sh PARTITION TYPES
 Partition types are identified on disk by particular strings or magic
 values.
 The
 .Nm
-utility uses symbolic names for common partition types to avoid the
-user needing to know these values or other details of the partitioning
+utility uses symbolic names for common partition types so the user
+does not need to know these values or other details of the partitioning
 scheme in question.
 The
 .Nm
 utility also allows the user to specify scheme-specific partition types
 for partition types that do not have symbolic names.
-The symbolic names currently understood are:
+Symbolic names currently understood are:
 .Bl -tag -width ".Cm freebsd-vinum"
 .It Cm bios-boot
 The system partition dedicated to second stage of the boot loader program.
-Usually it used by GRUB 2 loader when the partition table is GPT.
+Usually it is used by the GRUB 2 loader for GPT partitioning schemes.
 The scheme-specific type is
 .Qq Li "!21686148-6449-6E6F-744E-656564454649" .
 .It Cm efi
 The system partition for computers that use the Extensible Firmware
 Interface (EFI).
-In such cases, the GPT partitioning scheme is being used and the
+In such cases, the GPT partitioning scheme is used and the
 actual partition type for the system partition can also be specified as
 .Qq Li "!c12a7328-f81f-11d2-ba4b-00a0c93ec93ab" .
 .It Cm freebsd
 A
 .Fx
-partition that uses the
+partition subdivided into filesystems with a
 .Bx
-disklabel to sub-divide the
-partition into file systems.
+disklabel.
 This is a legacy partition type and should not be used for the APM
 or GPT schemes.
 The scheme-specific types are
@@ -581,7 +604,7 @@ for GPT, and tag 0x0901 for VTOC8.
 .It Cm freebsd-ufs
 A
 .Fx
-partition that contains a UFS or UFS2 file system.
+partition that contains a UFS or UFS2 filesystem.
 The scheme-specific types are
 .Qq Li "!FreeBSD-UFS"
 for APM,
@@ -606,7 +629,7 @@ for APM,
 .Qq Li "!516e7cba-6ecf-11d6-8ff8-00022d09712b"
 for GPT, and 0x0904 for VTOC8.
 .It Cm mbr
-A partition that is sub-partitioned by a master boot record (MBR).
+A partition that is sub-partitioned by a Master Boot Record (MBR).
 This type is known as
 .Qq Li "!024dee41-33e7-11d3-9d69-0008c781f39f"
 by GPT.
@@ -712,27 +735,28 @@ The scheme-specific attributes for PC98:
 .Fx
 supports several partitioning schemes and each scheme uses different
 bootstrap code.
-The bootstrap code is located in the specific disk area for each partitioning
-scheme and also it might have different size.
+The bootstrap code is located in a specific disk area for each partitioning
+scheme, and may vary in size for different schemes.
 .Pp
-The bootstrap code could be separated into two types.
-The first one is embedded in the partitioning scheme's metadata, the second
-type is located on the specific partition.
-The embedding bootstrap code should be done only with the
+Bootstrap code can be separated into two types.
+The first type is embedded in the partitioning scheme's metadata, while the
+second type is located on a specific partition.
+Embedding bootstrap code should only be done with the
 .Cm gpart bootcode
-command with
+command with the
 .Fl b Ar bootcode
 option.
-The GEOM PART class has knowlege on how to embed bootstrap code into specific
-partitioning scheme metadata without damage.
+The GEOM PART class knows how to safely embed bootstrap code into
+specific partitioning scheme metadata without causing any damage.
 .Pp
-The Master Boot Record (MBR) uses 512-bytes bootstrap code image, embedded into
-partition table's metadata area.
+The Master Boot Record (MBR) uses a 512-byte bootstrap code image, embedded
+into the partition table's metadata area.
 There are two variants of this bootstrap code:
 .Pa /boot/mbr
 and
 .Pa /boot/boot0 .
-The first one searches partition with
+.Pa /boot/mbr
+searches for a partition with the
 .Cm active
 attribute (see the
 .Sx ATTRIBUTES
@@ -740,9 +764,10 @@ section) in the partition table.
 Then it runs next bootstrap stage.
 The
 .Pa /boot/boot0
-image contains a boot manager with some additional interactive functions.
+image contains a boot manager with some additional interactive functions
+for multi-booting from a user-selected partition.
 .Pp
-The BSD disklabel is usually created on top of the MBR partition (slice)
+A BSD disklabel is usually created inside an MBR partition (slice)
 with type
 .Cm freebsd
 (see the
@@ -750,17 +775,17 @@ with type
 section).
 It uses 8 KB size bootstrap code image
 .Pa /boot/boot ,
-embedded into partition table's metadata area.
+embedded into the partition table's metadata area.
 .Pp
 Both types of bootstrap code are used to boot from the GUID Partition Table.
-First of all, a protective MBR is embedded into first disk sector from the
+First, a protective MBR is embedded into the first disk sector from the
 .Pa /boot/pmbr
 image.
-It searches the
+It searches the GPT
 .Cm freebsd-boot
 partition (see the
 .Sx "PARTITION TYPES"
-section) in the GPT and runs next bootstrap stage from it.
+section) in the GPT and runs the next bootstrap stage from it.
 The
 .Cm freebsd-boot
 partition should be smaller than 545 KB.
@@ -768,37 +793,39 @@ There are two variants of bootstrap code to write to this partition:
 .Pa /boot/gptboot
 and
 .Pa /boot/gptzfsboot .
-The first one is used to boot from UFS.
-It searches in the GPT partition with type
-.Cm freebsd-ufs ,
-and it runs the third bootstrap stage (
-.Pa /boot/loader )
-if it is found.
+.Pa /boot/gptboot
+is used to boot from UFS.
+It searches
+.Cm freebsd-ufs
+GPT partitions and starts
+.Pa /boot/loader
+.Pq the third bootstrap stage
+if found.
 The
 .Pa /boot/gptzfsboot
 is used to boot from ZFS.
-It searches partition with type
+It searches
 .Cm freebsd-zfs
-and starts
+GPT partitions and starts
 .Pa /boot/zfsloader
-from it.
+if found.
 .Pp
 The VTOC8 scheme does not support embedding bootstrap code.
 Instead, the 8 KBytes bootstrap code image
 .Pa /boot/boot1
-should be written with
+should be written with the
 .Cm gpart bootcode
-command with
+command with the
 .Fl p Ar bootcode
 option to all sufficiently large VTOC8 partitions.
 To do this the
 .Fl i Ar index
-option could be ommited.
+option could be omitted.
 .Pp
 The APM scheme also does not support embedding bootstrap code.
 Instead, the 800 KBytes bootstrap code image
 .Pa /boot/boot1.hfs
-should be written with
+should be written with the
 .Cm gpart bootcode
 command to a partition of type
 .Cm freebsd-boot ,
@@ -829,21 +856,22 @@ action or reverted with the
 action.
 .Sh RECOVERING
 The GEOM PART class supports recovering of partition tables only for GPT.
-The GUID partition table has a primary and secondary (backup) copy of
-metadata for redundance, these are stored at the begining and the end
-of the device respectively.
-As a result of having two copies, it is acceptable to have some corruption
-within the metadata that is not fatal to the working of GPT.
-When the kernel detects corrupt metadata it marks this table as corrupt and
-reports the corruption.
-Any operations on corrupt tables are prohibited except for
+The GPT primary metadata is stored at the beginning of the device.
+For redundancy, a secondary
+.Pq backup
+copy of the metadata is stored at the end of the device.
+As a result of having two copies, some corruption of metadata is not
+fatal to the working of GPT.
+When the kernel detects corrupt metadata, it marks this table as corrupt
+and reports the problem.
 .Cm destroy
 and
-.Cm recover .
+.Cm recover
+are the only operations allowed on corrupt tables.
 .Pp
 If the first sector of a provider is corrupt, the kernel can not detect GPT
-even if partition table itself is not corrupt.
-You can rewrite the protective MBR using the
+even if the partition table itself is not corrupt.
+The protective MBR can be rewritten using the
 .Xr dd 1
 command, to restore the ability to detect the GPT.
 The copy of the protective MBR is usually located in the
@@ -874,7 +902,7 @@ will report about corrupt tables.
 If the size of the device has changed (e.g.\& volume expansion) the
 secondary GPT header will no longer be located in the last sector.
 This is not a metadata corruption, but it is dangerous because any
-corruption of the primary GPT will lead to loss of partition table.
+corruption of the primary GPT will lead to loss of the partition table.
 This problem is reported by the kernel with the message:
 .Bd -literal -offset indent
 GEOM: provider: the secondary GPT header is not in the last LBA.
@@ -913,13 +941,13 @@ rejected with a diagnostic message:
 .Sh EXIT STATUS
 Exit status is 0 on success, and 1 if the command fails.
 .Sh EXAMPLES
-Create GPT scheme on
+Create a GPT scheme on
 .Pa ad0 :
 .Bd -literal -offset indent
 /sbin/gpart create -s GPT ad0
 .Ed
 .Pp
-Embed GPT bootstrap code into protective MBR:
+Embed GPT bootstrap code into a protective MBR:
 .Bd -literal -offset indent
 /sbin/gpart bootcode -b /boot/pmbr ad0
 .Ed
@@ -935,9 +963,11 @@ This partition must be larger than
 .Pa /boot/gptboot ,
 or the GPT boot you are planning to write, but smaller than 545 KB.
 A size of 15 blocks (7680 bytes) would be sufficient for
-booting from UFS but let's use 128 blocks (64 KB) here in
-this example, in order to reserve some space for potential
-future need (e.g.\& from a ZFS partition).
+booting from UFS but 128 blocks (64 KB) is used in
+this example to reserve some space for potential
+future need (e.g.\& a larger
+.Pa /boot/gptzfsboot
+for booting from a ZFS partition).
 .Bd -literal -offset indent
 /sbin/gpart add -b 34 -s 128 -t freebsd-boot ad0
 /sbin/gpart bootcode -p /boot/gptboot -i 1 ad0
@@ -945,17 +975,17 @@ future need (e.g.\& from a ZFS partition).
 .Pp
 Create a 512MB-sized
 .Cm freebsd-ufs
-partition that would contain UFS where the system boots from:
+partition to contain a UFS filesystem from which the system can boot.
 .Bd -literal -offset indent
 /sbin/gpart add -b 162 -s 1048576 -t freebsd-ufs ad0
 .Ed
 .Pp
-Create MBR scheme on
+Create an MBR scheme on
 .Pa ada0 ,
-then create 30GB-sized
+then create a 30GB-sized
 .Fx
 slice, mark it active and
-install
+install the
 .Nm boot0
 boot manager:
 .Bd -literal -offset indent
@@ -965,16 +995,16 @@ boot manager:
 /sbin/gpart bootcode -b /boot/boot0 ada0
 .Ed
 .Pp
-Now create
+Now create a
 .Bx
 scheme
 .Pf ( Bx
-label) with ability to have up to 20 partitions:
+label) with space for up to 20 partitions:
 .Bd -literal -offset indent
 /sbin/gpart create -s BSD -n 20 ada0s1
 .Ed
 .Pp
-Create 1GB-sized UFS partition and 4GB-sized swap partition:
+Create a 1GB-sized UFS partition and a 4GB-sized swap partition:
 .Bd -literal -offset indent
 /sbin/gpart add -t freebsd-ufs -s 1G ada0s1
 /sbin/gpart add -t freebsd-swap -s 4G ada0s1
@@ -987,7 +1017,7 @@ label:
 /sbin/gpart bootcode -b /boot/boot ada0s1
 .Ed
 .Pp
-Create VTOC8 scheme on
+Create a VTOC8 scheme on
 .Pa da0 :
 .Bd -literal -offset indent
 /sbin/gpart create -s VTOC8 da0
@@ -995,36 +1025,36 @@ Create VTOC8 scheme on
 .Pp
 Create a 512MB-sized
 .Cm freebsd-ufs
-partition that would contain UFS where the system boots from:
+partition to contain a UFS filesystem from which the system can boot.
 .Bd -literal -offset indent
 /sbin/gpart add -s 512M -t freebsd-ufs da0
 .Ed
 .Pp
 Create a 15GB-sized
 .Cm freebsd-ufs
-partition that would contain UFS and aligned on 4KB boundaries:
+partition to contain a UFS filesystem and aligned on 4KB boundaries:
 .Bd -literal -offset indent
 /sbin/gpart add -s 15G -t freebsd-ufs -a 4k da0
 .Ed
 .Pp
-After having created all required partitions, embed bootstrap code into them:
+After creating all required partitions, embed bootstrap code into them:
 .Bd -literal -offset indent
 /sbin/gpart bootcode -p /boot/boot1 da0
 .Ed
 .Pp
-Create backup of partition table from
+Create a backup of the partition table from
 .Pa da0 :
 .Bd -literal -offset indent
 /sbin/gpart backup da0 > da0.backup
 .Ed
 .Pp
-Restore partition table from backup to
+Restore the partition table from the backup to
 .Pa da0 :
 .Bd -literal -offset indent
 /sbin/gpart restore -l da0 < /mnt/da0.backup
 .Ed
 .Pp
-Clone partition table from
+Clone the partition table from
 .Pa ada0
 to
 .Pa ada1
diff --git a/sbin/growfs/growfs.c b/sbin/growfs/growfs.c
index 148f8ca..18b4a05 100644
--- a/sbin/growfs/growfs.c
+++ b/sbin/growfs/growfs.c
@@ -116,7 +116,7 @@ union dinode {
 	} while (0)
 static ufs2_daddr_t 	inoblk;			/* inode block address */
 static char		inobuf[MAXBSIZE];	/* inode block */
-ino_t			maxino;			/* last valid inode */
+static ino_t		maxino;			/* last valid inode */
 static int		unlabeled;     /* unlabeled partition, e.g. vinum volume etc. */
 
 /*
diff --git a/sbin/gvinum/gvinum.c b/sbin/gvinum/gvinum.c
index 2039761..0544d00 100644
--- a/sbin/gvinum/gvinum.c
+++ b/sbin/gvinum/gvinum.c
@@ -557,7 +557,7 @@ find_pattern(char *line, char *pattern)
 	return (NULL);
 }
 
-/* Find a free name for an object given a a prefix. */
+/* Find a free name for an object given a prefix. */
 char *
 find_name(const char *prefix, int type, int namelen)
 {
diff --git a/sbin/hastd/hast.conf.5 b/sbin/hastd/hast.conf.5
index 1eb66e3..9ecfa1d 100644
--- a/sbin/hastd/hast.conf.5
+++ b/sbin/hastd/hast.conf.5
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 20, 2011
+.Dd October 27, 2011
 .Dt HAST.CONF 5
 .Os
 .Sh NAME
@@ -224,9 +224,6 @@ completes.
 This is the fastest and the most dangerous replication mode.
 This mode should be used when replicating to a distant node where
 latency is too high for other modes.
-The
-.Ic async
-replication mode is currently not implemented.
 .El
 .It Ic checksum Aq algorithm
 .Pp
diff --git a/sbin/hastd/parse.y b/sbin/hastd/parse.y
index 6174a2b..72d24b4 100644
--- a/sbin/hastd/parse.y
+++ b/sbin/hastd/parse.y
@@ -301,11 +301,9 @@ yy_config_parse(const char *config, bool exitonerror)
 			 */
 			curres->hr_replication = depth0_replication;
 		}
-		if (curres->hr_replication == HAST_REPLICATION_MEMSYNC ||
-		    curres->hr_replication == HAST_REPLICATION_ASYNC) {
+		if (curres->hr_replication == HAST_REPLICATION_MEMSYNC) {
 			pjdlog_warning("Replication mode \"%s\" is not implemented, falling back to \"%s\".",
-			    curres->hr_replication == HAST_REPLICATION_MEMSYNC ?
-			    "memsync" : "async", "fullsync");
+			    "memsync", "fullsync");
 			curres->hr_replication = HAST_REPLICATION_FULLSYNC;
 		}
 		if (curres->hr_checksum == -1) {
diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c
index 21e888a..be1d7bb 100644
--- a/sbin/hastd/primary.c
+++ b/sbin/hastd/primary.c
@@ -89,6 +89,15 @@ struct hio {
 	 * Structure used to communicate with GEOM Gate class.
 	 */
 	struct g_gate_ctl_io	 hio_ggio;
+	/*
+	 * Request was already confirmed to GEOM Gate.
+	 */
+	bool			 hio_done;
+	/*
+	 * Remember replication from the time the request was initiated,
+	 * so we won't get confused when replication changes on reload.
+	 */
+	int			 hio_replication;
 	TAILQ_ENTRY(hio)	*hio_next;
 };
 #define	hio_free_next	hio_next[0]
@@ -1021,7 +1030,7 @@ remote_close(struct hast_resource *res, int ncomp)
 
 	rw_wlock(&hio_remote_lock[ncomp]);
 	/*
-	 * A race is possible between dropping rlock and acquiring wlock -
+	 * Check for a race between dropping rlock and acquiring wlock -
 	 * another thread can close connection in-between.
 	 */
 	if (!ISCONNECTED(res, ncomp)) {
@@ -1056,6 +1065,42 @@ remote_close(struct hast_resource *res, int ncomp)
 }
 
 /*
+ * Acknowledge write completion to the kernel, but don't update activemap yet.
+ */
+static void
+write_complete(struct hast_resource *res, struct hio *hio)
+{
+	struct g_gate_ctl_io *ggio;
+	unsigned int ncomp;
+
+	PJDLOG_ASSERT(!hio->hio_done);
+
+	ggio = &hio->hio_ggio;
+	PJDLOG_ASSERT(ggio->gctl_cmd == BIO_WRITE);
+
+	/*
+	 * Bump local count if this is first write after
+	 * connection failure with remote node.
+	 */
+	ncomp = 1;
+	rw_rlock(&hio_remote_lock[ncomp]);
+	if (!ISCONNECTED(res, ncomp)) {
+		mtx_lock(&metadata_lock);
+		if (res->hr_primary_localcnt == res->hr_secondary_remotecnt) {
+			res->hr_primary_localcnt++;
+			pjdlog_debug(1, "Increasing localcnt to %ju.",
+			    (uintmax_t)res->hr_primary_localcnt);
+			(void)metadata_write(res);
+		}
+		mtx_unlock(&metadata_lock);
+	}
+	rw_unlock(&hio_remote_lock[ncomp]);
+	if (ioctl(res->hr_ggatefd, G_GATE_CMD_DONE, ggio) < 0)
+		primary_exit(EX_OSERR, "G_GATE_CMD_DONE failed");
+	hio->hio_done = true;
+}
+
+/*
  * Thread receives ggate I/O requests from the kernel and passes them to
  * appropriate threads:
  * WRITE - always goes to both local_send and remote_send threads
@@ -1075,8 +1120,6 @@ ggate_recv_thread(void *arg)
 	unsigned int ii, ncomp, ncomps;
 	int error;
 
-	ncomps = HAST_NCOMPONENTS;
-
 	for (;;) {
 		pjdlog_debug(2, "ggate_recv: Taking free request.");
 		QUEUE_TAKE2(hio, free);
@@ -1085,6 +1128,8 @@ ggate_recv_thread(void *arg)
 		ggio->gctl_unit = res->hr_ggateunit;
 		ggio->gctl_length = MAXPHYS;
 		ggio->gctl_error = 0;
+		hio->hio_done = false;
+		hio->hio_replication = res->hr_replication;
 		pjdlog_debug(2,
 		    "ggate_recv: (%p) Waiting for request from the kernel.",
 		    hio);
@@ -1117,11 +1162,16 @@ ggate_recv_thread(void *arg)
 			primary_exitx(EX_OSERR, "G_GATE_CMD_START failed: %s.",
 			    strerror(error));
 		}
+
+		ncomp = 0;
+		ncomps = HAST_NCOMPONENTS;
+
 		for (ii = 0; ii < ncomps; ii++)
 			hio->hio_errors[ii] = EINVAL;
 		reqlog(LOG_DEBUG, 2, ggio,
 		    "ggate_recv: (%p) Request received from the kernel: ",
 		    hio);
+
 		/*
 		 * Inform all components about new write request.
 		 * For read request prefer local component unless the given
@@ -1130,10 +1180,7 @@ ggate_recv_thread(void *arg)
 		switch (ggio->gctl_cmd) {
 		case BIO_READ:
 			res->hr_stat_read++;
-			pjdlog_debug(2,
-			    "ggate_recv: (%p) Moving request to the send queue.",
-			    hio);
-			refcount_init(&hio->hio_countdown, 1);
+			ncomps = 1;
 			mtx_lock(&metadata_lock);
 			if (res->hr_syncsrc == HAST_SYNCSRC_UNDEF ||
 			    res->hr_syncsrc == HAST_SYNCSRC_PRIMARY) {
@@ -1155,17 +1202,13 @@ ggate_recv_thread(void *arg)
 				ncomp = 1;
 			}
 			mtx_unlock(&metadata_lock);
-			QUEUE_INSERT1(hio, send, ncomp);
 			break;
 		case BIO_WRITE:
 			res->hr_stat_write++;
-			if (res->hr_resuid == 0) {
-				/*
-				 * This is first write, initialize localcnt and
-				 * resuid.
-				 */
+			if (res->hr_resuid == 0 &&
+			    res->hr_primary_localcnt == 0) {
+				/* This is first write. */
 				res->hr_primary_localcnt = 1;
-				(void)init_resuid(res);
 			}
 			for (;;) {
 				mtx_lock(&range_lock);
@@ -1201,25 +1244,19 @@ ggate_recv_thread(void *arg)
 				(void)hast_activemap_flush(res);
 			}
 			mtx_unlock(&res->hr_amp_lock);
-			/* FALLTHROUGH */
+			break;
 		case BIO_DELETE:
+			res->hr_stat_delete++;
+			break;
 		case BIO_FLUSH:
-			switch (ggio->gctl_cmd) {
-			case BIO_DELETE:
-				res->hr_stat_delete++;
-				break;
-			case BIO_FLUSH:
-				res->hr_stat_flush++;
-				break;
-			}
-			pjdlog_debug(2,
-			    "ggate_recv: (%p) Moving request to the send queue.",
-			    hio);
-			refcount_init(&hio->hio_countdown, ncomps);
-			for (ii = 0; ii < ncomps; ii++)
-				QUEUE_INSERT1(hio, send, ii);
+			res->hr_stat_flush++;
 			break;
 		}
+		pjdlog_debug(2,
+		    "ggate_recv: (%p) Moving request to the send queues.", hio);
+		refcount_init(&hio->hio_countdown, ncomps);
+		for (ii = ncomp; ii < ncomps; ii++)
+			QUEUE_INSERT1(hio, send, ii);
 	}
 	/* NOTREACHED */
 	return (NULL);
@@ -1288,6 +1325,11 @@ local_send_thread(void *arg)
 				    ret, (intmax_t)ggio->gctl_length);
 			} else {
 				hio->hio_errors[ncomp] = 0;
+				if (hio->hio_replication ==
+				    HAST_REPLICATION_ASYNC) {
+					ggio->gctl_error = 0;
+					write_complete(res, hio);
+				}
 			}
 			break;
 		case BIO_DELETE:
@@ -1322,18 +1364,18 @@ local_send_thread(void *arg)
 			}
 			break;
 		}
-		if (refcount_release(&hio->hio_countdown)) {
-			if (ISSYNCREQ(hio)) {
-				mtx_lock(&sync_lock);
-				SYNCREQDONE(hio);
-				mtx_unlock(&sync_lock);
-				cv_signal(&sync_cond);
-			} else {
-				pjdlog_debug(2,
-				    "local_send: (%p) Moving request to the done queue.",
-				    hio);
-				QUEUE_INSERT2(hio, done);
-			}
+		if (!refcount_release(&hio->hio_countdown))
+			continue;
+		if (ISSYNCREQ(hio)) {
+			mtx_lock(&sync_lock);
+			SYNCREQDONE(hio);
+			mtx_unlock(&sync_lock);
+			cv_signal(&sync_cond);
+		} else {
+			pjdlog_debug(2,
+			    "local_send: (%p) Moving request to the done queue.",
+			    hio);
+			QUEUE_INSERT2(hio, done);
 		}
 	}
 	/* NOTREACHED */
@@ -1456,9 +1498,6 @@ remote_send_thread(void *arg)
 			/* Move failed request immediately to the done queue. */
 			goto done_queue;
 		}
-		pjdlog_debug(2,
-		    "remote_send: (%p) Moving request to the recv queue.",
-		    hio);
 		/*
 		 * Protect connection from disappearing.
 		 */
@@ -1473,6 +1512,9 @@ remote_send_thread(void *arg)
 		 * in different order we can get reply before we move request
 		 * to recv queue.
 		 */
+		pjdlog_debug(2,
+		    "remote_send: (%p) Moving request to the recv queue.",
+		    hio);
 		mtx_lock(&hio_recv_list_lock[ncomp]);
 		wakeup = TAILQ_EMPTY(&hio_recv_list[ncomp]);
 		TAILQ_INSERT_TAIL(&hio_recv_list[ncomp], hio, hio_next[ncomp]);
@@ -1492,7 +1534,8 @@ remote_send_thread(void *arg)
 			 * it immediately to the done queue.
 			 */
 			mtx_lock(&hio_recv_list_lock[ncomp]);
-			TAILQ_REMOVE(&hio_recv_list[ncomp], hio, hio_next[ncomp]);
+			TAILQ_REMOVE(&hio_recv_list[ncomp], hio,
+			    hio_next[ncomp]);
 			mtx_unlock(&hio_recv_list_lock[ncomp]);
 			goto done_queue;
 		}
@@ -1558,6 +1601,7 @@ remote_recv_thread(void *arg)
 			    &hio_recv_list_lock[ncomp]);
 		}
 		mtx_unlock(&hio_recv_list_lock[ncomp]);
+
 		rw_rlock(&hio_remote_lock[ncomp]);
 		if (!ISCONNECTED(res, ncomp)) {
 			rw_unlock(&hio_remote_lock[ncomp]);
@@ -1602,16 +1646,16 @@ remote_recv_thread(void *arg)
 			nv_free(nv);
 			continue;
 		}
+		ggio = &hio->hio_ggio;
 		error = nv_get_int16(nv, "error");
 		if (error != 0) {
 			/* Request failed on remote side. */
 			hio->hio_errors[ncomp] = error;
-			reqlog(LOG_WARNING, 0, &hio->hio_ggio,
+			reqlog(LOG_WARNING, 0, ggio,
 			    "Remote request failed (%s): ", strerror(error));
 			nv_free(nv);
 			goto done_queue;
 		}
-		ggio = &hio->hio_ggio;
 		switch (ggio->gctl_cmd) {
 		case BIO_READ:
 			rw_rlock(&hio_remote_lock[ncomp]);
@@ -1642,18 +1686,18 @@ remote_recv_thread(void *arg)
 		hio->hio_errors[ncomp] = 0;
 		nv_free(nv);
 done_queue:
-		if (refcount_release(&hio->hio_countdown)) {
-			if (ISSYNCREQ(hio)) {
-				mtx_lock(&sync_lock);
-				SYNCREQDONE(hio);
-				mtx_unlock(&sync_lock);
-				cv_signal(&sync_cond);
-			} else {
-				pjdlog_debug(2,
-				    "remote_recv: (%p) Moving request to the done queue.",
-				    hio);
-				QUEUE_INSERT2(hio, done);
-			}
+		if (!refcount_release(&hio->hio_countdown))
+			continue;
+		if (ISSYNCREQ(hio)) {
+			mtx_lock(&sync_lock);
+			SYNCREQDONE(hio);
+			mtx_unlock(&sync_lock);
+			cv_signal(&sync_cond);
+		} else {
+			pjdlog_debug(2,
+			    "remote_recv: (%p) Moving request to the done queue.",
+			    hio);
+			QUEUE_INSERT2(hio, done);
 		}
 	}
 	/* NOTREACHED */
@@ -1669,7 +1713,7 @@ ggate_send_thread(void *arg)
 	struct hast_resource *res = arg;
 	struct g_gate_ctl_io *ggio;
 	struct hio *hio;
-	unsigned int ii, ncomp, ncomps;
+	unsigned int ii, ncomps;
 
 	ncomps = HAST_NCOMPONENTS;
 
@@ -1719,28 +1763,14 @@ ggate_send_thread(void *arg)
 			if (range_sync_wait)
 				cv_signal(&range_sync_cond);
 			mtx_unlock(&range_lock);
-			/*
-			 * Bump local count if this is first write after
-			 * connection failure with remote node.
-			 */
-			ncomp = 1;
-			rw_rlock(&hio_remote_lock[ncomp]);
-			if (!ISCONNECTED(res, ncomp)) {
-				mtx_lock(&metadata_lock);
-				if (res->hr_primary_localcnt ==
-				    res->hr_secondary_remotecnt) {
-					res->hr_primary_localcnt++;
-					pjdlog_debug(1,
-					    "Increasing localcnt to %ju.",
-					    (uintmax_t)res->hr_primary_localcnt);
-					(void)metadata_write(res);
-				}
-				mtx_unlock(&metadata_lock);
+			if (!hio->hio_done)
+				write_complete(res, hio);
+		} else {
+			if (ioctl(res->hr_ggatefd, G_GATE_CMD_DONE, ggio) < 0) {
+				primary_exit(EX_OSERR,
+				    "G_GATE_CMD_DONE failed");
 			}
-			rw_unlock(&hio_remote_lock[ncomp]);
 		}
-		if (ioctl(res->hr_ggatefd, G_GATE_CMD_DONE, ggio) < 0)
-			primary_exit(EX_OSERR, "G_GATE_CMD_DONE failed");
 		pjdlog_debug(2,
 		    "ggate_send: (%p) Moving request to the free queue.", hio);
 		QUEUE_INSERT2(hio, free);
@@ -1893,6 +1923,8 @@ sync_thread(void *arg __unused)
 		ggio->gctl_offset = offset;
 		ggio->gctl_length = length;
 		ggio->gctl_error = 0;
+		hio->hio_done = false;
+		hio->hio_replication = res->hr_replication;
 		for (ii = 0; ii < ncomps; ii++)
 			hio->hio_errors[ii] = EINVAL;
 		reqlog(LOG_DEBUG, 2, ggio, "sync: (%p) Sending sync request: ",
@@ -1965,7 +1997,7 @@ sync_thread(void *arg __unused)
 		}
 		mtx_unlock(&metadata_lock);
 
-		pjdlog_debug(2, "sync: (%p) Moving request to the send queues.",
+		pjdlog_debug(2, "sync: (%p) Moving request to the send queue.",
 		    hio);
 		refcount_init(&hio->hio_countdown, 1);
 		QUEUE_INSERT1(hio, send, ncomp);
@@ -2081,8 +2113,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 	 * Don't bother if we need to reconnect.
 	 */
 	if ((modified & MODIFIED_TIMEOUT) != 0 &&
-	    (modified & (MODIFIED_REMOTEADDR | MODIFIED_SOURCEADDR |
-	    MODIFIED_REPLICATION)) == 0) {
+	    (modified & (MODIFIED_REMOTEADDR | MODIFIED_SOURCEADDR)) == 0) {
 		for (ii = 0; ii < ncomps; ii++) {
 			if (!ISREMOTE(ii))
 				continue;
@@ -2104,8 +2135,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 			}
 		}
 	}
-	if ((modified & (MODIFIED_REMOTEADDR | MODIFIED_SOURCEADDR |
-	    MODIFIED_REPLICATION)) != 0) {
+	if ((modified & (MODIFIED_REMOTEADDR | MODIFIED_SOURCEADDR)) != 0) {
 		for (ii = 0; ii < ncomps; ii++) {
 			if (!ISREMOTE(ii))
 				continue;
diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c
index c761143..7a50f1e 100644
--- a/sbin/hastd/secondary.c
+++ b/sbin/hastd/secondary.c
@@ -67,7 +67,6 @@ __FBSDID("$FreeBSD$");
 struct hio {
 	uint64_t	 hio_seq;
 	int		 hio_error;
-	struct nv	*hio_nv;
 	void		*hio_data;
 	uint8_t		 hio_cmd;
 	uint64_t	 hio_offset;
@@ -128,6 +127,17 @@ static void *send_thread(void *arg);
 } while (0)
 
 static void
+hio_clear(struct hio *hio)
+{
+
+	hio->hio_seq = 0;
+	hio->hio_error = 0;
+	hio->hio_cmd = HIO_UNDEF;
+	hio->hio_offset = 0;
+	hio->hio_length = 0;
+}
+
+static void
 init_environment(void)
 {
 	struct hio *hio;
@@ -156,13 +166,13 @@ init_environment(void)
 			    "Unable to allocate memory (%zu bytes) for hio request.",
 			    sizeof(*hio));
 		}
-		hio->hio_error = 0;
 		hio->hio_data = malloc(MAXPHYS);
 		if (hio->hio_data == NULL) {
 			pjdlog_exitx(EX_TEMPFAIL,
 			    "Unable to allocate memory (%zu bytes) for gctl_data.",
 			    (size_t)MAXPHYS);
 		}
+		hio_clear(hio);
 		TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_next);
 	}
 }
@@ -268,6 +278,7 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 	} else if (res->hr_resuid != resuid) {
 		char errmsg[256];
 
+		free(map);
 		(void)snprintf(errmsg, sizeof(errmsg),
 		    "Resource unique ID mismatch (primary=%ju, secondary=%ju).",
 		    (uintmax_t)resuid, (uintmax_t)res->hr_resuid);
@@ -280,13 +291,13 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 		nv_free(nvout);
 		exit(EX_CONFIG);
 	} else if (
-	    /* Is primary is out-of-date? */
+	    /* Is primary out-of-date? */
 	    (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
 	     res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
-	    /* Nodes are more or less in sync? */
+	    /* Are the nodes more or less in sync? */
 	    (res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
 	     res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
-	    /* Is secondary is out-of-date? */
+	    /* Is secondary out-of-date? */
 	    (res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
 	     res->hr_secondary_remotecnt < res->hr_primary_localcnt)) {
 		/*
@@ -315,11 +326,17 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 		/*
 		 * Not good, we have split-brain condition.
 		 */
+		free(map);
 		pjdlog_error("Split-brain detected, exiting.");
 		nv_add_string(nvout, "Split-brain condition!", "errmsg");
-		free(map);
-		map = NULL;
-		mapsize = 0;
+		if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
+			pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
+			    res->hr_remoteaddr);
+		}
+		nv_free(nvout);
+		/* Exit on split-brain. */
+		event_send(res, EVENT_SPLITBRAIN);
+		exit(EX_CONFIG);
 	} else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
 	    res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ {
 		/*
@@ -358,12 +375,6 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 	if (proto_recv(res->hr_remotein, NULL, 0) == -1)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
 #endif
-	if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
-	     res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
-		/* Exit on split-brain. */
-		event_send(res, EVENT_SPLITBRAIN);
-		exit(EX_CONFIG);
-	}
 }
 
 void
@@ -508,15 +519,23 @@ reqlog(int loglevel, int debuglevel, int error, struct hio *hio, const char *fmt
 }
 
 static int
-requnpack(struct hast_resource *res, struct hio *hio)
+requnpack(struct hast_resource *res, struct hio *hio, struct nv *nv)
 {
 
-	hio->hio_cmd = nv_get_uint8(hio->hio_nv, "cmd");
+	hio->hio_cmd = nv_get_uint8(nv, "cmd");
 	if (hio->hio_cmd == 0) {
 		pjdlog_error("Header contains no 'cmd' field.");
 		hio->hio_error = EINVAL;
 		goto end;
 	}
+	if (hio->hio_cmd != HIO_KEEPALIVE) {
+		hio->hio_seq = nv_get_uint64(nv, "seq");
+		if (hio->hio_seq == 0) {
+			pjdlog_error("Header contains no 'seq' field.");
+			hio->hio_error = EINVAL;
+			goto end;
+		}
+	}
 	switch (hio->hio_cmd) {
 	case HIO_FLUSH:
 	case HIO_KEEPALIVE:
@@ -524,14 +543,14 @@ requnpack(struct hast_resource *res, struct hio *hio)
 	case HIO_READ:
 	case HIO_WRITE:
 	case HIO_DELETE:
-		hio->hio_offset = nv_get_uint64(hio->hio_nv, "offset");
-		if (nv_error(hio->hio_nv) != 0) {
+		hio->hio_offset = nv_get_uint64(nv, "offset");
+		if (nv_error(nv) != 0) {
 			pjdlog_error("Header is missing 'offset' field.");
 			hio->hio_error = EINVAL;
 			goto end;
 		}
-		hio->hio_length = nv_get_uint64(hio->hio_nv, "length");
-		if (nv_error(hio->hio_nv) != 0) {
+		hio->hio_length = nv_get_uint64(nv, "length");
+		if (nv_error(nv) != 0) {
 			pjdlog_error("Header is missing 'length' field.");
 			hio->hio_error = EINVAL;
 			goto end;
@@ -600,16 +619,18 @@ recv_thread(void *arg)
 {
 	struct hast_resource *res = arg;
 	struct hio *hio;
+	struct nv *nv;
 
 	for (;;) {
 		pjdlog_debug(2, "recv: Taking free request.");
 		QUEUE_TAKE(free, hio);
 		pjdlog_debug(2, "recv: (%p) Got request.", hio);
-		if (hast_proto_recv_hdr(res->hr_remotein, &hio->hio_nv) < 0) {
+		if (hast_proto_recv_hdr(res->hr_remotein, &nv) < 0) {
 			secondary_exit(EX_TEMPFAIL,
 			    "Unable to receive request header");
 		}
-		if (requnpack(res, hio) != 0) {
+		if (requnpack(res, hio, nv) != 0) {
+			nv_free(nv);
 			pjdlog_debug(2,
 			    "recv: (%p) Moving request to the send queue.",
 			    hio);
@@ -629,23 +650,30 @@ recv_thread(void *arg)
 		case HIO_FLUSH:
 			res->hr_stat_flush++;
 			break;
+		case HIO_KEEPALIVE:
+			break;
+		default:
+			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
+			    hio->hio_cmd);
 		}
 		reqlog(LOG_DEBUG, 2, -1, hio,
 		    "recv: (%p) Got request header: ", hio);
 		if (hio->hio_cmd == HIO_KEEPALIVE) {
+			nv_free(nv);
 			pjdlog_debug(2,
 			    "recv: (%p) Moving request to the free queue.",
 			    hio);
-			nv_free(hio->hio_nv);
+			hio_clear(hio);
 			QUEUE_INSERT(free, hio);
 			continue;
 		} else if (hio->hio_cmd == HIO_WRITE) {
-			if (hast_proto_recv_data(res, res->hr_remotein,
-			    hio->hio_nv, hio->hio_data, MAXPHYS) < 0) {
+			if (hast_proto_recv_data(res, res->hr_remotein, nv,
+			    hio->hio_data, MAXPHYS) < 0) {
 				secondary_exit(EX_TEMPFAIL,
 				    "Unable to receive request data");
 			}
 		}
+		nv_free(nv);
 		pjdlog_debug(2, "recv: (%p) Moving request to the disk queue.",
 		    hio);
 		QUEUE_INSERT(disk, hio);
@@ -752,6 +780,9 @@ disk_thread(void *arg)
 				hio->hio_error = 0;
 			}
 			break;
+		default:
+			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
+			    hio->hio_cmd);
 		}
 		if (logerror && hio->hio_error != 0) {
 			reqlog(LOG_ERR, 0, hio->hio_error, hio,
@@ -783,7 +814,7 @@ send_thread(void *arg)
 		reqlog(LOG_DEBUG, 2, -1, hio, "send: (%p) Got request: ", hio);
 		nvout = nv_alloc();
 		/* Copy sequence number. */
-		nv_add_uint64(nvout, nv_get_uint64(hio->hio_nv, "seq"), "seq");
+		nv_add_uint64(nvout, hio->hio_seq, "seq");
 		switch (hio->hio_cmd) {
 		case HIO_READ:
 			if (hio->hio_error == 0) {
@@ -814,8 +845,7 @@ send_thread(void *arg)
 		nv_free(nvout);
 		pjdlog_debug(2, "send: (%p) Moving request to the free queue.",
 		    hio);
-		nv_free(hio->hio_nv);
-		hio->hio_error = 0;
+		hio_clear(hio);
 		QUEUE_INSERT(free, hio);
 	}
 	/* NOTREACHED */
diff --git a/sbin/ifconfig/af_inet.c b/sbin/ifconfig/af_inet.c
index 19e27c9..edb9b80 100644
--- a/sbin/ifconfig/af_inet.c
+++ b/sbin/ifconfig/af_inet.c
@@ -104,8 +104,7 @@ in_getaddr(const char *s, int which)
 	struct netent *np;
 
 	sin->sin_len = sizeof(*sin);
-	if (which != MASK)
-		sin->sin_family = AF_INET;
+	sin->sin_family = AF_INET;
 
 	if (which == ADDR) {
 		char *p = NULL;
diff --git a/sbin/ifconfig/ifconfig.8 b/sbin/ifconfig/ifconfig.8
index 280baaa..c6b8ea3 100644
--- a/sbin/ifconfig/ifconfig.8
+++ b/sbin/ifconfig/ifconfig.8
@@ -28,7 +28,7 @@
 .\"     From: @(#)ifconfig.8	8.3 (Berkeley) 1/5/94
 .\" $FreeBSD$
 .\"
-.Dd September 13, 2011
+.Dd November 12, 2011
 .Dt IFCONFIG 8
 .Os
 .Sh NAME
@@ -1088,6 +1088,38 @@ The
 specifies the number of beacon intervals between DTIM
 and must be in the range 1 to 15.
 By default DTIM is 1 (i.e., DTIM occurs at each beacon).
+.It Cm quiet
+Enable the use of quiet IE.  Hostap will use this to silent other
+stations to reduce interference for radar detection when
+operating on 5Ghz frequency and doth support is enabled.
+Use
+.Fl quiet
+to disable this functionality.
+.It Cm quiet_period Ar period
+Set the QUIET
+.Ar period
+to the number of beacon intervals between the start of regularly
+scheduled quiet intervals defined by Quiet element.
+.It Cm quiet_count Ar count
+Set the QUIET
+.Ar count
+to the number of TBTTs until the beacon interval during which the
+next quiet interval shall start.  A value of 1 indicates the quiet
+interval will start during the beacon interval starting at the next
+TBTT. A value 0 is reserved.
+.It Cm quiet_offset Ar offset
+Set the QUIET
+.Ar offset
+to the offset of the start of the quiet interval from the TBTT
+specified by the Quiet count, expressed in TUs.
+The value of the
+.Ar offset
+shall be less than one beacon interval.
+.It Cm quiet_duration Ar dur
+Set the QUIET
+.Ar dur
+to the duration of the Quiet interval, expressed in TUs.
+The value should be less than beacon interval.
 .It Cm dturbo
 Enable the use of Atheros Dynamic Turbo mode when communicating with
 another Dynamic Turbo-capable station.
diff --git a/sbin/ifconfig/ifieee80211.c b/sbin/ifconfig/ifieee80211.c
index 77fcc8c..a84c5bf 100644
--- a/sbin/ifconfig/ifieee80211.c
+++ b/sbin/ifconfig/ifieee80211.c
@@ -1337,6 +1337,36 @@ set80211pureg(const char *val, int d, int s, const struct afswtch *rafp)
 }
 
 static void
+set80211quiet(const char *val, int d, int s, const struct afswtch *rafp)
+{
+	set80211(s, IEEE80211_IOC_QUIET, d, 0, NULL);
+}
+
+static
+DECL_CMD_FUNC(set80211quietperiod, val, d)
+{
+	set80211(s, IEEE80211_IOC_QUIET_PERIOD, atoi(val), 0, NULL);
+}
+
+static
+DECL_CMD_FUNC(set80211quietcount, val, d)
+{
+	set80211(s, IEEE80211_IOC_QUIET_COUNT, atoi(val), 0, NULL);
+}
+
+static
+DECL_CMD_FUNC(set80211quietduration, val, d)
+{
+	set80211(s, IEEE80211_IOC_QUIET_DUR, atoi(val), 0, NULL);
+}
+
+static
+DECL_CMD_FUNC(set80211quietoffset, val, d)
+{
+	set80211(s, IEEE80211_IOC_QUIET_OFFSET, atoi(val), 0, NULL);
+}
+
+static void
 set80211bgscan(const char *val, int d, int s, const struct afswtch *rafp)
 {
 	set80211(s, IEEE80211_IOC_BGSCAN, d, 0, NULL);
@@ -5161,6 +5191,12 @@ static struct cmd ieee80211_cmds[] = {
 	DEF_CMD_ARG("bgscanidle",	set80211bgscanidle),
 	DEF_CMD_ARG("bgscanintvl",	set80211bgscanintvl),
 	DEF_CMD_ARG("scanvalid",	set80211scanvalid),
+	DEF_CMD("quiet",        1,      set80211quiet),
+	DEF_CMD("-quiet",       0,      set80211quiet),
+	DEF_CMD_ARG("quiet_count",      set80211quietcount),
+	DEF_CMD_ARG("quiet_period",     set80211quietperiod),
+	DEF_CMD_ARG("quiet_dur",        set80211quietduration),
+	DEF_CMD_ARG("quiet_offset",     set80211quietoffset),
 	DEF_CMD_ARG("roam:rssi",	set80211roamrssi),
 	DEF_CMD_ARG("roam:rate",	set80211roamrate),
 	DEF_CMD_ARG("mcastrate",	set80211mcastrate),
diff --git a/sbin/init/init.c b/sbin/init/init.c
index 3f6b345..cce74bb 100644
--- a/sbin/init/init.c
+++ b/sbin/init/init.c
@@ -126,14 +126,14 @@ static state_func_t death_single(void);
 
 static state_func_t run_script(const char *);
 
-enum { AUTOBOOT, FASTBOOT } runcom_mode = AUTOBOOT;
+static enum { AUTOBOOT, FASTBOOT } runcom_mode = AUTOBOOT;
 #define FALSE	0
 #define TRUE	1
 
-int Reboot = FALSE;
-int howto = RB_AUTOBOOT;
+static int Reboot = FALSE;
+static int howto = RB_AUTOBOOT;
 
-int devfs;
+static int devfs;
 
 static void transition(state_t);
 static state_t requested_transition;
diff --git a/sbin/ipfw/ipfw.8 b/sbin/ipfw/ipfw.8
index 9d00287..0270802 100644
--- a/sbin/ipfw/ipfw.8
+++ b/sbin/ipfw/ipfw.8
@@ -1,7 +1,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 20, 2011
+.Dd November 10, 2011
 .Dt IPFW 8
 .Os
 .Sh NAME
@@ -769,7 +769,7 @@ To enable
 .Cm fwd
 a custom kernel needs to be compiled with the option
 .Cd "options IPFIREWALL_FORWARD" .
-.It Cm nat Ar nat_nr
+.It Cm nat Ar nat_nr | tablearg
 Pass packet to a
 nat instance
 (for network address translation, address redirect, etc.):
diff --git a/sbin/ipfw/main.c b/sbin/ipfw/main.c
index 433ce7d..e665b45 100644
--- a/sbin/ipfw/main.c
+++ b/sbin/ipfw/main.c
@@ -87,7 +87,7 @@ help(void)
  * which includes the program name and a NULL entry at the end.
  * If we are called with a single string, we split it on whitespace.
  * Also, arguments with a trailing ',' are joined to the next one.
- * The pointers (av[]) and data are in a a single chunk of memory.
+ * The pointers (av[]) and data are in a single chunk of memory.
  * av[0] points to the original program name, all other entries
  * point into the allocated chunk.
  */
diff --git a/sbin/md5/md5.1 b/sbin/md5/md5.1
index 2397d9e..bc8ece4 100644
--- a/sbin/md5/md5.1
+++ b/sbin/md5/md5.1
@@ -8,18 +8,22 @@
 .Sh SYNOPSIS
 .Nm md5
 .Op Fl pqrtx
+.Op Fl c Ar string
 .Op Fl s Ar string
 .Op Ar
 .Nm sha1
 .Op Fl pqrtx
+.Op Fl c Ar string
 .Op Fl s Ar string
 .Op Ar
 .Nm sha256
 .Op Fl pqrtx
+.Op Fl c Ar string
 .Op Fl s Ar string
 .Op Ar
 .Nm rmd160
 .Op Fl pqrtx
+.Op Fl c Ar string
 .Op Fl s Ar string
 .Op Ar
 .Sh DESCRIPTION
@@ -73,6 +77,9 @@ precede any files named on the command line.
 The hexadecimal checksum of each file listed on the command line is printed
 after the options are processed.
 .Bl -tag -width indent
+.It Fl c Ar string
+Compare files to this md5 string.
+(Note that this option is not yet useful if multiple files are specified.)
 .It Fl s Ar string
 Print a checksum of the given
 .Ar string .
@@ -101,7 +108,8 @@ The
 and
 .Nm rmd160
 utilities exit 0 on success,
-and 1 if at least one of the input files could not be read.
+1 if at least one of the input files could not be read,
+and 2 if at least one file does not have the same hash as the -c option.
 .Sh SEE ALSO
 .Xr cksum 1 ,
 .Xr md5 3 ,
diff --git a/sbin/md5/md5.c b/sbin/md5/md5.c
index 9436aa2..2e0a0d0 100644
--- a/sbin/md5/md5.c
+++ b/sbin/md5/md5.c
@@ -44,6 +44,8 @@ __FBSDID("$FreeBSD$");
 int qflag;
 int rflag;
 int sflag;
+unsigned char* checkAgainst;
+int	checksFailed;
 
 typedef void (DIGEST_Init)(void *);
 typedef void (DIGEST_Update)(void *, const unsigned char *, size_t);
@@ -138,8 +140,13 @@ main(int argc, char *argv[])
  		digest = 0;
 
 	failed = 0;
-	while ((ch = getopt(argc, argv, "pqrs:tx")) != -1)
+	checkAgainst = NULL;
+	checksFailed = 0;
+	while ((ch = getopt(argc, argv, "c:pqrs:tx")) != -1)
 		switch (ch) {
+		case 'c':
+			checkAgainst = optarg;
+			break;
 		case 'p':
 			MDFilter(&Algorithm[digest], 1);
 			break;
@@ -173,12 +180,19 @@ main(int argc, char *argv[])
 				failed++;
 			} else {
 				if (qflag)
-					printf("%s\n", p);
+					printf("%s", p);
 				else if (rflag)
-					printf("%s %s\n", p, *argv);
+					printf("%s %s", p, *argv);
 				else
-					printf("%s (%s) = %s\n",
+					printf("%s (%s) = %s",
 					    Algorithm[digest].name, *argv, p);
+				if (checkAgainst && strcmp(checkAgainst,p))
+				{
+					checksFailed++;
+					if (!qflag)
+						printf(" [ Failed ]");
+				}
+				printf("\n");
 			}
 		} while (*++argv);
 	} else if (!sflag && (optind == 1 || qflag || rflag))
@@ -186,6 +200,8 @@ main(int argc, char *argv[])
 
 	if (failed != 0)
 		return (1);
+	if (checksFailed != 0)
+		return (2);
 
 	return (0);
 }
@@ -198,12 +214,20 @@ MDString(Algorithm_t *alg, const char *string)
 	size_t len = strlen(string);
 	char buf[HEX_DIGEST_LENGTH];
 
+	alg->Data(string,len,buf);
 	if (qflag)
-		printf("%s\n", alg->Data(string, len, buf));
+		printf("%s", buf);
 	else if (rflag)
-		printf("%s \"%s\"\n", alg->Data(string, len, buf), string);
+		printf("%s \"%s\"", buf, string);
 	else
-		printf("%s (\"%s\") = %s\n", alg->name, string, alg->Data(string, len, buf));
+		printf("%s (\"%s\") = %s", alg->name, string, buf);
+	if (checkAgainst && strcmp(buf,checkAgainst))
+	{
+		checksFailed++;
+		if (!qflag)
+			printf(" [ failed ]");
+	}
+	printf("\n");
 }
 /*
  * Measures the time to digest TEST_BLOCK_COUNT TEST_BLOCK_LEN-byte blocks.
diff --git a/sbin/mount/mount.c b/sbin/mount/mount.c
index 3f6d2ea..cf142e4 100644
--- a/sbin/mount/mount.c
+++ b/sbin/mount/mount.c
@@ -66,7 +66,7 @@ __FBSDID("$FreeBSD$");
 #define MOUNT_META_OPTION_FSTAB		"fstab"
 #define MOUNT_META_OPTION_CURRENT	"current"
 
-int debug, fstab_style, verbose;
+static int debug, fstab_style, verbose;
 
 struct cpa {
 	char	**a;
diff --git a/sbin/mount/mount_fs.c b/sbin/mount/mount_fs.c
index cb25bc5..526d493 100644
--- a/sbin/mount/mount_fs.c
+++ b/sbin/mount/mount_fs.c
@@ -58,7 +58,7 @@ static const char rcsid[] =
 #include "extern.h"
 #include "mntopts.h"
 
-struct mntopt mopts[] = {
+static struct mntopt mopts[] = {
 	MOPT_STDOPTS,
 	MOPT_END
 };
diff --git a/sbin/mount_cd9660/mount_cd9660.c b/sbin/mount_cd9660/mount_cd9660.c
index 922f6cf..3b2827e 100644
--- a/sbin/mount_cd9660/mount_cd9660.c
+++ b/sbin/mount_cd9660/mount_cd9660.c
@@ -68,7 +68,7 @@ static const char rcsid[] =
 
 #include "mntopts.h"
 
-struct mntopt mopts[] = {
+static struct mntopt mopts[] = {
 	MOPT_STDOPTS,
 	MOPT_UPDATE,
 	MOPT_END
diff --git a/sbin/mount_nfs/mount_nfs.8 b/sbin/mount_nfs/mount_nfs.8
index be9e309..bdb0707 100644
--- a/sbin/mount_nfs/mount_nfs.8
+++ b/sbin/mount_nfs/mount_nfs.8
@@ -318,6 +318,10 @@ tune the timeout
 interval.)
 .It Cm udp
 Use UDP transport.
+.It Cm wcommitsize Ns = Ns Aq Ar value
+Set the maximum pending write commit size to the specified value.
+This determines the maximum amount of pending write data that the NFS
+client is willing to cache for each file.
 .It Cm wsize Ns = Ns Aq Ar value
 Set the write data size to the specified value.
 Ditto the comments w.r.t.\& the
diff --git a/sbin/mount_nfs/mount_nfs.c b/sbin/mount_nfs/mount_nfs.c
index 5386723..9da3137 100644
--- a/sbin/mount_nfs/mount_nfs.c
+++ b/sbin/mount_nfs/mount_nfs.c
@@ -612,6 +612,13 @@ fallback_mount(struct iovec *iov, int iovlen, int mntflags)
 		}
 		args.flags |= NFSMNT_ACDIRMAX;
 	}
+	if (findopt(iov, iovlen, "wcommitsize", &opt, NULL) == 0) {
+		ret = sscanf(opt, "%d", &args.wcommitsize);
+		if (ret != 1 || args.wcommitsize < 0) {
+			errx(1, "illegal wcommitsize: %s", opt);
+		}
+		args.flags |= NFSMNT_WCOMMITSIZE;
+	}
 	if (findopt(iov, iovlen, "deadthresh", &opt, NULL) == 0) {
 		ret = sscanf(opt, "%d", &args.deadthresh);
 		if (ret != 1 || args.deadthresh <= 0) {
diff --git a/sbin/mount_nullfs/mount_nullfs.c b/sbin/mount_nullfs/mount_nullfs.c
index abca9fa..4f84954 100644
--- a/sbin/mount_nullfs/mount_nullfs.c
+++ b/sbin/mount_nullfs/mount_nullfs.c
@@ -57,7 +57,7 @@ static const char rcsid[] =
 
 #include "mntopts.h"
 
-struct mntopt mopts[] = {
+static struct mntopt mopts[] = {
 	MOPT_STDOPTS,
 	MOPT_END
 };
diff --git a/sbin/mount_udf/mount_udf.c b/sbin/mount_udf/mount_udf.c
index 0175f20..033db70 100644
--- a/sbin/mount_udf/mount_udf.c
+++ b/sbin/mount_udf/mount_udf.c
@@ -61,7 +61,7 @@
 
 #include "mntopts.h"
 
-struct mntopt mopts[] = {
+static struct mntopt mopts[] = {
 	MOPT_STDOPTS,
 	MOPT_UPDATE,
 	MOPT_END
diff --git a/sbin/natd/natd.c b/sbin/natd/natd.c
index 05f7463..3dbfef0 100644
--- a/sbin/natd/natd.c
+++ b/sbin/natd/natd.c
@@ -71,8 +71,8 @@ struct instance {
 static LIST_HEAD(, instance) root = LIST_HEAD_INITIALIZER(root);
 
 struct libalias *mla;
-struct instance *mip;
-int ninstance = 1;
+static struct instance *mip;
+static int ninstance = 1;
 
 /* 
  * Default values for input and output
diff --git a/sbin/newfs/mkfs.c b/sbin/newfs/mkfs.c
index 0503ebf..c40026a 100644
--- a/sbin/newfs/mkfs.c
+++ b/sbin/newfs/mkfs.c
@@ -804,7 +804,7 @@ initcg(int cylno, time_t utime)
  */
 #define ROOTLINKCNT 3
 
-struct direct root_dir[] = {
+static struct direct root_dir[] = {
 	{ ROOTINO, sizeof(struct direct), DT_DIR, 1, "." },
 	{ ROOTINO, sizeof(struct direct), DT_DIR, 2, ".." },
 	{ ROOTINO + 1, sizeof(struct direct), DT_DIR, 5, ".snap" },
@@ -812,7 +812,7 @@ struct direct root_dir[] = {
 
 #define SNAPLINKCNT 2
 
-struct direct snap_dir[] = {
+static struct direct snap_dir[] = {
 	{ ROOTINO + 1, sizeof(struct direct), DT_DIR, 1, "." },
 	{ ROOTINO, sizeof(struct direct), DT_DIR, 2, ".." },
 };
diff --git a/sbin/newfs/newfs.h b/sbin/newfs/newfs.h
index 326c518..d25baa7 100644
--- a/sbin/newfs/newfs.h
+++ b/sbin/newfs/newfs.h
@@ -81,6 +81,7 @@ extern int	Nflag;		/* run mkfs without writing file system */
 extern int	Oflag;		/* build UFS1 format file system */
 extern int	Rflag;		/* regression test */
 extern int	Uflag;		/* enable soft updates for file system */
+extern int	jflag;		/* enable soft updates journaling for filesys */
 extern int	Xflag;		/* exit in middle of newfs for testing */
 extern int	Jflag;		/* enable gjournal for file system */
 extern int	lflag;		/* enable multilabel MAC for file system */
diff --git a/sbin/newfs_msdos/newfs_msdos.c b/sbin/newfs_msdos/newfs_msdos.c
index 6d8fa7e..c7ec59b 100644
--- a/sbin/newfs_msdos/newfs_msdos.c
+++ b/sbin/newfs_msdos/newfs_msdos.c
@@ -99,7 +99,7 @@ static const char rcsid[] =
 struct bs {
     u_int8_t bsJump[3];		/* bootstrap entry point */
     u_int8_t bsOemName[8];		/* OEM name and version */
-};
+} __packed;
 
 struct bsbpb {
     u_int8_t bpbBytesPerSec[2];		/* bytes per sector */
@@ -114,7 +114,7 @@ struct bsbpb {
     u_int8_t bpbHeads[2];		/* drive heads */
     u_int8_t bpbHiddenSecs[4];		/* hidden sectors */
     u_int8_t bpbHugeSectors[4];		/* big total sectors */
-};
+} __packed;
 
 struct bsxbpb {
     u_int8_t bpbBigFATsecs[4];		/* big sectors per FAT */
@@ -124,7 +124,7 @@ struct bsxbpb {
     u_int8_t bpbFSInfo[2];		/* file system info sector */
     u_int8_t bpbBackup[2];		/* backup boot sector */
     u_int8_t bpbReserved[12];			/* reserved */
-};
+} __packed;
 
 struct bsx {
     u_int8_t exDriveNumber;		/* drive number */
@@ -133,7 +133,7 @@ struct bsx {
     u_int8_t exVolumeID[4];		/* volume ID number */
     u_int8_t exVolumeLabel[11]; 	/* volume label */
     u_int8_t exFileSysType[8];		/* file system type */
-};
+} __packed;
 
 struct de {
     u_int8_t deName[11];	/* name and extension */
@@ -143,7 +143,7 @@ struct de {
     u_int8_t deMDate[2];	/* creation date */
     u_int8_t deStartCluster[2];	/* starting cluster */
     u_int8_t deFileSize[4];	/* size */
-};
+} __packed;
 
 struct bpb {
     u_int bpbBytesPerSec;		/* bytes per sector */
diff --git a/sbin/rcorder/rcorder.c b/sbin/rcorder/rcorder.c
index 9a9959b..5eedc48 100644
--- a/sbin/rcorder/rcorder.c
+++ b/sbin/rcorder/rcorder.c
@@ -73,9 +73,9 @@ int debug = 0;
 #define KEYWORDS_STR	"# KEYWORDS:"
 #define KEYWORDS_LEN	(sizeof(KEYWORDS_STR) - 1)
 
-int exit_code;
-int file_count;
-char **file_list;
+static int exit_code;
+static int file_count;
+static char **file_list;
 
 typedef int bool;
 #define TRUE 1
diff --git a/sbin/recoverdisk/recoverdisk.c b/sbin/recoverdisk/recoverdisk.c
index 3930ef1..3a92d1d 100644
--- a/sbin/recoverdisk/recoverdisk.c
+++ b/sbin/recoverdisk/recoverdisk.c
@@ -24,7 +24,7 @@
 #include <time.h>
 #include <unistd.h>
 
-volatile sig_atomic_t aborting = 0;
+static volatile sig_atomic_t aborting = 0;
 static size_t bigsize = 1024 * 1024;
 static size_t medsize;
 static size_t minsize = 512;
diff --git a/sbin/shutdown/shutdown.c b/sbin/shutdown/shutdown.c
index 8e722ca..7bd93a0 100644
--- a/sbin/shutdown/shutdown.c
+++ b/sbin/shutdown/shutdown.c
@@ -67,7 +67,7 @@ __FBSDID("$FreeBSD$");
 #define	M		*60
 #define	S		*1
 #define	NOLOG_TIME	5*60
-struct interval {
+static struct interval {
 	int timeleft, timetowait;
 } tlist[] = {
 	{ 10 H,  5 H },
diff --git a/sbin/swapon/swapon.c b/sbin/swapon/swapon.c
index ed13dc5..5b9a0ed 100644
--- a/sbin/swapon/swapon.c
+++ b/sbin/swapon/swapon.c
@@ -60,7 +60,7 @@ static void usage(void);
 static int swap_on_off(char *name, int ignoreebusy);
 static void swaplist(int, int, int);
 
-enum { SWAPON, SWAPOFF, SWAPCTL } orig_prog, which_prog = SWAPCTL;
+static enum { SWAPON, SWAPOFF, SWAPCTL } orig_prog, which_prog = SWAPCTL;
 
 int
 main(int argc, char **argv)
diff --git a/sbin/tunefs/tunefs.c b/sbin/tunefs/tunefs.c
index c219607..11ffbe6 100644
--- a/sbin/tunefs/tunefs.c
+++ b/sbin/tunefs/tunefs.c
@@ -70,7 +70,7 @@ __FBSDID("$FreeBSD$");
 /* the optimization warning string template */
 #define	OPTWARN	"should optimize for %s with minfree %s %d%%"
 
-struct uufsd disk;
+static struct uufsd disk;
 #define	sblock disk.d_fs
 
 void usage(void);
@@ -553,7 +553,7 @@ sbdirty(void)
 	disk.d_fs.fs_clean = 0;
 }
 
-int blocks;
+static int blocks;
 static char clrbuf[MAXBSIZE];
 
 static ufs2_daddr_t
diff --git a/sbin/umount/umount.c b/sbin/umount/umount.c
index 6c65aec..f179a4c 100644
--- a/sbin/umount/umount.c
+++ b/sbin/umount/umount.c
@@ -63,9 +63,9 @@ static const char rcsid[] =
 
 typedef enum { FIND, REMOVE, CHECKUNIQUE } dowhat;
 
-struct  addrinfo *nfshost_ai = NULL;
-int	fflag, vflag;
-char   *nfshost;
+static struct addrinfo *nfshost_ai = NULL;
+static int	fflag, vflag;
+static char	*nfshost;
 
 struct statfs *checkmntlist(char *);
 int	 checkvfsname (const char *, char **);
diff --git a/secure/usr.bin/openssl/man/ca.1 b/secure/usr.bin/openssl/man/ca.1
index aecc18c..15095cf 100644
--- a/secure/usr.bin/openssl/man/ca.1
+++ b/secure/usr.bin/openssl/man/ca.1
@@ -205,7 +205,7 @@ section for information on the required format.
 .IP "\fB\-infiles\fR" 4
 .IX Item "-infiles"
 if present this should be the last option, all subsequent arguments
-are assumed to the names of files containing certificate requests.
+are assumed to be the names of files containing certificate requests.
 .IP "\fB\-out filename\fR" 4
 .IX Item "-out filename"
 the output file to output certificates to. The default is standard
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index b38a01c..0d5a780 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -253,6 +253,7 @@ MAN=	aac.4 \
 	net80211.4 \
 	netgraph.4 \
 	netintro.4 \
+	netmap.4 \
 	${_nfe.4} \
 	${_nfsmb.4} \
 	ng_async.4 \
@@ -347,6 +348,7 @@ MAN=	aac.4 \
 	pts.4 \
 	pty.4 \
 	puc.4 \
+	${_qlxgb.4} \
 	ral.4 \
 	random.4 \
 	rc.4 \
@@ -376,6 +378,7 @@ MAN=	aac.4 \
 	send.4 \
 	ses.4 \
 	sf.4 \
+	sfxge.4 \
 	sge.4 \
 	si.4 \
 	siba.4 \
@@ -713,6 +716,10 @@ _xen.4=		xen.4
 MLINKS+=lindev.4 full.4
 .endif
 
+.if ${MACHINE_CPUARCH} == "amd64"
+_qlxgb.4=	qlxgb.4
+.endif
+
 .if ${MACHINE_CPUARCH} == "powerpc"
 _atp.4=		atp.4
 .endif
diff --git a/share/man/man4/altq.4 b/share/man/man4/altq.4
index 7cfced4..90fe440 100644
--- a/share/man/man4/altq.4
+++ b/share/man/man4/altq.4
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 30, 2010
+.Dd November 8, 2011
 .Dt ALTQ 4
 .Os
 .Sh NAME
@@ -150,6 +150,7 @@ They have been applied to the following hardware drivers:
 .Xr nge 4 ,
 .Xr npe 4 ,
 .Xr nve 4 ,
+.Xr qlxgb 4 ,
 .Xr ral 4 ,
 .Xr re 4 ,
 .Xr rl 4 ,
@@ -160,6 +161,7 @@ They have been applied to the following hardware drivers:
 .Xr sk 4 ,
 .Xr ste 4 ,
 .Xr stge 4 ,
+.Xr ti 4 ,
 .Xr txp 4 ,
 .Xr udav 4 ,
 .Xr ural 4 ,
diff --git a/share/man/man4/esp.4 b/share/man/man4/esp.4
index 897ee8e..6ebc85b 100644
--- a/share/man/man4/esp.4
+++ b/share/man/man4/esp.4
@@ -1,5 +1,5 @@
 .\"
-.\" Copyright (c) 2004 Tom Rhodes
+.\" Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,158 +25,91 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 18, 2005
+.Dd November 1, 2011
 .Dt ESP 4
 .Os
 .Sh NAME
 .Nm esp
-.Nd Qlogic FEPS and FAS366 SCSI driver
+.Nd Emulex ESP, NCR 53C9x and QLogic FAS families based SCSI controllers
 .Sh SYNOPSIS
-To compile this driver into the kernel,
-place the following lines in your
+To compile this driver into the kernel, place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
 .Cd "device scbus"
 .Cd "device esp"
 .Ed
-.Sh HARDWARE
+.Pp
+Alternatively, to load the driver as a module at boot time, place the
+following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+if_esp_load="YES"
+.Ed
+.Sh DESCRIPTION
 The
 .Nm
 driver provides support for the
-.Tn Qlogic
-FAS216 and FAS408
+.Tn AMD
+Am53C974, the
+.Tn Emulex
+ESP100, ESP100A, ESP200 and ESP406, the
+.Tn NCR
+53C90, 53C94 and 53C96 as well as the
+.Tn QLogic
+FAS100A, FAS216, FAS366 and FAS408
 .Tn SCSI
 controller chips found in a wide variety of systems and peripheral boards.
-This includes the
-.Tn Qlogic
-.Tn SCSI
-cards found in most
-.Tn Sun
-.Tn Ultra 1e
-and
-.Tn Ultra 2
-machines.
-.Pp
-For
-.Tn Qlogic
-.Tn PCI
-.Tn SCSI
-host adapters, the
-.Xr isp 4
-driver should be used in place of the
-.Nm
-driver.
-.Sh IMPLEMENTATION NOTES
-During the boot sequence, the
+.Sh HARDWARE
+Controllers supported by the
 .Nm
-driver
-searches the linked list for LUN info by LUN ID.
-At this point the driver will attach on that instance,
-and then attempt to attach every sub-device.
-.Pp
-There are at least 4 variations of the
-configuration 3 register.
-A second config register must be loaded to inquire the chip revision,
-otherwise the
-.Fn ncr53c9x_reset
-function will not set the defaults correctly.
-.Pp
-The drive attach routine attempts to set the appropriate
-bit to put the chip into Fast
-.Tn SCSI
-mode so that it does not
-have to be figured out each time.
-This will then be stored in the
-.Tn NCR
-registers for later use.
+driver include:
 .Pp
-The
-.Nm
-driver makes use of the
-.Tn LSI
-64854
-.Tn DMA
-engine which contains three independent
-channels designed to interface with
-an NCR539X
-.Tn SCSI
-controller;
-an AM7990 Ethernet controller;
-and certain types of parallel port hardware.
-As such this driver may eventually replace the
-.Xr amd 4
-driver for network hardware.
-.Sh DIAGNOSTICS
-The following driver specific error messages
-may be reported:
-.Bl -diag
-.It "esp: cannot map %d segments"
-The device was unable to map segments while mapping the
-.Tn SCSI
-devices in
-.Tn DMA
-mode.
-.It "unknown variant %d, devices not attached"
-An unknown version of the card was found and devices will
-not be attached.
-.It "Unhandled function code %d"
-The device returned a function code which the driver is
-unable to handle, most likely during negotiation.
-.It "esp:%d:%d: selection failed %d left in FIFO"
-The target left the command phase prematurely.
-If a command did not reach the device then it will
-be reported as shown above.
-.It "invalid state: %d"
-The device has reached an invalid state.
-.El
-.Sh SEE ALSO
-.Xr amd 4 ,
-.Xr isp 4 ,
-.Xr scbus 4 ,
-.Xr camcontrol 8
-.Pp
-.Bl -item -compact
+.Bl -bullet -compact
+.It
+MELCO IFC-DP (PC-98)
+.It
+Sun ESP family
+.It
+Sun FAS family
 .It
-.Pa http://www.qlc.com/
+Tekram DC390
 .It
-.Pa http://www.sun.com/
+Tekram DC390T
 .El
+.Sh SEE ALSO
+.Xr cd 4 ,
+.Xr ch 4 ,
+.Xr da 4 ,
+.Xr intro 4 ,
+.Xr pci 4 ,
+.Xr sa 4 ,
+.Xr sbus 4 ,
+.Xr scsi 4 ,
+.Xr camcontrol 8
 .Sh HISTORY
 The
 .Nm
-driver was written for
-.Nx
-by
-.An Eric S. Hvozda .
+driver first appeared in
+.Nx 1.3 .
+The first
+.Fx
+version to include it was
+.Fx 5.3 .
+.Sh AUTHORS
+.An -nosplit
 The
 .Nm
-driver was then ported to
-.Fx 5.3
+driver was ported to
+.Fx
 by
-.An Scott Long Aq scottl@FreeBSD.org .
-.Sh AUTHORS
-This manual page was written by
-.An Tom Rhodes Aq trhodes@FreeBSD.org
-instead of porting the original from
-.Nx .
+.An Scott Long
+.Aq scottl@FreeBSD.org
+and later on considerably improved by
+.An Marius Strobl
+.Aq marius@FreeBSD.org .
 .Sh BUGS
 The
 .Nm
-driver includes the sbus front-end but lacks the
-.Tn PCI
-front-end.
-Thus it can only support the
-.Tn FEPS/FAS366
-.Tn SCSI
-found in
-.Tn Sun
-.Tn Ultra 1e
-and
-.Tn Ultra 2
-machines at this time.
-.Pp
-This driver should eventually replace the
-.Xr amd 4
-driver, but that requires porting the
-.Tn PCI
-front-end mentioned above.
+driver should read the EEPROM settings of
+.Tn Tekram
+controllers.
diff --git a/share/man/man4/ip.4 b/share/man/man4/ip.4
index ea9172b..4e7d8b0 100644
--- a/share/man/man4/ip.4
+++ b/share/man/man4/ip.4
@@ -32,7 +32,7 @@
 .\"     @(#)ip.4	8.2 (Berkeley) 11/30/93
 .\" $FreeBSD$
 .\"
-.Dd June 1, 2009
+.Dd November 14, 2011
 .Dt IP 4
 .Os
 .Sh NAME
@@ -847,3 +847,10 @@ The
 .Vt ip_mreqn
 structure appeared in
 .Tn Linux 2.4 .
+.Sh BUGS
+Before
+.Fx 10.0
+packets received on raw IP sockets had the
+.Va ip_hl
+subtracted from the
+.Va ip_len field.
diff --git a/share/man/man4/mps.4 b/share/man/man4/mps.4
index e22bf15..2899747 100644
--- a/share/man/man4/mps.4
+++ b/share/man/man4/mps.4
@@ -60,9 +60,9 @@ driver provides support for LSI Logic Fusion-MPT 2
 .Tn SAS
 controllers.
 .Sh HARDWARE
-The following controllers are supported by the
+The
 .Nm
-driver
+driver supports the following controllers:
 .Pp
 .Bl -bullet -compact
 .It
diff --git a/share/man/man4/mtio.4 b/share/man/man4/mtio.4
index 0b60b1e..f3dd602 100644
--- a/share/man/man4/mtio.4
+++ b/share/man/man4/mtio.4
@@ -35,7 +35,7 @@
 .\"     @(#)mtio.4	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd February 11, 1996
+.Dd November 11, 2011
 .Dt MTIO 4
 .Os
 .Sh NAME
@@ -97,7 +97,7 @@ The following definitions are from
 /* structure for MTIOCTOP - mag tape op command */
 struct mtop {
 	short	mt_op;		/* operations defined below */
-	daddr_t	mt_count;	/* how many of them */
+	int32_t	mt_count;	/* how many of them */
 };
 
 /* operations */
@@ -165,19 +165,25 @@ struct mtget {
 	short	mt_dsreg;	/* ``drive status'' register */
 	short	mt_erreg;	/* ``error'' register */
 /* end device-dependent registers */
+	/*
+	 * Note that the residual count, while maintained, may be
+	 * be nonsense because the size of the residual may (greatly)
+	 * exceed 32 K-bytes. Use the MTIOCERRSTAT ioctl to get a
+	 * more accurate count.
+	 */
 	short	mt_resid;	/* residual count */
 #if defined (__FreeBSD__)
-	daddr_t mt_blksiz;	/* presently operating blocksize */
-	daddr_t mt_density;	/* presently operating density */
+	int32_t mt_blksiz;	/* presently operating blocksize */
+	int32_t mt_density;	/* presently operating density */
 	u_int32_t mt_comp;	/* presently operating compression */
-	daddr_t mt_blksiz0;	/* blocksize for mode 0 */
-	daddr_t mt_blksiz1;	/* blocksize for mode 1 */
-	daddr_t mt_blksiz2;	/* blocksize for mode 2 */
-	daddr_t mt_blksiz3;	/* blocksize for mode 3 */
-	daddr_t mt_density0;	/* density for mode 0 */
-	daddr_t mt_density1;	/* density for mode 1 */
-	daddr_t mt_density2;	/* density for mode 2 */
-	daddr_t mt_density3;	/* density for mode 3 */
+	int32_t mt_blksiz0;	/* blocksize for mode 0 */
+	int32_t mt_blksiz1;	/* blocksize for mode 1 */
+	int32_t mt_blksiz2;	/* blocksize for mode 2 */
+	int32_t mt_blksiz3;	/* blocksize for mode 3 */
+	int32_t mt_density0;	/* density for mode 0 */
+	int32_t mt_density1;	/* density for mode 1 */
+	int32_t mt_density2;	/* density for mode 2 */
+	int32_t mt_density3;	/* density for mode 3 */
 /* the following are not yet implemented */
 	u_int32_t mt_comp0;	/* compression type for mode 0 */
 	u_int32_t mt_comp1;	/* compression type for mode 1 */
@@ -185,8 +191,8 @@ struct mtget {
 	u_int32_t mt_comp3;	/* compression type for mode 3 */
 /* end not yet implemented */
 #endif
-	daddr_t	mt_fileno;	/* relative file number of current position */
-	daddr_t	mt_blkno;	/* relative block number of current position */
+	int32_t	mt_fileno;	/* relative file number of current position */
+	int32_t	mt_blkno;	/* relative block number of current position */
 };
 
 /* structure for MTIOCERRSTAT - tape get error status command */
@@ -198,10 +204,10 @@ struct scsi_tape_errors {
 	 * of issuing an MTIOCERRSTAT unlatches and clears them.
 	 */
 	u_int8_t io_sense[32];	/* Last Sense Data For Data I/O */
-	u_int32_t io_resid;	/* residual count from last Data I/O */
+	int32_t io_resid;	/* residual count from last Data I/O */
 	u_int8_t io_cdb[16];	/* Command that Caused the Last Data Sense */
 	u_int8_t ctl_sense[32];	/* Last Sense Data For Control I/O */
-	u_int32_t ctl_resid;	/* residual count from last Control I/O */
+	int32_t ctl_resid;	/* residual count from last Control I/O */
 	u_int8_t ctl_cdb[16];	/* Command that Caused the Last Control Sense */
 	/*
 	 * These are the read and write cumulative error counters.
@@ -276,20 +282,6 @@ union mterrstat {
 #define	DEFTAPE	"/dev/nsa0"
 #endif
 
-#ifdef	_KERNEL
-/*
- * minor device number
- */
-
-#define	T_UNIT		003		/* unit selection */
-#define	T_NOREWIND	004		/* no rewind on close */
-#define	T_DENSEL	030		/* density select */
-#define	T_800BPI	000		/* select  800 bpi */
-#define	T_1600BPI	010		/* select 1600 bpi */
-#define	T_6250BPI	020		/* select 6250 bpi */
-#define	T_BADBPI	030		/* undefined selection */
-#endif
-
 #endif /* !_SYS_MTIO_H_ */
 .Ed
 .Sh FILES
diff --git a/share/man/man4/mvs.4 b/share/man/man4/mvs.4
index 23eed9d..a7bbe80 100644
--- a/share/man/man4/mvs.4
+++ b/share/man/man4/mvs.4
@@ -109,7 +109,8 @@ hardware command queues (up to 31 command per port),
 Native Command Queuing, SATA interface Power Management, device hot-plug
 and Message Signaled Interrupts.
 .Pp
-Same hardware is also supported by atamarvell and ataadaptec drivers from
+The same hardware is also supported by the atamarvell and ataadaptec
+drivers from the
 .Xr ata 4
 subsystem.
 If both drivers are loaded at the same time, this one will be
@@ -118,9 +119,9 @@ given precedence as the more functional of the two.
 The
 .Nm
 driver supports the following controllers:
-.Bl -tag -compact
-.It Gen-I (SATA 1.5Gbps):
-.Bl -bullet -compact
+.Pp
+Gen-I (SATA 1.5Gbps):
+.Bl -bullet -compact -offset "xxxxxx"
 .It
 88SX5040
 .It
@@ -130,8 +131,9 @@ driver supports the following controllers:
 .It
 88SX5081
 .El
-.It Gen-II (SATA 3Gbps, NCQ, PMP):
-.Bl -bullet -compact
+.Pp
+Gen-II (SATA 3Gbps, NCQ, PMP):
+.Bl -bullet -compact -offset "xxxxxx"
 .It
 88SX6040
 .It
@@ -141,8 +143,9 @@ driver supports the following controllers:
 .It
 88SX6081
 .El
-.It Gen-IIe (SATA 3Gbps, NCQ, PMP with FBS):
-.Bl -bullet -compact
+.Pp
+Gen-IIe (SATA 3Gbps, NCQ, PMP with FBS):
+.Bl -bullet -compact -offset "xxxxxx"
 .It
 88SX6042
 .It
@@ -154,11 +157,10 @@ driver supports the following controllers:
 .It
 MV78100 SoC
 .El
-.El
-Note, that this hardware supports command queueing and FIS-based switching
-only for ATA DMA commands. ATAPI and non-DMA ATA commands executed one by one
-for each port.
 .Pp
+Note, that this hardware supports command queueing and FIS-based switching
+only for ATA DMA commands.
+ATAPI and non-DMA ATA commands executed one by one for each port.
 .Sh SEE ALSO
 .Xr ada 4 ,
 .Xr ata 4 ,
@@ -172,4 +174,4 @@ The
 driver first appeared in
 .Fx 8.1 .
 .Sh AUTHORS
-.An Alexander Motin Aq mav@FreeBSD.org .
+.An Alexander Motin Aq mav@FreeBSD.org
diff --git a/share/man/man4/netmap.4 b/share/man/man4/netmap.4
new file mode 100644
index 0000000..8b646f9
--- /dev/null
+++ b/share/man/man4/netmap.4
@@ -0,0 +1,300 @@
+.\" Copyright (c) 2011 Matteo Landi, Luigi Rizzo, Universita` di Pisa
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\" 
+.\" This document is derived in part from the enet man page (enet.4)
+.\" distributed with 4.3BSD Unix.
+.\"
+.\" $FreeBSD$
+.\" $Id: netmap.4 9662 2011-11-16 13:18:06Z luigi $: stable/8/share/man/man4/bpf.4 181694 2008-08-13 17:45:06Z ed $
+.\"
+.Dd November 16, 2011
+.Dt NETMAP 4
+.Os
+.Sh NAME
+.Nm netmap
+.Nd a framework for fast packet I/O
+.Sh SYNOPSIS
+.Cd device netmap
+.Sh DESCRIPTION
+.Nm
+is a framework for fast and safe access to network devices
+(reaching 14.88 Mpps at less than 1 GHz).
+.Nm
+uses memory mapped buffers and metadata
+(buffer indexes and lengths) to communicate with the kernel,
+which is in charge of validating information through 
+.Pa ioctl()
+and
+.Pa select()/poll().
+.Nm
+can exploit the parallelism in multiqueue devices and
+multicore systems.
+.Pp
+.Pp
+.Nm
+requires explicit support in device drivers.
+For a list of supported devices, see the end of this manual page.
+.Sh OPERATION
+.Nm
+clients must first open the
+.Pa open("/dev/netmap") ,
+and then issue an
+.Pa ioctl(...,NIOCREGIF,...)
+to bind the file descriptor to a network device.
+.Pp
+When a device is put in
+.Nm
+mode, its data path is disconnected from the host stack.
+The processes owning the file descriptor 
+can exchange packets with the device, or with the host stack,
+through an mmapped memory region that contains pre-allocated
+buffers and metadata.
+.Pp
+Non blocking I/O is done with special
+.Pa ioctl()'s ,
+whereas the file descriptor can be passed to
+.Pa select()/poll()
+to be notified about incoming packet or available transmit buffers.
+.Ss Data structures
+All data structures for all devices in
+.Nm
+mode are in a memory
+region shared by the kernel and all processes
+who open
+.Pa /dev/netmap
+(NOTE: visibility may be restricted in future implementations).
+All references between the shared data structure
+are relative (offsets or indexes). Some macros help converting
+them into actual pointers.
+.Pp
+The data structures in shared memory are the following:
+.Pp
+.Bl -tag -width XXX
+.It Dv struct netmap_if (one per interface)
+indicates the number of rings supported by an interface, their
+sizes, and the offsets of the
+.Pa netmap_rings
+associated to the interface.
+The offset of a
+.Pa struct netmap_if
+in the shared memory region is indicated by the
+.Pa nr_offset
+field in the structure returned by the
+.Pa NIOCREGIF
+(see below).
+.Bd -literal
+struct netmap_if {
+    char ni_name[IFNAMSIZ]; /* name of the interface. */
+    const u_int ni_num_queues; /* number of hw ring pairs */
+    const ssize_t   ring_ofs[]; /* offset of tx and rx rings */
+};
+.Ed
+.It Dv struct netmap_ring (one per ring)
+contains the index of the current read or write slot (cur),
+the number of slots available for reception or transmission (avail),
+and an array of
+.Pa slots
+describing the buffers.
+There is one ring pair for each of the N hardware ring pairs
+supported by the card (numbered 0..N-1), plus
+one ring pair (numbered N) for packets from/to the host stack.
+.Bd -literal
+struct netmap_ring {
+    const ssize_t buf_ofs;
+    const uint32_t num_slots; /* number of slots in the ring. */
+    uint32_t avail; /* number of usable slots */
+    uint32_t cur; /* 'current' index for the user side */
+
+    const uint16_t nr_buf_size;
+    uint16_t flags;
+    struct netmap_slot slot[0]; /* array of slots. */
+}
+.Ed
+.It Dv struct netmap_slot (one per packet)
+contains the metadata for a packet: a buffer index (buf_idx),
+a buffer length (len), and some flags.
+.Bd -literal
+struct netmap_slot {
+    uint32_t buf_idx; /* buffer index */
+    uint16_t len;   /* packet length */
+    uint16_t flags; /* buf changed, etc. */
+#define NS_BUF_CHANGED  0x0001  /* must resync, buffer changed */
+#define NS_REPORT       0x0002  /* tell hw to report results
+                                 * e.g. by generating an interrupt
+                                 */
+};
+.Ed
+.It Dv packet buffers
+are fixed size (approximately 2k) buffers allocated by the kernel
+that contain packet data. Buffers addresses are computed through
+macros.
+.El
+.Pp
+Some macros support the access to objects in the shared memory
+region. In particular:
+.Bd -literal
+struct netmap_if *nifp;
+...
+struct netmap_ring *txring = NETMAP_TXRING(nifp, i);
+struct netmap_ring *rxring = NETMAP_RXRING(nifp, i);
+int i = txring->slot[txring->cur].buf_idx;
+char *buf = NETMAP_BUF(txring, i);
+.Ed
+.Ss IOCTLS
+.Pp
+.Nm
+supports some ioctl() to synchronize the state of the rings
+between the kernel and the user processes, plus some
+to query and configure the interface.
+The former do not require any argument, whereas the latter
+use a
+.Pa struct netmap_req
+defined as follows:
+.Bd -literal
+struct nmreq {
+        char      nr_name[IFNAMSIZ];
+        uint32_t  nr_offset;      /* nifp offset in the shared region */
+        uint32_t  nr_memsize;     /* size of the shared region */
+        uint32_t  nr_numdescs;    /* descriptors per queue */
+        uint16_t  nr_numqueues;
+        uint16_t  nr_ringid;      /* ring(s) we care about */
+#define NETMAP_HW_RING  0x4000    /* low bits indicate one hw ring */
+#define NETMAP_SW_RING  0x2000    /* we process the sw ring */
+#define NETMAP_NO_TX_POLL 0x1000  /* no gratuitous txsync on poll */
+#define NETMAP_RING_MASK 0xfff    /* the actual ring number */
+};
+
+.Ed
+A device descriptor obtained through
+.Pa /dev/netmap
+also supports the ioctl supported by network devices.
+.Pp
+The netmap-specific
+.Xr ioctl 2
+command codes below are defined in
+.In net/netmap.h
+and are:
+.Bl -tag -width XXXX
+.It Dv NIOCGINFO
+returns information about the interface named in nr_name.
+On return, nr_memsize indicates the size of the shared netmap
+memory region (this is device-independent),
+nr_numslots indicates how many buffers are in a ring,
+nr_numrings indicates the number of rings supported by the hardware.
+.Pp
+If the device does not support netmap, the ioctl returns EINVAL.
+.It Dv NIOCREGIF
+puts the interface named in nr_name into netmap mode, disconnecting
+it from the host stack, and/or defines which rings are controlled
+through this file descriptor.
+On return, it gives the same info as NIOCGINFO, and nr_ringid
+indicates the identity of the rings controlled through the file
+descriptor.
+.Pp
+Possible values for nr_ringid are
+.Bl -tag -width XXXXX
+.It 0
+default, all hardware rings
+.It NETMAP_SW_RING
+the ``host rings'' connecting to the host stack
+.It NETMAP_HW_RING + i
+the i-th hardware ring
+.El
+By default, a
+.Nm poll
+or
+.Nm select
+call pushes out any pending packets on the transmit ring, even if
+no write events are specified.
+The feature can be disabled by or-ing
+.Nm NETMAP_NO_TX_SYNC
+to nr_ringid.
+But normally you should keep this feature unless you are using
+separate file descriptors for the send and receive rings, because
+otherwise packets are pushed out only if NETMAP_TXSYNC is called,
+or the send queue is full.
+.Pp
+.Pa NIOCREGIF
+can be used multiple times to change the association of a
+file descriptor to a ring pair, always within the same device.
+.It Dv NIOCUNREGIF
+brings an interface back to normal mode.
+.It Dv NIOCTXSYNC
+tells the hardware of new packets to transmit, and updates the
+number of slots available for transmission.
+.It Dv NIOCRXSYNC
+tells the hardware of consumed packets, and asks for newly available
+packets.
+.El
+.Ss SYSTEM CALLS
+.Nm
+uses
+.Nm select
+and
+.Nm poll
+to wake up processes when significant events occur.
+.Sh EXAMPLES
+The following code implements a traffic generator
+.Pp
+.Bd -literal -compact
+#include <net/netmap.h>
+#include <net/netmap_user.h>
+struct netmap_if *nifp;
+struct netmap_ring *ring;
+struct netmap_request nmr;
+
+fd = open("/dev/netmap", O_RDWR);
+bzero(&nmr, sizeof(nmr));
+strcpy(nmr.nm_name, "ix0");
+ioctl(fd, NIOCREG, &nmr);
+p = mmap(0, nmr.memsize, fd);
+nifp = NETMAP_IF(p, nmr.offset);
+ring = NETMAP_TXRING(nifp, 0);
+fds.fd = fd;
+fds.events = POLLOUT;
+for (;;) {
+    poll(list, 1, -1);
+    while (ring->avail-- > 0) {
+        i = ring->cur;
+        buf = NETMAP_BUF(ring, ring->slot[i].buf_index);
+        ... prepare packet in buf ...
+        ring->slot[i].len = ... packet length ...
+        ring->cur = NETMAP_RING_NEXT(ring, i);
+    }
+}
+.Ed
+.Sh SUPPORTED INTERFACES
+.Nm
+supports the following interfaces:
+.Xr em 4 ,
+.Xr ixgbe 4 ,
+.Xr re 4 ,
+.Sh AUTHORS
+The
+.Nm
+framework has been designed and implemented by
+.An Luigi Rizzo
+and
+.An Matteo Landi
+in 2011 at the Universita` di Pisa.
diff --git a/share/man/man4/qlxgb.4 b/share/man/man4/qlxgb.4
new file mode 100644
index 0000000..c76386f
--- /dev/null
+++ b/share/man/man4/qlxgb.4
@@ -0,0 +1,93 @@
+.\"-
+.\" Copyright (c) 2011 "Bjoern A. Zeeb" <bz@FreeBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd November 3, 2011
+.Dt QLXGB 4
+.Os
+.Sh NAME
+.Nm qlxgb
+.Nd "QLogic 10 Gigabit Ethernet & CNA Adapter Driver"
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device qlxgb"
+.Ed
+.Pp
+To load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+if_qlxgb_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver supports IPv4 checksum offload,
+TCP and UDP checksum offload for both IPv4 and IPv6,
+Large Segment Offload for both IPv4 and IPv6,
+Jumbo frames, VLAN Tag, and
+Receive Side scaling.
+For further hardware information, see
+.Pa http://www.qlogic.com/ .
+.Sh HARDWARE
+The
+.Nm
+driver supports 10 Gigabit Ethernet & CNA Adapter based on the following
+chipsets:
+.Pp
+.Bl -bullet -compact
+.It
+QLogic 3200 series
+.It
+QLogic 8200 series
+.El
+.Sh SUPPORT
+For support questions please contact your QLogic approved reseller or
+QLogic Technical Support at
+.Pa http://support.qlogic.com ,
+or by E-mail at
+.Aq support@qlogic.com .
+.Sh SEE ALSO
+.Xr altq 4 ,
+.Xr arp 4 ,
+.Xr netintro 4 ,
+.Xr ng_ether 4 ,
+.Xr ifconfig 8
+.Sh HISTORY
+The
+.Nm
+device driver first appeared in
+.Fx 10.0 .
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+driver was written by
+.An David C Somayajulu
+at QLogic Corporation.
diff --git a/share/man/man4/sfxge.4 b/share/man/man4/sfxge.4
new file mode 100644
index 0000000..33063d9
--- /dev/null
+++ b/share/man/man4/sfxge.4
@@ -0,0 +1,96 @@
+.\" Copyright (c) 2011 Solarflare Communications, Inc.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd November 16, 2011
+.Dt SFXGE 4
+.Os
+.Sh NAME
+.Nm sfxge
+.Nd "Solarflare 10Gb Ethernet adapter driver"
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device sfxge"
+.Ed
+.Pp
+To load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+sfxge_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver provides support for 10Gb Ethernet adapters based on
+Solarflare SFC9000 family controllers.
+The driver supports jumbo
+frames, transmit/receive checksum offload, TCP Segmentation Offload
+(TSO), Large Receive Offload (LRO), VLAN checksum offload, VLAN TSO,
+and Receive Side Scaling (RSS) using MSI-X interrupts.  
+.Pp
+The driver allocates 1 receive queue, transmit queue, event queue and
+IRQ per CPU up to a maximum of 64.
+IRQ affinities should be spread out using
+.Xr cpuset 8 .
+Interrupt moderation may be controlled through the sysctl
+.Va dev.sfxge.%d.int_mod
+(units are microseconds).
+.Pp
+For more information on configuring this device, see
+.Xr ifconfig 8 .
+.Pp
+A large number of MAC, PHY and data path statistics are available
+under the sysctl
+.Va dev.sfxge.%d.stats .
+The adapter's VPD
+fields including its serial number are available under the sysctl
+.Va dev.sfxge.%d.vpd .
+.Sh HARDWARE
+The
+.Nm
+driver supports all 10Gb Ethernet adapters based on Solarflare SFC9000
+family controllers.
+.Sh SUPPORT
+For general information and support,
+go to the Solarflare support website at:
+.Pa https://support.solarflare.com .
+.Sh SEE ALSO
+.Xr arp 4 ,
+.Xr netintro 4 ,
+.Xr ng_ether 4 ,
+.Xr vlan 4 ,
+.Xr cpuset 8 ,
+.Xr ifconfig 8
+.Sh AUTHORS
+The
+.Nm
+driver was written by
+.An Philip Paeps
+and
+.An Solarflare Communications, Inc.
diff --git a/share/man/man4/tcp.4 b/share/man/man4/tcp.4
index 213e94a..e2d4ad3 100644
--- a/share/man/man4/tcp.4
+++ b/share/man/man4/tcp.4
@@ -38,7 +38,7 @@
 .\"     From: @(#)tcp.4	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd September 15, 2011
+.Dd November 14, 2011
 .Dt TCP 4
 .Os
 .Sh NAME
@@ -290,14 +290,6 @@ That of 2 results in any
 packets to closed ports being logged.
 Any value unlisted above disables the logging
 (default is 0, i.e., the logging is disabled).
-.It Va slowstart_flightsize
-The number of packets allowed to be in-flight during the
-.Tn TCP
-slow-start phase on a non-local network.
-.It Va local_slowstart_flightsize
-The number of packets allowed to be in-flight during the
-.Tn TCP
-slow-start phase to local machines in the same subnet.
 .It Va msl
 The Maximum Segment Lifetime, in milliseconds, for a packet.
 .It Va keepinit
@@ -411,15 +403,6 @@ maximum segment size.
 This helps throughput in general, but
 particularly affects short transfers and high-bandwidth large
 propagation-delay connections.
-.Pp
-When this feature is enabled, the
-.Va slowstart_flightsize
-and
-.Va local_slowstart_flightsize
-settings are not observed for new
-connection slow starts, but they are still used for slow starts
-that occur when the connection has been idle and starts sending
-again.
 .It Va sack.enable
 Enable support for RFC 2018, TCP Selective Acknowledgment option,
 which allows the receiver to inform the sender about all successfully
diff --git a/share/man/man4/ti.4 b/share/man/man4/ti.4
index 269b6cf..2ef606c 100644
--- a/share/man/man4/ti.4
+++ b/share/man/man4/ti.4
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 16, 2005
+.Dd November 14, 2011
 .Dt TI 4
 .Os
 .Sh NAME
@@ -42,7 +42,7 @@ place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
 .Cd "device ti"
-.Cd "options TI_PRIVATE_JUMBOS"
+.Cd "options TI_SF_BUF_JUMBO"
 .Cd "options TI_JUMBO_HDRSPLIT"
 .Ed
 .Pp
@@ -97,13 +97,14 @@ for more discussion on zero copy receive and header splitting.
 .Pp
 The
 .Nm
-driver normally uses jumbo receive buffers allocated by the
+driver uses UMA backed jumbo receive buffers, but can be configured
+to use
 .Xr sendfile 2
-buffer allocator, but can be configured to use its own private pool of
-jumbo buffers that are contiguous instead of buffers from the jumbo
-allocator, which are made up of multiple page sized chunks.
-To turn on private jumbos, use the
-.Dv TI_PRIVATE_JUMBOS
+buffer allocator.
+To turn on
+.Xr sendfile 2
+buffer allocator, use the
+.Dv TI_SF_BUF_JUMBO
 option.
 .Pp
 Support for vlans is also available using the
@@ -201,6 +202,93 @@ NEC Gigabit Ethernet
 .It
 Silicon Graphics PCI Gigabit Ethernet adapter
 .El
+.Sh LOADER TUNABLES
+Tunables can be set at the
+.Xr loader 8
+prompt before booting the kernel or stored in
+.Xr loader.conf 5 .
+.Bl -tag -width "xxxxxx"
+.It Va hw.ti.%d.dac
+If this tunable is set to 0 it will disable DAC (Dual Address Cycle).
+The default value is 1 which means driver will use full 64bit
+DMA addressing.
+.El
+.Sh SYSCTL VARIABLES
+The following variables are available as both
+.Xr sysctl 8
+variables and
+.Xr loader 8
+tunables.
+The interface has to be brought down and up again before a
+change takes effect when any of the following tunables are
+changed.
+The one microsecond clock tick referenced below is a nominal
+time and the actual hardware may not provide granularity to
+this level.
+For example, on Tigon 2 (revision 6) cards with release 12.0
+the clock granularity is 5 microseconds.
+.Bl -tag -width "xxxxxx"
+.It Va dev.ti.%d.rx_coal_ticks
+This value, receive coalesced ticks, controls the number of clock
+ticks (of 1 microseconds each) that must elapse before the NIC DMAs
+the receive return producer pointer to the Host and generates an
+interrupt.
+This parameter works in conjunction with the rx_max_coal_bds,
+receive max coalesced BDs, tunable parameter.
+The NIC will return the receive return producer pointer to the Host
+when either of the thresholds is exceeded.
+A value of 0 means that this parameter is ignored and receive BDs
+will only be returned when the receive max coalesced BDs value is
+reached.
+The default valus is 170.
+.It Va dev.ti.%d.rx_max_coal_bds
+This value, receive max coalesced BDs, controls the number of
+receive buffer descriptors that will be coalesced before the NIC
+updates the receive return ring producer index.
+If this value is set to 0 it will disable receive buffer descriptor
+coalescing.
+The default valus is 64.
+.It Va dev.ti.%d.ti_tx_coal_ticks
+This value, send coalesced ticks, controls the number of clock
+ticks (of 1 microseconds each) that must elapse before the NIC DMAs
+the send consumer pointer to the Host and generates an interrupt.
+This parameter works in conjunction with the tx_max_coal_bds,
+send max coalesced BDs, tunable parameter.
+The NIC will return the send consumer pointer to the Host when
+either of the thresholds is exceeded.
+A value of 0 means that this parameter is ignored and send BDs will
+only be returned when the send max coalesced BDs value is reached.
+The default value is 2000.
+.It Va dev.ti.%d.tx_max_coal_bds
+This value, send max coalesced BDs, controls the number of send
+buffer descriptors that will be coalesced before the NIC updates
+the send consumer index.
+If this valus is set to 0 it will disable send buffer descriptor
+coalescing.
+The default valus is 32.
+.It Va dev.ti.%d.tx_buf_ratio
+This value controls the ratio of the remaining memory in the NIC
+that should be devoted to transmit buffer vs. receive buffer.
+The lower 7 bits are used to indicate the ratio in 1/64th increments.
+For example, setting this value to 16 will set the transmit buffer
+to 1/4 of the remaining buffer space.
+In no cases will the transmit or receive buffer be reduced below
+68 KB.
+For a 1 MB NIC the approximate total space for data buffers is
+800 KB.
+For a 512 KB NIC that number is 300 KB.
+The default value is 21.
+.It Va dev.ti.%d.stat_ticks
+The value, stat ticks, controls the number of clock ticks
+(of 1 microseconds each) that must elapse before the NIC DMAs
+the statistics block to the Host and generates a STATS_UPDATED
+event.
+If set to zero then statistics are never DMAed to the Host.
+It is recommended that this value be set to a high enough
+frequency to not mislead someone reading statistics refreshes.
+Several times a second is enough.
+The default value is 2000000 (2 seconds).
+.El
 .Sh IOCTLS
 In addition to the standard
 .Xr socket 2
@@ -311,6 +399,7 @@ the network connection (cable).
 .El
 .Sh SEE ALSO
 .Xr sendfile 2 ,
+.Xr altq 4 ,
 .Xr arp 4 ,
 .Xr netintro 4 ,
 .Xr ng_ether 4 ,
diff --git a/share/man/man4/u3g.4 b/share/man/man4/u3g.4
index 215270c..87e3824 100644
--- a/share/man/man4/u3g.4
+++ b/share/man/man4/u3g.4
@@ -18,7 +18,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 7, 2008
+.Dd November 20, 2011
 .Dt U3G 4
 .Os
 .Sh NAME
@@ -29,6 +29,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device u3g"
 .Ed
 .Pp
diff --git a/share/man/man4/uark.4 b/share/man/man4/uark.4
index 7001a7f..945983c 100644
--- a/share/man/man4/uark.4
+++ b/share/man/man4/uark.4
@@ -16,7 +16,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 15, 2006
+.Dd November 20, 2011
 .Dt UARK 4
 .Os
 .Sh NAME
@@ -27,6 +27,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uark"
 .Ed
 .Pp
diff --git a/share/man/man4/ubsa.4 b/share/man/man4/ubsa.4
index 4eb45f6..df9002c 100644
--- a/share/man/man4/ubsa.4
+++ b/share/man/man4/ubsa.4
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 28, 2008
+.Dd November 20, 2011
 .Dt UBSA 4
 .Os
 .Sh NAME
@@ -39,6 +39,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device ubsa"
 .Ed
 .Pp
diff --git a/share/man/man4/ubser.4 b/share/man/man4/ubser.4
index 6e6fee30..556d4ec 100644
--- a/share/man/man4/ubser.4
+++ b/share/man/man4/ubser.4
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UBSER 4
 .Os
 .Sh NAME
@@ -39,6 +39,8 @@ To compile this driver into the kernel,
 place the following line in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device ubser"
 .Ed
 .Pp
diff --git a/share/man/man4/uchcom.4 b/share/man/man4/uchcom.4
index 4452b53..d7c0abc 100644
--- a/share/man/man4/uchcom.4
+++ b/share/man/man4/uchcom.4
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 18, 2009
+.Dd November 20, 2011
 .Dt UCHCOM 4
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uchcom"
 .Ed
 .Pp
diff --git a/share/man/man4/ucycom.4 b/share/man/man4/ucycom.4
index 6d76564..063d419 100644
--- a/share/man/man4/ucycom.4
+++ b/share/man/man4/ucycom.4
@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UCYCOM 4
 .Os
 .Sh NAME
@@ -38,6 +38,8 @@ To compile this driver into the kernel,
 place the following line in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device ucycom"
 .Ed
 .Pp
diff --git a/share/man/man4/ufoma.4 b/share/man/man4/ufoma.4
index 1cbde0f..77d192f 100644
--- a/share/man/man4/ufoma.4
+++ b/share/man/man4/ufoma.4
@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UFOMA 4
 .Os
 .Sh NAME
@@ -38,6 +38,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device ufoma"
 .Ed
 .Pp
diff --git a/share/man/man4/uftdi.4 b/share/man/man4/uftdi.4
index 9512560..db75c68 100644
--- a/share/man/man4/uftdi.4
+++ b/share/man/man4/uftdi.4
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UFTDI 4
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uftdi"
 .Ed
 .Pp
diff --git a/share/man/man4/uipaq.4 b/share/man/man4/uipaq.4
index 3362268..310cbfe 100644
--- a/share/man/man4/uipaq.4
+++ b/share/man/man4/uipaq.4
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 24, 2008
+.Dd November 20, 2011
 .Dt UIPAQ 4
 .Os
 .Sh NAME
@@ -41,6 +41,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uipaq"
 .Ed
 .Pp
diff --git a/share/man/man4/umcs.4 b/share/man/man4/umcs.4
index f57b084..af8bd5f 100644
--- a/share/man/man4/umcs.4
+++ b/share/man/man4/umcs.4
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 10, 2010
+.Dd November 20, 2011
 .Dt UMCS 4
 .Os
 .Sh NAME
@@ -39,6 +39,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device umcs"
 .Ed
 .Pp
diff --git a/share/man/man4/umct.4 b/share/man/man4/umct.4
index 5f4ac56..e40e146 100644
--- a/share/man/man4/umct.4
+++ b/share/man/man4/umct.4
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UMCT 4
 .Os
 .Sh NAME
@@ -36,6 +36,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device umct"
 .Ed
 .Pp
diff --git a/share/man/man4/umodem.4 b/share/man/man4/umodem.4
index a683a9c..594f4b0 100644
--- a/share/man/man4/umodem.4
+++ b/share/man/man4/umodem.4
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UMODEM 4
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device umodem"
 .Ed
 .Pp
diff --git a/share/man/man4/uplcom.4 b/share/man/man4/uplcom.4
index 3ebed34..05b4ff3 100644
--- a/share/man/man4/uplcom.4
+++ b/share/man/man4/uplcom.4
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 11, 2011
+.Dd November 20, 2011
 .Dt UPLCOM 4
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uplcom"
 .Ed
 .Pp
diff --git a/share/man/man4/uslcom.4 b/share/man/man4/uslcom.4
index 40e6fc8..a2a2dcc 100644
--- a/share/man/man4/uslcom.4
+++ b/share/man/man4/uslcom.4
@@ -16,7 +16,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 31, 2007
+.Dd November 20, 2011
 .Dt USLCOM 4
 .Os
 .Sh NAME
@@ -27,6 +27,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uslcom"
 .Ed
 .Pp
diff --git a/share/man/man4/uvisor.4 b/share/man/man4/uvisor.4
index 1a81aeb..2397e45 100644
--- a/share/man/man4/uvisor.4
+++ b/share/man/man4/uvisor.4
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UVISOR 4
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uvisor"
 .Ed
 .Pp
diff --git a/share/man/man4/uvscom.4 b/share/man/man4/uvscom.4
index 77a7f4e..be40518 100644
--- a/share/man/man4/uvscom.4
+++ b/share/man/man4/uvscom.4
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 22, 2006
+.Dd November 20, 2011
 .Dt UVSCOM 4
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ To compile this driver into the kernel,
 place the following lines in your
 kernel configuration file:
 .Bd -ragged -offset indent
+.Cd "device usb"
+.Cd "device ucom"
 .Cd "device uvscom"
 .Ed
 .Pp
diff --git a/share/man/man4/virtio.4 b/share/man/man4/virtio.4
new file mode 100644
index 0000000..7787051
--- /dev/null
+++ b/share/man/man4/virtio.4
@@ -0,0 +1,91 @@
+.\" Copyright (c) 2011 Bryan Venteicher
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd July 4, 2011
+.Dt VIRTIO 4
+.Os
+.Sh NAME
+.Nm virtio 
+.Nd VirtIO Device Support
+.Sh SYNOPSIS
+To compile VirtIO device support into the kernel, place the following lines
+in your kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device virtio"
+.Cd "device virtio_pci"
+.Ed
+.Pp
+Alternatively, to load VirtIO support as modules at boot time, place the
+following lines in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+virtio_load="YES"
+virtio_pci_load="YES"
+.Ed
+.Sh DESCRIPTION
+VirtIO is a specification for para-virtualized I/O in a virtual machine (VM).
+Traditionally, the hypervisor emulated real devices such as an Ethernet
+interface or disk controller to provide the VM with I/O. This emulation is
+often inefficient.
+.Pp
+VirtIO defines an interface for efficient I/O between the hypervisor and VM.
+The 
+.Xr virtio 4 
+module provides a shared memory transport called a virtqueue.
+The
+.Xr virtio_pci 4
+device driver represents an emulated PCI device that the hypervisor makes
+available to the VM. This device provides the probing, configuration, and 
+interrupt notifications need to interact with the hypervisor.
+.Fx
+supports the following VirtIO devices:
+.Bl -hang -offset indent -width xxxxxxxx
+.It Nm Ethernet
+An emulated Ethernet device is provided by the
+.Xr if_vtnet 4
+device driver.
+.It Nm Block
+An emulated disk controller is provided by the
+.Xr virtio_blk 4
+device driver.
+.It Nm Balloon
+A pseudo-device to allow the VM to release memory back to the hypervisor is
+provided by the
+.Xr virtio_balloon 4
+device driver.
+.El
+.Sh SEE ALSO
+.Xr if_vtnet 4 ,
+.Xr virtio_blk 4 ,
+.Xr virtio_balloon 4
+.Sh HISTORY
+Support for VirtIO first appeared in
+.Fx 9.0 .
+.Sh AUTHORS
+.An -nosplit
+.Fx
+support for VirtIO was first added by
+.An Bryan Venteicher Aq bryanv@daemoninthecloset.org .
diff --git a/share/man/man4/virtio_balloon.4 b/share/man/man4/virtio_balloon.4
new file mode 100644
index 0000000..4053f78
--- /dev/null
+++ b/share/man/man4/virtio_balloon.4
@@ -0,0 +1,64 @@
+.\" Copyright (c) 2011 Bryan Venteicher
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd July 4, 2011
+.Dt VIRTIO_BALLOON 4
+.Os
+.Sh NAME
+.Nm virtio_balloon
+.Nd VirtIO Memory Balloon driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device virtio_balloon"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+virtio_balloon_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+device driver provides support for VirtIO memory balloon devices.
+.Pp
+The memory balloon allows the guest to, at the request of the
+hypervisor, return memory allocated to the hypervisor so it can
+be made available to other guests. The hypervisor can later
+signal the balloon to return the memory.
+.Sh SEE ALSO
+.Xr virtio 4
+.Sh HISTORY
+The
+.Nm
+driver was written by
+.An Bryan Venteicher Aq bryanv@daemoninthecloset.org .
+It first appeared in
+.Fx 9.0 .
diff --git a/share/man/man4/virtio_blk.4 b/share/man/man4/virtio_blk.4
new file mode 100644
index 0000000..4250141
--- /dev/null
+++ b/share/man/man4/virtio_blk.4
@@ -0,0 +1,70 @@
+.\" Copyright (c) 2011 Bryan Venteicher
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd July 4, 2011
+.Dt VIRTIO_BLK 4
+.Os
+.Sh NAME
+.Nm virtio_blk
+.Nd VirtIO Block driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device virtio_blk"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+virtio_blk_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+device driver provides support for VirtIO block devices.
+.Pp
+.Sh LOADER TUNABLES
+Tunables can be set at the
+.Xr loader 8
+prompt before booting the kernel or stored in
+.Xr loader.conf 5 .
+.Bl -tag -width "xxxxxx"
+.It Va hw.vtblk.no_ident
+This tunable disables retrieving the device identification string
+from the hypervisor. The default value is 0.
+.El
+.Sh SEE ALSO
+.Xr virtio 4
+.Sh HISTORY
+The
+.Nm
+driver was written by
+.An Bryan Venteicher Aq bryanv@daemoninthecloset.org .
+It first appeared in
+.Fx 9.0 .
diff --git a/share/man/man4/vtnet.4 b/share/man/man4/vtnet.4
new file mode 100644
index 0000000..900d12d
--- /dev/null
+++ b/share/man/man4/vtnet.4
@@ -0,0 +1,98 @@
+.\" Copyright (c) 2011 Bryan Venteicher
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd July 4, 2011
+.Dt VTNET 4
+.Os
+.Sh NAME
+.Nm vtnet
+.Nd VirtIO Ethernet driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device if_vtnet"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+if_vtnet_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+device driver provides support for VirtIO Ethernet devices.
+.Pp
+If the hypervisor advertises the appreciate features, the
+.Nm
+driver supports TCP/UDP checksum offload for both transmit and receive,
+TCP segmentation offload (TSO), TCP large receive offload (LRO), and
+hardware VLAN tag stripping/insertion features, as well as a multicast
+hash filter, as well as Jumbo Frames (up to 9216 bytes), which can be
+configured via the interface MTU setting.
+Selecting an MTU larger than 1500 bytes with the
+.Xr ifconfig 8
+utility configures the adapter to receive and transmit Jumbo Frames.
+.Pp
+For more information on configuring this device, see
+.Xr ifconfig 8 .
+.El
+.Sh LOADER TUNABLES
+Tunables can be set at the
+.Xr loader 8
+prompt before booting the kernel or stored in
+.Xr loader.conf 5 .
+.Bl -tag -width "xxxxxx"
+.It Va hw.vtnet.csum_disable
+This tunable disables receive and send checksum offload. The default
+value is 0.
+.It Va hw.vtnet.tso_disable
+This tunable disables TSO. The default value is 0.
+.It Va hw.vtnet.lro_disable
+This tunable disables LRO. The default value is 0.
+.El
+.Sh SEE ALSO
+.Xr arp 4 ,
+.Xr netintro 4 ,
+.Xr ng_ether 4 ,
+.Xr vlan 4 ,
+.Xr virtio 4 ,
+.Xr ifconfig 8
+.Sh HISTORY
+The
+.Nm
+driver was written by
+.An Bryan Venteicher Aq bryanv@daemoninthecloset.org .
+It first appeared in
+.Fx 9.0 .
+.Sh CAVEATS
+The
+.Nm
+driver only supports LRO when the hypervisor advertises the
+mergeable buffer feature.
diff --git a/share/man/man7/release.7 b/share/man/man7/release.7
index f2ca0c8..f527c24 100644
--- a/share/man/man7/release.7
+++ b/share/man/man7/release.7
@@ -149,6 +149,11 @@ when invoked by the script.
 .It Ev SVNROOT
 The location of the FreeBSD SVN source repository. Defaults to
 .Pa svn://svn.freebsd.org/base .
+.It Ev RELSTRING
+Optional base name for generated media images (e.g. FreeBSD-9.0-RC2-amd64).
+Defaults to the output of 
+.Ic `uname -s`-`uname -r`-`uname -p`
+within the chroot.
 .El
 .Sh MAKEFILE TARGETS
 The release makefile
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index 3026ef8..0870355 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -724,7 +724,8 @@ MLINKS+=kobj.9 DEFINE_CLASS.9 \
 	kobj.9 kobj_class_free.9 \
 	kobj.9 kobj_create.9 \
 	kobj.9 kobj_delete.9 \
-	kobj.9 kobj_init.9
+	kobj.9 kobj_init.9 \
+	kobj.9 kobj_init_static.9
 MLINKS+=kproc.9 kproc_create.9 \
 	kproc.9 kproc_exit.9 \
 	kproc.9 kproc_resume.9 \
diff --git a/share/man/man9/VOP_VPTOCNP.9 b/share/man/man9/VOP_VPTOCNP.9
index 6bcbd25..f2d5ddc 100644
--- a/share/man/man9/VOP_VPTOCNP.9
+++ b/share/man/man9/VOP_VPTOCNP.9
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 7, 2008
+.Dd November 19, 2011
 .Dt VOP_VPTOCNP 9
 .Os
 .Sh NAME
@@ -65,9 +65,9 @@ is not a directory, then
 .Nm
 returns ENOENT.
 .Sh LOCKS
-The vnode should be locked on entry and will still be locked on exit.  The
-parent directory vnode will be unlocked on a successful exit.  However, it
-will have its hold count incremented.
+The vnode should be locked on entry and will still be locked on exit.
+The parent directory vnode will be unlocked on a successful exit.
+However, it will have its use count incremented.
 .Sh RETURN VALUES
 Zero is returned on success, otherwise an error code is returned.
 .Sh ERRORS
diff --git a/share/man/man9/kobj.9 b/share/man/man9/kobj.9
index aed1244..0e1745e 100644
--- a/share/man/man9/kobj.9
+++ b/share/man/man9/kobj.9
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 4, 2000
+.Dd November 14, 2011
 .Dt KOBJ 9
 .Os
 .Sh NAME
@@ -48,6 +48,8 @@
 .Ft void
 .Fn kobj_init "kobj_t obj" "kobj_class_t cls"
 .Ft void
+.Fn kobj_init_static "kobj_t obj" "kobj_class_t cls"
+.Ft void
 .Fn kobj_delete "kobj_t obj" "struct malloc_type *mtype"
 .Fn DEFINE_CLASS name "kobj_method_t *methods" "size_t size"
 .Sh DESCRIPTION
@@ -88,10 +90,14 @@ Objects created in this way should be freed by calling
 Clients which would like to manage the allocation of memory
 themselves should call
 .Fn kobj_init
+or
+.Fn kobj_init_static
 with a pointer to the memory for the object and the class which
 implements it.
 It is also possible to use
 .Fn kobj_init
+and
+.Fn kobj_init_static
 to change the class for an object.
 This should be done with care as the classes must agree on the layout
 of the object.
@@ -109,13 +115,19 @@ A client should not normally need to call these since a class
 will automatically be compiled the first time it is used.
 If a class is to be used before
 .Xr malloc 9
-is initialised,
+and
+.Xr mutex 9
+are initialised,
 then
 .Fn kobj_class_compile_static
 should be called with the class and a pointer to a statically
 allocated
 .Vt kobj_ops
 structure before the class is used to initialise any objects.
+In that case, also
+.Fn kobj_init_static
+should be used instead of
+.Fn kobj_init .
 .Pp
 To define a class, first define a simple array of
 .Vt kobj_method_t .
diff --git a/share/man/man9/kqueue.9 b/share/man/man9/kqueue.9
index c657088..8f0d109 100644
--- a/share/man/man9/kqueue.9
+++ b/share/man/man9/kqueue.9
@@ -1,4 +1,4 @@
-.\" Copyright 2006 John-Mark Gurney
+.\" Copyright 2006,2011 John-Mark Gurney
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -24,15 +24,16 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 28, 2006
+.Dd November 5, 2011
 .Dt KQUEUE 9
 .Os
 .Sh NAME
 .Nm kqueue_add_filteropts , kqueue_del_filteropts ,
 .Nm kqfd_register ,
 .Nm knote_fdclose ,
+.Nm knlist_init , knlist_init_mtx ,
 .Nm knlist_add , knlist_remove , knlist_remove_inevent , knlist_empty ,
-.Nm knlist_init , knlist_destroy , knlist_clear , knlist_delete ,
+.Nm knlist_clear , knlist_delete , knlist_destroy ,
 .Nm KNOTE_LOCKED , KNOTE_UNLOCKED
 .Nd "event delivery subsystem"
 .Sh SYNOPSIS
@@ -46,14 +47,6 @@
 .Ft void
 .Fn knote_fdclose "struct thread *td" "int fd"
 .Ft void
-.Fn knlist_add "struct knlist *knl" "struct knote *kn" "int islocked"
-.Ft void
-.Fn knlist_remove "struct knlist *knl" "struct knote *kn" "int islocked"
-.Ft void
-.Fn knlist_remove_inevent "struct knlist *knl" "struct knote *kn"
-.Ft int
-.Fn knlist_empty "struct knlist *knl"
-.Ft void
 .Fo knlist_init
 .Fa "struct knlist *knl"
 .Fa "void *lock"
@@ -62,12 +55,22 @@
 .Fa "int \*[lp]*kl_locked\*[rp]\*[lp]void *\*[rp]"
 .Fc
 .Ft void
-.Fn knlist_destroy "struct knlist *knl"
+.Fn knlist_init_mtx "struct knlist *knl" "struct mtx *lock"
+.Ft void
+.Fn knlist_add "struct knlist *knl" "struct knote *kn" "int islocked"
+.Ft void
+.Fn knlist_remove "struct knlist *knl" "struct knote *kn" "int islocked"
+.Ft void
+.Fn knlist_remove_inevent "struct knlist *knl" "struct knote *kn"
+.Ft int
+.Fn knlist_empty "struct knlist *knl"
 .Ft void
 .Fn knlist_clear "struct knlist *knl" "int islocked"
 .Ft void
 .Fn knlist_delete "struct knlist *knl" "struct thread *td" "int islocked"
 .Ft void
+.Fn knlist_destroy "struct knlist *knl"
+.Ft void
 .Fn KNOTE_LOCKED "struct knlist *knl" "long hint"
 .Ft void
 .Fn KNOTE_UNLOCKED "struct knlist *knl" "long hint"
@@ -135,7 +138,8 @@ bit of
 .Va kn_status
 in the
 .Vt knote .
-The function shall return 0 on success, or appropriate error for the failure.
+The function shall return 0 on success, or appropriate error for the failure,
+such as when the object is being destroyed, or does not exist.
 During
 .Va f_attach ,
 it is valid to change the
@@ -155,7 +159,13 @@ function will be called to detach the
 if the
 .Vt knote
 has not already been detached by a call to
-.Fn knlist_remove .
+.Fn knlist_remove ,
+.Fn knlist_remove_inevent
+or
+.Fn knlist_delete .
+The list
+.Fa lock
+will not be held when this function is called.
 .It Va f_event
 The
 .Va f_event
@@ -247,57 +257,120 @@ A
 is not required, but is commonly used.
 If used, the
 .Vt knlist
-must be initialized with the
+must be initialized with either
 .Fn knlist_init
-function.
-If
+or
+.Fn knlist_init_mtx .
+The
+.Vt knlist
+structure may be embedded into the object structure.
+The
+.Fa lock
+will be held over
+.Va f_event
+calls.
+.Pp
+For the
+.Fn knlist_init
+function, if
 .Fa lock
 is
 .Dv NULL ,
-an internal lock will be used and the remaining arguments will be ignored.
-The
+a shared global lock will be used and the remaining arguments must be
+.Dv NULL .
+The function pointers
 .Fa kl_lock , kl_unlock
 and
 .Fa kl_locked
-functions will be used to manipulate a
+will be used to manipulate the argument
 .Fa lock .
-If the argument is
+If any of the function pointers are
 .Dv NULL ,
-default routines operating on
-.Vt "struct mtx *"
-will be used.
-The
+a function operating on
+.Dv MTX_DEF
+style
+.Xr mutex 9
+locks will be used instead.
+.Pp
+The function
+.Fn knlist_init_mtx
+may be used to initalize a
 .Vt knlist
-structure may be embedded into the object structure.
-The
+when
 .Fa lock
-will be held over calls to
-.Va f_event .
-If
-.Dv NULL
-is passed for the mutex, a private mutex will be used.
+is a
+.Dv MTX_DEF
+style
+.Xr mutex 9
+lock.
+.Pp
 The function
 .Fn knlist_empty
-requires that a
+returns true when there are no
+.Vt knotes
+on the list.
+The function requires that the
 .Fa lock
-be held.
+be held when called.
+.Pp
 The function
 .Fn knlist_clear
-is used to remove all
+removes all
 .Vt knotes
-associated with the list.
+from the list.
 The
 .Fa islocked
-argument declares if
+argument declares if the
 .Fa lock
 has been acquired.
 All
 .Vt knotes
-will be marked as detached, and
+will have
 .Dv EV_ONESHOT
-will be set so that the
+set so that the
 .Vt knote
-will be deleted after the next scan.
+will be returned and removed durning the next scan.
+The
+.Va f_detach
+function will be called when the
+.Vt knote
+is deleted durning the next scan.
+This function must not be used when
+.Va f_isfd
+is set in
+.Vt "struct filterops" ,
+as the
+.Fa td
+argument of
+.Fn fdrop
+will be
+.Dv NULL .
+.Pp
+The function
+.Fn knlist_delete
+removes and deletes all
+.Vt knotes
+on the list.
+The function
+.Va f_detach
+will not be called, and the
+.Vt knote
+will not be returned on the next scan.
+Using this function could leak user land resources if a process uses the
+.Vt knote
+to track resources.
+.Pp
+Both the
+.Fn knlist_clear
+and
+.Fn knlist_delete
+functions may sleep.
+They also may release the
+.Fa lock
+to wait for other
+.Vt knotes
+to drain.
+.Pp
 The
 .Fn knlist_destroy
 function is used to destroy a
@@ -314,7 +387,9 @@ may be attached to the object.
 A
 .Vt knlist
 may be emptied by calling
-.Fn knlist_clear .
+.Fn knlist_clear
+or
+.Fn knlist_delete .
 .Pp
 The macros
 .Fn KNOTE_LOCKED
@@ -333,7 +408,7 @@ The macro
 .Fn KNOTE_LOCKED
 must be used if the lock associated with the
 .Fa knl
-passed in is held.
+is held.
 The function
 .Fn KNOTE_UNLOCKED
 will acquire the lock before iterating over the list of
diff --git a/share/man/man9/lock.9 b/share/man/man9/lock.9
index c94612a..049de42 100644
--- a/share/man/man9/lock.9
+++ b/share/man/man9/lock.9
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd June 16, 2009
+.Dd November 16, 2011
 .Dt LOCK 9
 .Os
 .Sh NAME
@@ -60,20 +60,20 @@
 .Ft void
 .Fn lockmgr_disown "struct lock *lkp"
 .Ft void
-.Fn lockmgr_printinfo "struct lock *lkp"
+.Fn lockmgr_printinfo "const struct lock *lkp"
 .Ft int
-.Fn lockmgr_recursed "struct lock *lkp"
+.Fn lockmgr_recursed "const struct lock *lkp"
 .Ft int
 .Fn lockmgr_rw "struct lock *lkp" "u_int flags" "struct rwlock *ilk"
 .Ft int
-.Fn lockmgr_waiters "struct lock *lkp"
+.Fn lockmgr_waiters "const struct lock *lkp"
 .Ft int
-.Fn lockstatus "struct lock *lkp"
+.Fn lockstatus "const struct lock *lkp"
 .Pp
 .Cd "options INVARIANTS"
 .Cd "options INVARIANT_SUPPORT"
 .Ft void
-.Fn lockmgr_assert "struct lock *lkp" "int what"
+.Fn lockmgr_assert "const struct lock *lkp" "int what"
 .Sh DESCRIPTION
 The
 .Fn lockinit
diff --git a/share/man/man9/mutex.9 b/share/man/man9/mutex.9
index d1e7438..628939f 100644
--- a/share/man/man9/mutex.9
+++ b/share/man/man9/mutex.9
@@ -28,7 +28,7 @@
 .\"	from BSDI $Id: mutex.4,v 1.1.2.3 1998/04/27 22:53:13 ewv Exp $
 .\" $FreeBSD$
 .\"
-.Dd December 21, 2006
+.Dd November 16, 2011
 .Dt MUTEX 9
 .Os
 .Sh NAME
@@ -83,16 +83,16 @@
 .Ft int
 .Fn mtx_sleep "void *chan" "struct mtx *mtx" "int priority" "const char *wmesg" "int timo"
 .Ft int
-.Fn mtx_initialized "struct mtx *mutex"
+.Fn mtx_initialized "const struct mtx *mutex"
 .Ft int
-.Fn mtx_owned "struct mtx *mutex"
+.Fn mtx_owned "const struct mtx *mutex"
 .Ft int
-.Fn mtx_recursed "struct mtx *mutex"
+.Fn mtx_recursed "const struct mtx *mutex"
 .Pp
 .Cd "options INVARIANTS"
 .Cd "options INVARIANT_SUPPORT"
 .Ft void
-.Fn mtx_assert "struct mtx *mutex" "int what"
+.Fn mtx_assert "const struct mtx *mutex" "int what"
 .In sys/kernel.h
 .Fn MTX_SYSINIT "name" "struct mtx *mtx" "const char *description" "int opts"
 .Sh DESCRIPTION
diff --git a/share/man/man9/rmlock.9 b/share/man/man9/rmlock.9
index 2e68b07..bc20d70 100644
--- a/share/man/man9/rmlock.9
+++ b/share/man/man9/rmlock.9
@@ -26,7 +26,7 @@
 .\" $FreeBSD$
 .\"
 .\" Based on rwlock.9 man page
-.Dd November 10, 2007
+.Dd November 16, 2011
 .Dt RMLOCK 9
 .Os
 .Sh NAME
@@ -63,7 +63,7 @@
 .Ft void
 .Fn rm_wunlock "struct rmlock *rm"
 .Ft int
-.Fn rm_wowned "struct rmlock *rm"
+.Fn rm_wowned "const struct rmlock *rm"
 .In sys/kernel.h
 .Fn RM_SYSINIT "name" "struct rmlock *rm" "const char *desc" "int opts"
 .Sh DESCRIPTION
@@ -208,7 +208,7 @@ This functions destroys a lock previously initialized with
 The
 .Fa rm
 lock must be unlocked.
-.It Fn rm_wowned "struct rmlock *rm"
+.It Fn rm_wowned "const struct rmlock *rm"
 This function returns a non-zero value if the current thread owns an
 exclusive lock on
 .Fa rm .
diff --git a/share/man/man9/rwlock.9 b/share/man/man9/rwlock.9
index 5039a42..f7da699 100644
--- a/share/man/man9/rwlock.9
+++ b/share/man/man9/rwlock.9
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 1, 2008
+.Dd November 16, 2011
 .Dt RWLOCK 9
 .Os
 .Sh NAME
@@ -78,14 +78,14 @@
 .Ft int
 .Fn rw_sleep "void *chan" "struct rwlock *rw" "int priority" "const char *wmesg" "int timo"
 .Ft int
-.Fn rw_initialized "struct rwlock *rw"
+.Fn rw_initialized "const struct rwlock *rw"
 .Ft int
-.Fn rw_wowned "struct rwlock *rw"
+.Fn rw_wowned "const struct rwlock *rw"
 .Pp
 .Cd "options INVARIANTS"
 .Cd "options INVARIANT_SUPPORT"
 .Ft void
-.Fn rw_assert "struct rwlock *rw" "int what"
+.Fn rw_assert "const struct rwlock *rw" "int what"
 .In sys/kernel.h
 .Fn RW_SYSINIT "name" "struct rwlock *rw" "const char *desc"
 .Sh DESCRIPTION
@@ -231,7 +231,7 @@ while waiting for an event.
 For more details on the parameters to this function,
 see
 .Xr sleep 9 .
-.It Fn rw_initialized "struct rwlock *rw"
+.It Fn rw_initialized "const struct rwlock *rw"
 This function returns non-zero if
 .Fa rw
 has been initialized, and zero otherwise.
@@ -241,11 +241,11 @@ This functions destroys a lock previously initialized with
 The
 .Fa rw
 lock must be unlocked.
-.It Fn rw_wowned "struct rwlock *rw"
+.It Fn rw_wowned "const struct rwlock *rw"
 This function returns a non-zero value if the current thread owns an
 exclusive lock on
 .Fa rw .
-.It Fn rw_assert "struct rwlock *rw" "int what"
+.It Fn rw_assert "const struct rwlock *rw" "int what"
 This function allows assertions specified in
 .Fa what
 to be made about
diff --git a/share/man/man9/stack.9 b/share/man/man9/stack.9
index 2c2b762..fee6b8f 100644
--- a/share/man/man9/stack.9
+++ b/share/man/man9/stack.9
@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd June 24, 2009
+.Dd November 16, 2011
 .Dt STACK 9
 .Os
 .Sh NAME
@@ -46,21 +46,21 @@ In the kernel configuration file:
 .Ft int
 .Fn stack_put "struct stack *st" "vm_offset_t pc"
 .Ft void
-.Fn stack_copy "struct stack *src" "struct stack dst"
+.Fn stack_copy "const struct stack *src" "struct stack dst"
 .Ft void
 .Fn stack_zero "struct stack *st"
 .Ft void
-.Fn stack_print "struct stack *st"
+.Fn stack_print "const struct stack *st"
 .Ft void
-.Fn stack_print_ddb "struct stack *st"
+.Fn stack_print_ddb "const struct stack *st"
 .Ft void
-.Fn stack_print_short "struct stack *st"
+.Fn stack_print_short "const struct stack *st"
 .Ft void
-.Fn stack_print_short_ddb "struct stack *st"
+.Fn stack_print_short_ddb "const struct stack *st"
 .Ft void
-.Fn stack_sbuf_print "struct sbuf sb*" "struct stack *st"
+.Fn stack_sbuf_print "struct sbuf sb*" "const struct stack *st"
 .Ft void
-.Fn stack_sbuf_print_ddb "struct sbuf sb*" "struct stack *st"
+.Fn stack_sbuf_print_ddb "struct sbuf sb*" "const struct stack *st"
 .Ft void
 .Fn stack_save "struct stack *st"
 .Sh DESCRIPTION
diff --git a/share/man/man9/sx.9 b/share/man/man9/sx.9
index c75c6a7..b998749 100644
--- a/share/man/man9/sx.9
+++ b/share/man/man9/sx.9
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 28, 2009
+.Dd November 16, 2011
 .Dt SX 9
 .Os
 .Sh NAME
@@ -88,12 +88,12 @@
 .Ft "struct thread *"
 .Fn sx_xholder "struct sx *sx"
 .Ft int
-.Fn sx_xlocked "struct sx *sx"
+.Fn sx_xlocked "const struct sx *sx"
 .Pp
 .Cd "options INVARIANTS"
 .Cd "options INVARIANT_SUPPORT"
 .Ft void
-.Fn sx_assert "struct sx *sx" "int what"
+.Fn sx_assert "const struct sx *sx" "int what"
 .In sys/kernel.h
 .Fn SX_SYSINIT "name" "struct sx *sx" "const char *description"
 .Sh DESCRIPTION
diff --git a/share/misc/bsd-family-tree b/share/misc/bsd-family-tree
index 1f4ec11..822e5f2 100644
--- a/share/misc/bsd-family-tree
+++ b/share/misc/bsd-family-tree
@@ -248,6 +248,9 @@ FreeBSD 5.2           |      |                 |                       |
  |    8.2      7.4    |      |                 |               DragonFly 2.10.1
  |     v              |      |             OpenBSD 4.9                 |
  |                    |      |                 |                       |
+ |                    |      |                 |                       |
+ |                    |      |             OpenBSD 5.0                 |
+ |                    |      |                 |                       |
 FreeBSD 9 -current    |  NetBSD -current  OpenBSD -current             |
  |                    |      |                 |                       |
  v                    v      v                 v                       v
@@ -534,6 +537,7 @@ FreeBSD 7.4		2011-02-24 [FBD]
 FreeBSD 8.2		2011-02-24 [FBD]
 DragonFly 2.10.1	2011-04-26 [DFB]
 OpenBSD 4.9		2011-05-01 [OBD]
+OpenBSD 5.0		2011-11-01 [OBD]
 
 Bibliography
 ------------------------
diff --git a/share/misc/committers-ports.dot b/share/misc/committers-ports.dot
index 2569169..35587e2 100644
--- a/share/misc/committers-ports.dot
+++ b/share/misc/committers-ports.dot
@@ -156,11 +156,13 @@ pgollucci [label="Philip M. Gollucci\npgollucci@FreeBSD.org\n2008/07/21"]
 rafan [label="Rong-En Fan\nrafan@FreeBSD.org\n2006/06/23"]
 rakuco [label="Raphael Kubo da Costa\nrakuco@FreeBSD.org\n2011/08/22"]
 rene [label="Rene Ladan\nrene@FreeBSD.org\n2010/04/11"]
+rm [label="Ruslan Mahmatkhanov\nrm@FreeBSD.org\n2011/11/06"]
 rnoland [label="Robert Noland\nrnoland@FreeBSD.org\n2008/07/21"]
 romain [label="Romain Tartiere\nromain@FreeBSD.org\n2010/01/24"]
 sahil [label="Sahil Tandon\nsahil@FreeBSD.org\n2010/04/11"]
 sat [label="Andrew Pantyukhin\nsat@FreeBSD.org\n2006/05/06"]
 sbz [label="Sofian Brabez\nsbz@FreeBSD.org\n2011/03/14"]
+scheidell [label="Michael Scheidell\nscheidell@FreeBSD.org\n2011/11/06"]
 sem [label="Sergey Matveychuk\nsem@FreeBSD.org\n2004/07/07"]
 sergei [label="Sergei Kolobov\nsergei@FreeBSD.org\n2003/10/21"]
 shaun [label="Shaun Amott\nshaun@FreeBSD.org\n2006/06/19"]
@@ -268,6 +270,7 @@ flz -> romain
 gabor -> lippe
 gabor -> pgj
 gabor -> stephen
+gabor -> scheidell
 
 garga -> acm
 garga -> alepulver
@@ -280,6 +283,7 @@ garga -> xride
 
 glarkin -> avl
 glarkin -> cs
+glarkin -> rm
 
 glewis -> hq
 glewis -> jkim
@@ -352,6 +356,7 @@ miwi -> mandree
 miwi -> mva
 miwi -> nox
 miwi -> pawel
+miwi -> rm
 miwi -> sbz
 miwi -> sylvio
 miwi -> tabthorpe
@@ -366,6 +371,7 @@ nork -> ale
 
 novel -> alexbl
 novel -> ehaupt
+novel -> rm
 
 obrien -> mharo
 obrien -> gerald
diff --git a/share/misc/committers-src.dot b/share/misc/committers-src.dot
index 8f97775..a0c2dd8 100644
--- a/share/misc/committers-src.dot
+++ b/share/misc/committers-src.dot
@@ -167,6 +167,7 @@ jls [label="Jordan Sissel\njls@FreeBSD.org\n2006/12/06"]
 joerg [label="Joerg Wunsch\njoerg@FreeBSD.org\n1993/11/14"]
 jon [label="Jonathan Chen\njon@FreeBSD.org\n2000/10/17"]
 jonathan [label="Jonathan Anderson\njonathan@FreeBSD.org\n2010/10/07"]
+jpaetzel [label="Josh Paetzel\njpaetzel@FreeBSD.org\n2011/01/21"]
 julian [label="Julian Elischer\njulian@FreeBSD.org\n1993/??/??"]
 kaiw [label="Kai Wang\nkaiw@FreeBSD.org\n2007/09/26"]
 kan [label="Alexander Kabaev\nkan@FreeBSD.org\n2002/07/21"]
@@ -237,6 +238,7 @@ stas [label="Stanislav Sedov\nstas@FreeBSD.org\n2008/08/22"]
 suz [label="SUZUKI Shinsuke\nsuz@FreeBSD.org\n2002/03/26"]
 syrinx [label="Shteryana Shopova\nsyrinx@FreeBSD.org\n2006/10/07"]
 takawata [label="Takanori Watanabe\ntakawata@FreeBSD.org\n2000/07/06"]
+theraven [label="David Chisnall\ntheraven@FreeBSD.org\n2011/11/11"]
 thompsa [label="Andrew Thompson\nthompsa@FreeBSD.org\n2005/05/25"]
 ticso [label="Bernd Walter\nticso@FreeBSD.org\n2002/01/31"]
 tijl [label="Tijl Coosemans\ntijl@FreeBSD.org\n2010/07/16"]
@@ -303,6 +305,7 @@ brian -> joe
 
 brooks -> bushman
 brooks -> jamie
+brooks -> theraven
 
 bz -> anchie
 bz -> jamie
@@ -332,6 +335,8 @@ dds -> versus
 
 dfr -> zml
 
+dim -> theraven
+
 dwmalone -> fanf
 dwmalone -> peadar
 dwmalone -> snb
@@ -443,6 +448,7 @@ kan -> kib
 
 kib -> ae
 kib -> dchagin
+kib -> jpaetzel
 kib -> lulf
 kib -> melifaro
 kib -> pho
diff --git a/share/mk/bsd.lib.mk b/share/mk/bsd.lib.mk
index 943010c..264f889 100644
--- a/share/mk/bsd.lib.mk
+++ b/share/mk/bsd.lib.mk
@@ -67,23 +67,29 @@ PICFLAG=-fpic
 
 PO_FLAG=-pg
 
+.c.o:
+	${CC} ${STATIC_CFLAGS} ${CFLAGS} -c ${.IMPSRC} -o ${.TARGET}
+
 .c.po:
-	${CC} ${PO_FLAG} ${PO_CFLAGS} -c ${.IMPSRC} -o ${.TARGET}
+	${CC} ${PO_FLAG} ${STATIC_CFLAGS} ${PO_CFLAGS} -c ${.IMPSRC} -o ${.TARGET}
 	@[ -z "${CTFCONVERT}" -o -n "${NO_CTF}" ] || \
 		(${ECHO} ${CTFCONVERT} ${CTFFLAGS} ${.TARGET} && \
 		${CTFCONVERT} ${CTFFLAGS} ${.TARGET})
 
 .c.So:
-	${CC} ${PICFLAG} -DPIC ${CFLAGS} -c ${.IMPSRC} -o ${.TARGET}
+	${CC} ${PICFLAG} -DPIC ${SHARED_CFLAGS} ${CFLAGS} -c ${.IMPSRC} -o ${.TARGET}
 	@[ -z "${CTFCONVERT}" -o -n "${NO_CTF}" ] || \
 		(${ECHO} ${CTFCONVERT} ${CTFFLAGS} ${.TARGET} && \
 		${CTFCONVERT} ${CTFFLAGS} ${.TARGET})
 
+.cc.o:
+	${CXX} ${STATIC_CXXFLAGS} ${CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}
+
 .cc.po .C.po .cpp.po .cxx.po:
-	${CXX} ${PO_FLAG} ${PO_CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}
+	${CXX} ${PO_FLAG} ${STATIC_CXXFLAGS} ${PO_CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}
 
 .cc.So .C.So .cpp.So .cxx.So:
-	${CXX} ${PICFLAG} -DPIC ${CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}
+	${CXX} ${PICFLAG} -DPIC ${SHARED_CXXFLAGS} ${CXXFLAGS} -c ${.IMPSRC} -o ${.TARGET}
 
 .f.po:
 	${FC} -pg ${FFLAGS} -o ${.TARGET} -c ${.IMPSRC}
diff --git a/share/mk/bsd.own.mk b/share/mk/bsd.own.mk
index 3058970..5743fc4 100644
--- a/share/mk/bsd.own.mk
+++ b/share/mk/bsd.own.mk
@@ -428,15 +428,14 @@ __T=${TARGET_ARCH}
 .else
 __T=${MACHINE_ARCH}
 .endif
-# Clang is only for x86 and 32-bit powerpc right now, by default.
-.if ${__T} == "amd64" || ${__T} == "i386" || ${__T} == "powerpc"
+# Clang is only for x86 and powerpc right now, by default.
+.if ${__T} == "amd64" || ${__T} == "i386" || ${__T:Mpowerpc*}
 __DEFAULT_YES_OPTIONS+=CLANG
 .else
 __DEFAULT_NO_OPTIONS+=CLANG
 .endif
-# FDT is needed only for arm, mips and powerpc (and not powerpc64)
-.if ${__T} == "arm" || ${__T} == "armeb" || ${__T} == "powerpc" || \
-    ${__T:Mmips*}
+# FDT is needed only for arm, mips and powerpc
+.if ${__T:Marm*} || ${__T:Mpowerpc*} || ${__T:Mmips*}
 __DEFAULT_YES_OPTIONS+=FDT
 .else
 __DEFAULT_NO_OPTIONS+=FDT
diff --git a/share/monetdef/zh_CN.UTF-8.src b/share/monetdef/zh_CN.UTF-8.src
index 136d66c..c130595 100644
--- a/share/monetdef/zh_CN.UTF-8.src
+++ b/share/monetdef/zh_CN.UTF-8.src
@@ -18,9 +18,9 @@ CNY
 # negative_sign
 -
 # int_frac_digits
-0
+2
 # frac_digits
-0
+2
 # p_cs_precedes
 1
 # p_sep_by_space
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 0818800..fab3111 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -2047,7 +2047,8 @@ int
 fill_fpregs(struct thread *td, struct fpreg *fpregs)
 {
 
-	KASSERT(td == curthread || TD_IS_SUSPENDED(td),
+	KASSERT(td == curthread || TD_IS_SUSPENDED(td) ||
+	    P_SHOULDSTOP(td->td_proc),
 	    ("not suspended thread %p", td));
 	fpugetregs(td);
 	fill_fpregs_xmm(&td->td_pcb->pcb_user_save, fpregs);
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 5cce565..7247314 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -1092,7 +1092,7 @@ start_ap(int apic_id)
 u_int xhits_gbl[MAXCPU];
 u_int xhits_pg[MAXCPU];
 u_int xhits_rng[MAXCPU];
-SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
+static SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
     sizeof(xhits_gbl), "IU", "");
 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 43df8ee..f7c0d2d 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -182,7 +182,7 @@ static vm_paddr_t dmaplimit;
 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
 pt_entry_t pg_nx;
 
-SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
+static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
 
 static int pat_works = 1;
 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1,
@@ -805,7 +805,7 @@ pmap_shpgperproc_proc(SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_vm_pmap, OID_AUTO, shpgperproc, CTLTYPE_INT|CTLFLAG_RW, 
     &shpgperproc, 0, pmap_shpgperproc_proc, "IU", "Page share factor per proc");
 
-SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
     "2MB page mapping counters");
 
 static u_long pmap_pde_demotions;
@@ -824,7 +824,7 @@ static u_long pmap_pde_promotions;
 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
     &pmap_pde_promotions, 0, "2MB page promotions");
 
-SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD, 0,
     "1GB page mapping counters");
 
 static u_long pmap_pdpe_demotions;
@@ -1635,7 +1635,6 @@ int
 pmap_pinit(pmap_t pmap)
 {
 	vm_page_t pml4pg;
-	static vm_pindex_t color;
 	int i;
 
 	PMAP_LOCK_INIT(pmap);
@@ -1643,8 +1642,8 @@ pmap_pinit(pmap_t pmap)
 	/*
 	 * allocate the page directory page
 	 */
-	while ((pml4pg = vm_page_alloc(NULL, color++, VM_ALLOC_NOOBJ |
-	    VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
+	while ((pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
+	    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
 		VM_WAIT;
 
 	pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
@@ -2188,7 +2187,6 @@ get_pv_entry(pmap_t pmap, int try)
 {
 	static const struct timeval printinterval = { 60, 0 };
 	static struct timeval lastprint;
-	static vm_pindex_t colour;
 	struct vpgqueues *pq;
 	int bit, field;
 	pv_entry_t pv;
@@ -2228,7 +2226,7 @@ retry:
 		}
 	}
 	/* No free items, allocate another chunk */
-	m = vm_page_alloc(NULL, colour, (pq == &vm_page_queues[PQ_ACTIVE] ?
+	m = vm_page_alloc(NULL, 0, (pq == &vm_page_queues[PQ_ACTIVE] ?
 	    VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ |
 	    VM_ALLOC_WIRED);
 	if (m == NULL) {
@@ -2255,7 +2253,6 @@ retry:
 	}
 	PV_STAT(pc_chunk_count++);
 	PV_STAT(pc_chunk_allocs++);
-	colour++;
 	dump_add_page(m->phys_addr);
 	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 	pc->pc_pmap = pmap;
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index 9c72a69..16b9c1a 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -125,7 +125,7 @@ void dblfault_handler(struct trapframe *frame);
 static int trap_pfault(struct trapframe *, int);
 static void trap_fatal(struct trapframe *, vm_offset_t);
 
-#define MAX_TRAP_MSG		30
+#define MAX_TRAP_MSG		33
 static char *trap_msg[] = {
 	"",					/*  0 unused */
 	"privileged instruction fault",		/*  1 T_PRIVINFLT */
@@ -158,6 +158,9 @@ static char *trap_msg[] = {
 	"machine check trap",			/* 28 T_MCHK */
 	"SIMD floating-point exception",	/* 29 T_XMMFLT */
 	"reserved (unknown) fault",		/* 30 T_RESERVED */
+	"",					/* 31 unused (reserved) */
+	"DTrace pid return trap",		/* 32 T_DTRACE_RET */
+	"DTrace fasttrap probe trap",		/* 33 T_DTRACE_PROBE */
 };
 
 #ifdef KDB
@@ -245,28 +248,26 @@ trap(struct trapframe *frame)
 	 * handled the trap and modified the trap frame so that this
 	 * function can return normally.
 	 */
-	if (dtrace_trap_func != NULL)
-		if ((*dtrace_trap_func)(frame, type))
-			goto out;
 	if (type == T_DTRACE_PROBE || type == T_DTRACE_RET ||
 	    type == T_BPTFLT) {
 		struct reg regs;
-		
+
 		fill_frame_regs(frame, &regs);
 		if (type == T_DTRACE_PROBE &&
 		    dtrace_fasttrap_probe_ptr != NULL &&
 		    dtrace_fasttrap_probe_ptr(&regs) == 0)
-				goto out;
-		if (type == T_BPTFLT &&
+			goto out;
+		else if (type == T_BPTFLT &&
 		    dtrace_pid_probe_ptr != NULL &&
 		    dtrace_pid_probe_ptr(&regs) == 0)
-				goto out;
-		if (type == T_DTRACE_RET &&
+			goto out;
+		else if (type == T_DTRACE_RET &&
 		    dtrace_return_probe_ptr != NULL &&
 		    dtrace_return_probe_ptr(&regs) == 0)
 			goto out;
-
 	}
+	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type))
+		goto out;
 #endif
 
 	if ((frame->tf_rflags & PSL_I) == 0) {
@@ -899,9 +900,9 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
 #include "../../kern/subr_syscall.c"
 
 /*
- *	syscall -	system call request C handler
- *
- *	A system call is essentially treated as a trap.
+ * System call handler for native binaries.  The trap frame is already
+ * set up by the assembler trampoline and a pointer to it is saved in
+ * td_frame.
  */
 void
 amd64_syscall(struct thread *td, int traced)
diff --git a/sys/amd64/amd64/uma_machdep.c b/sys/amd64/amd64/uma_machdep.c
index 1329668..3583975 100644
--- a/sys/amd64/amd64/uma_machdep.c
+++ b/sys/amd64/amd64/uma_machdep.c
@@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
 void *
 uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 {
-	static vm_pindex_t colour;
 	vm_page_t m;
 	vm_paddr_t pa;
 	void *va;
@@ -50,13 +49,13 @@ uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 
 	*flags = UMA_SLAB_PRIV;
 	if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
-		pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
+		pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
 	else
-		pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED;
+		pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
 	if (wait & M_ZERO)
 		pflags |= VM_ALLOC_ZERO;
 	for (;;) {
-		m = vm_page_alloc(NULL, colour++, pflags | VM_ALLOC_NOOBJ);
+		m = vm_page_alloc(NULL, 0, pflags);
 		if (m == NULL) {
 			if (wait & M_NOWAIT)
 				return (NULL);
diff --git a/sys/amd64/conf/DEFAULTS b/sys/amd64/conf/DEFAULTS
index 2c221cb..e39a9e4 100644
--- a/sys/amd64/conf/DEFAULTS
+++ b/sys/amd64/conf/DEFAULTS
@@ -22,3 +22,6 @@ options 	GEOM_PART_EBR_COMPAT
 options 	GEOM_PART_MBR
 
 options 	NEW_PCIB
+
+# Allow mounting non-MPSAFE filesystems
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC
index 43c20f3..2b55c88 100644
--- a/sys/amd64/conf/GENERIC
+++ b/sys/amd64/conf/GENERIC
@@ -65,9 +65,11 @@ options 	MAC			# TrustedBSD MAC Framework
 #options 	KDTRACE_HOOKS		# Kernel DTrace hooks
 options 	INCLUDE_CONFIG_FILE     # Include this file in kernel
 
-# Debugging for use in -current
+# Debugging support.  Always need this:
 options 	KDB			# Enable kernel debugger support.
-options		KDB_TRACE
+# For minimum debugger support (stable branch) use:
+options 	KDB_TRACE		# Print a stack trace for a panic.
+# For full debugger support use this instead:
 options 	DDB			# Support DDB.
 options 	GDB			# Support remote GDB.
 options 	DEADLKRES		# Enable the deadlock resolver
@@ -105,7 +107,7 @@ options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
 device		ahd		# AHA39320/29320 and onboard AIC79xx devices
 options 	AHD_REG_PRETTY_PRINT	# Print register bitfields in debug
 					# output.  Adds ~215k to driver.
-device		amd		# AMD 53C974 (Tekram DC-390(T))
+device		esp		# AMD Am53C974 (Tekram DC-390(T))
 device		hptiop		# Highpoint RocketRaid 3xxx series
 device		isp		# Qlogic family
 #device		ispfw		# Firmware for QLogic HBAs- normally a module
@@ -351,3 +353,8 @@ device		snd_hda		# Intel High Definition Audio
 device		snd_ich		# Intel, NVidia and other ICH AC'97 Audio
 device		snd_uaudio	# USB Audio
 device		snd_via8233	# VIA VT8233x Audio
+
+# MMC/SD
+device		mmc		# MMC/SD bus
+device		mmcsd		# MMC/SD memory card
+device		sdhci		# Generic PCI SD Host Controller
diff --git a/sys/amd64/include/_stdint.h b/sys/amd64/include/_stdint.h
index ee4e592..d755bb3 100644
--- a/sys/amd64/include/_stdint.h
+++ b/sys/amd64/include/_stdint.h
@@ -150,8 +150,8 @@
 #define	PTRDIFF_MAX	INT64_MAX
 
 /* Limits of sig_atomic_t. */
-#define	SIG_ATOMIC_MIN	INT32_MIN
-#define	SIG_ATOMIC_MAX	INT32_MAX
+#define	SIG_ATOMIC_MIN	LONG_MIN
+#define	SIG_ATOMIC_MAX	LONG_MAX
 
 /* Limit of size_t. */
 #define	SIZE_MAX	UINT64_MAX
diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h
index 22dc95a..2796511 100644
--- a/sys/amd64/include/segments.h
+++ b/sys/amd64/include/segments.h
@@ -214,7 +214,7 @@ struct region_descriptor {
 #define	IDT_XF		19	/* #XF: SIMD Floating-Point Exception */
 #define	IDT_IO_INTS	NRSVIDT	/* Base of IDT entries for I/O interrupts. */
 #define	IDT_SYSCALL	0x80	/* System Call Interrupt Vector */
-#define	IDT_DTRACE_RET	0x92	/* DTrace pid provider Interrupt Vector */
+#define	IDT_DTRACE_RET	0x20	/* DTrace pid provider Interrupt Vector */
 
 /*
  * Entries in the Global Descriptor Table (GDT)
diff --git a/sys/amd64/include/signal.h b/sys/amd64/include/signal.h
index 228e2d9..0374339 100644
--- a/sys/amd64/include/signal.h
+++ b/sys/amd64/include/signal.h
@@ -47,18 +47,14 @@ typedef long sig_atomic_t;
 #include <machine/trap.h>	/* codes for SIGILL, SIGFPE */
 
 /*
- * Only the kernel should need these old type definitions.
- */
-/*
  * Information pushed on stack when a signal is delivered.
  * This is used by the kernel to restore state following
  * execution of the signal handler.  It is also made available
  * to the handler to allow it to restore state properly if
  * a non-standard exit is performed.
- */
-/*
- * The sequence of the fields/registers in struct sigcontext should match
- * those in mcontext_t.
+ *
+ * The sequence of the fields/registers after sc_mask in struct
+ * sigcontext must match those in mcontext_t and struct trapframe.
  */
 struct sigcontext {
 	struct __sigset sc_mask;	/* signal mask to restore */
@@ -93,8 +89,8 @@ struct sigcontext {
 	long	sc_ss;
 	long	sc_len;			/* sizeof(mcontext_t) */
 	/*
-	 * XXX - See <machine/ucontext.h> and <machine/fpu.h> for
-	 *       the following fields.
+	 * See <machine/ucontext.h> and <machine/fpu.h> for the following
+	 * fields.
 	 */
 	long	sc_fpformat;
 	long	sc_ownedfp;
diff --git a/sys/amd64/include/trap.h b/sys/amd64/include/trap.h
index d8e36b5..a395d62 100644
--- a/sys/amd64/include/trap.h
+++ b/sys/amd64/include/trap.h
@@ -62,8 +62,8 @@
 #define	T_MCHK		28	/* machine check trap */
 #define	T_XMMFLT	29	/* SIMD floating-point exception */
 #define	T_RESERVED	30	/* reserved (unknown) */
-#define	T_DTRACE_RET	31	/* DTrace pid return */
-#define	T_DTRACE_PROBE	32	/* DTrace fasttrap probe */
+#define	T_DTRACE_RET	32	/* DTrace pid return */
+#define	T_DTRACE_PROBE	33	/* DTrace fasttrap probe */
 
 /* XXX most of the following codes aren't used, but could be. */
 
diff --git a/sys/amd64/include/ucontext.h b/sys/amd64/include/ucontext.h
index c5bbd65..75b7bd2 100644
--- a/sys/amd64/include/ucontext.h
+++ b/sys/amd64/include/ucontext.h
@@ -41,12 +41,13 @@
 
 typedef struct __mcontext {
 	/*
-	 * The first 24 fields must match the definition of
-	 * sigcontext. So that we can support sigcontext
-	 * and ucontext_t at the same time.
+	 * The definition of mcontext_t must match the layout of
+	 * struct sigcontext after the sc_mask member.  This is so
+	 * that we can support sigcontext and ucontext_t at the same
+	 * time.
 	 */
-	__register_t	mc_onstack;		/* XXX - sigcontext compat. */
-	__register_t	mc_rdi;			/* machine state (struct trapframe) */
+	__register_t	mc_onstack;	/* XXX - sigcontext compat. */
+	__register_t	mc_rdi;		/* machine state (struct trapframe) */
 	__register_t	mc_rsi;
 	__register_t	mc_rdx;
 	__register_t	mc_rcx;
diff --git a/sys/amd64/linux32/linux32_proto.h b/sys/amd64/linux32/linux32_proto.h
index 0fa4358..cc9b97d 100644
--- a/sys/amd64/linux32/linux32_proto.h
+++ b/sys/amd64/linux32/linux32_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 #ifndef _LINUX_SYSPROTO_H_
@@ -12,6 +12,7 @@
 #include <sys/signal.h>
 #include <sys/acl.h>
 #include <sys/cpuset.h>
+#include <sys/_ffcounter.h>
 #include <sys/_semaphore.h>
 #include <sys/ucontext.h>
 
@@ -131,7 +132,7 @@ struct linux_utime_args {
 };
 struct linux_access_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char flags_l_[PADL_(l_int)]; l_int flags; char flags_r_[PADR_(l_int)];
+	char amode_l_[PADL_(l_int)]; l_int amode; char amode_r_[PADR_(l_int)];
 };
 struct linux_nice_args {
 	char inc_l_[PADL_(l_int)]; l_int inc; char inc_r_[PADR_(l_int)];
@@ -942,7 +943,7 @@ struct linux_linkat_args {
 	char oldname_l_[PADL_(const char *)]; const char * oldname; char oldname_r_[PADR_(const char *)];
 	char newdfd_l_[PADL_(l_int)]; l_int newdfd; char newdfd_r_[PADR_(l_int)];
 	char newname_l_[PADL_(const char *)]; const char * newname; char newname_r_[PADR_(const char *)];
-	char flags_l_[PADL_(l_int)]; l_int flags; char flags_r_[PADR_(l_int)];
+	char flag_l_[PADL_(l_int)]; l_int flag; char flag_r_[PADR_(l_int)];
 };
 struct linux_symlinkat_args {
 	char oldname_l_[PADL_(const char *)]; const char * oldname; char oldname_r_[PADR_(const char *)];
@@ -963,7 +964,8 @@ struct linux_fchmodat_args {
 struct linux_faccessat_args {
 	char dfd_l_[PADL_(l_int)]; l_int dfd; char dfd_r_[PADR_(l_int)];
 	char filename_l_[PADL_(const char *)]; const char * filename; char filename_r_[PADR_(const char *)];
-	char mode_l_[PADL_(l_int)]; l_int mode; char mode_r_[PADR_(l_int)];
+	char amode_l_[PADL_(l_int)]; l_int amode; char amode_r_[PADR_(l_int)];
+	char flag_l_[PADL_(int)]; int flag; char flag_r_[PADR_(int)];
 };
 struct linux_pselect6_args {
 	register_t dummy;
diff --git a/sys/amd64/linux32/linux32_syscall.h b/sys/amd64/linux32/linux32_syscall.h
index edd9a03..7b74cd4 100644
--- a/sys/amd64/linux32/linux32_syscall.h
+++ b/sys/amd64/linux32/linux32_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 #define	LINUX_SYS_exit	1
diff --git a/sys/amd64/linux32/linux32_syscalls.c b/sys/amd64/linux32/linux32_syscalls.c
index 5a95e8e..c83a923 100644
--- a/sys/amd64/linux32/linux32_syscalls.c
+++ b/sys/amd64/linux32/linux32_syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 const char *linux_syscallnames[] = {
diff --git a/sys/amd64/linux32/linux32_sysent.c b/sys/amd64/linux32/linux32_sysent.c
index 3339116..549bf8e 100644
--- a/sys/amd64/linux32/linux32_sysent.c
+++ b/sys/amd64/linux32/linux32_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/amd64/linux32/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 #include "opt_compat.h"
diff --git a/sys/amd64/linux32/linux32_systrace_args.c b/sys/amd64/linux32/linux32_systrace_args.c
index 42f0b33..2a5e835 100644
--- a/sys/amd64/linux32/linux32_systrace_args.c
+++ b/sys/amd64/linux32/linux32_systrace_args.c
@@ -230,7 +230,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 33: {
 		struct linux_access_args *p = params;
 		uarg[0] = (intptr_t) p->path; /* char * */
-		iarg[1] = p->flags; /* l_int */
+		iarg[1] = p->amode; /* l_int */
 		*n_args = 2;
 		break;
 	}
@@ -1996,7 +1996,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		uarg[1] = (intptr_t) p->oldname; /* const char * */
 		iarg[2] = p->newdfd; /* l_int */
 		uarg[3] = (intptr_t) p->newname; /* const char * */
-		iarg[4] = p->flags; /* l_int */
+		iarg[4] = p->flag; /* l_int */
 		*n_args = 5;
 		break;
 	}
@@ -2033,8 +2033,9 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		struct linux_faccessat_args *p = params;
 		iarg[0] = p->dfd; /* l_int */
 		uarg[1] = (intptr_t) p->filename; /* const char * */
-		iarg[2] = p->mode; /* l_int */
-		*n_args = 3;
+		iarg[2] = p->amode; /* l_int */
+		iarg[3] = p->flag; /* int */
+		*n_args = 4;
 		break;
 	}
 	/* linux_pselect6 */
@@ -2095,7 +2096,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	};
 }
 static void
-systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 {
 	const char *p = NULL;
 	switch (sysnum) {
@@ -5064,6 +5065,9 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 		case 2:
 			p = "l_int";
 			break;
+		case 3:
+			p = "int";
+			break;
 		default:
 			break;
 		};
@@ -5124,3 +5128,1168 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 	if (p != NULL)
 		strlcpy(desc, p, descsz);
 }
+static void
+systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+{
+	const char *p = NULL;
+	switch (sysnum) {
+#define	nosys	linux_nosys
+	/* sys_exit */
+	case 1:
+		if (ndx == 0 || ndx == 1)
+			p = "void";
+		break;
+	/* linux_fork */
+	case 2:
+	/* read */
+	case 3:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* write */
+	case 4:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_open */
+	case 5:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* close */
+	case 6:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_waitpid */
+	case 7:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_creat */
+	case 8:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_link */
+	case 9:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_unlink */
+	case 10:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_execve */
+	case 11:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chdir */
+	case 12:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_time */
+	case 13:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mknod */
+	case 14:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chmod */
+	case 15:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lchown16 */
+	case 16:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_stat */
+	case 18:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lseek */
+	case 19:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getpid */
+	case 20:
+	/* linux_mount */
+	case 21:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_oldumount */
+	case 22:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setuid16 */
+	case 23:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getuid16 */
+	case 24:
+	/* linux_stime */
+	case 25:
+	/* linux_ptrace */
+	case 26:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_alarm */
+	case 27:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pause */
+	case 29:
+	/* linux_utime */
+	case 30:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_access */
+	case 33:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_nice */
+	case 34:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sync */
+	case 36:
+	/* linux_kill */
+	case 37:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rename */
+	case 38:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mkdir */
+	case 39:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rmdir */
+	case 40:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* dup */
+	case 41:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pipe */
+	case 42:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_times */
+	case 43:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_brk */
+	case 45:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setgid16 */
+	case 46:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getgid16 */
+	case 47:
+	/* linux_signal */
+	case 48:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_geteuid16 */
+	case 49:
+	/* linux_getegid16 */
+	case 50:
+	/* acct */
+	case 51:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_umount */
+	case 52:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ioctl */
+	case 54:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fcntl */
+	case 55:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setpgid */
+	case 57:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_olduname */
+	case 59:
+	/* umask */
+	case 60:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chroot */
+	case 61:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ustat */
+	case 62:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* dup2 */
+	case 63:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getppid */
+	case 64:
+	/* getpgrp */
+	case 65:
+	/* setsid */
+	case 66:
+	/* linux_sigaction */
+	case 67:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sgetmask */
+	case 68:
+	/* linux_ssetmask */
+	case 69:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setreuid16 */
+	case 70:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setregid16 */
+	case 71:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigsuspend */
+	case 72:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigpending */
+	case 73:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sethostname */
+	case 74:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setrlimit */
+	case 75:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_old_getrlimit */
+	case 76:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getrusage */
+	case 77:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_gettimeofday */
+	case 78:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_settimeofday */
+	case 79:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getgroups16 */
+	case 80:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setgroups16 */
+	case 81:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_old_select */
+	case 82:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_symlink */
+	case 83:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lstat */
+	case 84:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readlink */
+	case 85:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* swapon */
+	case 87:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_reboot */
+	case 88:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readdir */
+	case 89:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mmap */
+	case 90:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munmap */
+	case 91:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_truncate */
+	case 92:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ftruncate */
+	case 93:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchmod */
+	case 94:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchown */
+	case 95:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getpriority */
+	case 96:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setpriority */
+	case 97:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_statfs */
+	case 99:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstatfs */
+	case 100:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_socketcall */
+	case 102:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_syslog */
+	case 103:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setitimer */
+	case 104:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getitimer */
+	case 105:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newstat */
+	case 106:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newlstat */
+	case 107:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newfstat */
+	case 108:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_uname */
+	case 109:
+	/* linux_iopl */
+	case 110:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_vhangup */
+	case 111:
+	/* linux_wait4 */
+	case 114:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_swapoff */
+	case 115:
+	/* linux_sysinfo */
+	case 116:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ipc */
+	case 117:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fsync */
+	case 118:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigreturn */
+	case 119:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clone */
+	case 120:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setdomainname */
+	case 121:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newuname */
+	case 122:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_adjtimex */
+	case 124:
+	/* linux_mprotect */
+	case 125:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigprocmask */
+	case 126:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_create_module */
+	case 127:
+	/* linux_init_module */
+	case 128:
+	/* linux_delete_module */
+	case 129:
+	/* linux_get_kernel_syms */
+	case 130:
+	/* linux_quotactl */
+	case 131:
+	/* getpgid */
+	case 132:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchdir */
+	case 133:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_bdflush */
+	case 134:
+	/* linux_sysfs */
+	case 135:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_personality */
+	case 136:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsuid16 */
+	case 138:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsgid16 */
+	case 139:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_llseek */
+	case 140:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getdents */
+	case 141:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_select */
+	case 142:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* flock */
+	case 143:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_msync */
+	case 144:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readv */
+	case 145:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_writev */
+	case 146:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getsid */
+	case 147:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fdatasync */
+	case 148:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sysctl */
+	case 149:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mlock */
+	case 150:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlock */
+	case 151:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mlockall */
+	case 152:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlockall */
+	case 153:
+	/* sched_setparam */
+	case 154:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_getparam */
+	case 155:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_setscheduler */
+	case 156:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_getscheduler */
+	case 157:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_yield */
+	case 158:
+	/* linux_sched_get_priority_max */
+	case 159:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_get_priority_min */
+	case 160:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_rr_get_interval */
+	case 161:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_nanosleep */
+	case 162:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mremap */
+	case 163:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setresuid16 */
+	case 164:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getresuid16 */
+	case 165:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_query_module */
+	case 167:
+	/* poll */
+	case 168:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_nfsservctl */
+	case 169:
+	/* linux_setresgid16 */
+	case 170:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getresgid16 */
+	case 171:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_prctl */
+	case 172:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigreturn */
+	case 173:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigaction */
+	case 174:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigprocmask */
+	case 175:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigpending */
+	case 176:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigtimedwait */
+	case 177:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigqueueinfo */
+	case 178:
+	/* linux_rt_sigsuspend */
+	case 179:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pread */
+	case 180:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pwrite */
+	case 181:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chown16 */
+	case 182:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getcwd */
+	case 183:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_capget */
+	case 184:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_capset */
+	case 185:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigaltstack */
+	case 186:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sendfile */
+	case 187:
+	/* linux_vfork */
+	case 190:
+	/* linux_getrlimit */
+	case 191:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mmap2 */
+	case 192:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_truncate64 */
+	case 193:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ftruncate64 */
+	case 194:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_stat64 */
+	case 195:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lstat64 */
+	case 196:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstat64 */
+	case 197:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lchown */
+	case 198:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getuid */
+	case 199:
+	/* linux_getgid */
+	case 200:
+	/* geteuid */
+	case 201:
+	/* getegid */
+	case 202:
+	/* setreuid */
+	case 203:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setregid */
+	case 204:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getgroups */
+	case 205:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setgroups */
+	case 206:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchown */
+	case 207:
+	/* setresuid */
+	case 208:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresuid */
+	case 209:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setresgid */
+	case 210:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresgid */
+	case 211:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chown */
+	case 212:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setuid */
+	case 213:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setgid */
+	case 214:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsuid */
+	case 215:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsgid */
+	case 216:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pivot_root */
+	case 217:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mincore */
+	case 218:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* madvise */
+	case 219:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getdents64 */
+	case 220:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fcntl64 */
+	case 221:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_gettid */
+	case 224:
+	/* linux_setxattr */
+	case 226:
+	/* linux_lsetxattr */
+	case 227:
+	/* linux_fsetxattr */
+	case 228:
+	/* linux_getxattr */
+	case 229:
+	/* linux_lgetxattr */
+	case 230:
+	/* linux_fgetxattr */
+	case 231:
+	/* linux_listxattr */
+	case 232:
+	/* linux_llistxattr */
+	case 233:
+	/* linux_flistxattr */
+	case 234:
+	/* linux_removexattr */
+	case 235:
+	/* linux_lremovexattr */
+	case 236:
+	/* linux_fremovexattr */
+	case 237:
+	/* linux_tkill */
+	case 238:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sys_futex */
+	case 240:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_setaffinity */
+	case 241:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_getaffinity */
+	case 242:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_set_thread_area */
+	case 243:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fadvise64 */
+	case 250:
+	/* linux_exit_group */
+	case 252:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lookup_dcookie */
+	case 253:
+	/* linux_epoll_create */
+	case 254:
+	/* linux_epoll_ctl */
+	case 255:
+	/* linux_epoll_wait */
+	case 256:
+	/* linux_remap_file_pages */
+	case 257:
+	/* linux_set_tid_address */
+	case 258:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_timer_create */
+	case 259:
+	/* linux_timer_settime */
+	case 260:
+	/* linux_timer_gettime */
+	case 261:
+	/* linux_timer_getoverrun */
+	case 262:
+	/* linux_timer_delete */
+	case 263:
+	/* linux_clock_settime */
+	case 264:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_gettime */
+	case 265:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_getres */
+	case 266:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_nanosleep */
+	case 267:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_statfs64 */
+	case 268:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstatfs64 */
+	case 269:
+	/* linux_tgkill */
+	case 270:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_utimes */
+	case 271:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fadvise64_64 */
+	case 272:
+	/* linux_mbind */
+	case 274:
+	/* linux_get_mempolicy */
+	case 275:
+	/* linux_set_mempolicy */
+	case 276:
+	/* linux_mq_open */
+	case 277:
+	/* linux_mq_unlink */
+	case 278:
+	/* linux_mq_timedsend */
+	case 279:
+	/* linux_mq_timedreceive */
+	case 280:
+	/* linux_mq_notify */
+	case 281:
+	/* linux_mq_getsetattr */
+	case 282:
+	/* linux_kexec_load */
+	case 283:
+	/* linux_waitid */
+	case 284:
+	/* linux_add_key */
+	case 286:
+	/* linux_request_key */
+	case 287:
+	/* linux_keyctl */
+	case 288:
+	/* linux_ioprio_set */
+	case 289:
+	/* linux_ioprio_get */
+	case 290:
+	/* linux_inotify_init */
+	case 291:
+	/* linux_inotify_add_watch */
+	case 292:
+	/* linux_inotify_rm_watch */
+	case 293:
+	/* linux_migrate_pages */
+	case 294:
+	/* linux_openat */
+	case 295:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mkdirat */
+	case 296:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mknodat */
+	case 297:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fchownat */
+	case 298:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_futimesat */
+	case 299:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstatat64 */
+	case 300:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_unlinkat */
+	case 301:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_renameat */
+	case 302:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_linkat */
+	case 303:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_symlinkat */
+	case 304:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readlinkat */
+	case 305:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fchmodat */
+	case 306:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_faccessat */
+	case 307:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pselect6 */
+	case 308:
+	/* linux_ppoll */
+	case 309:
+	/* linux_unshare */
+	case 310:
+	/* linux_set_robust_list */
+	case 311:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_get_robust_list */
+	case 312:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_splice */
+	case 313:
+	/* linux_sync_file_range */
+	case 314:
+	/* linux_tee */
+	case 315:
+	/* linux_vmsplice */
+	case 316:
+	default:
+		break;
+	};
+	if (p != NULL)
+		strlcpy(desc, p, descsz);
+}
diff --git a/sys/amd64/linux32/linux32_sysvec.c b/sys/amd64/linux32/linux32_sysvec.c
index 6c0f0be..4ff5483 100644
--- a/sys/amd64/linux32/linux32_sysvec.c
+++ b/sys/amd64/linux32/linux32_sysvec.c
@@ -977,7 +977,7 @@ linux_copyout_strings(struct image_params *imgp)
 	return ((register_t *)stack_base);
 }
 
-SYSCTL_NODE(_compat, OID_AUTO, linux32, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_compat, OID_AUTO, linux32, CTLFLAG_RW, 0,
     "32-bit Linux emulation");
 
 static u_long	linux32_maxdsiz = LINUX32_MAXDSIZ;
diff --git a/sys/amd64/linux32/syscalls.master b/sys/amd64/linux32/syscalls.master
index f00100e..1c09084 100644
--- a/sys/amd64/linux32/syscalls.master
+++ b/sys/amd64/linux32/syscalls.master
@@ -86,7 +86,7 @@
 				    struct l_utimbuf *times); }
 31	AUE_NULL	UNIMPL	stty
 32	AUE_NULL	UNIMPL	gtty
-33	AUE_ACCESS	STD	{ int linux_access(char *path, l_int flags); }
+33	AUE_ACCESS	STD	{ int linux_access(char *path, l_int amode); }
 34	AUE_NICE	STD	{ int linux_nice(l_int inc); }
 35	AUE_NULL	UNIMPL	ftime
 36	AUE_SYNC	NOPROTO	{ int sync(void); }
@@ -483,14 +483,14 @@
 302	AUE_RENAMEAT	STD	{ int linux_renameat(l_int olddfd, const char *oldname, \
 					l_int newdfd, const char *newname); }
 303	AUE_LINKAT	STD	{ int linux_linkat(l_int olddfd, const char *oldname, \
-					l_int newdfd, const char *newname, l_int flags); }
+					l_int newdfd, const char *newname, l_int flag); }
 304	AUE_SYMLINKAT	STD	{ int linux_symlinkat(const char *oldname, l_int newdfd, \
 					const char *newname); }
 305	AUE_READLINKAT	STD	{ int linux_readlinkat(l_int dfd, const char *path, \
 					char *buf, l_int bufsiz); }
 306	AUE_FCHMODAT	STD	{ int linux_fchmodat(l_int dfd, const char *filename, \
 					l_mode_t mode); }
-307	AUE_FACCESSAT	STD	{ int linux_faccessat(l_int dfd, const char *filename, l_int mode); }
+307	AUE_FACCESSAT	STD	{ int linux_faccessat(l_int dfd, const char *filename, l_int amode, int flag); }
 308	AUE_NULL	STD	{ int linux_pselect6(void); }
 309	AUE_NULL	STD	{ int linux_ppoll(void); }
 310	AUE_NULL	STD	{ int linux_unshare(void); }
diff --git a/sys/arm/arm/busdma_machdep.c b/sys/arm/arm/busdma_machdep.c
index b7aa14f..cf8dd51 100644
--- a/sys/arm/arm/busdma_machdep.c
+++ b/sys/arm/arm/busdma_machdep.c
@@ -126,7 +126,7 @@ static int total_bpages;
 static int busdma_zonecount;
 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
 
-SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 	   "Total bounce pages");
 
diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c
index 3ff111c..1f31e3d 100644
--- a/sys/arm/arm/vm_machdep.c
+++ b/sys/arm/arm/vm_machdep.c
@@ -526,7 +526,7 @@ extern uma_zone_t l2zone;
 
 struct mtx smallalloc_mtx;
 
-MALLOC_DEFINE(M_VMSMALLALLOC, "vm_small_alloc", "VM Small alloc data");
+static MALLOC_DEFINE(M_VMSMALLALLOC, "vm_small_alloc", "VM Small alloc data");
 
 vm_offset_t alloc_firstaddr;
 
diff --git a/sys/arm/at91/at91.c b/sys/arm/at91/at91.c
index e979dcb..3d55fd3 100644
--- a/sys/arm/at91/at91.c
+++ b/sys/arm/at91/at91.c
@@ -367,11 +367,15 @@ at91_setup_intr(device_t dev, device_t child,
     driver_intr_t *intr, void *arg, void **cookiep)    
 {
 	struct at91_softc *sc = device_get_softc(dev);
+	int error;
 
 	if (rman_get_start(ires) == sc->sc_irq_system && filt == NULL)
 		panic("All system interrupt ISRs must be FILTER");
-	BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags, filt, 
-	    intr, arg, cookiep);
+	error = BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags,
+	    filt, intr, arg, cookiep);
+	if (error)
+		return (error);
+
 	bus_space_write_4(sc->sc_st, sc->sc_aic_sh, IC_IECR,
 	    1 << rman_get_start(ires));
 	return (0);
diff --git a/sys/arm/conf/DB-88F6XXX b/sys/arm/conf/DB-88F6XXX
index fec1c4e..1f20484 100644
--- a/sys/arm/conf/DB-88F6XXX
+++ b/sys/arm/conf/DB-88F6XXX
@@ -66,6 +66,10 @@ device		mii
 device		e1000phy
 device		bpf
 
+device		cesa			# Marvell security engine
+device		crypto
+device		cryptodev
+
 # USB
 options 	USB_DEBUG	# enable debug msgs
 device		usb
diff --git a/sys/arm/conf/DEFAULTS b/sys/arm/conf/DEFAULTS
index 591a0a1..3546d4e 100644
--- a/sys/arm/conf/DEFAULTS
+++ b/sys/arm/conf/DEFAULTS
@@ -9,3 +9,5 @@ device		mem
 
 options 	GEOM_PART_BSD
 options 	GEOM_PART_MBR
+
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/arm/conf/GUMSTIX b/sys/arm/conf/GUMSTIX
index 8017a7a..dbd658e 100644
--- a/sys/arm/conf/GUMSTIX
+++ b/sys/arm/conf/GUMSTIX
@@ -70,6 +70,7 @@ options 	PREEMPTION
 device		loop
 device		ether
 device		mii
+device		mii_bitbang
 device		smc
 device		smcphy
 device		uart
diff --git a/sys/arm/conf/SHEEVAPLUG b/sys/arm/conf/SHEEVAPLUG
index 5392c95..1962bfc 100644
--- a/sys/arm/conf/SHEEVAPLUG
+++ b/sys/arm/conf/SHEEVAPLUG
@@ -60,6 +60,10 @@ options		HZ=1000
 options		DEVICE_POLLING
 device		vlan
 
+device		cesa			# Marvell security engine
+device		crypto
+device		cryptodev
+
 # USB
 options 	USB_DEBUG	# enable debug msgs
 device		usb
diff --git a/sys/arm/econa/econa.c b/sys/arm/econa/econa.c
index eb6d04e..f96dfc3 100644
--- a/sys/arm/econa/econa.c
+++ b/sys/arm/econa/econa.c
@@ -592,12 +592,15 @@ econa_setup_intr(device_t dev, device_t child,
     struct resource *ires, int flags, driver_filter_t *filt,
     driver_intr_t *intr, void *arg, void **cookiep)
 {
+	int error;
 
 	if (rman_get_start(ires) == ECONA_IRQ_SYSTEM && filt == NULL)
 		panic("All system interrupt ISRs must be FILTER");
 
-	BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags, filt,
-	    intr, arg, cookiep);
+	error = BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags,
+	    filt, intr, arg, cookiep);
+	if (error)
+		return (error);
 
 	arm_unmask_irq(rman_get_start(ires));
 
diff --git a/sys/arm/mv/files.mv b/sys/arm/mv/files.mv
index ca54a22..7c1ef49 100644
--- a/sys/arm/mv/files.mv
+++ b/sys/arm/mv/files.mv
@@ -28,6 +28,7 @@ arm/mv/mv_sata.c		optional	ata | atamvsata
 arm/mv/timer.c			standard
 arm/mv/twsi.c			optional	iicbus
 
+dev/cesa/cesa.c			optional	cesa
 dev/mge/if_mge.c		optional	mge
 dev/mvs/mvs_soc.c		optional	mvs
 dev/uart/uart_dev_ns8250.c	optional	uart
diff --git a/sys/arm/xscale/i80321/iq31244_7seg.c b/sys/arm/xscale/i80321/iq31244_7seg.c
index 7cd1bc3..3ac090d 100644
--- a/sys/arm/xscale/i80321/iq31244_7seg.c
+++ b/sys/arm/xscale/i80321/iq31244_7seg.c
@@ -334,7 +334,7 @@ static const uint8_t snakemap[][2] = {
 	{ ~SEG_B,	~SEG_F },
 };
 
-SYSCTL_NODE(_hw, OID_AUTO, sevenseg, CTLFLAG_RD, 0, "7 seg");
+static SYSCTL_NODE(_hw, OID_AUTO, sevenseg, CTLFLAG_RD, 0, "7 seg");
 static int freq = 20;
 SYSCTL_INT(_hw_sevenseg, OID_AUTO, freq, CTLFLAG_RW, &freq, 0, 
     "7 Seg update frequency");
diff --git a/sys/arm/xscale/i80321/iq80321.c b/sys/arm/xscale/i80321/iq80321.c
index 04ef8b9..7a875bf 100644
--- a/sys/arm/xscale/i80321/iq80321.c
+++ b/sys/arm/xscale/i80321/iq80321.c
@@ -354,8 +354,12 @@ iq80321_setup_intr(device_t dev, device_t child,
     struct resource *ires, int flags, driver_filter_t *filt, 
     driver_intr_t *intr, void *arg, void **cookiep)
 {
-	BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags, filt, intr, 
-	    arg, cookiep);
+	int error;
+
+	error = BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags,
+	    filt, intr, arg, cookiep);
+	if (error)
+		return (error);
 	intr_enabled |= 1 << rman_get_start(ires);
 	i80321_set_intrmask();
 	
diff --git a/sys/arm/xscale/i8134x/i81342.c b/sys/arm/xscale/i8134x/i81342.c
index 2d8783e..44ec3fd 100644
--- a/sys/arm/xscale/i8134x/i81342.c
+++ b/sys/arm/xscale/i8134x/i81342.c
@@ -429,10 +429,12 @@ i81342_setup_intr(device_t dev, device_t child, struct resource *ires,
     int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, 
     void **cookiep)
 {
-	
+	int error;
 
-	BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags, filt, intr,
-	    arg, cookiep);
+	error = BUS_SETUP_INTR(device_get_parent(dev), child, ires, flags,
+	    filt, intr, arg, cookiep);
+	if (error)
+		return (error);
 	arm_unmask_irq(rman_get_start(ires));
 	return (0);
 }
diff --git a/sys/arm/xscale/i8134x/iq81342_7seg.c b/sys/arm/xscale/i8134x/iq81342_7seg.c
index eeecd41..cc0d871 100644
--- a/sys/arm/xscale/i8134x/iq81342_7seg.c
+++ b/sys/arm/xscale/i8134x/iq81342_7seg.c
@@ -335,7 +335,7 @@ static const uint8_t snakemap[][2] = {
 	{ SEG_B,	SEG_F },
 };
 
-SYSCTL_NODE(_hw, OID_AUTO, sevenseg, CTLFLAG_RD, 0, "7 seg");
+static SYSCTL_NODE(_hw, OID_AUTO, sevenseg, CTLFLAG_RD, 0, "7 seg");
 static int freq = 20;
 SYSCTL_INT(_hw_sevenseg, OID_AUTO, freq, CTLFLAG_RW, &freq, 0, 
     "7 Seg update frequency");
diff --git a/sys/arm/xscale/ixp425/avila_ata.c b/sys/arm/xscale/ixp425/avila_ata.c
index 03a534f..7194d03 100644
--- a/sys/arm/xscale/ixp425/avila_ata.c
+++ b/sys/arm/xscale/ixp425/avila_ata.c
@@ -259,17 +259,11 @@ static int
 ata_avila_detach(device_t dev)
 {
 	struct ata_avila_softc *sc = device_get_softc(dev);
-	device_t *children;
-	int nc;
 
 	/* XXX quiesce gpio? */
 
 	/* detach & delete all children */
-	if (device_get_children(dev, &children, &nc) == 0) {
-		if (nc > 0)
-			device_delete_child(dev, children[0]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
 
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_rid, sc->sc_irq);
diff --git a/sys/arm/xscale/ixp425/if_npe.c b/sys/arm/xscale/ixp425/if_npe.c
index d4a1f10..9ac78a2 100644
--- a/sys/arm/xscale/ixp425/if_npe.c
+++ b/sys/arm/xscale/ixp425/if_npe.c
@@ -250,7 +250,8 @@ static int	npe_setloopback(struct npe_softc *, int ena);
 /* NB: all tx done processing goes through one queue */
 static int tx_doneqid = -1;
 
-SYSCTL_NODE(_hw, OID_AUTO, npe, CTLFLAG_RD, 0, "IXP4XX NPE driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, npe, CTLFLAG_RD, 0,
+    "IXP4XX NPE driver parameters");
 
 static int npe_debug = 0;
 SYSCTL_INT(_hw_npe, OID_AUTO, debug, CTLFLAG_RW, &npe_debug,
diff --git a/sys/arm/xscale/ixp425/ixp425.c b/sys/arm/xscale/ixp425/ixp425.c
index 78d2042..1b56642 100644
--- a/sys/arm/xscale/ixp425/ixp425.c
+++ b/sys/arm/xscale/ixp425/ixp425.c
@@ -639,9 +639,12 @@ ixp425_setup_intr(device_t dev, device_t child,
     driver_intr_t *intr, void *arg, void **cookiep)    
 {
 	uint32_t mask, mask2;
+	int error;
 
-	BUS_SETUP_INTR(device_get_parent(dev), child, res, flags, filt, intr,
-	     arg, cookiep);
+	error = BUS_SETUP_INTR(device_get_parent(dev), child, res, flags,
+	    filt, intr, arg, cookiep);
+	if (error)
+		return (error);
 
 	get_masks(res, &mask, &mask2);
 	update_masks(intr_enabled | mask, intr_enabled2 | mask2);
diff --git a/sys/arm/xscale/pxa/pxa_obio.c b/sys/arm/xscale/pxa/pxa_obio.c
index e2f8166..0f7e4a6 100644
--- a/sys/arm/xscale/pxa/pxa_obio.c
+++ b/sys/arm/xscale/pxa/pxa_obio.c
@@ -173,11 +173,14 @@ pxa_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
     driver_filter_t *filter, driver_intr_t *ithread, void *arg, void **cookiep)
 {
 	struct	obio_softc *sc;
+	int error;
 
 	sc = (struct obio_softc *)device_get_softc(dev);
 
-	BUS_SETUP_INTR(device_get_parent(dev), child, irq, flags, filter,
-	    ithread, arg, cookiep);
+	error = BUS_SETUP_INTR(device_get_parent(dev), child, irq, flags,
+	    filter, ithread, arg, cookiep);
+	if (error)
+		return (error);
 	arm_unmask_irq(rman_get_start(irq));
 	return (0);
 }
diff --git a/sys/arm/xscale/pxa/pxa_smi.c b/sys/arm/xscale/pxa/pxa_smi.c
index b2b904c..f25dac7 100644
--- a/sys/arm/xscale/pxa/pxa_smi.c
+++ b/sys/arm/xscale/pxa/pxa_smi.c
@@ -42,7 +42,8 @@ __FBSDID("$FreeBSD$");
 #include <arm/xscale/pxa/pxavar.h>
 #include <arm/xscale/pxa/pxareg.h>
 
-MALLOC_DEFINE(M_PXASMI, "PXA SMI", "Data for static memory interface devices.");
+static MALLOC_DEFINE(M_PXASMI, "PXA SMI",
+    "Data for static memory interface devices.");
 
 struct pxa_smi_softc {
 	struct resource	*ps_res[1];
diff --git a/sys/arm/xscale/pxa/pxa_space.c b/sys/arm/xscale/pxa/pxa_space.c
index 342e23a..f728e1d 100644
--- a/sys/arm/xscale/pxa/pxa_space.c
+++ b/sys/arm/xscale/pxa/pxa_space.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <arm/xscale/pxa/pxareg.h>
 #include <arm/xscale/pxa/pxavar.h>
 
-MALLOC_DEFINE(M_PXATAG, "PXA bus_space tags", "Bus_space tags for PXA");
+static MALLOC_DEFINE(M_PXATAG, "PXA bus_space tags", "Bus_space tags for PXA");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
diff --git a/sys/boot/common/loader.8 b/sys/boot/common/loader.8
index 51c0ab9..c36072c 100644
--- a/sys/boot/common/loader.8
+++ b/sys/boot/common/loader.8
@@ -443,7 +443,7 @@ Sets the list of binaries which the kernel will try to run as the initial
 process.
 The first matching binary is used.
 The default list is
-.Dq Li /sbin/init:/sbin/oinit:/sbin/init.bak:\:/rescue/init:/stand/sysinstall .
+.Dq Li /sbin/init:/sbin/oinit:/sbin/init.bak:\:/rescue/init .
 .It Va init_script
 If set to a valid file name in the root file system,
 instructs
diff --git a/sys/boot/fdt/dts/db88f6281.dts b/sys/boot/fdt/dts/db88f6281.dts
index 3b87593..fd4cdad 100644
--- a/sys/boot/fdt/dts/db88f6281.dts
+++ b/sys/boot/fdt/dts/db88f6281.dts
@@ -239,6 +239,8 @@
 			reg = <0x30000 0x10000>;
 			interrupts = <22>;
 			interrupt-parent = <&PIC>;
+
+			sram-handle = <&SRAM>;
 		};
 
 		usb@50000 {
diff --git a/sys/boot/fdt/dts/p1020rdb.dts b/sys/boot/fdt/dts/p1020rdb.dts
new file mode 100644
index 0000000..ec5118c
--- /dev/null
+++ b/sys/boot/fdt/dts/p1020rdb.dts
@@ -0,0 +1,627 @@
+/*
+ * P1020 RDB Device Tree Source
+ *
+ * Copyright 2009 Freescale Semiconductor Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/* $FreeBSD$ */
+
+/dts-v1/;
+
+/ {
+	model = "fsl,P1020";
+	compatible = "fsl,P1020RDB";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		serial0 = &serial0;
+		serial1 = &serial1;
+		ethernet0 = &enet0;
+		ethernet1 = &enet1;
+		ethernet2 = &enet2;
+		pci0 = &pci0;
+		pci1 = &pci1;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		PowerPC,P1020@0 {
+			device_type = "cpu";
+			reg = <0x0>;
+			next-level-cache = <&L2>;
+		};
+
+		PowerPC,P1020@1 {
+			device_type = "cpu";
+			reg = <0x1>;
+			next-level-cache = <&L2>;
+		};
+	};
+
+	memory {
+		device_type = "memory";
+	};
+
+	localbus@ffe05000 {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		compatible = "fsl,p1020-elbc", "fsl,elbc", "simple-bus";
+		reg = <0 0xffe05000 0 0x1000>;
+		interrupts = <19 2>;
+		interrupt-parent = <&mpic>;
+
+		/* NOR, NAND Flashes and Vitesse 5 port L2 switch */
+		ranges = <0x0 0x0 0x0 0xef000000 0x01000000
+			  0x1 0x0 0x0 0xffa00000 0x00040000
+			  0x2 0x0 0x0 0xffb00000 0x00020000>;
+
+		nor@0,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "cfi-flash";
+			reg = <0x0 0x0 0x1000000>;
+			bank-width = <2>;
+			device-width = <1>;
+
+			partition@0 {
+				/* This location must not be altered  */
+				/* 256KB for Vitesse 7385 Switch firmware */
+				reg = <0x0 0x00040000>;
+				label = "NOR (RO) Vitesse-7385 Firmware";
+				read-only;
+			};
+
+			partition@40000 {
+				/* 256KB for DTB Image */
+				reg = <0x00040000 0x00040000>;
+				label = "NOR (RO) DTB Image";
+				read-only;
+			};
+
+			partition@80000 {
+				/* 3.5 MB for Linux Kernel Image */
+				reg = <0x00080000 0x00380000>;
+				label = "NOR (RO) Linux Kernel Image";
+				read-only;
+			};
+
+			partition@400000 {
+				/* 11MB for JFFS2 based Root file System */
+				reg = <0x00400000 0x00b00000>;
+				label = "NOR (RW) JFFS2 Root File System";
+			};
+
+			partition@f00000 {
+				/* This location must not be altered  */
+				/* 512KB for u-boot Bootloader Image */
+				/* 512KB for u-boot Environment Variables */
+				reg = <0x00f00000 0x00100000>;
+				label = "NOR (RO) U-Boot Image";
+				read-only;
+			};
+		};
+
+		nand@1,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,p1020-fcm-nand",
+				     "fsl,elbc-fcm-nand";
+			reg = <0x1 0x0 0x40000>;
+
+			partition@0 {
+				/* This location must not be altered  */
+				/* 1MB for u-boot Bootloader Image */
+				reg = <0x0 0x00100000>;
+				label = "NAND (RO) U-Boot Image";
+				read-only;
+			};
+
+			partition@100000 {
+				/* 1MB for DTB Image */
+				reg = <0x00100000 0x00100000>;
+				label = "NAND (RO) DTB Image";
+				read-only;
+			};
+
+			partition@200000 {
+				/* 4MB for Linux Kernel Image */
+				reg = <0x00200000 0x00400000>;
+				label = "NAND (RO) Linux Kernel Image";
+				read-only;
+			};
+
+			partition@600000 {
+				/* 4MB for Compressed Root file System Image */
+				reg = <0x00600000 0x00400000>;
+				label = "NAND (RO) Compressed RFS Image";
+				read-only;
+			};
+
+			partition@a00000 {
+				/* 7MB for JFFS2 based Root file System */
+				reg = <0x00a00000 0x00700000>;
+				label = "NAND (RW) JFFS2 Root File System";
+			};
+
+			partition@1100000 {
+				/* 15MB for JFFS2 based Root file System */
+				reg = <0x01100000 0x00f00000>;
+				label = "NAND (RW) Writable User area";
+			};
+		};
+
+		L2switch@2,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "vitesse-7385";
+			reg = <0x2 0x0 0x20000>;
+		};
+
+	};
+
+	soc@ffe00000 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		device_type = "soc";
+		compatible = "fsl,p1020-immr", "simple-bus";
+		ranges = <0x0  0x0 0xffe00000 0x100000>;
+		bus-frequency = <0>;		// Filled out by uboot.
+
+		ecm-law@0 {
+			compatible = "fsl,ecm-law";
+			reg = <0x0 0x1000>;
+			fsl,num-laws = <12>;
+		};
+
+		ecm@1000 {
+			compatible = "fsl,p1020-ecm", "fsl,ecm";
+			reg = <0x1000 0x1000>;
+			interrupts = <16 2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		memory-controller@2000 {
+			compatible = "fsl,p1020-memory-controller";
+			reg = <0x2000 0x1000>;
+			interrupt-parent = <&mpic>;
+			interrupts = <16 2>;
+		};
+
+		i2c@3000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <0>;
+			compatible = "fsl-i2c";
+			reg = <0x3000 0x100>;
+			interrupts = <43 2>;
+			interrupt-parent = <&mpic>;
+			dfsrr;
+			rtc@68 {
+				compatible = "dallas,ds1339";
+				reg = <0x68>;
+			};
+		};
+
+		i2c@3100 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <1>;
+			compatible = "fsl-i2c";
+			reg = <0x3100 0x100>;
+			interrupts = <43 2>;
+			interrupt-parent = <&mpic>;
+			dfsrr;
+		};
+
+		serial0: serial@4500 {
+			cell-index = <0>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x4500 0x100>;
+			clock-frequency = <0>;
+			interrupts = <42 2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		serial1: serial@4600 {
+			cell-index = <1>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x4600 0x100>;
+			clock-frequency = <0>;
+			interrupts = <42 2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		spi@7000 {
+			cell-index = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl,espi";
+			reg = <0x7000 0x1000>;
+			interrupts = <59 0x2>;
+			interrupt-parent = <&mpic>;
+			mode = "cpu";
+
+			fsl_m25p80@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "fsl,espi-flash";
+				reg = <0>;
+				linux,modalias = "fsl_m25p80";
+				modal = "s25sl128b";
+				spi-max-frequency = <50000000>;
+				mode = <0>;
+
+				partition@0 {
+					/* 512KB for u-boot Bootloader Image */
+					reg = <0x0 0x00080000>;
+					label = "SPI (RO) U-Boot Image";
+					read-only;
+				};
+
+				partition@80000 {
+					/* 512KB for DTB Image */
+					reg = <0x00080000 0x00080000>;
+					label = "SPI (RO) DTB Image";
+					read-only;
+				};
+
+				partition@100000 {
+					/* 4MB for Linux Kernel Image */
+					reg = <0x00100000 0x00400000>;
+					label = "SPI (RO) Linux Kernel Image";
+					read-only;
+				};
+
+				partition@500000 {
+					/* 4MB for Compressed RFS Image */
+					reg = <0x00500000 0x00400000>;
+					label = "SPI (RO) Compressed RFS Image";
+					read-only;
+				};
+
+				partition@900000 {
+					/* 7MB for JFFS2 based RFS */
+					reg = <0x00900000 0x00700000>;
+					label = "SPI (RW) JFFS2 RFS";
+				};
+			};
+		};
+
+		gpio: gpio-controller@f000 {
+			#gpio-cells = <2>;
+			compatible = "fsl,mpc8572-gpio";
+			reg = <0xf000 0x100>;
+			interrupts = <47 0x2>;
+			interrupt-parent = <&mpic>;
+			gpio-controller;
+		};
+
+		L2: l2-cache-controller@20000 {
+			compatible = "fsl,p1020-l2-cache-controller";
+			reg = <0x20000 0x1000>;
+			cache-line-size = <32>;	// 32 bytes
+			cache-size = <0x40000>; // L2,256K
+			interrupt-parent = <&mpic>;
+			interrupts = <16 2>;
+		};
+
+		dma@21300 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,eloplus-dma";
+			reg = <0x21300 0x4>;
+			ranges = <0x0 0x21100 0x200>;
+			cell-index = <0>;
+			dma-channel@0 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x0 0x80>;
+				cell-index = <0>;
+				interrupt-parent = <&mpic>;
+				interrupts = <20 2>;
+			};
+			dma-channel@80 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x80 0x80>;
+				cell-index = <1>;
+				interrupt-parent = <&mpic>;
+				interrupts = <21 2>;
+			};
+			dma-channel@100 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x100 0x80>;
+				cell-index = <2>;
+				interrupt-parent = <&mpic>;
+				interrupts = <22 2>;
+			};
+			dma-channel@180 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x180 0x80>;
+				cell-index = <3>;
+				interrupt-parent = <&mpic>;
+				interrupts = <23 2>;
+			};
+		};
+
+		mdio@24000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl,etsec2-mdio";
+			reg = <0x24000 0x1000 0xb0030 0x4>;
+
+			phy0: ethernet-phy@0 {
+				interrupt-parent = <&mpic>;
+				interrupts = <3 1>;
+				reg = <0x0>;
+			};
+
+			phy1: ethernet-phy@1 {
+				interrupt-parent = <&mpic>;
+				interrupts = <2 1>;
+				reg = <0x1>;
+			};
+		};
+
+		mdio@25000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl,etsec2-tbi";
+			reg = <0x25000 0x1000 0xb1030 0x4>;
+
+			tbi0: tbi-phy@11 {
+				reg = <0x11>;
+				device_type = "tbi-phy";
+			};
+		};
+
+		enet0: ethernet@b0000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			device_type = "network";
+			model = "eTSEC";
+			compatible = "fsl,etsec2";
+			fsl,num_rx_queues = <0x8>;
+			fsl,num_tx_queues = <0x8>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupt-parent = <&mpic>;
+			fixed-link = <1 1 1000 0 0>;
+			phy-connection-type = "rgmii-id";
+
+			queue-group@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0xb0000 0x1000>;
+				interrupts = <29 2 30 2 34 2>;
+			};
+
+			queue-group@1 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0xb4000 0x1000>;
+				interrupts = <17 2 18 2 24 2>;
+			};
+		};
+
+		enet1: ethernet@b1000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			device_type = "network";
+			model = "eTSEC";
+			compatible = "fsl,etsec2";
+			fsl,num_rx_queues = <0x8>;
+			fsl,num_tx_queues = <0x8>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupt-parent = <&mpic>;
+			phy-handle = <&phy0>;
+			tbi-handle = <&tbi0>;
+			phy-connection-type = "sgmii";
+
+			queue-group@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0xb1000 0x1000>;
+				interrupts = <35 2 36 2 40 2>;
+			};
+
+			queue-group@1 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0xb5000 0x1000>;
+				interrupts = <51 2 52 2 67 2>;
+			};
+		};
+
+		enet2: ethernet@b2000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			device_type = "network";
+			model = "eTSEC";
+			compatible = "fsl,etsec2";
+			fsl,num_rx_queues = <0x8>;
+			fsl,num_tx_queues = <0x8>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupt-parent = <&mpic>;
+			phy-handle = <&phy1>;
+			phy-connection-type = "rgmii-id";
+
+			queue-group@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0xb2000 0x1000>;
+				interrupts = <31 2 32 2 33 2>;
+			};
+
+			queue-group@1 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0xb6000 0x1000>;
+				interrupts = <25 2 26 2 27 2>;
+			};
+		};
+
+		usb@22000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl-usb2-dr";
+			reg = <0x22000 0x1000>;
+			interrupt-parent = <&mpic>;
+			interrupts = <28 0x2>;
+			phy_type = "ulpi";
+		};
+
+		/* USB2 is shared with localbus, so it must be disabled
+		   by default. We can't put 'status = "disabled";' here
+		   since U-Boot doesn't clear the status property when
+		   it enables USB2. OTOH, U-Boot does create a new node
+		   when there isn't any. So, just comment it out.
+		usb@23000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl-usb2-dr";
+			reg = <0x23000 0x1000>;
+			interrupt-parent = <&mpic>;
+			interrupts = <46 0x2>;
+			phy_type = "ulpi";
+		};
+		*/
+
+		sdhci@2e000 {
+			compatible = "fsl,p1020-esdhc", "fsl,esdhc";
+			reg = <0x2e000 0x1000>;
+			interrupts = <72 0x2>;
+			interrupt-parent = <&mpic>;
+			/* Filled in by U-Boot */
+			clock-frequency = <0>;
+		};
+
+		crypto@30000 {
+			compatible = "fsl,sec3.1", "fsl,sec3.0", "fsl,sec2.4",
+				     "fsl,sec2.2", "fsl,sec2.1", "fsl,sec2.0";
+			reg = <0x30000 0x10000>;
+			interrupts = <45 2 58 2>;
+			interrupt-parent = <&mpic>;
+			fsl,num-channels = <4>;
+			fsl,channel-fifo-len = <24>;
+			fsl,exec-units-mask = <0xbfe>;
+			fsl,descriptor-types-mask = <0x3ab0ebf>;
+		};
+
+		mpic: pic@40000 {
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <2>;
+			reg = <0x40000 0x40000>;
+			compatible = "chrp,open-pic";
+			device_type = "open-pic";
+		};
+
+		msi@41600 {
+			compatible = "fsl,p1020-msi", "fsl,mpic-msi";
+			reg = <0x41600 0x80>;
+			msi-available-ranges = <0 0x100>;
+			interrupts = <
+				0xe0 0
+				0xe1 0
+				0xe2 0
+				0xe3 0
+				0xe4 0
+				0xe5 0
+				0xe6 0
+				0xe7 0>;
+			interrupt-parent = <&mpic>;
+		};
+
+		global-utilities@e0000 {	//global utilities block
+			compatible = "fsl,p1020-guts";
+			reg = <0xe0000 0x1000>;
+			fsl,has-rstcr;
+		};
+	};
+
+	pci0: pcie@ffe09000 {
+		compatible = "fsl,mpc8548-pcie";
+		device_type = "pci";
+		#interrupt-cells = <1>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0 0xffe09000 0 0x1000>;
+		bus-range = <0 255>;
+		ranges = <0x2000000 0x0 0xa0000000 0 0xa0000000 0x0 0x20000000
+			  0x1000000 0x0 0x00000000 0 0xffc30000 0x0 0x10000>;
+		clock-frequency = <33333333>;
+		interrupt-parent = <&mpic>;
+		interrupts = <16 2>;
+		pcie@0 {
+			reg = <0x0 0x0 0x0 0x0 0x0>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			ranges = <0x2000000 0x0 0xa0000000
+				  0x2000000 0x0 0xa0000000
+				  0x0 0x20000000
+
+				  0x1000000 0x0 0x0
+				  0x1000000 0x0 0x0
+				  0x0 0x100000>;
+		};
+	};
+
+	pci1: pcie@ffe0a000 {
+		compatible = "fsl,mpc8548-pcie";
+		device_type = "pci";
+		#interrupt-cells = <1>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0 0xffe0a000 0 0x1000>;
+		bus-range = <0 255>;
+		ranges = <0x2000000 0x0 0xc0000000 0 0xc0000000 0x0 0x20000000
+			  0x1000000 0x0 0x00000000 0 0xffc20000 0x0 0x10000>;
+		clock-frequency = <33333333>;
+		interrupt-parent = <&mpic>;
+		interrupts = <16 2>;
+		pcie@0 {
+			reg = <0x0 0x0 0x0 0x0 0x0>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			ranges = <0x2000000 0x0 0xc0000000
+				  0x2000000 0x0 0xc0000000
+				  0x0 0x20000000
+
+				  0x1000000 0x0 0x0
+				  0x1000000 0x0 0x0
+				  0x0 0x100000>;
+		};
+	};
+};
diff --git a/sys/boot/fdt/dts/p2020ds.dts b/sys/boot/fdt/dts/p2020ds.dts
new file mode 100644
index 0000000..7823429
--- /dev/null
+++ b/sys/boot/fdt/dts/p2020ds.dts
@@ -0,0 +1,754 @@
+/*
+ * P2020 DS Device Tree Source
+ *
+ * Copyright 2009 Freescale Semiconductor Inc.
+ *
+ *	Neither the name of Freescale Semiconductor, Inc nor the names of
+ *	its contributors may be used to endorse or promote products derived
+ *	from this software without specific prior written permission.
+ *
+ * Freescale hereby publishes it under the following licenses:
+ *
+ *   BSD License
+ *
+ *	Redistribution and use in source and binary forms, with or
+ *	without modification, are permitted provided that the following
+ *	conditions are met:
+ *
+ *	Redistributions of source code must retain the above copyright
+ *	notice, this list of conditions and the following disclaimer.
+ *
+ *	Redistributions in binary form must reproduce the above copyright
+ *	notice, this list of conditions and the following disclaimer in
+ *	the documentation and/or other materials provided with the
+ *	distribution.
+ *
+ *	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ *	CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ *	INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ *	MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ *	DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ *	BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *	EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ *	TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ *	ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ *	OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ *	OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *	POSSIBILITY OF SUCH DAMAGE.
+ *
+ *   GNU General Public License, version 2
+ *
+ *	This program is free software; you can redistribute it and/or
+ *	modify it under the terms of the GNU General Public License
+ *	as published by the Free Software Foundation; either version 2
+ *	of the License, or (at your option) any later version.
+ *
+ *	This program is distributed in the hope that it will be useful,
+ *	but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *	GNU General Public License for more details.
+ *
+ *	You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ *	MA  02110-1301, USA.
+ *
+ * You may select the license of your choice.
+ *------------------------------------------------------------------
+ */
+/* $FreeBSD$ */
+
+/dts-v1/;
+/ {
+	model = "fsl,P2020";
+	compatible = "fsl,P2020DS";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		ethernet0 = &enet0;
+		ethernet1 = &enet1;
+		ethernet2 = &enet2;
+		serial0 = &serial0;
+		serial1 = &serial1;
+		pci0 = &pci0;
+		pci1 = &pci1;
+		pci2 = &pci2;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		PowerPC,P2020@0 {
+			device_type = "cpu";
+			reg = <0x0>;
+			next-level-cache = <&L2>;
+		};
+
+		PowerPC,P2020@1 {
+			device_type = "cpu";
+			reg = <0x1>;
+			next-level-cache = <&L2>;
+		};
+	};
+
+	memory {
+		device_type = "memory";
+	};
+
+	localbus@ffe05000 {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		compatible = "fsl,elbc", "simple-bus";
+		reg = <0 0xffe05000 0 0x1000>;
+		interrupts = <19 2>;
+		interrupt-parent = <&mpic>;
+
+		ranges = <0x0 0x0 0x0 0xe8000000 0x08000000
+			  0x1 0x0 0x0 0xe0000000 0x08000000
+			  0x2 0x0 0x0 0xffa00000 0x00040000
+			  0x3 0x0 0x0 0xffdf0000 0x00008000
+			  0x4 0x0 0x0 0xffa40000 0x00040000
+			  0x5 0x0 0x0 0xffa80000 0x00040000
+			  0x6 0x0 0x0 0xffac0000 0x00040000>;
+
+		nor@0,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "cfi-flash";
+			reg = <0x0 0x0 0x8000000>;
+			bank-width = <2>;
+			device-width = <1>;
+
+			ramdisk@0 {
+				reg = <0x0 0x03000000>;
+				read-only;
+			};
+
+			diagnostic@3000000 {
+				reg = <0x03000000 0x00e00000>;
+				read-only;
+			};
+
+			dink@3e00000 {
+				reg = <0x03e00000 0x00200000>;
+				read-only;
+			};
+
+			kernel@4000000 {
+				reg = <0x04000000 0x00400000>;
+				read-only;
+			};
+
+			jffs2@4400000 {
+				reg = <0x04400000 0x03b00000>;
+			};
+
+			dtb@7f00000 {
+				reg = <0x07f00000 0x00080000>;
+				read-only;
+			};
+
+			u-boot@7f80000 {
+				reg = <0x07f80000 0x00080000>;
+				read-only;
+			};
+		};
+
+		nand@2,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,elbc-fcm-nand";
+			reg = <0x2 0x0 0x40000>;
+
+			u-boot@0 {
+				reg = <0x0 0x02000000>;
+				read-only;
+			};
+
+			jffs2@2000000 {
+				reg = <0x02000000 0x10000000>;
+			};
+
+			ramdisk@12000000 {
+				reg = <0x12000000 0x08000000>;
+				read-only;
+			};
+
+			kernel@1a000000 {
+				reg = <0x1a000000 0x04000000>;
+			};
+
+			dtb@1e000000 {
+				reg = <0x1e000000 0x01000000>;
+				read-only;
+			};
+
+			empty@1f000000 {
+				reg = <0x1f000000 0x21000000>;
+			};
+		};
+
+		nand@4,0 {
+			compatible = "fsl,elbc-fcm-nand";
+			reg = <0x4 0x0 0x40000>;
+		};
+
+		nand@5,0 {
+			compatible = "fsl,elbc-fcm-nand";
+			reg = <0x5 0x0 0x40000>;
+		};
+
+		nand@6,0 {
+			compatible = "fsl,elbc-fcm-nand";
+			reg = <0x6 0x0 0x40000>;
+		};
+	};
+
+	soc@ffe00000 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		device_type = "soc";
+		compatible = "fsl,p2020-immr", "simple-bus";
+		ranges = <0x0 0 0xffe00000 0x100000>;
+		bus-frequency = <0>;		// Filled out by uboot.
+
+		ecm-law@0 {
+			compatible = "fsl,ecm-law";
+			reg = <0x0 0x1000>;
+			fsl,num-laws = <12>;
+		};
+
+		ecm@1000 {
+			compatible = "fsl,p2020-ecm", "fsl,ecm";
+			reg = <0x1000 0x1000>;
+			interrupts = <17 2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		memory-controller@2000 {
+			compatible = "fsl,p2020-memory-controller";
+			reg = <0x2000 0x1000>;
+			interrupt-parent = <&mpic>;
+			interrupts = <18 2>;
+		};
+
+		i2c@3000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <0>;
+			compatible = "fsl-i2c";
+			reg = <0x3000 0x100>;
+			interrupts = <43 2>;
+			interrupt-parent = <&mpic>;
+			dfsrr;
+		};
+
+		i2c@3100 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <1>;
+			compatible = "fsl-i2c";
+			reg = <0x3100 0x100>;
+			interrupts = <43 2>;
+			interrupt-parent = <&mpic>;
+			dfsrr;
+		};
+
+		serial0: serial@4500 {
+			cell-index = <0>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x4500 0x100>;
+			clock-frequency = <0>;
+			interrupts = <42 2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		serial1: serial@4600 {
+			cell-index = <1>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x4600 0x100>;
+			clock-frequency = <0>;
+			interrupts = <42 2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		spi@7000 {
+			compatible = "fsl,espi";
+			reg = <0x7000 0x1000>;
+			interrupts = <59 0x2>;
+			interrupt-parent = <&mpic>;
+		};
+
+		dma@c300 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,eloplus-dma";
+			reg = <0xc300 0x4>;
+			ranges = <0x0 0xc100 0x200>;
+			cell-index = <1>;
+			dma-channel@0 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x0 0x80>;
+				cell-index = <0>;
+				interrupt-parent = <&mpic>;
+				interrupts = <76 2>;
+			};
+			dma-channel@80 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x80 0x80>;
+				cell-index = <1>;
+				interrupt-parent = <&mpic>;
+				interrupts = <77 2>;
+			};
+			dma-channel@100 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x100 0x80>;
+				cell-index = <2>;
+				interrupt-parent = <&mpic>;
+				interrupts = <78 2>;
+			};
+			dma-channel@180 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x180 0x80>;
+				cell-index = <3>;
+				interrupt-parent = <&mpic>;
+				interrupts = <79 2>;
+			};
+		};
+
+		gpio: gpio-controller@f000 {
+			#gpio-cells = <2>;
+			compatible = "fsl,mpc8572-gpio";
+			reg = <0xf000 0x100>;
+			interrupts = <47 0x2>;
+			interrupt-parent = <&mpic>;
+			gpio-controller;
+		};
+
+		L2: l2-cache-controller@20000 {
+			compatible = "fsl,p2020-l2-cache-controller";
+			reg = <0x20000 0x1000>;
+			cache-line-size = <32>;	// 32 bytes
+			cache-size = <0x80000>; // L2, 512k
+			interrupt-parent = <&mpic>;
+			interrupts = <16 2>;
+		};
+
+		dma@21300 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,eloplus-dma";
+			reg = <0x21300 0x4>;
+			ranges = <0x0 0x21100 0x200>;
+			cell-index = <0>;
+			dma-channel@0 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x0 0x80>;
+				cell-index = <0>;
+				interrupt-parent = <&mpic>;
+				interrupts = <20 2>;
+			};
+			dma-channel@80 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x80 0x80>;
+				cell-index = <1>;
+				interrupt-parent = <&mpic>;
+				interrupts = <21 2>;
+			};
+			dma-channel@100 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x100 0x80>;
+				cell-index = <2>;
+				interrupt-parent = <&mpic>;
+				interrupts = <22 2>;
+			};
+			dma-channel@180 {
+				compatible = "fsl,eloplus-dma-channel";
+				reg = <0x180 0x80>;
+				cell-index = <3>;
+				interrupt-parent = <&mpic>;
+				interrupts = <23 2>;
+			};
+		};
+
+		usb@22000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl-usb2-dr";
+			reg = <0x22000 0x1000>;
+			interrupt-parent = <&mpic>;
+			interrupts = <28 0x2>;
+			phy_type = "ulpi";
+		};
+
+		enet0: ethernet@24000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			cell-index = <0>;
+			device_type = "network";
+			model = "eTSEC";
+			compatible = "gianfar";
+			reg = <0x24000 0x1000>;
+			ranges = <0x0 0x24000 0x1000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <29 2 30 2 34 2>;
+			interrupt-parent = <&mpic>;
+			tbi-handle = <&tbi0>;
+			phy-handle = <&phy0>;
+			phy-connection-type = "rgmii-id";
+
+			mdio@520 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "fsl,gianfar-mdio";
+				reg = <0x520 0x20>;
+
+				phy0: ethernet-phy@0 {
+					interrupt-parent = <&mpic>;
+					interrupts = <3 1>;
+					reg = <0x0>;
+				};
+				phy1: ethernet-phy@1 {
+					interrupt-parent = <&mpic>;
+					interrupts = <3 1>;
+					reg = <0x1>;
+				};
+				phy2: ethernet-phy@2 {
+					interrupt-parent = <&mpic>;
+					interrupts = <3 1>;
+					reg = <0x2>;
+				};
+				tbi0: tbi-phy@11 {
+					reg = <0x11>;
+					device_type = "tbi-phy";
+				};
+			};
+		};
+
+		enet1: ethernet@25000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			cell-index = <1>;
+			device_type = "network";
+			model = "eTSEC";
+			compatible = "gianfar";
+			reg = <0x25000 0x1000>;
+			ranges = <0x0 0x25000 0x1000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <35 2 36 2 40 2>;
+			interrupt-parent = <&mpic>;
+			tbi-handle = <&tbi1>;
+			phy-handle = <&phy1>;
+			phy-connection-type = "rgmii-id";
+
+			mdio@520 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "fsl,gianfar-tbi";
+				reg = <0x520 0x20>;
+
+				tbi1: tbi-phy@11 {
+					reg = <0x11>;
+					device_type = "tbi-phy";
+				};
+			};
+		};
+
+		enet2: ethernet@26000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			cell-index = <2>;
+			device_type = "network";
+			model = "eTSEC";
+			compatible = "gianfar";
+			reg = <0x26000 0x1000>;
+			ranges = <0x0 0x26000 0x1000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <31 2 32 2 33 2>;
+			interrupt-parent = <&mpic>;
+			tbi-handle = <&tbi2>;
+			phy-handle = <&phy2>;
+			phy-connection-type = "rgmii-id";
+
+			mdio@520 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "fsl,gianfar-tbi";
+				reg = <0x520 0x20>;
+
+				tbi2: tbi-phy@11 {
+					reg = <0x11>;
+					device_type = "tbi-phy";
+				};
+			};
+		};
+
+		sdhci@2e000 {
+			compatible = "fsl,p2020-esdhc", "fsl,esdhc";
+			reg = <0x2e000 0x1000>;
+			interrupts = <72 0x2>;
+			interrupt-parent = <&mpic>;
+			/* Filled in by U-Boot */
+			clock-frequency = <0>;
+		};
+
+		crypto@30000 {
+			compatible = "fsl,sec3.1", "fsl,sec3.0", "fsl,sec2.4",
+				     "fsl,sec2.2", "fsl,sec2.1", "fsl,sec2.0";
+			reg = <0x30000 0x10000>;
+			interrupts = <45 2 58 2>;
+			interrupt-parent = <&mpic>;
+			fsl,num-channels = <4>;
+			fsl,channel-fifo-len = <24>;
+			fsl,exec-units-mask = <0xbfe>;
+			fsl,descriptor-types-mask = <0x3ab0ebf>;
+		};
+
+		mpic: pic@40000 {
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <2>;
+			reg = <0x40000 0x40000>;
+			compatible = "chrp,open-pic";
+			device_type = "open-pic";
+		};
+
+		msi@41600 {
+			compatible = "fsl,mpic-msi";
+			reg = <0x41600 0x80>;
+			msi-available-ranges = <0 0x100>;
+			interrupts = <
+				0xe0 0
+				0xe1 0
+				0xe2 0
+				0xe3 0
+				0xe4 0
+				0xe5 0
+				0xe6 0
+				0xe7 0>;
+			interrupt-parent = <&mpic>;
+		};
+
+		global-utilities@e0000 {	//global utilities block
+			compatible = "fsl,p2020-guts";
+			reg = <0xe0000 0x1000>;
+			fsl,has-rstcr;
+		};
+	};
+
+	pci0: pcie@ffe08000 {
+		compatible = "fsl,mpc8548-pcie";
+		device_type = "pci";
+		#interrupt-cells = <1>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0 0xffe08000 0 0x1000>;
+		bus-range = <0 255>;
+		ranges = <0x2000000 0x0 0x80000000 0 0x80000000 0x0 0x20000000
+			  0x1000000 0x0 0x00000000 0 0xffc00000 0x0 0x10000>;
+		clock-frequency = <33333333>;
+		interrupt-parent = <&mpic>;
+		interrupts = <24 2>;
+		interrupt-map-mask = <0xf800 0x0 0x0 0x7>;
+		interrupt-map = <
+			/* IDSEL 0x0 */
+			0000 0x0 0x0 0x1 &mpic 0x8 0x1
+			0000 0x0 0x0 0x2 &mpic 0x9 0x1
+			0000 0x0 0x0 0x3 &mpic 0xa 0x1
+			0000 0x0 0x0 0x4 &mpic 0xb 0x1
+			>;
+		pcie@0 {
+			reg = <0x0 0x0 0x0 0x0 0x0>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			ranges = <0x2000000 0x0 0x80000000
+				  0x2000000 0x0 0x80000000
+				  0x0 0x20000000
+
+				  0x1000000 0x0 0x0
+				  0x1000000 0x0 0x0
+				  0x0 0x10000>;
+		};
+	};
+
+	pci1: pcie@ffe09000 {
+		compatible = "fsl,mpc8548-pcie";
+		device_type = "pci";
+		#interrupt-cells = <1>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0 0xffe09000 0 0x1000>;
+		bus-range = <0 255>;
+		ranges = <0x2000000 0x0 0xa0000000 0 0xa0000000 0x0 0x20000000
+			  0x1000000 0x0 0x00000000 0 0xffc10000 0x0 0x10000>;
+		clock-frequency = <33333333>;
+		interrupt-parent = <&mpic>;
+		interrupts = <25 2>;
+		interrupt-map-mask = <0xff00 0x0 0x0 0x7>;
+		interrupt-map = <
+
+			// IDSEL 0x11 func 0 - PCI slot 1
+			0x8800 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8800 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 1 - PCI slot 1
+			0x8900 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8900 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 2 - PCI slot 1
+			0x8a00 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8a00 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 3 - PCI slot 1
+			0x8b00 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8b00 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 4 - PCI slot 1
+			0x8c00 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8c00 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 5 - PCI slot 1
+			0x8d00 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8d00 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 6 - PCI slot 1
+			0x8e00 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8e00 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x11 func 7 - PCI slot 1
+			0x8f00 0x0 0x0 0x1 &i8259 0x9 0x2
+			0x8f00 0x0 0x0 0x2 &i8259 0xa 0x2
+
+			// IDSEL 0x1d  Audio
+			0xe800 0x0 0x0 0x1 &i8259 0x6 0x2
+
+			// IDSEL 0x1e Legacy
+			0xf000 0x0 0x0 0x1 &i8259 0x7 0x2
+			0xf100 0x0 0x0 0x1 &i8259 0x7 0x2
+
+			// IDSEL 0x1f IDE/SATA
+			0xf800 0x0 0x0 0x1 &i8259 0xe 0x2
+			0xf900 0x0 0x0 0x1 &i8259 0x5 0x2
+			>;
+
+		pcie@0 {
+			reg = <0x0 0x0 0x0 0x0 0x0>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			ranges = <0x2000000 0x0 0xa0000000
+				  0x2000000 0x0 0xa0000000
+				  0x0 0x20000000
+
+				  0x1000000 0x0 0x0
+				  0x1000000 0x0 0x0
+				  0x0 0x10000>;
+			uli1575@0 {
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				#size-cells = <2>;
+				#address-cells = <3>;
+				ranges = <0x2000000 0x0 0xa0000000
+					  0x2000000 0x0 0xa0000000
+					  0x0 0x20000000
+
+					  0x1000000 0x0 0x0
+					  0x1000000 0x0 0x0
+					  0x0 0x10000>;
+				isa@1e {
+					device_type = "isa";
+					#interrupt-cells = <2>;
+					#size-cells = <1>;
+					#address-cells = <2>;
+					reg = <0xf000 0x0 0x0 0x0 0x0>;
+					ranges = <0x1 0x0 0x1000000 0x0 0x0
+						  0x1000>;
+					interrupt-parent = <&i8259>;
+
+					i8259: interrupt-controller@20 {
+						reg = <0x1 0x20 0x2
+						       0x1 0xa0 0x2
+						       0x1 0x4d0 0x2>;
+						interrupt-controller;
+						device_type = "interrupt-controller";
+						#address-cells = <0>;
+						#interrupt-cells = <2>;
+						compatible = "chrp,iic";
+						interrupts = <4 1>;
+						interrupt-parent = <&mpic>;
+					};
+
+					i8042@60 {
+						#size-cells = <0>;
+						#address-cells = <1>;
+						reg = <0x1 0x60 0x1 0x1 0x64 0x1>;
+						interrupts = <1 3 12 3>;
+						interrupt-parent =
+							<&i8259>;
+
+						keyboard@0 {
+							reg = <0x0>;
+							compatible = "pnpPNP,303";
+						};
+
+						mouse@1 {
+							reg = <0x1>;
+							compatible = "pnpPNP,f03";
+						};
+					};
+
+					rtc@70 {
+						compatible = "pnpPNP,b00";
+						reg = <0x1 0x70 0x2>;
+					};
+
+					gpio@400 {
+						reg = <0x1 0x400 0x80>;
+					};
+				};
+			};
+		};
+
+	};
+
+	pci2: pcie@ffe0a000 {
+		compatible = "fsl,mpc8548-pcie";
+		device_type = "pci";
+		#interrupt-cells = <1>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0 0xffe0a000 0 0x1000>;
+		bus-range = <0 255>;
+		ranges = <0x2000000 0x0 0xc0000000 0 0xc0000000 0x0 0x20000000
+			  0x1000000 0x0 0x00000000 0 0xffc20000 0x0 0x10000>;
+		clock-frequency = <33333333>;
+		interrupt-parent = <&mpic>;
+		interrupts = <26 2>;
+		interrupt-map-mask = <0xf800 0x0 0x0 0x7>;
+		interrupt-map = <
+			/* IDSEL 0x0 */
+			0000 0x0 0x0 0x1 &mpic 0x0 0x1
+			0000 0x0 0x0 0x2 &mpic 0x1 0x1
+			0000 0x0 0x0 0x3 &mpic 0x2 0x1
+			0000 0x0 0x0 0x4 &mpic 0x3 0x1
+			>;
+		pcie@0 {
+			reg = <0x0 0x0 0x0 0x0 0x0>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			ranges = <0x2000000 0x0 0xc0000000
+				  0x2000000 0x0 0xc0000000
+				  0x0 0x20000000
+
+				  0x1000000 0x0 0x0
+				  0x1000000 0x0 0x0
+				  0x0 0x10000>;
+		};
+	};
+};
diff --git a/sys/boot/fdt/dts/p3041ds.dts b/sys/boot/fdt/dts/p3041ds.dts
new file mode 100644
index 0000000..01875d9
--- /dev/null
+++ b/sys/boot/fdt/dts/p3041ds.dts
@@ -0,0 +1,753 @@
+/*
+ * P3041DS Device Tree Source
+ *
+ * Copyright 2010-2011 Freescale Semiconductor Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/* $FreeBSD$ */
+
+/dts-v1/;
+
+/ {
+	model = "fsl,P3041DS";
+	compatible = "fsl,P3041DS";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	interrupt-parent = <&mpic>;
+
+	aliases {
+		ccsr = &soc;
+
+		serial0 = &serial0;
+		serial1 = &serial1;
+		serial2 = &serial2;
+		serial3 = &serial3;
+		pci0 = &pci0;
+		pci1 = &pci1;
+		pci2 = &pci2;
+		pci3 = &pci3;
+		usb0 = &usb0;
+		usb1 = &usb1;
+		dma0 = &dma0;
+		dma1 = &dma1;
+		sdhc = &sdhc;
+		msi0 = &msi0;
+		msi1 = &msi1;
+		msi2 = &msi2;
+
+		crypto = &crypto;
+		sec_jr0 = &sec_jr0;
+		sec_jr1 = &sec_jr1;
+		sec_jr2 = &sec_jr2;
+		sec_jr3 = &sec_jr3;
+		rtic_a = &rtic_a;
+		rtic_b = &rtic_b;
+		rtic_c = &rtic_c;
+		rtic_d = &rtic_d;
+		sec_mon = &sec_mon;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: PowerPC,e500mc@0 {
+			device_type = "cpu";
+			reg = <0>;
+			next-level-cache = <&L2_0>;
+			L2_0: l2-cache {
+				next-level-cache = <&cpc>;
+			};
+		};
+		cpu1: PowerPC,e500mc@1 {
+			device_type = "cpu";
+			reg = <1>;
+			next-level-cache = <&L2_1>;
+			L2_1: l2-cache {
+				next-level-cache = <&cpc>;
+			};
+		};
+		cpu2: PowerPC,e500mc@2 {
+			device_type = "cpu";
+			reg = <2>;
+			next-level-cache = <&L2_2>;
+			L2_2: l2-cache {
+				next-level-cache = <&cpc>;
+			};
+		};
+		cpu3: PowerPC,e500mc@3 {
+			device_type = "cpu";
+			reg = <3>;
+			next-level-cache = <&L2_3>;
+			L2_3: l2-cache {
+				next-level-cache = <&cpc>;
+			};
+		};
+	};
+
+	memory {
+		device_type = "memory";
+	};
+
+	soc: soc@ffe000000 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		device_type = "soc";
+		compatible = "simple-bus";
+		ranges = <0x00000000 0xf 0xfe000000 0x1000000>;
+		reg = <0xf 0xfe000000 0 0x00001000>;
+
+		soc-sram-error {
+			compatible = "fsl,soc-sram-error";
+			interrupts = <16 2 1 29>;
+		};
+
+		corenet-law@0 {
+			compatible = "fsl,corenet-law";
+			reg = <0x0 0x1000>;
+			fsl,num-laws = <32>;
+		};
+
+		memory-controller@8000 {
+			compatible = "fsl,qoriq-memory-controller-v4.5", "fsl,qoriq-memory-controller";
+			reg = <0x8000 0x1000>;
+			interrupts = <16 2 1 23>;
+		};
+
+		cpc: l3-cache-controller@10000 {
+			compatible = "fsl,p3041-l3-cache-controller", "fsl,p4080-l3-cache-controller", "cache";
+			reg = <0x10000 0x1000>;
+			interrupts = <16 2 1 27>;
+		};
+
+		corenet-cf@18000 {
+			compatible = "fsl,corenet-cf";
+			reg = <0x18000 0x1000>;
+			interrupts = <16 2 1 31>;
+			fsl,ccf-num-csdids = <32>;
+			fsl,ccf-num-snoopids = <32>;
+		};
+
+		iommu@20000 {
+			compatible = "fsl,pamu-v1.0", "fsl,pamu";
+			reg = <0x20000 0x4000>;
+			interrupts = <
+				24 2 0 0
+				16 2 1 30>;
+		};
+
+		mpic: pic@40000 {
+			clock-frequency = <0>;
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <4>;
+			reg = <0x40000 0x40000>;
+			compatible = "fsl,mpic", "chrp,open-pic";
+			device_type = "open-pic";
+		};
+
+		msi0: msi@41600 {
+			compatible = "fsl,mpic-msi";
+			reg = <0x41600 0x200>;
+			msi-available-ranges = <0 0x100>;
+			interrupts = <
+				0xe0 0 0 0
+				0xe1 0 0 0
+				0xe2 0 0 0
+				0xe3 0 0 0
+				0xe4 0 0 0
+				0xe5 0 0 0
+				0xe6 0 0 0
+				0xe7 0 0 0>;
+		};
+
+		msi1: msi@41800 {
+			compatible = "fsl,mpic-msi";
+			reg = <0x41800 0x200>;
+			msi-available-ranges = <0 0x100>;
+			interrupts = <
+				0xe8 0 0 0
+				0xe9 0 0 0
+				0xea 0 0 0
+				0xeb 0 0 0
+				0xec 0 0 0
+				0xed 0 0 0
+				0xee 0 0 0
+				0xef 0 0 0>;
+		};
+
+		msi2: msi@41a00 {
+			compatible = "fsl,mpic-msi";
+			reg = <0x41a00 0x200>;
+			msi-available-ranges = <0 0x100>;
+			interrupts = <
+				0xf0 0 0 0
+				0xf1 0 0 0
+				0xf2 0 0 0
+				0xf3 0 0 0
+				0xf4 0 0 0
+				0xf5 0 0 0
+				0xf6 0 0 0
+				0xf7 0 0 0>;
+		};
+
+		guts: global-utilities@e0000 {
+			compatible = "fsl,qoriq-device-config-1.0";
+			reg = <0xe0000 0xe00>;
+			fsl,has-rstcr;
+			#sleep-cells = <1>;
+			fsl,liodn-bits = <12>;
+		};
+
+		pins: global-utilities@e0e00 {
+			compatible = "fsl,qoriq-pin-control-1.0";
+			reg = <0xe0e00 0x200>;
+			#sleep-cells = <2>;
+		};
+
+		clockgen: global-utilities@e1000 {
+			compatible = "fsl,p3041-clockgen", "fsl,qoriq-clockgen-1.0";
+			reg = <0xe1000 0x1000>;
+			clock-frequency = <0>;
+		};
+
+		rcpm: global-utilities@e2000 {
+			compatible = "fsl,qoriq-rcpm-1.0";
+			reg = <0xe2000 0x1000>;
+			#sleep-cells = <1>;
+		};
+
+		sfp: sfp@e8000 {
+			compatible = "fsl,p3041-sfp", "fsl,qoriq-sfp-1.0";
+			reg	   = <0xe8000 0x1000>;
+		};
+
+		serdes: serdes@ea000 {
+			compatible = "fsl,p3041-serdes";
+			reg	   = <0xea000 0x1000>;
+		};
+
+		dma0: dma@100300 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,p3041-dma", "fsl,eloplus-dma";
+			reg = <0x100300 0x4>;
+			ranges = <0x0 0x100100 0x200>;
+			cell-index = <0>;
+			dma-channel@0 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x0 0x80>;
+				cell-index = <0>;
+				interrupts = <28 2 0 0>;
+			};
+			dma-channel@80 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x80 0x80>;
+				cell-index = <1>;
+				interrupts = <29 2 0 0>;
+			};
+			dma-channel@100 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x100 0x80>;
+				cell-index = <2>;
+				interrupts = <30 2 0 0>;
+			};
+			dma-channel@180 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x180 0x80>;
+				cell-index = <3>;
+				interrupts = <31 2 0 0>;
+			};
+		};
+
+		dma1: dma@101300 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "fsl,p3041-dma", "fsl,eloplus-dma";
+			reg = <0x101300 0x4>;
+			ranges = <0x0 0x101100 0x200>;
+			cell-index = <1>;
+			dma-channel@0 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x0 0x80>;
+				cell-index = <0>;
+				interrupts = <32 2 0 0>;
+			};
+			dma-channel@80 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x80 0x80>;
+				cell-index = <1>;
+				interrupts = <33 2 0 0>;
+			};
+			dma-channel@100 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x100 0x80>;
+				cell-index = <2>;
+				interrupts = <34 2 0 0>;
+			};
+			dma-channel@180 {
+				compatible = "fsl,p3041-dma-channel",
+						"fsl,eloplus-dma-channel";
+				reg = <0x180 0x80>;
+				cell-index = <3>;
+				interrupts = <35 2 0 0>;
+			};
+		};
+
+		spi@110000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "fsl,p3041-espi", "fsl,mpc8536-espi";
+			reg = <0x110000 0x1000>;
+			interrupts = <53 0x2 0 0>;
+			fsl,espi-num-chipselects = <4>;
+
+			flash@0 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "spansion,s25sl12801";
+				reg = <0>;
+				spi-max-frequency = <40000000>; /* input clock */
+				partition@u-boot {
+					label = "u-boot";
+					reg = <0x00000000 0x00100000>;
+					read-only;
+				};
+				partition@kernel {
+					label = "kernel";
+					reg = <0x00100000 0x00500000>;
+					read-only;
+				};
+				partition@dtb {
+					label = "dtb";
+					reg = <0x00600000 0x00100000>;
+					read-only;
+				};
+				partition@fs {
+					label = "file system";
+					reg = <0x00700000 0x00900000>;
+				};
+			};
+		};
+
+		sdhc: sdhc@114000 {
+			compatible = "fsl,p3041-esdhc", "fsl,esdhc";
+			reg = <0x114000 0x1000>;
+			interrupts = <48 2 0 0>;
+			sdhci,auto-cmd12;
+			clock-frequency = <0>;
+		};
+
+		i2c@118000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <0>;
+			compatible = "fsl-i2c";
+			reg = <0x118000 0x100>;
+			interrupts = <38 2 0 0>;
+			dfsrr;
+		};
+
+		i2c@118100 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <1>;
+			compatible = "fsl-i2c";
+			reg = <0x118100 0x100>;
+			interrupts = <38 2 0 0>;
+			dfsrr;
+			eeprom@51 {
+				compatible = "at24,24c256";
+				reg = <0x51>;
+			};
+			eeprom@52 {
+				compatible = "at24,24c256";
+				reg = <0x52>;
+			};
+		};
+
+		i2c@119000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <2>;
+			compatible = "fsl-i2c";
+			reg = <0x119000 0x100>;
+			interrupts = <39 2 0 0>;
+			dfsrr;
+		};
+
+		i2c@119100 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			cell-index = <3>;
+			compatible = "fsl-i2c";
+			reg = <0x119100 0x100>;
+			interrupts = <39 2 0 0>;
+			dfsrr;
+			rtc@68 {
+				compatible = "dallas,ds3232";
+				reg = <0x68>;
+				interrupts = <0x1 0x1 0 0>;
+			};
+		};
+
+		serial0: serial@11c500 {
+			cell-index = <0>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x11c500 0x100>;
+			clock-frequency = <0>;
+			interrupts = <36 2 0 0>;
+		};
+
+		serial1: serial@11c600 {
+			cell-index = <1>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x11c600 0x100>;
+			clock-frequency = <0>;
+			interrupts = <36 2 0 0>;
+		};
+
+		serial2: serial@11d500 {
+			cell-index = <2>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x11d500 0x100>;
+			clock-frequency = <0>;
+			interrupts = <37 2 0 0>;
+		};
+
+		serial3: serial@11d600 {
+			cell-index = <3>;
+			device_type = "serial";
+			compatible = "ns16550";
+			reg = <0x11d600 0x100>;
+			clock-frequency = <0>;
+			interrupts = <37 2 0 0>;
+		};
+
+		gpio0: gpio@130000 {
+			compatible = "fsl,p3041-gpio", "fsl,qoriq-gpio";
+			reg = <0x130000 0x1000>;
+			interrupts = <55 2 0 0>;
+			#gpio-cells = <2>;
+			gpio-controller;
+		};
+
+		usb0: usb@210000 {
+			compatible = "fsl,p3041-usb2-mph",
+					"fsl,mpc85xx-usb2-mph", "fsl-usb2-mph";
+			reg = <0x210000 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <44 0x2 0 0>;
+			phy_type = "utmi";
+			port0;
+		};
+
+		usb1: usb@211000 {
+			compatible = "fsl,p3041-usb2-dr",
+					"fsl,mpc85xx-usb2-dr", "fsl-usb2-dr";
+			reg = <0x211000 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <45 0x2 0 0>;
+			dr_mode = "host";
+			phy_type = "utmi";
+		};
+
+		sata@220000 {
+			compatible = "fsl,p3041-sata", "fsl,pq-sata-v2";
+			reg = <0x220000 0x1000>;
+			interrupts = <68 0x2 0 0>;
+		};
+
+		sata@221000 {
+			compatible = "fsl,p3041-sata", "fsl,pq-sata-v2";
+			reg = <0x221000 0x1000>;
+			interrupts = <69 0x2 0 0>;
+		};
+
+		crypto: crypto@300000 {
+			compatible = "fsl,sec-v4.2", "fsl,sec-v4.0";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0x300000 0x10000>;
+			ranges = <0 0x300000 0x10000>;
+			interrupts = <92 2 0 0>;
+
+			sec_jr0: jr@1000 {
+				compatible = "fsl,sec-v4.2-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg = <0x1000 0x1000>;
+				interrupts = <88 2 0 0>;
+			};
+
+			sec_jr1: jr@2000 {
+				compatible = "fsl,sec-v4.2-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg = <0x2000 0x1000>;
+				interrupts = <89 2 0 0>;
+			};
+
+			sec_jr2: jr@3000 {
+				compatible = "fsl,sec-v4.2-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg = <0x3000 0x1000>;
+				interrupts = <90 2 0 0>;
+			};
+
+			sec_jr3: jr@4000 {
+				compatible = "fsl,sec-v4.2-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg = <0x4000 0x1000>;
+				interrupts = <91 2 0 0>;
+			};
+
+			rtic@6000 {
+				compatible = "fsl,sec-v4.2-rtic",
+					     "fsl,sec-v4.0-rtic";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0x6000 0x100>;
+				ranges = <0x0 0x6100 0xe00>;
+
+				rtic_a: rtic-a@0 {
+					compatible = "fsl,sec-v4.2-rtic-memory",
+						     "fsl,sec-v4.0-rtic-memory";
+					reg = <0x00 0x20 0x100 0x80>;
+				};
+
+				rtic_b: rtic-b@20 {
+					compatible = "fsl,sec-v4.2-rtic-memory",
+						     "fsl,sec-v4.0-rtic-memory";
+					reg = <0x20 0x20 0x200 0x80>;
+				};
+
+				rtic_c: rtic-c@40 {
+					compatible = "fsl,sec-v4.2-rtic-memory",
+						     "fsl,sec-v4.0-rtic-memory";
+					reg = <0x40 0x20 0x300 0x80>;
+				};
+
+				rtic_d: rtic-d@60 {
+					compatible = "fsl,sec-v4.2-rtic-memory",
+						     "fsl,sec-v4.0-rtic-memory";
+					reg = <0x60 0x20 0x500 0x80>;
+				};
+			};
+		};
+
+		sec_mon: sec_mon@314000 {
+			compatible = "fsl,sec-v4.2-mon", "fsl,sec-v4.0-mon";
+			reg = <0x314000 0x1000>;
+			interrupts = <93 2 0 0>;
+		};
+	};
+
+	localbus@ffe124000 {
+		compatible = "fsl,p3041-elbc", "fsl,elbc", "simple-bus";
+		reg = <0xf 0xfe124000 0 0x1000>;
+		interrupts = <25 2 0 0>;
+		#address-cells = <2>;
+		#size-cells = <1>;
+
+		ranges = <0 0 0xf 0xe8000000 0x08000000
+			  3 0 0xf 0xffdf0000 0x00008000>;
+
+		flash@0,0 {
+			compatible = "cfi-flash";
+			reg = <0 0 0x08000000>;
+			bank-width = <2>;
+			device-width = <2>;
+		};
+
+		board-control@3,0 {
+			compatible = "fsl,p3041ds-pixis";
+			reg = <3 0 0x20>;
+		};
+	};
+
+	pci0: pcie@ffe200000 {
+		compatible = "fsl,p3041-pcie", "fsl,qoriq-pcie-v2.2";
+		device_type = "pci";
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0xf 0xfe200000 0 0x1000>;
+		bus-range = <0x0 0xff>;
+		ranges = <0x02000000 0 0xe0000000 0xc 0x00000000 0x0 0x20000000
+			  0x01000000 0 0x00000000 0xf 0xf8000000 0x0 0x00010000>;
+		clock-frequency = <0x1fca055>;
+		fsl,msi = <&msi0>;
+		interrupts = <16 2 1 15>;
+		pcie@0 {
+			reg = <0 0 0 0 0>;
+			#interrupt-cells = <1>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			interrupts = <16 2 1 15>;
+			interrupt-map-mask = <0xf800 0 0 7>;
+			interrupt-map = <
+				/* IDSEL 0x0 */
+				0000 0 0 1 &mpic 40 1 0 0
+				0000 0 0 2 &mpic 1 1 0 0
+				0000 0 0 3 &mpic 2 1 0 0
+				0000 0 0 4 &mpic 3 1 0 0
+				>;
+			ranges = <0x02000000 0 0xe0000000
+				  0x02000000 0 0xe0000000
+				  0 0x20000000
+
+				  0x01000000 0 0x00000000
+				  0x01000000 0 0x00000000
+				  0 0x00010000>;
+		};
+	};
+
+	pci1: pcie@ffe201000 {
+		compatible = "fsl,p3041-pcie", "fsl,qoriq-pcie-v2.2";
+		device_type = "pci";
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0xf 0xfe201000 0 0x1000>;
+		bus-range = <0 0xff>;
+		ranges = <0x02000000 0x0 0xe0000000 0xc 0x20000000 0x0 0x20000000
+			  0x01000000 0x0 0x00000000 0xf 0xf8010000 0x0 0x00010000>;
+		clock-frequency = <0x1fca055>;
+		fsl,msi = <&msi1>;
+		interrupts = <16 2 1 14>;
+		pcie@0 {
+			reg = <0 0 0 0 0>;
+			#interrupt-cells = <1>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			interrupts = <16 2 1 14>;
+			interrupt-map-mask = <0xf800 0 0 7>;
+			interrupt-map = <
+				/* IDSEL 0x0 */
+				0000 0 0 1 &mpic 41 1 0 0
+				0000 0 0 2 &mpic 5 1 0 0
+				0000 0 0 3 &mpic 6 1 0 0
+				0000 0 0 4 &mpic 7 1 0 0
+				>;
+			ranges = <0x02000000 0 0xe0000000
+				  0x02000000 0 0xe0000000
+				  0 0x20000000
+
+				  0x01000000 0 0x00000000
+				  0x01000000 0 0x00000000
+				  0 0x00010000>;
+		};
+	};
+
+	pci2: pcie@ffe202000 {
+		compatible = "fsl,p3041-pcie", "fsl,qoriq-pcie-v2.2";
+		device_type = "pci";
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0xf 0xfe202000 0 0x1000>;
+		bus-range = <0x0 0xff>;
+		ranges = <0x02000000 0 0xe0000000 0xc 0x40000000 0 0x20000000
+			  0x01000000 0 0x00000000 0xf 0xf8020000 0 0x00010000>;
+		clock-frequency = <0x1fca055>;
+		fsl,msi = <&msi2>;
+		interrupts = <16 2 1 13>;
+		pcie@0 {
+			reg = <0 0 0 0 0>;
+			#interrupt-cells = <1>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			interrupts = <16 2 1 13>;
+			interrupt-map-mask = <0xf800 0 0 7>;
+			interrupt-map = <
+				/* IDSEL 0x0 */
+				0000 0 0 1 &mpic 42 1 0 0
+				0000 0 0 2 &mpic 9 1 0 0
+				0000 0 0 3 &mpic 10 1 0 0
+				0000 0 0 4 &mpic 11 1 0 0
+				>;
+			ranges = <0x02000000 0 0xe0000000
+				  0x02000000 0 0xe0000000
+				  0 0x20000000
+
+				  0x01000000 0 0x00000000
+				  0x01000000 0 0x00000000
+				  0 0x00010000>;
+		};
+	};
+
+	pci3: pcie@ffe203000 {
+		compatible = "fsl,p3041-pcie", "fsl,qoriq-pcie-v2.2";
+		device_type = "pci";
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0xf 0xfe203000 0 0x1000>;
+		bus-range = <0x0 0xff>;
+		ranges = <0x02000000 0 0xe0000000 0xc 0x60000000 0 0x20000000
+			  0x01000000 0 0x00000000 0xf 0xf8030000 0 0x00010000>;
+		clock-frequency = <0x1fca055>;
+		fsl,msi = <&msi2>;
+		interrupts = <16 2 1 12>;
+		pcie@0 {
+			reg = <0 0 0 0 0>;
+			#interrupt-cells = <1>;
+			#size-cells = <2>;
+			#address-cells = <3>;
+			device_type = "pci";
+			interrupts = <16 2 1 12>;
+			interrupt-map-mask = <0xf800 0 0 7>;
+			interrupt-map = <
+				/* IDSEL 0x0 */
+				0000 0 0 1 &mpic 43 1 0 0
+				0000 0 0 2 &mpic 0 1 0 0
+				0000 0 0 3 &mpic 4 1 0 0
+				0000 0 0 4 &mpic 8 1 0 0
+				>;
+			ranges = <0x02000000 0 0xe0000000
+				  0x02000000 0 0xe0000000
+				  0 0x20000000
+
+				  0x01000000 0 0x00000000
+				  0x01000000 0 0x00000000
+				  0 0x00010000>;
+		};
+	};
+};
diff --git a/sys/boot/fdt/dts/sheevaplug.dts b/sys/boot/fdt/dts/sheevaplug.dts
index 251c8df..cbfde62 100644
--- a/sys/boot/fdt/dts/sheevaplug.dts
+++ b/sys/boot/fdt/dts/sheevaplug.dts
@@ -236,6 +236,8 @@
 			reg = <0x30000 0x10000>;
 			interrupts = <22>;
 			interrupt-parent = <&PIC>;
+
+			sram-handle = <&SRAM>;
 		};
 
 		usb@50000 {
diff --git a/sys/boot/forth/loader.conf b/sys/boot/forth/loader.conf
index c8abe58..7a20f91 100644
--- a/sys/boot/forth/loader.conf
+++ b/sys/boot/forth/loader.conf
@@ -47,7 +47,7 @@ bitmap_type="splash_image_data" # and place it on the module_path
 				# escape to the loader prompt, set to
 				# "NO" to disable autobooting
 #beastie_disable="NO"		# Turn the beastie boot menu on and off
-#loader_logo="fbsdbw"		# Desired logo: fbsdbw, beastiebw, beastie, none
+#loader_logo="orbbw"		# Desired logo: orbbw, orb, fbsdbw, beastiebw, beastie, none
 #comconsole_speed="9600"	# Set the current serial console speed
 #console="vidconsole"		# A comma separated list of console(s)
 #currdev="disk1s1a"		# Set the current device
@@ -81,7 +81,7 @@ module_path="/boot/modules"	# Set the module search path
 #boot_serial=""		# -h: Use serial console
 #boot_single=""		# -s: Start system in single-user mode
 #boot_verbose=""	# -v: Causes extra debugging information to be printed
-#init_path="/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init:/stand/sysinstall"
+#init_path="/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init"
 			# Sets the list of init candidates
 #init_shell="/bin/sh"	# The shell binary used by init(8).
 #init_script=""		# Initial script to run by init(8) before chrooting.
diff --git a/sys/boot/i386/libi386/bioscd.c b/sys/boot/i386/libi386/bioscd.c
index dd50f7d..9babcdf 100644
--- a/sys/boot/i386/libi386/bioscd.c
+++ b/sys/boot/i386/libi386/bioscd.c
@@ -117,7 +117,6 @@ bc_bios2unit(int biosdev)
 	int i;
     
 	DEBUG("looking for bios device 0x%x", biosdev);
-	printf("looking for bios device 0x%x, nbcinfo=%d\n", biosdev, nbcinfo);
 	for (i = 0; i < nbcinfo; i++) {
 		DEBUG("bc unit %d is BIOS device 0x%x", i, bcinfo[i].bc_unit);
 		if (bcinfo[i].bc_unit == biosdev)
@@ -149,7 +148,6 @@ bc_init(void)
 int
 bc_add(int biosdev)
 {
-	printf("bc_add(%d)\n", biosdev);
 
 	if (nbcinfo >= MAXBCDEV)
 		return (-1);
@@ -161,10 +159,8 @@ bc_add(int biosdev)
 	v86.ds = VTOPSEG(&bcinfo[nbcinfo].bc_sp);
 	v86.esi = VTOPOFF(&bcinfo[nbcinfo].bc_sp);
 	v86int();
-	if ((v86.eax & 0xff00) != 0) {
-                printf("CD probe failed, eax=0x%08x\n", v86.eax);
+	if ((v86.eax & 0xff00) != 0)
 		return (-1);
-        }
 
 	printf("BIOS CD is cd%d\n", nbcinfo);
 	nbcinfo++;
diff --git a/sys/boot/i386/loader/Makefile b/sys/boot/i386/loader/Makefile
index f4babff..9128e62 100644
--- a/sys/boot/i386/loader/Makefile
+++ b/sys/boot/i386/loader/Makefile
@@ -56,7 +56,7 @@ CFLAGS+=	-DLOADER_GZIP_SUPPORT
 CFLAGS+=	-DLOADER_GPT_SUPPORT
 .endif
 
-# Always add MI sources 
+# Always add MI sources
 .PATH:		${.CURDIR}/../../common
 .include	"${.CURDIR}/../../common/Makefile.inc"
 CFLAGS+=	-I${.CURDIR}/../../common
@@ -101,7 +101,7 @@ FILES=	${LOADER}
 FILESMODE_${LOADER}= ${BINMODE} -b
 
 .if !defined(LOADER_ONLY)
-.PATH: ${.CURDIR}/../../forth 
+.PATH: ${.CURDIR}/../../forth
 FILES+=	loader.help loader.4th support.4th loader.conf
 FILES+= screen.4th frames.4th beastie.4th
 FILES+= brand.4th check-password.4th color.4th delay.4th
@@ -117,7 +117,7 @@ FILES+= menu.rc
 .endif
 
 # XXX crt0.o needs to be first for pxeboot(8) to work
-OBJS=	${BTXCRT} 
+OBJS=	${BTXCRT}
 
 DPADD=	${LIBFICL} ${LIBFIREWIRE} ${LIBZFS} ${LIBI386} ${LIBSTAND}
 LDADD=	${LIBFICL} ${LIBFIREWIRE} ${LIBZFS} ${LIBI386} ${LIBSTAND}
diff --git a/sys/boot/ia64/common/exec.c b/sys/boot/ia64/common/exec.c
index 65886fa..b721f97 100644
--- a/sys/boot/ia64/common/exec.c
+++ b/sys/boot/ia64/common/exec.c
@@ -187,7 +187,7 @@ mmu_setup_paged(struct bootinfo *bi)
 	pa = ia64_va2pa(ia64_text_start, &ia64_text_size);
 	ia64_text_size = sz;	/* XXX */
 	shft = sz2shft(ia64_text_start, ia64_text_size);
-	shft = mmu_wire(ia64_text_start, (uintptr_t)pa, shft, PTE_AR_RX);
+	shft = mmu_wire(ia64_text_start, (uintptr_t)pa, shft, PTE_AR_RWX);
 	ia64_copyin(&shft, (uintptr_t)&bi->bi_text_mapped, 4);
 
 	/* Wire as much of the data segment as well. */
diff --git a/sys/boot/ofw/Makefile.inc b/sys/boot/ofw/Makefile.inc
index dab65e5..e67c0dc 100644
--- a/sys/boot/ofw/Makefile.inc
+++ b/sys/boot/ofw/Makefile.inc
@@ -2,7 +2,7 @@
 
 .if ${MACHINE_ARCH} == "powerpc64"
 CFLAGS+=	-m32 -mcpu=powerpc
-LDFLAGS+=	-m elf32ppc
+LDFLAGS+=	-m elf32ppc_fbsd
 .endif
 
 .include "../Makefile.inc"
diff --git a/sys/boot/powerpc/Makefile.inc b/sys/boot/powerpc/Makefile.inc
index dab65e5..e67c0dc 100644
--- a/sys/boot/powerpc/Makefile.inc
+++ b/sys/boot/powerpc/Makefile.inc
@@ -2,7 +2,7 @@
 
 .if ${MACHINE_ARCH} == "powerpc64"
 CFLAGS+=	-m32 -mcpu=powerpc
-LDFLAGS+=	-m elf32ppc
+LDFLAGS+=	-m elf32ppc_fbsd
 .endif
 
 .include "../Makefile.inc"
diff --git a/sys/boot/uboot/Makefile.inc b/sys/boot/uboot/Makefile.inc
index dab65e5..e67c0dc 100644
--- a/sys/boot/uboot/Makefile.inc
+++ b/sys/boot/uboot/Makefile.inc
@@ -2,7 +2,7 @@
 
 .if ${MACHINE_ARCH} == "powerpc64"
 CFLAGS+=	-m32 -mcpu=powerpc
-LDFLAGS+=	-m elf32ppc
+LDFLAGS+=	-m elf32ppc_fbsd
 .endif
 
 .include "../Makefile.inc"
diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c
index b1d5ea8..f004ee82 100644
--- a/sys/cam/ata/ata_da.c
+++ b/sys/cam/ata/ata_da.c
@@ -327,7 +327,7 @@ static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 
-SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
             "CAM Direct Access Disk driver");
 SYSCTL_INT(_kern_cam_ada, OID_AUTO, legacy_aliases, CTLFLAG_RW,
            &ada_legacy_aliases, 0, "Create legacy-like device aliases");
@@ -378,7 +378,7 @@ static struct periph_driver adadriver =
 
 PERIPHDRIVER_DECLARE(ada, adadriver);
 
-MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
+static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 
 static int
 adaopen(struct disk *dp)
diff --git a/sys/cam/ata/ata_pmp.c b/sys/cam/ata/ata_pmp.c
index 91b9186..13fc760 100644
--- a/sys/cam/ata/ata_pmp.c
+++ b/sys/cam/ata/ata_pmp.c
@@ -129,7 +129,7 @@ static	void		pmpdone(struct cam_periph *periph,
 static int pmp_retry_count = PMP_DEFAULT_RETRY;
 static int pmp_default_timeout = PMP_DEFAULT_TIMEOUT;
 
-SYSCTL_NODE(_kern_cam, OID_AUTO, pmp, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_kern_cam, OID_AUTO, pmp, CTLFLAG_RD, 0,
             "CAM Direct Access Disk driver");
 SYSCTL_INT(_kern_cam_pmp, OID_AUTO, retry_count, CTLFLAG_RW,
            &pmp_retry_count, 0, "Normal I/O retry count");
@@ -147,7 +147,7 @@ static struct periph_driver pmpdriver =
 
 PERIPHDRIVER_DECLARE(pmp, pmpdriver);
 
-MALLOC_DEFINE(M_ATPMP, "ata_pmp", "ata_pmp buffers");
+static MALLOC_DEFINE(M_ATPMP, "ata_pmp", "ata_pmp buffers");
 
 static void
 pmpinit(void)
diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c
index 80dba39..3854b5c 100644
--- a/sys/cam/cam_periph.c
+++ b/sys/cam/cam_periph.c
@@ -87,7 +87,7 @@ static int nperiph_drivers;
 static int initialized = 0;
 struct periph_driver **periph_drivers;
 
-MALLOC_DEFINE(M_CAMPERIPH, "CAM periph", "CAM peripheral buffers");
+static MALLOC_DEFINE(M_CAMPERIPH, "CAM periph", "CAM peripheral buffers");
 
 static int periph_selto_delay = 1000;
 TUNABLE_INT("kern.cam.periph_selto_delay", &periph_selto_delay);
diff --git a/sys/cam/cam_queue.c b/sys/cam/cam_queue.c
index 328f0b8..d6d953d 100644
--- a/sys/cam/cam_queue.c
+++ b/sys/cam/cam_queue.c
@@ -40,9 +40,9 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_queue.h>
 #include <cam/cam_debug.h>
 
-MALLOC_DEFINE(M_CAMQ, "CAM queue", "CAM queue buffers");
-MALLOC_DEFINE(M_CAMDEVQ, "CAM dev queue", "CAM dev queue buffers");
-MALLOC_DEFINE(M_CAMCCBQ, "CAM ccb queue", "CAM ccb queue buffers");
+static MALLOC_DEFINE(M_CAMQ, "CAM queue", "CAM queue buffers");
+static MALLOC_DEFINE(M_CAMDEVQ, "CAM dev queue", "CAM dev queue buffers");
+static MALLOC_DEFINE(M_CAMCCBQ, "CAM ccb queue", "CAM ccb queue buffers");
 
 static __inline int
 		queue_cmp(cam_pinfo **queue_array, int i, int j);
diff --git a/sys/cam/cam_sim.c b/sys/cam/cam_sim.c
index 59148a9..d6ac740 100644
--- a/sys/cam/cam_sim.c
+++ b/sys/cam/cam_sim.c
@@ -43,7 +43,7 @@ __FBSDID("$FreeBSD$");
 
 #define CAM_PATH_ANY (u_int32_t)-1
 
-MALLOC_DEFINE(M_CAMSIM, "CAM SIM", "CAM SIM buffers");
+static MALLOC_DEFINE(M_CAMSIM, "CAM SIM", "CAM SIM buffers");
 
 struct cam_devq *
 cam_simq_alloc(u_int32_t max_sim_transactions)
diff --git a/sys/cam/cam_sim.h b/sys/cam/cam_sim.h
index 0b55f6e..398d540 100644
--- a/sys/cam/cam_sim.h
+++ b/sys/cam/cam_sim.h
@@ -104,7 +104,8 @@ struct cam_sim {
 	u_int32_t		flags;
 #define	CAM_SIM_REL_TIMEOUT_PENDING	0x01
 #define	CAM_SIM_MPSAFE			0x02
-#define CAM_SIM_ON_DONEQ		0x04
+#define	CAM_SIM_ON_DONEQ		0x04
+#define	CAM_SIM_POLLED			0x08
 	struct callout		callout;
 	struct cam_devq 	*devq;	/* Device Queue to use for this SIM */
 	int			refcount; /* References to the SIM. */
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index 10b89c7..04cc99a 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -2957,6 +2957,9 @@ xpt_polled_action(union ccb *start_ccb)
 
 	mtx_assert(sim->mtx, MA_OWNED);
 
+	/* Don't use ISR for this SIM while polling. */
+	sim->flags |= CAM_SIM_POLLED;
+
 	/*
 	 * Steal an opening so that no other queued requests
 	 * can get it before us while we simulate interrupts.
@@ -2996,6 +2999,9 @@ xpt_polled_action(union ccb *start_ccb)
 	} else {
 		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
 	}
+
+	/* We will use CAM ISR for this SIM again. */
+	sim->flags &= ~CAM_SIM_POLLED;
 }
 
 /*
@@ -4224,7 +4230,7 @@ xpt_done(union ccb *done_ccb)
 		TAILQ_INSERT_TAIL(&sim->sim_doneq, &done_ccb->ccb_h,
 		    sim_links.tqe);
 		done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
-		if ((sim->flags & CAM_SIM_ON_DONEQ) == 0) {
+		if ((sim->flags & (CAM_SIM_ON_DONEQ | CAM_SIM_POLLED)) == 0) {
 			mtx_lock(&cam_simq_lock);
 			first = TAILQ_EMPTY(&cam_simq);
 			TAILQ_INSERT_TAIL(&cam_simq, sim, links);
diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c
index 53d352e..77dbcef 100644
--- a/sys/cam/scsi/scsi_cd.c
+++ b/sys/cam/scsi/scsi_cd.c
@@ -306,8 +306,9 @@ static int cd_retry_count = CD_DEFAULT_RETRY;
 static int changer_min_busy_seconds = CHANGER_MIN_BUSY_SECONDS;
 static int changer_max_busy_seconds = CHANGER_MAX_BUSY_SECONDS;
 
-SYSCTL_NODE(_kern_cam, OID_AUTO, cd, CTLFLAG_RD, 0, "CAM CDROM driver");
-SYSCTL_NODE(_kern_cam_cd, OID_AUTO, changer, CTLFLAG_RD, 0, "CD Changer");
+static SYSCTL_NODE(_kern_cam, OID_AUTO, cd, CTLFLAG_RD, 0, "CAM CDROM driver");
+static SYSCTL_NODE(_kern_cam_cd, OID_AUTO, changer, CTLFLAG_RD, 0,
+    "CD Changer");
 SYSCTL_INT(_kern_cam_cd, OID_AUTO, retry_count, CTLFLAG_RW,
            &cd_retry_count, 0, "Normal I/O retry count");
 TUNABLE_INT("kern.cam.cd.retry_count", &cd_retry_count);
@@ -336,7 +337,7 @@ static struct mtx changerq_mtx;
 static STAILQ_HEAD(changerlist, cdchanger) changerq;
 static int num_changers;
 
-MALLOC_DEFINE(M_SCSICD, "scsi_cd", "scsi_cd buffers");
+static MALLOC_DEFINE(M_SCSICD, "scsi_cd", "scsi_cd buffers");
 
 static void
 cdinit(void)
diff --git a/sys/cam/scsi/scsi_ch.c b/sys/cam/scsi/scsi_ch.c
index 4d33cd9..8b04b4f 100644
--- a/sys/cam/scsi/scsi_ch.c
+++ b/sys/cam/scsi/scsi_ch.c
@@ -218,7 +218,7 @@ static struct cdevsw ch_cdevsw = {
 	.d_name =	"ch",
 };
 
-MALLOC_DEFINE(M_SCSICH, "scsi_ch", "scsi_ch buffers");
+static MALLOC_DEFINE(M_SCSICH, "scsi_ch", "scsi_ch buffers");
 
 static void
 chinit(void)
diff --git a/sys/cam/scsi/scsi_ch.h b/sys/cam/scsi/scsi_ch.h
index d9ca946..5ee2244 100644
--- a/sys/cam/scsi/scsi_ch.h
+++ b/sys/cam/scsi/scsi_ch.h
@@ -359,7 +359,7 @@ struct page_device_capabilities {
  * Some of these fields can be a little confusing, so an explanation
  * is in order.
  *
- * Each component within a a medium changer apparatus is called an
+ * Each component within a medium changer apparatus is called an
  * "element".
  *
  * The "medium transport element address" is the address of the first
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index ab46cf6..2bbbf32 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -606,7 +606,7 @@ static int da_retry_count = DA_DEFAULT_RETRY;
 static int da_default_timeout = DA_DEFAULT_TIMEOUT;
 static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
 
-SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD, 0,
             "CAM Direct Access Disk driver");
 SYSCTL_INT(_kern_cam_da, OID_AUTO, retry_count, CTLFLAG_RW,
            &da_retry_count, 0, "Normal I/O retry count");
@@ -642,7 +642,7 @@ static struct periph_driver dadriver =
 
 PERIPHDRIVER_DECLARE(da, dadriver);
 
-MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers");
+static MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers");
 
 static int
 daopen(struct disk *dp)
diff --git a/sys/cam/scsi/scsi_low.c b/sys/cam/scsi/scsi_low.c
index 60253cd..ec101ff 100644
--- a/sys/cam/scsi/scsi_low.c
+++ b/sys/cam/scsi/scsi_low.c
@@ -145,7 +145,7 @@ __FBSDID("$FreeBSD$");
 #define	SCSI_LOW_DISK_LFLAGS	0x0000ffff
 #define	SCSI_LOW_DISK_TFLAGS	0xffff0000
 
-MALLOC_DEFINE(M_SCSILOW, "SCSI low", "SCSI low buffers");
+static MALLOC_DEFINE(M_SCSILOW, "SCSI low", "SCSI low buffers");
 
 /**************************************************************
  * Declarations
diff --git a/sys/cam/scsi/scsi_sa.c b/sys/cam/scsi/scsi_sa.c
index 9a504b5..4999109 100644
--- a/sys/cam/scsi/scsi_sa.c
+++ b/sys/cam/scsi/scsi_sa.c
@@ -102,7 +102,7 @@ __FBSDID("$FreeBSD$");
  * Driver states
  */
 
-MALLOC_DEFINE(M_SCSISA, "SCSI sa", "SCSI sequential access buffers");
+static MALLOC_DEFINE(M_SCSISA, "SCSI sa", "SCSI sequential access buffers");
 
 typedef enum {
 	SA_STATE_NORMAL, SA_STATE_ABNORMAL
diff --git a/sys/cam/scsi/scsi_ses.c b/sys/cam/scsi/scsi_ses.c
index 2275654..dcc9e4e 100644
--- a/sys/cam/scsi/scsi_ses.c
+++ b/sys/cam/scsi/scsi_ses.c
@@ -52,7 +52,7 @@ __FBSDID("$FreeBSD$");
 
 #include <opt_ses.h>
 
-MALLOC_DEFINE(M_SCSISES, "SCSI SES", "SCSI SES buffers");
+static MALLOC_DEFINE(M_SCSISES, "SCSI SES", "SCSI SES buffers");
 
 /*
  * Platform Independent Driver Internal Definitions for SES devices.
@@ -714,7 +714,7 @@ ses_log(struct ses_softc *ssc, const char *fmt, ...)
 /*
  * Is this a device that supports enclosure services?
  *
- * It's a a pretty simple ruleset- if it is device type 0x0D (13), it's
+ * It's a pretty simple ruleset- if it is device type 0x0D (13), it's
  * an SES device. If it happens to be an old UNISYS SEN device, we can
  * handle that too.
  */
diff --git a/sys/cam/scsi/scsi_targ_bh.c b/sys/cam/scsi/scsi_targ_bh.c
index d5887a1..7f37157 100644
--- a/sys/cam/scsi/scsi_targ_bh.c
+++ b/sys/cam/scsi/scsi_targ_bh.c
@@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
 
-MALLOC_DEFINE(M_SCSIBH, "SCSI bh", "SCSI blackhole buffers");
+static MALLOC_DEFINE(M_SCSIBH, "SCSI bh", "SCSI blackhole buffers");
 
 typedef enum {
 	TARGBH_STATE_NORMAL,
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c
index a880961..65fc902 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c
@@ -1594,7 +1594,7 @@ zfsctl_snapshot_vptocnp(struct vop_vptocnp_args *ap)
 		*ap->a_buflen -= len;
 		bcopy(sep->se_name, ap->a_buf + *ap->a_buflen, len);
 		mutex_exit(&sdp->sd_lock);
-		vhold(dvp);
+		vref(dvp);
 		*ap->a_vpp = dvp;
 	}
 	VN_RELE(dvp);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c
index a14a9fe..d22edbf 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c
@@ -875,6 +875,14 @@ zvol_open(struct g_provider *pp, int flag, int count)
 	zvol_state_t *zv;
 	int err = 0;
 
+	if (MUTEX_HELD(&spa_namespace_lock)) {
+		/*
+		 * If the spa_namespace_lock is being held, it means that ZFS
+		 * is trying to open ZVOL as its VDEV. This is not supported.
+		 */
+		return (EOPNOTSUPP);
+	}
+
 	mutex_enter(&spa_namespace_lock);
 
 	zv = pp->private;
diff --git a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
index 85d9649..d0db808 100644
--- a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
+++ b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
@@ -97,7 +97,7 @@ uwrite(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr)
 
 	return (proc_ops(UIO_WRITE, p, kaddr, uaddr, len));
 }
-#endif
+#endif /* sun */
 #ifdef __i386__
 #define	r_rax	r_eax
 #define	r_rbx	r_ebx
@@ -1380,6 +1380,7 @@ fasttrap_pid_probe(struct reg *rp)
 	{
 		int ret = 0;
 		uintptr_t addr = 0;
+
 #ifdef __amd64
 		if (p->p_model == DATAMODEL_NATIVE) {
 			addr = rp->r_rsp - sizeof (uintptr_t);
@@ -1558,7 +1559,7 @@ fasttrap_pid_probe(struct reg *rp)
 		 *	------------------------	-----
 		 * a:	<original instruction>		<= 15
 		 *	jmp	<pc + tp->ftt_size>	    5
-		 * b:	<original instrction>		<= 15
+		 * b:	<original instruction>		<= 15
 		 *	int	T_DTRACE_RET		    2
 		 *					-----
 		 *					<= 37
@@ -1731,12 +1732,16 @@ fasttrap_pid_probe(struct reg *rp)
 
 		ASSERT(i <= sizeof (scratch));
 
+
+#if defined(sun)
 		if (fasttrap_copyout(scratch, (char *)addr, i)) {
+#else
+		if (uwrite(curproc, scratch, i, addr)) {
+#endif
 			fasttrap_sigtrap(p, curthread, pc);
 			new_pc = pc;
 			break;
 		}
-
 		if (tp->ftt_retids != NULL) {
 			curthread->t_dtrace_step = 1;
 			curthread->t_dtrace_ret = 1;
diff --git a/sys/cddl/dev/cyclic/cyclic.c b/sys/cddl/dev/cyclic/cyclic.c
index e7b7f50..efb0687 100644
--- a/sys/cddl/dev/cyclic/cyclic.c
+++ b/sys/cddl/dev/cyclic/cyclic.c
@@ -339,7 +339,7 @@ static kmem_cache_t *cyclic_id_cache;
 static cyc_id_t *cyclic_id_head;
 static cyc_backend_t cyclic_backend;
 
-MALLOC_DEFINE(M_CYCLIC, "cyclic", "Cyclic timer subsystem");
+static MALLOC_DEFINE(M_CYCLIC, "cyclic", "Cyclic timer subsystem");
 
 static __inline hrtime_t
 cyc_gethrtime(void)
diff --git a/sys/cddl/dev/dtrace/i386/dtrace_asm.S b/sys/cddl/dev/dtrace/i386/dtrace_asm.S
index 787a3c8..1a92818 100644
--- a/sys/cddl/dev/dtrace/i386/dtrace_asm.S
+++ b/sys/cddl/dev/dtrace/i386/dtrace_asm.S
@@ -125,11 +125,11 @@ invop_leave:
 	movl	8(%esp), %eax		/* load calling EIP */
 	incl	%eax			/* increment over LOCK prefix */
 	movl	%eax, -8(%ebx)		/* store calling EIP */
-	movl	%ebx, -4(%esp)		/* temporarily store new %esp */
+	subl	$8, %ebx		/* adjust for three pushes, one pop */
+	movl	%ebx, 8(%esp)		/* temporarily store new %esp */
 	popl	%ebx			/* pop off temp */
 	popl	%eax			/* pop off temp */
-	movl	-12(%esp), %esp		/* set stack pointer */
-	subl	$8, %esp		/* adjust for three pushes, one pop */
+	movl	(%esp), %esp		/* set stack pointer */
 	iret				/* return from interrupt */
 invop_nop:
 	/*
diff --git a/sys/cddl/dev/fbt/fbt.c b/sys/cddl/dev/fbt/fbt.c
index f3ddc83..be15d6e 100644
--- a/sys/cddl/dev/fbt/fbt.c
+++ b/sys/cddl/dev/fbt/fbt.c
@@ -61,7 +61,7 @@
 #include <sys/dtrace.h>
 #include <sys/dtrace_bsd.h>
 
-MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
+static MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
 
 #define	FBT_PUSHL_EBP		0x55
 #define	FBT_MOVL_ESP_EBP0_V0	0x8b
diff --git a/sys/cddl/dev/systrace/systrace.c b/sys/cddl/dev/systrace/systrace.c
index cc48747..ff63129 100644
--- a/sys/cddl/dev/systrace/systrace.c
+++ b/sys/cddl/dev/systrace/systrace.c
@@ -213,6 +213,7 @@ systrace_probe(u_int32_t id, int sysnum, struct sysent *sysent, void *params,
 	/* Process the probe using the converted argments. */
 	dtrace_probe(id, uargs[0], uargs[1], uargs[2], uargs[3], uargs[4]);
 }
+
 #endif
 
 static void
@@ -220,8 +221,12 @@ systrace_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *des
 {
 	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
 
-	systrace_setargdesc(sysnum, desc->dtargd_ndx, desc->dtargd_native,
-	    sizeof(desc->dtargd_native));
+	if (SYSTRACE_ISENTRY((uintptr_t)parg))
+		systrace_entry_setargdesc(sysnum, desc->dtargd_ndx, 
+		    desc->dtargd_native, sizeof(desc->dtargd_native));
+	else
+		systrace_return_setargdesc(sysnum, desc->dtargd_ndx, 
+		    desc->dtargd_native, sizeof(desc->dtargd_native));
 
 	if (desc->dtargd_native[0] == '\0')
 		desc->dtargd_ndx = DTRACE_ARGNONE;
diff --git a/sys/compat/freebsd32/freebsd32.h b/sys/compat/freebsd32/freebsd32.h
index 5d1a269..1d8f149 100644
--- a/sys/compat/freebsd32/freebsd32.h
+++ b/sys/compat/freebsd32/freebsd32.h
@@ -52,8 +52,8 @@ struct timeval32 {
 } while (0)
 
 struct timespec32 {
-	u_int32_t tv_sec;
-	u_int32_t tv_nsec;
+	int32_t tv_sec;
+	int32_t tv_nsec;
 };
 #define TS_CP(src,dst,fld) do {			\
 	CP((src).fld,(dst).fld,tv_sec);		\
diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c
index fc2932b..9390bc6 100644
--- a/sys/compat/freebsd32/freebsd32_misc.c
+++ b/sys/compat/freebsd32/freebsd32_misc.c
@@ -2828,10 +2828,16 @@ int
 freebsd32_posix_fallocate(struct thread *td,
     struct freebsd32_posix_fallocate_args *uap)
 {
-	struct posix_fallocate_args ap;
 
-	ap.fd = uap->fd;
-	ap.offset = PAIR32TO64(off_t, uap->offset);
-	ap.len = PAIR32TO64(off_t, uap->len);
-	return (sys_posix_fallocate(td, &ap));
+	return (kern_posix_fallocate(td, uap->fd,
+	    PAIR32TO64(off_t, uap->offset), PAIR32TO64(off_t, uap->len)));
+}
+
+int
+freebsd32_posix_fadvise(struct thread *td,
+    struct freebsd32_posix_fadvise_args *uap)
+{
+
+	return (kern_posix_fadvise(td, uap->fd, PAIR32TO64(off_t, uap->offset),
+	    PAIR32TO64(off_t, uap->len), uap->advice));
 }
diff --git a/sys/compat/freebsd32/freebsd32_proto.h b/sys/compat/freebsd32/freebsd32_proto.h
index ae2036a..1d6310b 100644
--- a/sys/compat/freebsd32/freebsd32_proto.h
+++ b/sys/compat/freebsd32/freebsd32_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #ifndef _FREEBSD32_SYSPROTO_H_
@@ -12,6 +12,7 @@
 #include <sys/signal.h>
 #include <sys/acl.h>
 #include <sys/cpuset.h>
+#include <sys/_ffcounter.h>
 #include <sys/_semaphore.h>
 #include <sys/ucontext.h>
 
@@ -580,6 +581,14 @@ struct freebsd32_posix_fallocate_args {
 	char len1_l_[PADL_(uint32_t)]; uint32_t len1; char len1_r_[PADR_(uint32_t)];
 	char len2_l_[PADL_(uint32_t)]; uint32_t len2; char len2_r_[PADR_(uint32_t)];
 };
+struct freebsd32_posix_fadvise_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char offset1_l_[PADL_(uint32_t)]; uint32_t offset1; char offset1_r_[PADR_(uint32_t)];
+	char offset2_l_[PADL_(uint32_t)]; uint32_t offset2; char offset2_r_[PADR_(uint32_t)];
+	char len1_l_[PADL_(uint32_t)]; uint32_t len1; char len1_r_[PADR_(uint32_t)];
+	char len2_l_[PADL_(uint32_t)]; uint32_t len2; char len2_r_[PADR_(uint32_t)];
+	char advice_l_[PADL_(int)]; int advice; char advice_r_[PADR_(int)];
+};
 #if !defined(PAD64_REQUIRED) && defined(__powerpc__)
 #define PAD64_REQUIRED
 #endif
@@ -690,6 +699,7 @@ int	freebsd32_msgctl(struct thread *, struct freebsd32_msgctl_args *);
 int	freebsd32_shmctl(struct thread *, struct freebsd32_shmctl_args *);
 int	freebsd32_pselect(struct thread *, struct freebsd32_pselect_args *);
 int	freebsd32_posix_fallocate(struct thread *, struct freebsd32_posix_fallocate_args *);
+int	freebsd32_posix_fadvise(struct thread *, struct freebsd32_posix_fadvise_args *);
 
 #ifdef COMPAT_43
 
@@ -1065,6 +1075,7 @@ int	freebsd7_freebsd32_shmctl(struct thread *, struct freebsd7_freebsd32_shmctl_
 #define	FREEBSD32_SYS_AUE_freebsd32_shmctl	AUE_SHMCTL
 #define	FREEBSD32_SYS_AUE_freebsd32_pselect	AUE_SELECT
 #define	FREEBSD32_SYS_AUE_freebsd32_posix_fallocate	AUE_NULL
+#define	FREEBSD32_SYS_AUE_freebsd32_posix_fadvise	AUE_NULL
 
 #undef PAD_
 #undef PADL_
diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h
index fe4dbc1..d21b583 100644
--- a/sys/compat/freebsd32/freebsd32_syscall.h
+++ b/sys/compat/freebsd32/freebsd32_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #define	FREEBSD32_SYS_syscall	0
@@ -207,6 +207,9 @@
 #define	FREEBSD32_SYS_freebsd32_clock_settime	233
 #define	FREEBSD32_SYS_freebsd32_clock_getres	234
 #define	FREEBSD32_SYS_freebsd32_nanosleep	240
+#define	FREEBSD32_SYS_ffclock_getcounter	241
+#define	FREEBSD32_SYS_ffclock_setestimate	242
+#define	FREEBSD32_SYS_ffclock_getestimate	243
 #define	FREEBSD32_SYS_minherit	250
 #define	FREEBSD32_SYS_rfork	251
 #define	FREEBSD32_SYS_openbsd_poll	252
@@ -424,4 +427,5 @@
 #define	FREEBSD32_SYS_rctl_add_rule	528
 #define	FREEBSD32_SYS_rctl_remove_rule	529
 #define	FREEBSD32_SYS_freebsd32_posix_fallocate	530
+#define	FREEBSD32_SYS_freebsd32_posix_fadvise	531
 #define	FREEBSD32_SYS_MAXSYSCALL	532
diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c
index c9308bb..c58a414 100644
--- a/sys/compat/freebsd32/freebsd32_syscalls.c
+++ b/sys/compat/freebsd32/freebsd32_syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 const char *freebsd32_syscallnames[] = {
@@ -251,9 +251,9 @@ const char *freebsd32_syscallnames[] = {
 	"#238",			/* 238 = timer_gettime */
 	"#239",			/* 239 = timer_getoverrun */
 	"freebsd32_nanosleep",			/* 240 = freebsd32_nanosleep */
-	"#241",			/* 241 = nosys */
-	"#242",			/* 242 = nosys */
-	"#243",			/* 243 = nosys */
+	"ffclock_getcounter",			/* 241 = ffclock_getcounter */
+	"ffclock_setestimate",			/* 242 = ffclock_setestimate */
+	"ffclock_getestimate",			/* 243 = ffclock_getestimate */
 	"#244",			/* 244 = nosys */
 	"#245",			/* 245 = nosys */
 	"#246",			/* 246 = nosys */
@@ -554,5 +554,5 @@ const char *freebsd32_syscallnames[] = {
 	"rctl_add_rule",			/* 528 = rctl_add_rule */
 	"rctl_remove_rule",			/* 529 = rctl_remove_rule */
 	"freebsd32_posix_fallocate",			/* 530 = freebsd32_posix_fallocate */
-	"#531",			/* 531 = posix_fadvise */
+	"freebsd32_posix_fadvise",			/* 531 = freebsd32_posix_fadvise */
 };
diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c
index 3f54930..4371d16 100644
--- a/sys/compat/freebsd32/freebsd32_sysent.c
+++ b/sys/compat/freebsd32/freebsd32_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 226364 2011-10-14 11:46:46Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #include "opt_compat.h"
@@ -288,9 +288,9 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 238 = timer_gettime */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 239 = timer_getoverrun */
 	{ AS(freebsd32_nanosleep_args), (sy_call_t *)freebsd32_nanosleep, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 240 = freebsd32_nanosleep */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 241 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 242 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 243 = nosys */
+	{ AS(ffclock_getcounter_args), (sy_call_t *)sys_ffclock_getcounter, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 241 = ffclock_getcounter */
+	{ AS(ffclock_setestimate_args), (sy_call_t *)sys_ffclock_setestimate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 242 = ffclock_setestimate */
+	{ AS(ffclock_getestimate_args), (sy_call_t *)sys_ffclock_getestimate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 243 = ffclock_getestimate */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 244 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 245 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 246 = nosys */
@@ -591,5 +591,5 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(rctl_add_rule_args), (sy_call_t *)sys_rctl_add_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 528 = rctl_add_rule */
 	{ AS(rctl_remove_rule_args), (sy_call_t *)sys_rctl_remove_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 529 = rctl_remove_rule */
 	{ AS(freebsd32_posix_fallocate_args), (sy_call_t *)freebsd32_posix_fallocate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 530 = freebsd32_posix_fallocate */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 531 = posix_fadvise */
+	{ AS(freebsd32_posix_fadvise_args), (sy_call_t *)freebsd32_posix_fadvise, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 531 = freebsd32_posix_fadvise */
 };
diff --git a/sys/compat/freebsd32/freebsd32_systrace_args.c b/sys/compat/freebsd32/freebsd32_systrace_args.c
index a858b27..7f6bf68 100644
--- a/sys/compat/freebsd32/freebsd32_systrace_args.c
+++ b/sys/compat/freebsd32/freebsd32_systrace_args.c
@@ -248,7 +248,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 33: {
 		struct access_args *p = params;
 		uarg[0] = (intptr_t) p->path; /* char * */
-		iarg[1] = p->flags; /* int */
+		iarg[1] = p->amode; /* int */
 		*n_args = 2;
 		break;
 	}
@@ -1203,6 +1203,27 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 2;
 		break;
 	}
+	/* ffclock_getcounter */
+	case 241: {
+		struct ffclock_getcounter_args *p = params;
+		uarg[0] = (intptr_t) p->ffcount; /* ffcounter * */
+		*n_args = 1;
+		break;
+	}
+	/* ffclock_setestimate */
+	case 242: {
+		struct ffclock_setestimate_args *p = params;
+		uarg[0] = (intptr_t) p->cest; /* struct ffclock_estimate * */
+		*n_args = 1;
+		break;
+	}
+	/* ffclock_getestimate */
+	case 243: {
+		struct ffclock_getestimate_args *p = params;
+		uarg[0] = (intptr_t) p->cest; /* struct ffclock_estimate * */
+		*n_args = 1;
+		break;
+	}
 	/* minherit */
 	case 250: {
 		struct minherit_args *p = params;
@@ -1868,7 +1889,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 376: {
 		struct eaccess_args *p = params;
 		uarg[0] = (intptr_t) p->path; /* char * */
-		iarg[1] = p->flags; /* int */
+		iarg[1] = p->amode; /* int */
 		*n_args = 2;
 		break;
 	}
@@ -2702,7 +2723,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		struct faccessat_args *p = params;
 		iarg[0] = p->fd; /* int */
 		uarg[1] = (intptr_t) p->path; /* char * */
-		iarg[2] = p->mode; /* int */
+		iarg[2] = p->amode; /* int */
 		iarg[3] = p->flag; /* int */
 		*n_args = 4;
 		break;
@@ -3034,13 +3055,25 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 5;
 		break;
 	}
+	/* freebsd32_posix_fadvise */
+	case 531: {
+		struct freebsd32_posix_fadvise_args *p = params;
+		iarg[0] = p->fd; /* int */
+		uarg[1] = p->offset1; /* uint32_t */
+		uarg[2] = p->offset2; /* uint32_t */
+		uarg[3] = p->len1; /* uint32_t */
+		uarg[4] = p->len2; /* uint32_t */
+		iarg[5] = p->advice; /* int */
+		*n_args = 6;
+		break;
+	}
 	default:
 		*n_args = 0;
 		break;
 	};
 }
 static void
-systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 {
 	const char *p = NULL;
 	switch (sysnum) {
@@ -4928,6 +4961,36 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
+	/* ffclock_getcounter */
+	case 241:
+		switch(ndx) {
+		case 0:
+			p = "ffcounter *";
+			break;
+		default:
+			break;
+		};
+		break;
+	/* ffclock_setestimate */
+	case 242:
+		switch(ndx) {
+		case 0:
+			p = "struct ffclock_estimate *";
+			break;
+		default:
+			break;
+		};
+		break;
+	/* ffclock_getestimate */
+	case 243:
+		switch(ndx) {
+		case 0:
+			p = "struct ffclock_estimate *";
+			break;
+		default:
+			break;
+		};
+		break;
 	/* minherit */
 	case 250:
 		switch(ndx) {
@@ -8093,6 +8156,1783 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
+	/* freebsd32_posix_fadvise */
+	case 531:
+		switch(ndx) {
+		case 0:
+			p = "int";
+			break;
+		case 1:
+			p = "uint32_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		case 3:
+			p = "uint32_t";
+			break;
+		case 4:
+			p = "uint32_t";
+			break;
+		case 5:
+			p = "int";
+			break;
+		default:
+			break;
+		};
+		break;
+	default:
+		break;
+	};
+	if (p != NULL)
+		strlcpy(desc, p, descsz);
+}
+static void
+systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+{
+	const char *p = NULL;
+	switch (sysnum) {
+#if !defined(PAD64_REQUIRED) && defined(__powerpc__)
+#define PAD64_REQUIRED
+#endif
+	/* nosys */
+	case 0:
+	/* sys_exit */
+	case 1:
+		if (ndx == 0 || ndx == 1)
+			p = "void";
+		break;
+	/* fork */
+	case 2:
+	/* read */
+	case 3:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* write */
+	case 4:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* open */
+	case 5:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* close */
+	case 6:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_wait4 */
+	case 7:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* link */
+	case 9:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* unlink */
+	case 10:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chdir */
+	case 12:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchdir */
+	case 13:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mknod */
+	case 14:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chmod */
+	case 15:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chown */
+	case 16:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* obreak */
+	case 17:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpid */
+	case 20:
+	/* mount */
+	case 21:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* unmount */
+	case 22:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setuid */
+	case 23:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getuid */
+	case 24:
+	/* geteuid */
+	case 25:
+	/* ptrace */
+	case 26:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_recvmsg */
+	case 27:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sendmsg */
+	case 28:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_recvfrom */
+	case 29:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* accept */
+	case 30:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpeername */
+	case 31:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getsockname */
+	case 32:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* access */
+	case 33:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chflags */
+	case 34:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchflags */
+	case 35:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sync */
+	case 36:
+	/* kill */
+	case 37:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getppid */
+	case 39:
+	/* dup */
+	case 41:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pipe */
+	case 42:
+	/* getegid */
+	case 43:
+	/* profil */
+	case 44:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktrace */
+	case 45:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getgid */
+	case 47:
+	/* getlogin */
+	case 49:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setlogin */
+	case 50:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* acct */
+	case 51:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sigaltstack */
+	case 53:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_ioctl */
+	case 54:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* reboot */
+	case 55:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* revoke */
+	case 56:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* symlink */
+	case 57:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* readlink */
+	case 58:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_execve */
+	case 59:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* umask */
+	case 60:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chroot */
+	case 61:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msync */
+	case 65:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* vfork */
+	case 66:
+	/* sbrk */
+	case 69:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sstk */
+	case 70:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ovadvise */
+	case 72:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munmap */
+	case 73:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_mprotect */
+	case 74:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* madvise */
+	case 75:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mincore */
+	case 78:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getgroups */
+	case 79:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setgroups */
+	case 80:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpgrp */
+	case 81:
+	/* setpgid */
+	case 82:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_setitimer */
+	case 83:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* swapon */
+	case 85:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_getitimer */
+	case 86:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getdtablesize */
+	case 89:
+	/* dup2 */
+	case 90:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fcntl */
+	case 92:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_select */
+	case 93:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fsync */
+	case 95:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setpriority */
+	case 96:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* socket */
+	case 97:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* connect */
+	case 98:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpriority */
+	case 100:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* bind */
+	case 104:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setsockopt */
+	case 105:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* listen */
+	case 106:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_gettimeofday */
+	case 116:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_getrusage */
+	case 117:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getsockopt */
+	case 118:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_readv */
+	case 120:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_writev */
+	case 121:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_settimeofday */
+	case 122:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchown */
+	case 123:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchmod */
+	case 124:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setreuid */
+	case 126:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setregid */
+	case 127:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rename */
+	case 128:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* flock */
+	case 131:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkfifo */
+	case 132:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sendto */
+	case 133:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shutdown */
+	case 134:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* socketpair */
+	case 135:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkdir */
+	case 136:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rmdir */
+	case 137:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_utimes */
+	case 138:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_adjtime */
+	case 140:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setsid */
+	case 147:
+	/* quotactl */
+	case 148:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getfh */
+	case 161:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sysarch */
+	case 165:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rtprio */
+	case 166:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_semsys */
+	case 169:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_msgsys */
+	case 170:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_shmsys */
+	case 171:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ntp_adjtime */
+	case 176:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setgid */
+	case 181:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setegid */
+	case 182:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* seteuid */
+	case 183:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_stat */
+	case 188:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_fstat */
+	case 189:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_lstat */
+	case 190:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pathconf */
+	case 191:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fpathconf */
+	case 192:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getrlimit */
+	case 194:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setrlimit */
+	case 195:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_getdirentries */
+	case 196:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nosys */
+	case 198:
+	/* freebsd32_sysctl */
+	case 202:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mlock */
+	case 203:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlock */
+	case 204:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* undelete */
+	case 205:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_futimes */
+	case 206:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpgid */
+	case 207:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* poll */
+	case 209:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lkmnosys */
+	case 210:
+	/* lkmnosys */
+	case 211:
+	/* lkmnosys */
+	case 212:
+	/* lkmnosys */
+	case 213:
+	/* lkmnosys */
+	case 214:
+	/* lkmnosys */
+	case 215:
+	/* lkmnosys */
+	case 216:
+	/* lkmnosys */
+	case 217:
+	/* lkmnosys */
+	case 218:
+	/* lkmnosys */
+	case 219:
+	/* semget */
+	case 221:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* semop */
+	case 222:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msgget */
+	case 225:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_msgsnd */
+	case 226:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_msgrcv */
+	case 227:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmat */
+	case 228:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmdt */
+	case 230:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmget */
+	case 231:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_clock_gettime */
+	case 232:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_clock_settime */
+	case 233:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_clock_getres */
+	case 234:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_nanosleep */
+	case 240:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ffclock_getcounter */
+	case 241:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ffclock_setestimate */
+	case 242:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ffclock_getestimate */
+	case 243:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* minherit */
+	case 250:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rfork */
+	case 251:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* openbsd_poll */
+	case 252:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* issetugid */
+	case 253:
+	/* lchown */
+	case 254:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_read */
+	case 255:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_write */
+	case 256:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_lio_listio */
+	case 257:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getdents */
+	case 272:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lchmod */
+	case 274:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lchown */
+	case 275:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_lutimes */
+	case 276:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msync */
+	case 277:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nstat */
+	case 278:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nfstat */
+	case 279:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nlstat */
+	case 280:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_preadv */
+	case 289:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_pwritev */
+	case 290:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* fhopen */
+	case 298:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fhstat */
+	case 299:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modnext */
+	case 300:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_modstat */
+	case 301:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modfnext */
+	case 302:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modfind */
+	case 303:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldload */
+	case 304:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldunload */
+	case 305:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldfind */
+	case 306:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldnext */
+	case 307:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_kldstat */
+	case 308:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldfirstmod */
+	case 309:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getsid */
+	case 310:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setresuid */
+	case 311:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setresgid */
+	case 312:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_return */
+	case 314:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_suspend */
+	case 315:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_cancel */
+	case 316:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_error */
+	case 317:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_oaio_read */
+	case 318:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_oaio_write */
+	case 319:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_olio_listio */
+	case 320:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* yield */
+	case 321:
+	/* mlockall */
+	case 324:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlockall */
+	case 325:
+	/* __getcwd */
+	case 326:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_setparam */
+	case 327:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_getparam */
+	case 328:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_setscheduler */
+	case 329:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_getscheduler */
+	case 330:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_yield */
+	case 331:
+	/* sched_get_priority_max */
+	case 332:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_get_priority_min */
+	case 333:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_rr_get_interval */
+	case 334:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* utrace */
+	case 335:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldsym */
+	case 337:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_jail */
+	case 338:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigprocmask */
+	case 340:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigsuspend */
+	case 341:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigpending */
+	case 343:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sigtimedwait */
+	case 345:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sigwaitinfo */
+	case 346:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_get_file */
+	case 347:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_set_file */
+	case 348:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_get_fd */
+	case 349:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_set_fd */
+	case 350:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_delete_file */
+	case 351:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_delete_fd */
+	case 352:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_aclcheck_file */
+	case 353:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_aclcheck_fd */
+	case 354:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattrctl */
+	case 355:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_set_file */
+	case 356:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_get_file */
+	case 357:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_delete_file */
+	case 358:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_waitcomplete */
+	case 359:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresuid */
+	case 360:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresgid */
+	case 361:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kqueue */
+	case 362:
+	/* freebsd32_kevent */
+	case 363:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_set_fd */
+	case 371:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_get_fd */
+	case 372:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_delete_fd */
+	case 373:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __setugid */
+	case 374:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* eaccess */
+	case 376:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_nmount */
+	case 378:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kenv */
+	case 390:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lchflags */
+	case 391:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* uuidgen */
+	case 392:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sendfile */
+	case 393:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getfsstat */
+	case 395:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* statfs */
+	case 396:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fstatfs */
+	case 397:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fhstatfs */
+	case 398:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_close */
+	case 400:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_post */
+	case 401:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_wait */
+	case 402:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_trywait */
+	case 403:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_ksem_init */
+	case 404:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_ksem_open */
+	case 405:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_unlink */
+	case 406:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_getvalue */
+	case 407:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_destroy */
+	case 408:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_set_link */
+	case 412:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_get_link */
+	case 413:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_delete_link */
+	case 414:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sigaction */
+	case 416:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_sigreturn */
+	case 417:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_getcontext */
+	case 421:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_setcontext */
+	case 422:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_swapcontext */
+	case 423:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_get_link */
+	case 425:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_set_link */
+	case 426:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_delete_link */
+	case 427:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_aclcheck_link */
+	case 428:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigwait */
+	case 429:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_exit */
+	case 431:
+		if (ndx == 0 || ndx == 1)
+			p = "void";
+		break;
+	/* thr_self */
+	case 432:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_kill */
+	case 433:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_umtx_lock */
+	case 434:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_umtx_unlock */
+	case 435:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail_attach */
+	case 436:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_list_fd */
+	case 437:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_list_file */
+	case 438:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_list_link */
+	case 439:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_ksem_timedwait */
+	case 441:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_thr_suspend */
+	case 442:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_wake */
+	case 443:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldunloadf */
+	case 444:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* audit */
+	case 445:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* auditon */
+	case 446:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getauid */
+	case 447:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setauid */
+	case 448:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getaudit */
+	case 449:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setaudit */
+	case 450:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getaudit_addr */
+	case 451:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setaudit_addr */
+	case 452:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* auditctl */
+	case 453:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_umtx_op */
+	case 454:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_thr_new */
+	case 455:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigqueue */
+	case 456:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_kmq_open */
+	case 457:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_kmq_setattr */
+	case 458:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_kmq_timedreceive */
+	case 459:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_kmq_timedsend */
+	case 460:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_notify */
+	case 461:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_unlink */
+	case 462:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* abort2 */
+	case 463:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_set_name */
+	case 464:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_aio_fsync */
+	case 465:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rtprio_thread */
+	case 466:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_peeloff */
+	case 471:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_generic_sendmsg */
+	case 472:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_generic_sendmsg_iov */
+	case 473:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_generic_recvmsg */
+	case 474:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+#ifdef PAD64_REQUIRED
+	/* freebsd32_pread */
+	case 475:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_pwrite */
+	case 476:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_mmap */
+	case 477:
+		if (ndx == 0 || ndx == 1)
+			p = "caddr_t";
+		break;
+	/* freebsd32_lseek */
+	case 478:
+		if (ndx == 0 || ndx == 1)
+			p = "off_t";
+		break;
+	/* freebsd32_truncate */
+	case 479:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_ftruncate */
+	case 480:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+#else
+	/* freebsd32_pread */
+	case 475:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_pwrite */
+	case 476:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd32_mmap */
+	case 477:
+		if (ndx == 0 || ndx == 1)
+			p = "caddr_t";
+		break;
+	/* freebsd32_lseek */
+	case 478:
+		if (ndx == 0 || ndx == 1)
+			p = "off_t";
+		break;
+	/* freebsd32_truncate */
+	case 479:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_ftruncate */
+	case 480:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+#endif
+	/* thr_kill2 */
+	case 481:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shm_open */
+	case 482:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shm_unlink */
+	case 483:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cpuset */
+	case 484:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+#ifdef PAD64_REQUIRED
+	/* freebsd32_cpuset_setid */
+	case 485:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+#else
+	/* freebsd32_cpuset_setid */
+	case 485:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+#endif
+	/* freebsd32_cpuset_getid */
+	case 486:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_cpuset_getaffinity */
+	case 487:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_cpuset_setaffinity */
+	case 488:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* faccessat */
+	case 489:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchmodat */
+	case 490:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchownat */
+	case 491:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_fexecve */
+	case 492:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_fstatat */
+	case 493:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_futimesat */
+	case 494:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linkat */
+	case 495:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkdirat */
+	case 496:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkfifoat */
+	case 497:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mknodat */
+	case 498:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* openat */
+	case 499:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* readlinkat */
+	case 500:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* renameat */
+	case 501:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* symlinkat */
+	case 502:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* unlinkat */
+	case 503:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* posix_openpt */
+	case 504:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_jail_get */
+	case 506:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_jail_set */
+	case 507:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail_remove */
+	case 508:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* closefrom */
+	case 509:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_semctl */
+	case 510:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_msgctl */
+	case 511:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_shmctl */
+	case 512:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lpathconf */
+	case 513:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cap_new */
+	case 514:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cap_getrights */
+	case 515:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cap_enter */
+	case 516:
+	/* cap_getmode */
+	case 517:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_pselect */
+	case 522:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getloginclass */
+	case 523:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setloginclass */
+	case 524:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_get_racct */
+	case 525:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_get_rules */
+	case 526:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_get_limits */
+	case 527:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_add_rule */
+	case 528:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_remove_rule */
+	case 529:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_posix_fallocate */
+	case 530:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_posix_fadvise */
+	case 531:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
 	default:
 		break;
 	};
diff --git a/sys/compat/freebsd32/syscalls.master b/sys/compat/freebsd32/syscalls.master
index 9428b6c..b80fd9c 100644
--- a/sys/compat/freebsd32/syscalls.master
+++ b/sys/compat/freebsd32/syscalls.master
@@ -113,7 +113,7 @@
 				    int *alen); }
 32	AUE_GETSOCKNAME	NOPROTO	{ int getsockname(int fdes, caddr_t asa, \
 				    int *alen); }
-33	AUE_ACCESS	NOPROTO	{ int access(char *path, int flags); }
+33	AUE_ACCESS	NOPROTO	{ int access(char *path, int amode); }
 34	AUE_CHFLAGS	NOPROTO	{ int chflags(char *path, int flags); }
 35	AUE_FCHFLAGS	NOPROTO	{ int fchflags(int fd, int flags); }
 36	AUE_SYNC	NOPROTO	{ int sync(void); }
@@ -447,9 +447,11 @@
 240	AUE_NULL	STD	{ int freebsd32_nanosleep( \
 				    const struct timespec32 *rqtp, \
 				    struct timespec32 *rmtp); }
-241	AUE_NULL	UNIMPL	nosys
-242	AUE_NULL	UNIMPL	nosys
-243	AUE_NULL	UNIMPL	nosys
+241	AUE_NULL	NOPROTO	{ int ffclock_getcounter(ffcounter *ffcount); }
+242	AUE_NULL	NOPROTO	{ int ffclock_setestimate( \
+				    struct ffclock_estimate *cest); }
+243	AUE_NULL	NOPROTO	{ int ffclock_getestimate( \
+				    struct ffclock_estimate *cest); }
 244	AUE_NULL	UNIMPL	nosys
 245	AUE_NULL	UNIMPL	nosys
 246	AUE_NULL	UNIMPL	nosys
@@ -671,7 +673,7 @@
 				    const char *attrname); }
 374	AUE_NULL	NOPROTO	{ int __setugid(int flag); }
 375	AUE_NULL	UNIMPL	nfsclnt
-376	AUE_EACCESS	NOPROTO	{ int eaccess(char *path, int flags); }
+376	AUE_EACCESS	NOPROTO	{ int eaccess(char *path, int amode); }
 377	AUE_NULL	UNIMPL	afs_syscall
 378	AUE_NMOUNT	STD	{ int freebsd32_nmount(struct iovec32 *iovp, \
 				    unsigned int iovcnt, int flags); }
@@ -911,7 +913,7 @@
 				    u_int32_t id1, u_int32_t id2, \
 				    size_t cpusetsize, \
 				    const cpuset_t *mask); }
-489	AUE_FACCESSAT	NOPROTO	{ int faccessat(int fd, char *path, int mode, \
+489	AUE_FACCESSAT	NOPROTO	{ int faccessat(int fd, char *path, int amode, \
 				    int flag); }
 490	AUE_FCHMODAT	NOPROTO	{ int fchmodat(int fd, const char *path, \
 				    mode_t mode, int flag); }
@@ -991,4 +993,7 @@
 530	AUE_NULL	STD	{ int freebsd32_posix_fallocate(int fd,\
 				    uint32_t offset1, uint32_t offset2,\
 				    uint32_t len1, uint32_t len2); }
-531	AUE_NULL	UNIMPL	posix_fadvise
+531	AUE_NULL	STD	{ int freebsd32_posix_fadvise(int fd, \
+				    uint32_t offset1, uint32_t offset2,\
+				    uint32_t len1, uint32_t len2, \
+				    int advice); }
diff --git a/sys/compat/ia32/ia32_sysvec.c b/sys/compat/ia32/ia32_sysvec.c
index c7edac4..66a1ece 100644
--- a/sys/compat/ia32/ia32_sysvec.c
+++ b/sys/compat/ia32/ia32_sysvec.c
@@ -95,7 +95,7 @@ CTASSERT(sizeof(struct ia32_sigframe4) == 408);
 
 extern const char *freebsd32_syscallnames[];
 
-SYSCTL_NODE(_compat, OID_AUTO, ia32, CTLFLAG_RW, 0, "ia32 mode");
+static SYSCTL_NODE(_compat, OID_AUTO, ia32, CTLFLAG_RW, 0, "ia32 mode");
 
 static u_long	ia32_maxdsiz = IA32_MAXDSIZ;
 SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxdsiz, CTLFLAG_RW, &ia32_maxdsiz, 0, "");
diff --git a/sys/compat/linux/linux_file.c b/sys/compat/linux/linux_file.c
index 9cbc3a2..ffa2282 100644
--- a/sys/compat/linux/linux_file.c
+++ b/sys/compat/linux/linux_file.c
@@ -565,16 +565,16 @@ linux_access(struct thread *td, struct linux_access_args *args)
 	int error;
 
 	/* linux convention */
-	if (args->flags & ~(F_OK | X_OK | W_OK | R_OK))
+	if (args->amode & ~(F_OK | X_OK | W_OK | R_OK))
 		return (EINVAL);
 
 	LCONVPATHEXIST(td, args->path, &path);
 
 #ifdef DEBUG
 	if (ldebug(access))
-		printf(ARGS(access, "%s, %d"), path, args->flags);
+		printf(ARGS(access, "%s, %d"), path, args->amode);
 #endif
-	error = kern_access(td, path, UIO_SYSSPACE, args->flags);
+	error = kern_access(td, path, UIO_SYSSPACE, args->amode);
 	LFREEPATH(path);
 
 	return (error);
@@ -584,10 +584,12 @@ int
 linux_faccessat(struct thread *td, struct linux_faccessat_args *args)
 {
 	char *path;
-	int error, dfd;
+	int error, dfd, flag;
 
+	if (args->flag & ~LINUX_AT_EACCESS)
+		return (EINVAL);
 	/* linux convention */
-	if (args->mode & ~(F_OK | X_OK | W_OK | R_OK))
+	if (args->amode & ~(F_OK | X_OK | W_OK | R_OK))
 		return (EINVAL);
 
 	dfd = (args->dfd == LINUX_AT_FDCWD) ? AT_FDCWD : args->dfd;
@@ -595,11 +597,11 @@ linux_faccessat(struct thread *td, struct linux_faccessat_args *args)
 
 #ifdef DEBUG
 	if (ldebug(access))
-		printf(ARGS(access, "%s, %d"), path, args->mode);
+		printf(ARGS(access, "%s, %d"), path, args->amode);
 #endif
 
-	error = kern_accessat(td, dfd, path, UIO_SYSSPACE, 0 /* XXX */,
-	    args->mode);
+	flag = (args->flag & LINUX_AT_EACCESS) == 0 ? 0 : AT_EACCESS;
+	error = kern_accessat(td, dfd, path, UIO_SYSSPACE, flag, args->amode);
 	LFREEPATH(path);
 
 	return (error);
@@ -982,13 +984,9 @@ int
 linux_linkat(struct thread *td, struct linux_linkat_args *args)
 {
 	char *path, *to;
-	int error, olddfd, newdfd;
+	int error, olddfd, newdfd, follow;
 
-	/*
-	 * They really introduced flags argument which is forbidden to
-	 * use.
-	 */
-	if (args->flags != 0)
+	if (args->flag & ~LINUX_AT_SYMLINK_FOLLOW)
 		return (EINVAL);
 
 	olddfd = (args->olddfd == LINUX_AT_FDCWD) ? AT_FDCWD : args->olddfd;
@@ -1004,10 +1002,12 @@ linux_linkat(struct thread *td, struct linux_linkat_args *args)
 #ifdef DEBUG
 	if (ldebug(linkat))
 		printf(ARGS(linkat, "%i, %s, %i, %s, %i"), args->olddfd, path,
-			args->newdfd, to, args->flags);
+			args->newdfd, to, args->flag);
 #endif
 
-	error = kern_linkat(td, olddfd, newdfd, path, to, UIO_SYSSPACE, FOLLOW);
+	follow = (args->flag & LINUX_AT_SYMLINK_FOLLOW) == 0 ? NOFOLLOW :
+	    FOLLOW;
+	error = kern_linkat(td, olddfd, newdfd, path, to, UIO_SYSSPACE, follow);
 	LFREEPATH(path);
 	LFREEPATH(to);
 	return (error);
@@ -1493,7 +1493,7 @@ int
 linux_fchownat(struct thread *td, struct linux_fchownat_args *args)
 {
 	char *path;
-	int error, dfd, follow;
+	int error, dfd, flag;
 
 	if (args->flag & ~LINUX_AT_SYMLINK_NOFOLLOW)
 		return (EINVAL);
@@ -1506,10 +1506,10 @@ linux_fchownat(struct thread *td, struct linux_fchownat_args *args)
 		printf(ARGS(fchownat, "%s, %d, %d"), path, args->uid, args->gid);
 #endif
 
-	follow = (args->flag & LINUX_AT_SYMLINK_NOFOLLOW) == 0 ? 0 :
+	flag = (args->flag & LINUX_AT_SYMLINK_NOFOLLOW) == 0 ? 0 :
 	    AT_SYMLINK_NOFOLLOW;
 	error = kern_fchownat(td, dfd, path, UIO_SYSSPACE, args->uid, args->gid,
-	    follow);
+	    flag);
 	LFREEPATH(path);
 	return (error);
 }
diff --git a/sys/compat/linux/linux_file.h b/sys/compat/linux/linux_file.h
index e229cb6..d0d95ac 100644
--- a/sys/compat/linux/linux_file.h
+++ b/sys/compat/linux/linux_file.h
@@ -31,6 +31,8 @@
 
 #define	LINUX_AT_FDCWD			-100
 #define	LINUX_AT_SYMLINK_NOFOLLOW	0x100
+#define	LINUX_AT_EACCESS		0x200
 #define	LINUX_AT_REMOVEDIR		0x200
+#define	LINUX_AT_SYMLINK_FOLLOW		0x400
 
 #endif	/* !_LINUX_FILE_H_ */
diff --git a/sys/compat/linux/linux_futex.c b/sys/compat/linux/linux_futex.c
index 736bd28..44d68f4 100644
--- a/sys/compat/linux/linux_futex.c
+++ b/sys/compat/linux/linux_futex.c
@@ -65,8 +65,8 @@ __KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $")
 #include <compat/linux/linux_futex.h>
 #include <compat/linux/linux_util.h>
 
-MALLOC_DEFINE(M_FUTEX, "futex", "Linux futexes");
-MALLOC_DEFINE(M_FUTEX_WP, "futex wp", "Linux futexes wp");
+static MALLOC_DEFINE(M_FUTEX, "futex", "Linux futexes");
+static MALLOC_DEFINE(M_FUTEX_WP, "futex wp", "Linux futexes wp");
 
 struct futex;
 
diff --git a/sys/compat/linux/linux_mib.c b/sys/compat/linux/linux_mib.c
index 1bb8bb7..16f9ac3 100644
--- a/sys/compat/linux/linux_mib.c
+++ b/sys/compat/linux/linux_mib.c
@@ -66,7 +66,7 @@ static struct linux_prison lprison0 = {
 
 static unsigned linux_osd_jail_slot;
 
-SYSCTL_NODE(_compat, OID_AUTO, linux, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_compat, OID_AUTO, linux, CTLFLAG_RW, 0,
 	    "Linux mode");
 
 static int	linux_set_osname(struct thread *td, char *osname);
diff --git a/sys/compat/svr4/svr4_fcntl.c b/sys/compat/svr4/svr4_fcntl.c
index cbca7e2..b9d3ace 100644
--- a/sys/compat/svr4/svr4_fcntl.c
+++ b/sys/compat/svr4/svr4_fcntl.c
@@ -488,7 +488,7 @@ svr4_sys_access(td, uap)
 	int error;
 
 	CHECKALTEXIST(td, uap->path, &newpath);
-	error = kern_access(td, newpath, UIO_SYSSPACE, uap->flags);
+	error = kern_access(td, newpath, UIO_SYSSPACE, uap->amode);
 	free(newpath, M_TEMP);
 	return (error);
 }
diff --git a/sys/compat/svr4/svr4_proto.h b/sys/compat/svr4/svr4_proto.h
index 49735fe..0478d3c 100644
--- a/sys/compat/svr4/svr4_proto.h
+++ b/sys/compat/svr4/svr4_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/svr4/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/compat/svr4/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #ifndef _SVR4_SYSPROTO_H_
@@ -79,7 +79,7 @@ struct svr4_sys_utime_args {
 };
 struct svr4_sys_access_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char amode_l_[PADL_(int)]; int amode; char amode_r_[PADR_(int)];
 };
 struct svr4_sys_nice_args {
 	char prio_l_[PADL_(int)]; int prio; char prio_r_[PADR_(int)];
diff --git a/sys/compat/svr4/svr4_syscall.h b/sys/compat/svr4/svr4_syscall.h
index b9c5d53..1500591 100644
--- a/sys/compat/svr4/svr4_syscall.h
+++ b/sys/compat/svr4/svr4_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/svr4/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/compat/svr4/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #define	SVR4_SYS_exit	1
diff --git a/sys/compat/svr4/svr4_syscallnames.c b/sys/compat/svr4/svr4_syscallnames.c
index ec1376a..dffc0cd 100644
--- a/sys/compat/svr4/svr4_syscallnames.c
+++ b/sys/compat/svr4/svr4_syscallnames.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/svr4/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/compat/svr4/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 const char *svr4_syscallnames[] = {
diff --git a/sys/compat/svr4/svr4_sysent.c b/sys/compat/svr4/svr4_sysent.c
index cf4481e..1b18501 100644
--- a/sys/compat/svr4/svr4_sysent.c
+++ b/sys/compat/svr4/svr4_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/svr4/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/compat/svr4/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #include <sys/types.h>
diff --git a/sys/compat/svr4/syscalls.master b/sys/compat/svr4/syscalls.master
index 2d6afee..0997ab7 100644
--- a/sys/compat/svr4/syscalls.master
+++ b/sys/compat/svr4/syscalls.master
@@ -77,7 +77,7 @@
 31	AUE_NULL	UNIMPL	stty
 32	AUE_NULL	UNIMPL	gtty
 33	AUE_NULL	STD	{ int svr4_sys_access(char *path, \
-				    int flags); }
+				    int amode); }
 34	AUE_NULL	STD	{ int svr4_sys_nice(int prio); }
 35	AUE_NULL	UNIMPL	statfs
 36	AUE_NULL	NOPROTO	{ int sync(void); }
diff --git a/sys/compat/x86bios/x86bios.c b/sys/compat/x86bios/x86bios.c
index afa01f4..32b9929 100644
--- a/sys/compat/x86bios/x86bios.c
+++ b/sys/compat/x86bios/x86bios.c
@@ -67,7 +67,8 @@ __FBSDID("$FreeBSD$");
 
 static struct mtx x86bios_lock;
 
-SYSCTL_NODE(_debug, OID_AUTO, x86bios, CTLFLAG_RD, NULL, "x86bios debugging");
+static SYSCTL_NODE(_debug, OID_AUTO, x86bios, CTLFLAG_RD, NULL,
+    "x86bios debugging");
 static int x86bios_trace_call;
 TUNABLE_INT("debug.x86bios.call", &x86bios_trace_call);
 SYSCTL_INT(_debug_x86bios, OID_AUTO, call, CTLFLAG_RW, &x86bios_trace_call, 0,
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index ed94494..fa3e374 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -1104,6 +1104,9 @@ options 	XFS
 # unsuitable for inclusion on machines with untrusted local users.
 options 	VFS_AIO
 
+# Enable mounting of non-MPSAFE filesystems.
+options 	VFS_ALLOW_NONMPSAFE
+
 # Cryptographically secure random number generator; /dev/random
 device		random
 
@@ -1183,6 +1186,14 @@ options 	HZ=100
 
 options 	PPS_SYNC
 
+# Enable support for generic feed-forward clocks in the kernel.
+# The feed-forward clock support is an alternative to the feedback oriented
+# ntpd/system clock approach, and is to be used with a feed-forward
+# synchronization algorithm such as the RADclock:
+# More info here: http://www.synclab.org/radclock
+
+options 	FFCLOCK
+
 
 #####################################################################
 # SCSI DEVICES
@@ -1459,7 +1470,9 @@ options 	TEKEN_UTF8		# UTF-8 output handling
 #      such as the Tekram DC-390(T).
 # bt:  Most Buslogic controllers: including BT-445, BT-54x, BT-64x, BT-74x,
 #      BT-75x, BT-946, BT-948, BT-956, BT-958, SDC3211B, SDC3211F, SDC3222F
-# esp: NCR53c9x.  Only for SBUS hardware right now.
+# esp: Emulex ESP, NCR 53C9x and QLogic FAS families based controllers
+#      including the AMD Am53C974 (found on devices such as the Tekram
+#      DC-390(T)) and the Sun ESP and FAS families of controllers
 # isp: Qlogic ISP 1020, 1040 and 1040B PCI SCSI host adapters,
 #      ISP 1240 Dual Ultra SCSI, ISP 1080 and 1280 (Dual) Ultra2,
 #      ISP 12160 Ultra3 SCSI,
@@ -1846,13 +1859,15 @@ device		puc
 # MII bus support is required for many PCI Ethernet NICs,
 # namely those which use MII-compliant transceivers or implement
 # transceiver control interfaces that operate like an MII.  Adding
-# "device miibus" to the kernel config pulls in support for
-# the generic miibus API and all of the PHY drivers, including a
-# generic one for PHYs that aren't specifically handled by an
-# individual driver.  Support for specific PHYs may be built by adding
-# "device mii" then adding the appropriate PHY driver.
-device  	miibus		# MII support including all PHYs
+# "device miibus" to the kernel config pulls in support for the generic
+# miibus API, the common support for for bit-bang'ing the MII and all
+# of the PHY drivers, including a generic one for PHYs that aren't
+# specifically handled by an individual driver.  Support for specific
+# PHYs may be built by adding "device mii", "device mii_bitbang" if
+# needed by the NIC driver and then adding the appropriate PHY driver.
 device  	mii		# Minimal MII support
+device  	mii_bitbang	# Common module for bit-bang'ing the MII
+device  	miibus		# MII support w/ bit-bang'ing and all PHYs
 
 device  	acphy		# Altima Communications AC101
 device  	amphy		# AMD AM79c873 / Davicom DM910{1,2}
@@ -2135,12 +2150,12 @@ device		malo		# Marvell Libertas wireless NICs.
 device		mwl		# Marvell 88W8363 802.11n wireless NICs.
 device		ral		# Ralink Technology RT2500 wireless NICs.
 
-# Use "private" jumbo buffers allocated exclusively for the ti(4) driver.
-# This option is incompatible with the TI_JUMBO_HDRSPLIT option below.
-#options 	TI_PRIVATE_JUMBOS
+# Use sf_buf(9) interface for jumbo buffers on ti(4) controllers.
+#options 	TI_SF_BUF_JUMBO
 # Turn on the header splitting option for the ti(4) driver firmware.  This
 # only works for Tigon II chips, and has no effect for Tigon I chips.
-options 	TI_JUMBO_HDRSPLIT
+# This option requires the TI_SF_BUF_JUMBO option above.
+#options 	TI_JUMBO_HDRSPLIT
 
 #
 # Use header splitting feature on bce(4) adapters.
@@ -2809,7 +2824,7 @@ options 	UBSEC_RNDTEST	# enable rndtest support
 # Embedded system options:
 #
 # An embedded system might want to run something other than init.
-options 	INIT_PATH=/sbin/init:/stand/sysinstall
+options 	INIT_PATH=/sbin/init:/rescue/init
 
 # Debug options
 options 	BUS_DEBUG	# enable newbus debugging
diff --git a/sys/conf/files b/sys/conf/files
index 90e7b76..4d98f28 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1064,6 +1064,7 @@ dev/ep/if_ep_eisa.c		optional ep eisa
 dev/ep/if_ep_isa.c		optional ep isa
 dev/ep/if_ep_mca.c		optional ep mca
 dev/ep/if_ep_pccard.c		optional ep pccard
+dev/esp/esp_pci.c		optional esp pci
 dev/esp/ncr53c9x.c		optional esp
 dev/ex/if_ex.c			optional ex
 dev/ex/if_ex_isa.c		optional ex isa
@@ -1425,6 +1426,7 @@ dev/mii/ip1000phy.c		optional miibus | ip1000phy
 dev/mii/jmphy.c			optional miibus | jmphy
 dev/mii/lxtphy.c		optional miibus | lxtphy
 dev/mii/mii.c			optional miibus | mii
+dev/mii/mii_bitbang.c		optional miibus | mii_bitbang
 dev/mii/mii_physubr.c		optional miibus | mii
 dev/mii/miibus_if.m		optional miibus | mii
 dev/mii/mlphy.c			optional miibus | mlphy
@@ -1669,6 +1671,37 @@ dev/scd/scd.c			optional scd isa
 dev/scd/scd_isa.c		optional scd isa
 dev/sdhci/sdhci.c		optional sdhci pci
 dev/sf/if_sf.c			optional sf pci
+dev/sfxge/common/efx_bootcfg.c	optional sfxge inet pci
+dev/sfxge/common/efx_ev.c	optional sfxge inet pci
+dev/sfxge/common/efx_filter.c	optional sfxge inet pci
+dev/sfxge/common/efx_intr.c	optional sfxge inet pci
+dev/sfxge/common/efx_mac.c	optional sfxge inet pci
+dev/sfxge/common/efx_mcdi.c	optional sfxge inet pci
+dev/sfxge/common/efx_mon.c	optional sfxge inet pci
+dev/sfxge/common/efx_nic.c	optional sfxge inet pci
+dev/sfxge/common/efx_nvram.c	optional sfxge inet pci
+dev/sfxge/common/efx_phy.c	optional sfxge inet pci
+dev/sfxge/common/efx_port.c	optional sfxge inet pci
+dev/sfxge/common/efx_rx.c	optional sfxge inet pci
+dev/sfxge/common/efx_sram.c	optional sfxge inet pci
+dev/sfxge/common/efx_tx.c	optional sfxge inet pci
+dev/sfxge/common/efx_vpd.c	optional sfxge inet pci
+dev/sfxge/common/efx_wol.c	optional sfxge inet pci
+dev/sfxge/common/siena_mac.c	optional sfxge inet pci
+dev/sfxge/common/siena_mon.c	optional sfxge inet pci
+dev/sfxge/common/siena_nic.c	optional sfxge inet pci
+dev/sfxge/common/siena_nvram.c	optional sfxge inet pci
+dev/sfxge/common/siena_phy.c	optional sfxge inet pci
+dev/sfxge/common/siena_sram.c	optional sfxge inet pci
+dev/sfxge/common/siena_vpd.c	optional sfxge inet pci
+dev/sfxge/sfxge.c		optional sfxge inet pci
+dev/sfxge/sfxge_dma.c		optional sfxge inet pci
+dev/sfxge/sfxge_ev.c		optional sfxge inet pci
+dev/sfxge/sfxge_intr.c		optional sfxge inet pci
+dev/sfxge/sfxge_mcdi.c		optional sfxge inet pci
+dev/sfxge/sfxge_port.c		optional sfxge inet pci
+dev/sfxge/sfxge_rx.c		optional sfxge inet pci
+dev/sfxge/sfxge_tx.c		optional sfxge inet pci
 dev/sge/if_sge.c		optional sge pci
 dev/si/si.c			optional si
 dev/si/si2_z280.c		optional si
@@ -2308,6 +2341,7 @@ kern/kern_event.c		standard
 kern/kern_exec.c		standard
 kern/kern_exit.c		standard
 kern/kern_fail.c		standard
+kern/kern_ffclock.c		standard
 kern/kern_fork.c		standard
 kern/kern_gzio.c		optional gzio
 kern/kern_hhook.c		standard
@@ -2524,6 +2558,7 @@ libkern/fnmatch.c		standard
 libkern/gets.c			standard
 libkern/iconv.c			optional libiconv
 libkern/iconv_converter_if.m	optional libiconv
+libkern/iconv_ucs.c		optional libiconv
 libkern/iconv_xlat.c		optional libiconv
 libkern/iconv_xlat16.c		optional libiconv
 libkern/index.c			standard
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 401705a..aec05c1 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -208,6 +208,12 @@ dev/lindev/lindev.c		optional	lindev
 dev/nfe/if_nfe.c		optional	nfe pci
 dev/nve/if_nve.c		optional	nve pci
 dev/nvram/nvram.c		optional	nvram isa
+dev/qlxgb/qla_dbg.c		optional	qlxgb pci
+dev/qlxgb/qla_hw.c		optional	qlxgb pci
+dev/qlxgb/qla_ioctl.c		optional	qlxgb pci
+dev/qlxgb/qla_isr.c		optional	qlxgb pci
+dev/qlxgb/qla_misc.c		optional	qlxgb pci
+dev/qlxgb/qla_os.c		optional	qlxgb pci
 dev/sio/sio.c			optional	sio
 dev/sio/sio_isa.c		optional	sio isa
 dev/sio/sio_pccard.c		optional	sio pccard
diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh
index 36be77f..55a644e 100644
--- a/sys/conf/newvers.sh
+++ b/sys/conf/newvers.sh
@@ -88,7 +88,7 @@ v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} t=`date`
 i=`${MAKE:-make} -V KERN_IDENT`
 
 for dir in /bin /usr/bin /usr/local/bin; do
-	if [ -d "${SYSDIR}/.svn" -a -x "${dir}/svnversion" ] ; then
+	if [ -x "${dir}/svnversion" ] ; then
 		svnversion=${dir}/svnversion
 		break
 	fi
@@ -99,8 +99,12 @@ for dir in /bin /usr/bin /usr/local/bin; do
 done
 
 if [ -n "$svnversion" ] ; then
-    echo "$svnversion"
-	svn=" r`cd ${SYSDIR} && $svnversion`"
+	echo "$svnversion"
+	svn=`cd ${SYSDIR} && $svnversion`
+	case "$svn" in
+	[0-9]*)	svn=" r${svn}" ;;
+	*)	unset svn ;;
+	esac
 fi
 
 if [ -n "$git_cmd" ] ; then
diff --git a/sys/conf/options b/sys/conf/options
index a7f12124..02b908a 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -59,6 +59,7 @@ GDB
 KDB		opt_global.h
 KDB_TRACE	opt_kdb.h
 KDB_UNATTENDED	opt_kdb.h
+KLD_DEBUG	opt_kld.h
 SYSCTL_DEBUG	opt_sysctl.h
 
 # Miscellaneous options.
@@ -79,6 +80,7 @@ COMPRESS_USER_CORES opt_core.h
 CY_PCI_FASTINTR
 DEADLKRES	opt_watchdog.h
 DIRECTIO
+FFCLOCK
 FULL_PREEMPTION	opt_sched.h
 IPI_PREEMPTION	opt_sched.h
 GEOM_AES	opt_geom.h
@@ -184,6 +186,7 @@ SYSVSHM		opt_sysvipc.h
 SW_WATCHDOG	opt_watchdog.h
 TURNSTILE_PROFILING
 VFS_AIO
+VFS_ALLOW_NONMPSAFE
 VERBOSE_SYSINIT	opt_global.h
 WLCACHE		opt_wavelan.h
 WLDEBUG		opt_wavelan.h
@@ -524,7 +527,7 @@ NGATM_CCATM		opt_netgraph.h
 DRM_DEBUG		opt_drm.h
 
 ZERO_COPY_SOCKETS	opt_zero.h
-TI_PRIVATE_JUMBOS	opt_ti.h
+TI_SF_BUF_JUMBO		opt_ti.h
 TI_JUMBO_HDRSPLIT	opt_ti.h
 BCE_JUMBO_HDRSPLIT	opt_bce.h
 
diff --git a/sys/contrib/pf/net/if_pfsync.c b/sys/contrib/pf/net/if_pfsync.c
index 80714c2..f60f70b 100644
--- a/sys/contrib/pf/net/if_pfsync.c
+++ b/sys/contrib/pf/net/if_pfsync.c
@@ -3290,16 +3290,17 @@ void
 pfsyncintr(void *arg)
 {
 	struct pfsync_softc *sc = arg;
-	struct mbuf *m;
+	struct mbuf *m, *n;
 
 	CURVNET_SET(sc->sc_ifp->if_vnet);
 	pfsync_ints++;
 
-	for (;;) {
-		IF_DEQUEUE(&sc->sc_ifp->if_snd, m);
-		if (m == 0)
-			break;
+	IF_DEQUEUE_ALL(&sc->sc_ifp->if_snd, m);
+
+	for (; m != NULL; m = n) {
 
+		n = m->m_nextpkt;
+		m->m_nextpkt = NULL;
 		if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL)
 		    == 0)
 			V_pfsyncstats.pfsyncs_opackets++;
diff --git a/sys/contrib/pf/net/pf.c b/sys/contrib/pf/net/pf.c
index 9b8ed26..f8fc086 100644
--- a/sys/contrib/pf/net/pf.c
+++ b/sys/contrib/pf/net/pf.c
@@ -7176,11 +7176,15 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0,
 	}
 
 #ifdef __FreeBSD__
-	if (pd.pf_mtag->flags & PF_TAG_GENERATED)
+	if (pd.pf_mtag->flags & PF_TAG_GENERATED) {
+		PF_UNLOCK();
 #else
 	if (m->m_pkthdr.pf.flags & PF_TAG_GENERATED)
 #endif
 		return (PF_PASS);
+#ifdef __FreeBSD__
+	}
+#endif
 
 	/* We do IP header normalization and packet reassembly here */
 	if (pf_normalize_ip6(m0, dir, kif, &reason, &pd) != PF_PASS) {
diff --git a/sys/crypto/aesni/aesni_wrap.c b/sys/crypto/aesni/aesni_wrap.c
index 0f3eeb1..1492640 100644
--- a/sys/crypto/aesni/aesni_wrap.c
+++ b/sys/crypto/aesni/aesni_wrap.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
- * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -87,33 +87,33 @@ aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len,
 #define	AES_XTS_ALPHA		0x87	/* GF(2^128) generator polynomial */
 
 static void
-aesni_crypt_xts_block(int rounds, const void *key_schedule, uint8_t *tweak,
-    const uint8_t *from, uint8_t *to, int do_encrypt)
+aesni_crypt_xts_block(int rounds, const void *key_schedule, uint64_t *tweak,
+    const uint64_t *from, uint64_t *to, uint64_t *block, int do_encrypt)
 {
-	uint8_t block[AES_XTS_BLOCKSIZE];
-	u_int i, carry_in, carry_out;
+	int carry;
 
-	for (i = 0; i < AES_XTS_BLOCKSIZE; i++)
-		block[i] = from[i] ^ tweak[i];
+	block[0] = from[0] ^ tweak[0];
+	block[1] = from[1] ^ tweak[1];
 
 	if (do_encrypt)
-		aesni_enc(rounds - 1, key_schedule, block, to, NULL);
+		aesni_enc(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL);
 	else
-		aesni_dec(rounds - 1, key_schedule, block, to, NULL);
+		aesni_dec(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL);
 
-	for (i = 0; i < AES_XTS_BLOCKSIZE; i++)
-		to[i] ^= tweak[i];
+	to[0] ^= tweak[0];
+	to[1] ^= tweak[1];
 
 	/* Exponentiate tweak. */
-	carry_in = 0;
-	for (i = 0; i < AES_XTS_BLOCKSIZE; i++) {
-		carry_out = tweak[i] & 0x80;
-		tweak[i] = (tweak[i] << 1) | (carry_in ? 1 : 0);
-		carry_in = carry_out;
+	carry = ((tweak[0] & 0x8000000000000000ULL) > 0);
+	tweak[0] <<= 1;
+	if (tweak[1] & 0x8000000000000000ULL) {
+		uint8_t *twk = (uint8_t *)tweak;
+
+		twk[0] ^= AES_XTS_ALPHA;
 	}
-	if (carry_in)
-		tweak[0] ^= AES_XTS_ALPHA;
-	bzero(block, sizeof(block));
+	tweak[1] <<= 1;
+	if (carry)
+		tweak[1] |= 1;
 }
 
 static void
@@ -121,32 +121,33 @@ aesni_crypt_xts(int rounds, const void *data_schedule,
     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[AES_BLOCK_LEN], int do_encrypt)
 {
+	uint64_t block[AES_XTS_BLOCKSIZE / 8];
 	uint8_t tweak[AES_XTS_BLOCKSIZE];
-	uint64_t blocknum;
 	size_t i;
 
 	/*
 	 * Prepare tweak as E_k2(IV). IV is specified as LE representation
 	 * of a 64-bit block number which we allow to be passed in directly.
 	 */
-	bcopy(iv, &blocknum, AES_XTS_IVSIZE);
-	for (i = 0; i < AES_XTS_IVSIZE; i++) {
-		tweak[i] = blocknum & 0xff;
-		blocknum >>= 8;
-	}
+#if BYTE_ORDER == LITTLE_ENDIAN
+	bcopy(iv, tweak, AES_XTS_IVSIZE);
 	/* Last 64 bits of IV are always zero. */
 	bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
+#else
+#error Only LITTLE_ENDIAN architectures are supported.
+#endif
 	aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL);
 
 	len /= AES_XTS_BLOCKSIZE;
 	for (i = 0; i < len; i++) {
-		aesni_crypt_xts_block(rounds, data_schedule, tweak, from, to,
-		    do_encrypt);
+		aesni_crypt_xts_block(rounds, data_schedule, (uint64_t *)tweak,
+		    (const uint64_t *)from, (uint64_t *)to, block, do_encrypt);
 		from += AES_XTS_BLOCKSIZE;
 		to += AES_XTS_BLOCKSIZE;
 	}
 
 	bzero(tweak, sizeof(tweak));
+	bzero(block, sizeof(block));
 }
 
 static void
diff --git a/sys/dev/aac/aac.c b/sys/dev/aac/aac.c
index 45cfa02..68854ae 100644
--- a/sys/dev/aac/aac.c
+++ b/sys/dev/aac/aac.c
@@ -219,10 +219,10 @@ static struct cdevsw aac_cdevsw = {
 	.d_name =	"aac",
 };
 
-MALLOC_DEFINE(M_AACBUF, "aacbuf", "Buffers for the AAC driver");
+static MALLOC_DEFINE(M_AACBUF, "aacbuf", "Buffers for the AAC driver");
 
 /* sysctl node */
-SYSCTL_NODE(_hw, OID_AUTO, aac, CTLFLAG_RD, 0, "AAC driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, aac, CTLFLAG_RD, 0, "AAC driver parameters");
 
 /*
  * Device Interface
diff --git a/sys/dev/aac/aac_cam.c b/sys/dev/aac/aac_cam.c
index b5c3b26..2fafa97 100644
--- a/sys/dev/aac/aac_cam.c
+++ b/sys/dev/aac/aac_cam.c
@@ -104,7 +104,7 @@ static driver_t	aac_pass_driver = {
 DRIVER_MODULE(aacp, aac, aac_pass_driver, aac_pass_devclass, 0, 0);
 MODULE_DEPEND(aacp, cam, 1, 1, 1);
 
-MALLOC_DEFINE(M_AACCAM, "aaccam", "AAC CAM info");
+static MALLOC_DEFINE(M_AACCAM, "aaccam", "AAC CAM info");
 
 static void
 aac_cam_rescan(struct aac_softc *sc, uint32_t channel, uint32_t target_id)
diff --git a/sys/dev/acpi_support/acpi_wmi.c b/sys/dev/acpi_support/acpi_wmi.c
index 5b95cc9..2acc262 100644
--- a/sys/dev/acpi_support/acpi_wmi.c
+++ b/sys/dev/acpi_support/acpi_wmi.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/acpica/acpivar.h>
 #include "acpi_wmi_if.h"
 
-MALLOC_DEFINE(M_ACPIWMI, "acpiwmi", "ACPI-WMI mapping");
+static MALLOC_DEFINE(M_ACPIWMI, "acpiwmi", "ACPI-WMI mapping");
 
 #define _COMPONENT	ACPI_OEM
 ACPI_MODULE_NAME("ACPI_WMI");
diff --git a/sys/dev/acpica/Osd/OsdInterrupt.c b/sys/dev/acpica/Osd/OsdInterrupt.c
index 0ca2b55..3fb1e70 100644
--- a/sys/dev/acpica/Osd/OsdInterrupt.c
+++ b/sys/dev/acpica/Osd/OsdInterrupt.c
@@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
 #define _COMPONENT	ACPI_OS_SERVICES
 ACPI_MODULE_NAME("INTERRUPT")
 
-MALLOC_DEFINE(M_ACPIINTR, "acpiintr", "ACPI interrupt");
+static MALLOC_DEFINE(M_ACPIINTR, "acpiintr", "ACPI interrupt");
 
 struct acpi_intr {
 	SLIST_ENTRY(acpi_intr)	ai_link;
diff --git a/sys/dev/acpica/Osd/OsdMemory.c b/sys/dev/acpica/Osd/OsdMemory.c
index 4b7eca2..615c4b5 100644
--- a/sys/dev/acpica/Osd/OsdMemory.c
+++ b/sys/dev/acpica/Osd/OsdMemory.c
@@ -40,7 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
-MALLOC_DEFINE(M_ACPICA, "acpica", "ACPI CA memory pool");
+static MALLOC_DEFINE(M_ACPICA, "acpica", "ACPI CA memory pool");
 
 void *
 AcpiOsAllocate(ACPI_SIZE Size)
diff --git a/sys/dev/acpica/Osd/OsdSchedule.c b/sys/dev/acpica/Osd/OsdSchedule.c
index 32d49bc..fd1b931 100644
--- a/sys/dev/acpica/Osd/OsdSchedule.c
+++ b/sys/dev/acpica/Osd/OsdSchedule.c
@@ -65,7 +65,7 @@ TUNABLE_INT("debug.acpi.max_tasks", &acpi_max_tasks);
 static int acpi_max_threads = ACPI_MAX_THREADS;
 TUNABLE_INT("debug.acpi.max_threads", &acpi_max_threads);
 
-MALLOC_DEFINE(M_ACPITASK, "acpitask", "ACPI deferred task");
+static MALLOC_DEFINE(M_ACPITASK, "acpitask", "ACPI deferred task");
 
 struct acpi_task_ctx {
     struct task			at_task;
diff --git a/sys/dev/acpica/Osd/OsdSynch.c b/sys/dev/acpica/Osd/OsdSynch.c
index 5e2e777..84e331c 100644
--- a/sys/dev/acpica/Osd/OsdSynch.c
+++ b/sys/dev/acpica/Osd/OsdSynch.c
@@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$");
 #define	_COMPONENT	ACPI_OS_SERVICES
 ACPI_MODULE_NAME("SYNCH")
 
-MALLOC_DEFINE(M_ACPISEM, "acpisem", "ACPI semaphore");
+static MALLOC_DEFINE(M_ACPISEM, "acpisem", "ACPI semaphore");
 
 /*
  * Convert milliseconds to ticks.
diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
index 5047e61..bff74fb 100644
--- a/sys/dev/acpica/acpi.c
+++ b/sys/dev/acpica/acpi.c
@@ -68,7 +68,7 @@ __FBSDID("$FreeBSD$");
 
 #include <vm/vm_param.h>
 
-MALLOC_DEFINE(M_ACPIDEV, "acpidev", "ACPI devices");
+static MALLOC_DEFINE(M_ACPIDEV, "acpidev", "ACPI devices");
 
 /* Hooks for the ACPI CA debugging infrastructure */
 #define _COMPONENT	ACPI_BUS
diff --git a/sys/dev/acpica/acpi_cmbat.c b/sys/dev/acpica/acpi_cmbat.c
index 6c6525d..ba44da8 100644
--- a/sys/dev/acpica/acpi_cmbat.c
+++ b/sys/dev/acpica/acpi_cmbat.c
@@ -46,7 +46,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/acpica/acpivar.h>
 #include <dev/acpica/acpiio.h>
 
-MALLOC_DEFINE(M_ACPICMBAT, "acpicmbat", "ACPI control method battery data");
+static MALLOC_DEFINE(M_ACPICMBAT, "acpicmbat",
+    "ACPI control method battery data");
 
 /* Number of times to retry initialization before giving up. */
 #define ACPI_CMBAT_RETRY_MAX	6
diff --git a/sys/dev/acpica/acpi_ec.c b/sys/dev/acpica/acpi_ec.c
index 7bf1b58..5c71d3d 100644
--- a/sys/dev/acpica/acpi_ec.c
+++ b/sys/dev/acpica/acpi_ec.c
@@ -179,7 +179,7 @@ struct acpi_ec_softc {
 ACPI_SERIAL_DECL(ec, "ACPI embedded controller");
 
 SYSCTL_DECL(_debug_acpi);
-SYSCTL_NODE(_debug_acpi, OID_AUTO, ec, CTLFLAG_RD, NULL, "EC debugging");
+static SYSCTL_NODE(_debug_acpi, OID_AUTO, ec, CTLFLAG_RD, NULL, "EC debugging");
 
 static int	ec_burst_mode;
 TUNABLE_INT("debug.acpi.ec.burst", &ec_burst_mode);
diff --git a/sys/dev/acpica/acpi_pci_link.c b/sys/dev/acpica/acpi_pci_link.c
index d5d2d82..ba03d72 100644
--- a/sys/dev/acpica/acpi_pci_link.c
+++ b/sys/dev/acpica/acpi_pci_link.c
@@ -120,7 +120,7 @@ struct link_res_request {
 	int	link_index;
 };
 
-MALLOC_DEFINE(M_PCI_LINK, "pci_link", "ACPI PCI Link structures");
+static MALLOC_DEFINE(M_PCI_LINK, "pci_link", "ACPI PCI Link structures");
 
 static int pci_link_interrupt_weights[NUM_ACPI_INTERRUPTS];
 static int pci_link_bios_isa_irqs;
diff --git a/sys/dev/acpica/acpi_perf.c b/sys/dev/acpica/acpi_perf.c
index 31b1c63..3f047cc 100644
--- a/sys/dev/acpica/acpi_perf.c
+++ b/sys/dev/acpica/acpi_perf.c
@@ -135,7 +135,7 @@ static devclass_t acpi_perf_devclass;
 DRIVER_MODULE(acpi_perf, cpu, acpi_perf_driver, acpi_perf_devclass, 0, 0);
 MODULE_DEPEND(acpi_perf, acpi, 1, 1, 1);
 
-MALLOC_DEFINE(M_ACPIPERF, "acpi_perf", "ACPI Performance states");
+static MALLOC_DEFINE(M_ACPIPERF, "acpi_perf", "ACPI Performance states");
 
 static void
 acpi_perf_identify(driver_t *driver, device_t parent)
diff --git a/sys/dev/acpica/acpi_powerres.c b/sys/dev/acpica/acpi_powerres.c
index 9c43c88..b661594 100644
--- a/sys/dev/acpica/acpi_powerres.c
+++ b/sys/dev/acpica/acpi_powerres.c
@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
  * on, turned off, etc.
  */
 
-MALLOC_DEFINE(M_ACPIPWR, "acpipwr", "ACPI power resources");
+static MALLOC_DEFINE(M_ACPIPWR, "acpipwr", "ACPI power resources");
 
 /* Hooks for the ACPI CA debugging infrastructure */
 #define _COMPONENT	ACPI_POWERRES
diff --git a/sys/dev/acpica/acpi_smbat.c b/sys/dev/acpica/acpi_smbat.c
index bdaf945..0647772 100644
--- a/sys/dev/acpica/acpi_smbat.c
+++ b/sys/dev/acpica/acpi_smbat.c
@@ -63,7 +63,8 @@ static int	acpi_smbat_get_bst(device_t dev, struct acpi_bst *bst);
 ACPI_SERIAL_DECL(smbat, "ACPI Smart Battery");
 
 SYSCTL_DECL(_debug_acpi);
-SYSCTL_NODE(_debug_acpi, OID_AUTO, batt, CTLFLAG_RD, NULL, "Battery debugging");
+static SYSCTL_NODE(_debug_acpi, OID_AUTO, batt, CTLFLAG_RD, NULL,
+    "Battery debugging");
 
 /* On some laptops with smart batteries, enabling battery monitoring
  * software causes keystrokes from atkbd to be lost.  This has also been
diff --git a/sys/dev/acpica/acpi_thermal.c b/sys/dev/acpica/acpi_thermal.c
index 18996bd..32e5c2d 100644
--- a/sys/dev/acpica/acpi_thermal.c
+++ b/sys/dev/acpica/acpi_thermal.c
@@ -245,7 +245,7 @@ acpi_tz_attach(device_t dev)
 	SYSCTL_ADD_INT(&acpi_tz_sysctl_ctx,
 		       SYSCTL_CHILDREN(acpi_tz_sysctl_tree),
 		       OID_AUTO, "polling_rate", CTLFLAG_RW,
-		       &acpi_tz_polling_rate, 0, "monitor polling rate");
+		       &acpi_tz_polling_rate, 0, "monitor polling interval in seconds");
 	SYSCTL_ADD_INT(&acpi_tz_sysctl_ctx,
 		       SYSCTL_CHILDREN(acpi_tz_sysctl_tree), OID_AUTO,
 		       "user_override", CTLFLAG_RW, &acpi_tz_override, 0,
diff --git a/sys/dev/acpica/acpi_video.c b/sys/dev/acpica/acpi_video.c
index f8133aa..eac2d07 100644
--- a/sys/dev/acpica/acpi_video.c
+++ b/sys/dev/acpica/acpi_video.c
@@ -183,7 +183,7 @@ static struct acpi_video_output_queue crt_units, tv_units,
  */
 ACPI_SERIAL_DECL(video, "ACPI video");
 ACPI_SERIAL_DECL(video_output, "ACPI video output");
-MALLOC_DEFINE(M_ACPIVIDEO, "acpivideo", "ACPI video extension");
+static MALLOC_DEFINE(M_ACPIVIDEO, "acpivideo", "ACPI video extension");
 
 static int
 acpi_video_modevent(struct module *mod __unused, int evt, void *cookie __unused)
diff --git a/sys/dev/ae/if_ae.c b/sys/dev/ae/if_ae.c
index 96329fd..3b882ee 100644
--- a/sys/dev/ae/if_ae.c
+++ b/sys/dev/ae/if_ae.c
@@ -1431,7 +1431,7 @@ ae_tx_avail_size(ae_softc_t *sc)
 	else
 		avail = sc->txd_ack - sc->txd_cur;
 
-	return (avail - 4);	/* 4-byte header. */
+	return (avail);
 }
 
 static int
@@ -1448,7 +1448,7 @@ ae_encap(ae_softc_t *sc, struct mbuf **m_head)
 	len = m0->m_pkthdr.len;
 	
 	if ((sc->flags & AE_FLAG_TXAVAIL) == 0 ||
-	    ae_tx_avail_size(sc) < len) {
+	    len + sizeof(ae_txd_t) + 3 > ae_tx_avail_size(sc)) {
 #ifdef AE_DEBUG
 		if_printf(sc->ifp, "No free Tx available.\n");
 #endif
@@ -1457,11 +1457,10 @@ ae_encap(ae_softc_t *sc, struct mbuf **m_head)
 
 	hdr = (ae_txd_t *)(sc->txd_base + sc->txd_cur);
 	bzero(hdr, sizeof(*hdr));
-	sc->txd_cur = (sc->txd_cur + 4) % AE_TXD_BUFSIZE_DEFAULT; /* Header
-								     size. */
-	to_end = AE_TXD_BUFSIZE_DEFAULT - sc->txd_cur; /* Space available to
-							* the end of the ring
-							*/
+	/* Skip header size. */
+	sc->txd_cur = (sc->txd_cur + sizeof(ae_txd_t)) % AE_TXD_BUFSIZE_DEFAULT;
+	/* Space available to the end of the ring */
+	to_end = AE_TXD_BUFSIZE_DEFAULT - sc->txd_cur;
 	if (to_end >= len) {
 		m_copydata(m0, 0, len, (caddr_t)(sc->txd_base + sc->txd_cur));
 	} else {
@@ -1840,8 +1839,8 @@ ae_tx_intr(ae_softc_t *sc)
 		/*
 		 * Move txd ack and align on 4-byte boundary.
 		 */
-		sc->txd_ack = ((sc->txd_ack + le16toh(txd->len) + 4 + 3) & ~3) %
-		    AE_TXD_BUFSIZE_DEFAULT;
+		sc->txd_ack = ((sc->txd_ack + le16toh(txd->len) +
+		    sizeof(ae_txs_t) + 3) & ~3) % AE_TXD_BUFSIZE_DEFAULT;
 
 		if ((flags & AE_TXS_SUCCESS) != 0)
 			ifp->if_opackets++;
diff --git a/sys/dev/ahci/ahci.c b/sys/dev/ahci/ahci.c
index 0a20f2c..6aa131c 100644
--- a/sys/dev/ahci/ahci.c
+++ b/sys/dev/ahci/ahci.c
@@ -101,7 +101,7 @@ static void ahci_process_request_sense(device_t dev, union ccb *ccb);
 static void ahciaction(struct cam_sim *sim, union ccb *ccb);
 static void ahcipoll(struct cam_sim *sim);
 
-MALLOC_DEFINE(M_AHCI, "AHCI driver", "AHCI driver data buffers");
+static MALLOC_DEFINE(M_AHCI, "AHCI driver", "AHCI driver data buffers");
 
 static struct {
 	uint32_t	id;
@@ -498,13 +498,14 @@ ahci_attach(device_t dev)
 	}
 	/* Attach all channels on this controller */
 	for (unit = 0; unit < ctlr->channels; unit++) {
-		if ((ctlr->ichannels & (1 << unit)) == 0)
-			continue;
 		child = device_add_child(dev, "ahcich", -1);
-		if (child == NULL)
+		if (child == NULL) {
 			device_printf(dev, "failed to add channel device\n");
-		else
-			device_set_ivars(child, (void *)(intptr_t)unit);
+			continue;
+		}
+		device_set_ivars(child, (void *)(intptr_t)unit);
+		if ((ctlr->ichannels & (1 << unit)) == 0)
+			device_disable(child);
 	}
 	bus_generic_attach(dev);
 	return 0;
@@ -514,15 +515,11 @@ static int
 ahci_detach(device_t dev)
 {
 	struct ahci_controller *ctlr = device_get_softc(dev);
-	device_t *children;
-	int nchildren, i;
+	int i;
 
 	/* Detach & delete all children */
-	if (!device_get_children(dev, &children, &nchildren)) {
-		for (i = 0; i < nchildren; i++)
-			device_delete_child(dev, children[i]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
+
 	/* Free interrupts. */
 	for (i = 0; i < ctlr->numirqs; i++) {
 		if (ctlr->irqs[i].r_irq) {
diff --git a/sys/dev/amr/amr.c b/sys/dev/amr/amr.c
index 5ccf9b8..4cb42e2 100644
--- a/sys/dev/amr/amr.c
+++ b/sys/dev/amr/amr.c
@@ -175,7 +175,7 @@ static void	amr_init_sysctl(struct amr_softc *sc);
 static int	amr_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t addr,
 		    int32_t flag, struct thread *td);
 
-MALLOC_DEFINE(M_AMR, "amr", "AMR memory");
+static MALLOC_DEFINE(M_AMR, "amr", "AMR memory");
 
 /********************************************************************************
  ********************************************************************************
diff --git a/sys/dev/amr/amr_cam.c b/sys/dev/amr/amr_cam.c
index bcd036e..d03ac6d 100644
--- a/sys/dev/amr/amr_cam.c
+++ b/sys/dev/amr/amr_cam.c
@@ -109,7 +109,7 @@ static driver_t	amr_pass_driver = {
 DRIVER_MODULE(amrp, amr, amr_pass_driver, amr_pass_devclass, 0, 0);
 MODULE_DEPEND(amrp, cam, 1, 1, 1);
 
-MALLOC_DEFINE(M_AMRCAM, "amrcam", "AMR CAM memory");
+static MALLOC_DEFINE(M_AMRCAM, "amrcam", "AMR CAM memory");
 
 /***********************************************************************
  * Enqueue/dequeue functions
diff --git a/sys/dev/an/if_an.c b/sys/dev/an/if_an.c
index 0898961..f52f292 100644
--- a/sys/dev/an/if_an.c
+++ b/sys/dev/an/if_an.c
@@ -203,7 +203,8 @@ static char an_conf_cache[256];
 
 /* sysctl vars */
 
-SYSCTL_NODE(_hw, OID_AUTO, an, CTLFLAG_RD, 0, "Wireless driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, an, CTLFLAG_RD, 0,
+    "Wireless driver parameters");
 
 /* XXX violate ethernet/netgraph callback hooks */
 extern	void	(*ng_ether_attach_p)(struct ifnet *ifp);
diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c
index 34453ad..1e577b6 100644
--- a/sys/dev/ata/ata-all.c
+++ b/sys/dev/ata/ata-all.c
@@ -103,7 +103,7 @@ static int ata_dma = 1;
 static int atapi_dma = 1;
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, ata, CTLFLAG_RD, 0, "ATA driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, ata, CTLFLAG_RD, 0, "ATA driver parameters");
 TUNABLE_INT("hw.ata.ata_dma", &ata_dma);
 SYSCTL_INT(_hw_ata, OID_AUTO, ata_dma, CTLFLAG_RDTUN, &ata_dma, 0,
 	   "ATA disk DMA mode control");
diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c
index bcaf6c4..c872f62 100644
--- a/sys/dev/ata/ata-disk.c
+++ b/sys/dev/ata/ata-disk.c
@@ -172,8 +172,6 @@ ad_detach(device_t dev)
 {
     struct ad_softc *adp = device_get_ivars(dev);
     struct ata_device *atadev = device_get_softc(dev);
-    device_t *children;
-    int nchildren, i;
 
     /* check that we have a valid disk to detach */
     if (!device_get_ivars(dev))
@@ -183,12 +181,7 @@ ad_detach(device_t dev)
     callout_drain(&atadev->spindown_timer);
 
     /* detach & delete all children */
-    if (!device_get_children(dev, &children, &nchildren)) {
-	for (i = 0; i < nchildren; i++)
-	    if (children[i])
-		device_delete_child(dev, children[i]);
-	free(children, M_TEMP);
-    }
+    device_delete_all_children(dev);
 
     /* destroy disk from the system so we don't get any further requests */
     disk_destroy(adp->disk);
diff --git a/sys/dev/ata/ata-pci.c b/sys/dev/ata/ata-pci.c
index b0359e6..9fcd003 100644
--- a/sys/dev/ata/ata-pci.c
+++ b/sys/dev/ata/ata-pci.c
@@ -136,15 +136,10 @@ int
 ata_pci_detach(device_t dev)
 {
     struct ata_pci_controller *ctlr = device_get_softc(dev);
-    device_t *children;
-    int nchildren, i;
 
     /* detach & delete all children */
-    if (!device_get_children(dev, &children, &nchildren)) {
-	for (i = 0; i < nchildren; i++)
-	    device_delete_child(dev, children[i]);
-	free(children, M_TEMP);
-    }
+    device_delete_all_children(dev);
+
     if (ctlr->r_irq) {
 	bus_teardown_intr(dev, ctlr->r_irq, ctlr->handle);
 	bus_release_resource(dev, SYS_RES_IRQ, ctlr->r_irq_rid, ctlr->r_irq);
@@ -153,10 +148,20 @@ ata_pci_detach(device_t dev)
     }
     if (ctlr->chipdeinit != NULL)
 	ctlr->chipdeinit(dev);
-    if (ctlr->r_res2)
+    if (ctlr->r_res2) {
+#ifdef __sparc64__
+	bus_space_unmap(rman_get_bustag(ctlr->r_res2),
+	    rman_get_bushandle(ctlr->r_res2), rman_get_size(ctlr->r_res2));
+#endif
 	bus_release_resource(dev, ctlr->r_type2, ctlr->r_rid2, ctlr->r_res2);
-    if (ctlr->r_res1)
+    }
+    if (ctlr->r_res1) {
+#ifdef __sparc64__
+	bus_space_unmap(rman_get_bustag(ctlr->r_res1),
+	    rman_get_bushandle(ctlr->r_res1), rman_get_size(ctlr->r_res1));
+#endif
 	bus_release_resource(dev, ctlr->r_type1, ctlr->r_rid1, ctlr->r_res1);
+    }
 
     return 0;
 }
@@ -775,7 +780,6 @@ driver_t ata_pcichannel_driver = {
 
 DRIVER_MODULE(ata, atapci, ata_pcichannel_driver, ata_devclass, 0, 0);
 
-
 /*
  * misc support fucntions
  */
@@ -936,4 +940,3 @@ ata_mode2idx(int mode)
 	return (mode & ATA_MODE_MASK) + 5;
     return (mode & ATA_MODE_MASK) - ATA_PIO0;
 }
-
diff --git a/sys/dev/ata/chipsets/ata-promise.c b/sys/dev/ata/chipsets/ata-promise.c
index 2f8b2a5..6f2f57c 100644
--- a/sys/dev/ata/chipsets/ata-promise.c
+++ b/sys/dev/ata/chipsets/ata-promise.c
@@ -94,7 +94,6 @@ static void ata_promise_next_hpkt(struct ata_pci_controller *ctlr);
 #define PR_SATA		0x40
 #define PR_SATA2	0x80
 
-
 /*
  * Promise chipset support functions
  */
@@ -250,6 +249,14 @@ ata_promise_chipinit(device_t dev)
 						    &ctlr->r_rid1, RF_ACTIVE)))
 	    goto failnfree;
 
+#ifdef __sparc64__
+	if (ctlr->chip->cfg2 == PR_SX4X &&
+	    !bus_space_map(rman_get_bustag(ctlr->r_res1),
+	    rman_get_bushandle(ctlr->r_res1), rman_get_size(ctlr->r_res1),
+	    BUS_SPACE_MAP_LINEAR, NULL))
+		goto failnfree;
+#endif
+
 	ctlr->r_type2 = SYS_RES_MEMORY;
 	ctlr->r_rid2 = PCIR_BAR(3);
 	if (!(ctlr->r_res2 = bus_alloc_resource_any(dev, ctlr->r_type2,
diff --git a/sys/dev/ata/chipsets/ata-siliconimage.c b/sys/dev/ata/chipsets/ata-siliconimage.c
index 2882b17..52eae96 100644
--- a/sys/dev/ata/chipsets/ata-siliconimage.c
+++ b/sys/dev/ata/chipsets/ata-siliconimage.c
@@ -80,7 +80,6 @@ static void ata_siiprb_dmainit(device_t dev);
 #define SII_BUG		0x04
 #define SII_4CH		0x08
 
-
 /*
  * Silicon Image Inc. (SiI) (former CMD) chipset support functions
  */
@@ -141,6 +140,17 @@ ata_sii_chipinit(device_t dev)
 	    bus_release_resource(dev, ctlr->r_type1, ctlr->r_rid1,ctlr->r_res1);
 	    return ENXIO;
 	}
+#ifdef __sparc64__
+	if (!bus_space_map(rman_get_bustag(ctlr->r_res2),
+	    rman_get_bushandle(ctlr->r_res2), rman_get_size(ctlr->r_res2),
+	    BUS_SPACE_MAP_LINEAR, NULL)) {
+	    	bus_release_resource(dev, ctlr->r_type1, ctlr->r_rid1,
+		    ctlr->r_res1);
+		bus_release_resource(dev, ctlr->r_type2, ctlr->r_rid2,
+		    ctlr->r_res2);
+		return (ENXIO);
+	}
+#endif
 	ctlr->ch_attach = ata_siiprb_ch_attach;
 	ctlr->ch_detach = ata_siiprb_ch_detach;
 	ctlr->reset = ata_siiprb_reset;
@@ -432,7 +442,6 @@ ata_sii_setmode(device_t dev, int target, int mode)
 	return (mode);
 }
 
-
 struct ata_siiprb_dma_prdentry {
     u_int64_t addr;
     u_int32_t count;
diff --git a/sys/dev/ath/ah_osdep.c b/sys/dev/ath/ah_osdep.c
index 54dfb73..37b07c1 100644
--- a/sys/dev/ath/ah_osdep.c
+++ b/sys/dev/ath/ah_osdep.c
@@ -38,6 +38,8 @@
 #include <sys/bus.h>
 #include <sys/malloc.h>
 #include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 
 #include <machine/stdarg.h>
 
@@ -59,6 +61,17 @@
 #define	BUSTAG(ah)	((ah)->ah_st)
 #endif
 
+/*
+ * This lock is used to seralise register access for chips which have
+ * problems w/ SMP CPUs issuing concurrent PCI transactions.
+ *
+ * XXX This is a global lock for now; it should be pushed to
+ * a per-device lock in some platform-independent fashion.
+ */
+struct mtx ah_regser_mtx;
+MTX_SYSINIT(ah_regser, &ah_regser_mtx, "Atheros register access mutex",
+    MTX_SPIN);
+
 extern	void ath_hal_printf(struct ath_hal *, const char*, ...)
 		__printflike(2,3);
 extern	void ath_hal_vprintf(struct ath_hal *, const char*, __va_list)
@@ -76,7 +89,8 @@ extern	void DO_HALDEBUG(struct ath_hal *ah, u_int mask, const char* fmt, ...);
 
 /* NB: put this here instead of the driver to avoid circular references */
 SYSCTL_NODE(_hw, OID_AUTO, ath, CTLFLAG_RD, 0, "Atheros driver parameters");
-SYSCTL_NODE(_hw_ath, OID_AUTO, hal, CTLFLAG_RD, 0, "Atheros HAL parameters");
+static SYSCTL_NODE(_hw_ath, OID_AUTO, hal, CTLFLAG_RD, 0,
+    "Atheros HAL parameters");
 
 #ifdef AH_DEBUG
 int ath_hal_debug = 0;
@@ -85,7 +99,7 @@ SYSCTL_INT(_hw_ath_hal, OID_AUTO, debug, CTLFLAG_RW, &ath_hal_debug,
 TUNABLE_INT("hw.ath.hal.debug", &ath_hal_debug);
 #endif /* AH_DEBUG */
 
-MALLOC_DEFINE(M_ATH_HAL, "ath_hal", "ath hal data");
+static MALLOC_DEFINE(M_ATH_HAL, "ath_hal", "ath hal data");
 
 void*
 ath_hal_malloc(size_t size)
@@ -249,12 +263,16 @@ ath_hal_reg_write(struct ath_hal *ah, u_int32_t reg, u_int32_t val)
 			alq_post(ath_hal_alq, ale);
 		}
 	}
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_lock_spin(&ah_regser_mtx);
 #if _BYTE_ORDER == _BIG_ENDIAN
 	if (OS_REG_UNSWAPPED(reg))
 		bus_space_write_4(tag, h, reg, val);
 	else
 #endif
 		bus_space_write_stream_4(tag, h, reg, val);
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_unlock_spin(&ah_regser_mtx);
 }
 
 u_int32_t
@@ -264,12 +282,16 @@ ath_hal_reg_read(struct ath_hal *ah, u_int32_t reg)
 	bus_space_handle_t h = ah->ah_sh;
 	u_int32_t val;
 
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_lock_spin(&ah_regser_mtx);
 #if _BYTE_ORDER == _BIG_ENDIAN
 	if (OS_REG_UNSWAPPED(reg))
 		val = bus_space_read_4(tag, h, reg);
 	else
 #endif
 		val = bus_space_read_stream_4(tag, h, reg);
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_unlock_spin(&ah_regser_mtx);
 	if (ath_hal_alq) {
 		struct ale *ale = ath_hal_alq_get(ah);
 		if (ale) {
@@ -315,12 +337,16 @@ ath_hal_reg_write(struct ath_hal *ah, u_int32_t reg, u_int32_t val)
 	bus_space_tag_t tag = BUSTAG(ah);
 	bus_space_handle_t h = ah->ah_sh;
 
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_lock_spin(&ah_regser_mtx);
 #if _BYTE_ORDER == _BIG_ENDIAN
 	if (OS_REG_UNSWAPPED(reg))
 		bus_space_write_4(tag, h, reg, val);
 	else
 #endif
 		bus_space_write_stream_4(tag, h, reg, val);
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_unlock_spin(&ah_regser_mtx);
 }
 
 u_int32_t
@@ -330,12 +356,16 @@ ath_hal_reg_read(struct ath_hal *ah, u_int32_t reg)
 	bus_space_handle_t h = ah->ah_sh;
 	u_int32_t val;
 
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_lock_spin(&ah_regser_mtx);
 #if _BYTE_ORDER == _BIG_ENDIAN
 	if (OS_REG_UNSWAPPED(reg))
 		val = bus_space_read_4(tag, h, reg);
 	else
 #endif
 		val = bus_space_read_stream_4(tag, h, reg);
+	if (ah->ah_config.ah_serialise_reg_war)
+		mtx_unlock_spin(&ah_regser_mtx);
 	return val;
 }
 #endif /* AH_DEBUG || AH_REGOPS_FUNC */
diff --git a/sys/dev/ath/ath_hal/ah.c b/sys/dev/ath/ath_hal/ah.c
index 383ae8f..3af16aa 100644
--- a/sys/dev/ath/ath_hal/ah.c
+++ b/sys/dev/ath/ath_hal/ah.c
@@ -114,11 +114,15 @@ ath_hal_mac_name(struct ath_hal *ah)
 	case AR_XSREV_VERSION_SOWL:
 		return "9160";
 	case AR_XSREV_VERSION_MERLIN:
-		return "9280";
+		if (AH_PRIVATE(ah)->ah_ispcie)
+			return "9280";
+		return "9220";
 	case AR_XSREV_VERSION_KITE:
 		return "9285";
 	case AR_XSREV_VERSION_KIWI:
-		return "9287";
+		if (AH_PRIVATE(ah)->ah_ispcie)
+			return "9287";
+		return "9227";
 	}
 	return "????";
 }
@@ -661,6 +665,8 @@ ath_hal_getcapability(struct ath_hal *ah, HAL_CAPABILITY_TYPE type,
 		return pCap->halHasLongRxDescTsf ? HAL_OK : HAL_ENOTSUPP;
 	case HAL_CAP_BB_READ_WAR:		/* Baseband read WAR */
 		return pCap->halHasBBReadWar? HAL_OK : HAL_ENOTSUPP;
+	case HAL_CAP_SERIALISE_WAR:		/* PCI register serialisation */
+		return pCap->halSerialiseRegWar ? HAL_OK : HAL_ENOTSUPP;
 	default:
 		return HAL_EINVAL;
 	}
diff --git a/sys/dev/ath/ath_hal/ah.h b/sys/dev/ath/ath_hal/ah.h
index a5d9156..b195d02 100644
--- a/sys/dev/ath/ath_hal/ah.h
+++ b/sys/dev/ath/ath_hal/ah.h
@@ -150,6 +150,7 @@ typedef enum {
 	HAL_CAP_RXDESC_SELFLINK	= 242,	/* support a self-linked tail RX descriptor */
 	HAL_CAP_LONG_RXDESC_TSF	= 243,	/* hardware supports 32bit TSF in RX descriptor */
 	HAL_CAP_BB_READ_WAR	= 244,	/* baseband read WAR */
+	HAL_CAP_SERIALISE_WAR	= 245,	/* serialise register access on PCI */
 } HAL_CAPABILITY_TYPE;
 
 /* 
@@ -780,6 +781,8 @@ typedef struct
 	int ah_dma_beacon_response_time;/* in TU's */
 	int ah_sw_beacon_response_time;	/* in TU's */
 	int ah_additional_swba_backoff;	/* in TU's */
+	int ah_force_full_reset;	/* force full chip reset rather then warm reset */
+	int ah_serialise_reg_war;	/* force serialisation of register IO */
 } HAL_OPS_CONFIG;
 
 /*
@@ -810,6 +813,8 @@ struct ath_hal {
 
 	uint16_t	*ah_eepromdata;	/* eeprom buffer, if needed */
 
+	uint32_t	ah_intrstate[8];	/* last int state */
+
 	HAL_OPS_CONFIG ah_config;
 	const HAL_RATE_TABLE *__ahdecl(*ah_getRateTable)(struct ath_hal *,
 				u_int mode);
@@ -1023,6 +1028,9 @@ struct ath_hal {
 	    			struct ath_desc *);
 	void	  __ahdecl(*ah_set11nBurstDuration)(struct ath_hal *,
 	    			struct ath_desc *, u_int);
+	uint32_t  __ahdecl(*ah_get_mib_cycle_counts_pct) (struct ath_hal *,
+				uint32_t *, uint32_t *, uint32_t *, uint32_t *);
+
 	uint32_t  __ahdecl(*ah_get11nExtBusy)(struct ath_hal *);
 	void      __ahdecl(*ah_set11nMac2040)(struct ath_hal *,
 				HAL_HT_MACMODE);
diff --git a/sys/dev/ath/ath_hal/ah_devid.h b/sys/dev/ath/ath_hal/ah_devid.h
index c7a98dd..e6bea21 100644
--- a/sys/dev/ath/ath_hal/ah_devid.h
+++ b/sys/dev/ath/ath_hal/ah_devid.h
@@ -73,11 +73,11 @@
 
 /* AR5416 compatible devid's  */
 #define AR5416_DEVID_PCI	0x0023          /* AR5416 PCI (MB/CB) Owl */
-#define AR5416_DEVID_PCIE	0x0024          /* AR5416 PCI-E (XB) Owl */
+#define AR5416_DEVID_PCIE	0x0024          /* AR5418 PCI-E (XB) Owl */
 #define	AR5416_AR9130_DEVID     0x000b          /* AR9130 SoC WiMAC */
 #define AR9160_DEVID_PCI	0x0027          /* AR9160 PCI Sowl */
 #define AR9280_DEVID_PCI	0x0029          /* AR9280 PCI Merlin */
-#define AR9280_DEVID_PCIE	0x002a          /* AR9280 PCI-E Merlin */
+#define AR9280_DEVID_PCIE	0x002a          /* AR9220 PCI-E Merlin */
 #define AR9285_DEVID_PCIE	0x002b          /* AR9285 PCI-E Kite */
 #define	AR2427_DEVID_PCIE	0x002c		/* AR2427 PCI-E w/ 802.11n bonded out */
 #define	AR9287_DEVID_PCI	0x002d		/* AR9227 PCI Kiwi */
diff --git a/sys/dev/ath/ath_hal/ah_internal.h b/sys/dev/ath/ath_hal/ah_internal.h
index 4378c83..a253b65 100644
--- a/sys/dev/ath/ath_hal/ah_internal.h
+++ b/sys/dev/ath/ath_hal/ah_internal.h
@@ -210,7 +210,8 @@ typedef struct {
 			halHasRxSelfLinkedTail		: 1,
 			halSupportsFastClock5GHz	: 1,	/* Hardware supports 5ghz fast clock; check eeprom/channel before using */
 			halHasLongRxDescTsf		: 1,
-			halHasBBReadWar			: 1;
+			halHasBBReadWar			: 1,
+			halSerialiseRegWar		: 1;
 	uint32_t	halWirelessModes;
 	uint16_t	halTotalQueues;
 	uint16_t	halKeyCacheSize;
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416.h b/sys/dev/ath/ath_hal/ar5416/ar5416.h
index 35d9a4d..5006c31 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416.h
@@ -112,11 +112,13 @@ struct ath_hal_5416 {
 	int		ah_hangs;		/* h/w hangs state */
 	uint8_t		ah_keytype[AR5416_KEYTABLE_SIZE];
 	/*
-	 * Extension Channel Rx Clear State
+	 * Primary/Extension Channel Tx, Rx, Rx Clear State
 	 */
 	uint32_t	ah_cycleCount;
 	uint32_t	ah_ctlBusy;
 	uint32_t	ah_extBusy;
+	uint32_t	ah_rxBusy;
+	uint32_t	ah_txBusy;
 	uint32_t	ah_rx_chainmask;
 	uint32_t	ah_tx_chainmask;
 
@@ -194,6 +196,9 @@ extern	uint32_t ar5416GetCurRssi(struct ath_hal *ah);
 extern	HAL_BOOL ar5416SetAntennaSwitch(struct ath_hal *, HAL_ANT_SETTING);
 extern	HAL_BOOL ar5416SetDecompMask(struct ath_hal *, uint16_t, int);
 extern	void ar5416SetCoverageClass(struct ath_hal *, uint8_t, int);
+extern	uint32_t ar5416GetMibCycleCountsPct(struct ath_hal *ah,
+    uint32_t *rxc_pcnt, uint32_t *rxextc_pcnt, uint32_t *rxf_pcnt,
+    uint32_t *txf_pcnt);
 extern	uint32_t ar5416Get11nExtBusy(struct ath_hal *ah);
 extern	void ar5416Set11nMac2040(struct ath_hal *ah, HAL_HT_MACMODE mode);
 extern	HAL_HT_RXCLEAR ar5416Get11nRxClear(struct ath_hal *ah);
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_ani.c b/sys/dev/ath/ath_hal/ar5416/ar5416_ani.c
index 5af6b24..deaacd7 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_ani.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_ani.c
@@ -227,7 +227,7 @@ ar5416AniControl(struct ath_hal *ah, HAL_ANI_CMD cmd, int param)
 		u_int level = param;
 
 		HALDEBUG(ah, HAL_DEBUG_ANI, "%s: HAL_ANI_NOISE_IMMUNITY_LEVEL: set level = %d\n", __func__, level);
-		if (level >= params->maxNoiseImmunityLevel) {
+		if (level > params->maxNoiseImmunityLevel) {
 			HALDEBUG(ah, HAL_DEBUG_ANI,
 			    "%s: immunity level out of range (%u > %u)\n",
 			    __func__, level, params->maxNoiseImmunityLevel);
@@ -314,7 +314,7 @@ ar5416AniControl(struct ath_hal *ah, HAL_ANI_CMD cmd, int param)
 		u_int level = param;
 
 		HALDEBUG(ah, HAL_DEBUG_ANI, "%s: HAL_ANI_FIRSTEP_LEVEL: level = %d\n", __func__, level);
-		if (level >= params->maxFirstepLevel) {
+		if (level > params->maxFirstepLevel) {
 			HALDEBUG(ah, HAL_DEBUG_ANI,
 			    "%s: firstep level out of range (%u > %u)\n",
 			    __func__, level, params->maxFirstepLevel);
@@ -333,7 +333,7 @@ ar5416AniControl(struct ath_hal *ah, HAL_ANI_CMD cmd, int param)
 		u_int level = param;
 
 		HALDEBUG(ah, HAL_DEBUG_ANI, "%s: HAL_ANI_SPUR_IMMUNITY_LEVEL: level = %d\n", __func__, level);
-		if (level >= params->maxSpurImmunityLevel) {
+		if (level > params->maxSpurImmunityLevel) {
 			HALDEBUG(ah, HAL_DEBUG_ANI,
 			    "%s: spur immunity level out of range (%u > %u)\n",
 			    __func__, level, params->maxSpurImmunityLevel);
@@ -342,11 +342,6 @@ ar5416AniControl(struct ath_hal *ah, HAL_ANI_CMD cmd, int param)
 		OS_REG_RMW_FIELD(ah, AR_PHY_TIMING5,
 		    AR_PHY_TIMING5_CYCPWR_THR1, params->cycPwrThr1[level]);
 
-		/* Only set the ext channel cycpwr_thr1 field for ht/40 */
-		if (IEEE80211_IS_CHAN_HT40(AH_PRIVATE(ah)->ah_curchan))
-			OS_REG_RMW_FIELD(ah, AR_PHY_EXT_CCA,
-			    AR_PHY_EXT_TIMING5_CYCPWR_THR1, params->cycPwrThr1[level]);
-
 		if (level > aniState->spurImmunityLevel)
 			ahp->ah_stats.ast_ani_spurup++;
 		else if (level < aniState->spurImmunityLevel)
@@ -384,20 +379,30 @@ ar5416AniOfdmErrTrigger(struct ath_hal *ah)
 	aniState = ahp->ah_curani;
 	params = aniState->params;
 	/* First, raise noise immunity level, up to max */
-	if ((AH5416(ah)->ah_ani_function & (1 << HAL_ANI_NOISE_IMMUNITY_LEVEL)) &&
-	    (aniState->noiseImmunityLevel+1 < params->maxNoiseImmunityLevel)) {
-		ar5416AniControl(ah, HAL_ANI_NOISE_IMMUNITY_LEVEL, 
-				 aniState->noiseImmunityLevel + 1);
-		return;
+	if (aniState->noiseImmunityLevel+1 < params->maxNoiseImmunityLevel) {
+		if (ar5416AniControl(ah, HAL_ANI_NOISE_IMMUNITY_LEVEL, 
+				 aniState->noiseImmunityLevel + 1))
+			return;
 	}
 	/* then, raise spur immunity level, up to max */
-	if ((AH5416(ah)->ah_ani_function & (1 << HAL_ANI_SPUR_IMMUNITY_LEVEL)) &&
-	    (aniState->spurImmunityLevel+1 < params->maxSpurImmunityLevel)) {
-		ar5416AniControl(ah, HAL_ANI_SPUR_IMMUNITY_LEVEL,
-				 aniState->spurImmunityLevel + 1);
-		return;
+	if (aniState->spurImmunityLevel+1 < params->maxSpurImmunityLevel) {
+		if (ar5416AniControl(ah, HAL_ANI_SPUR_IMMUNITY_LEVEL,
+				 aniState->spurImmunityLevel + 1))
+			return;
 	}
 
+	/*
+	 * In the case of AP mode operation, we cannot bucketize beacons
+	 * according to RSSI.  Instead, raise Firstep level, up to max, and
+	 * simply return.
+	 */
+	if (AH_PRIVATE(ah)->ah_opmode == HAL_M_HOSTAP) {
+		if (aniState->firstepLevel < params->maxFirstepLevel) {
+			if (ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
+			    aniState->firstepLevel + 1))
+				return;
+		}
+	}
 	if (ANI_ENA_RSSI(ah)) {
 		int32_t rssi = BEACON_RSSI(ahp);
 		if (rssi > params->rssiThrHigh) {
@@ -418,9 +423,9 @@ ar5416AniOfdmErrTrigger(struct ath_hal *ah)
 			 * raise firstep level 
 			 */
 			if (aniState->firstepLevel+1 < params->maxFirstepLevel) {
-				ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
-						 aniState->firstepLevel + 1);
-				return;
+				if (ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
+						 aniState->firstepLevel + 1))
+					return;
 			}
 		} else if (rssi > params->rssiThrLow) {
 			/* 
@@ -432,9 +437,9 @@ ar5416AniOfdmErrTrigger(struct ath_hal *ah)
 				    HAL_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				    AH_TRUE);
 			if (aniState->firstepLevel+1 < params->maxFirstepLevel)
-				ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
-				     aniState->firstepLevel + 1);
-			return;
+				if (ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
+				     aniState->firstepLevel + 1))
+				return;
 		} else {
 			/* 
 			 * Beacon rssi is low, if in 11b/g mode, turn off ofdm
@@ -447,9 +452,9 @@ ar5416AniOfdmErrTrigger(struct ath_hal *ah)
 					    HAL_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 					    AH_FALSE);
 				if (aniState->firstepLevel > 0)
-					ar5416AniControl(ah,
-					     HAL_ANI_FIRSTEP_LEVEL, 0);
-				return;
+					if (ar5416AniControl(ah,
+					     HAL_ANI_FIRSTEP_LEVEL, 0))
+						return;
 			}
 		}
 	}
@@ -582,6 +587,16 @@ ar5416AniReset(struct ath_hal *ah, const struct ieee80211_channel *chan,
 		goto finish;
 	}
 
+	/*
+	 * Use a restrictive set of ANI parameters for hostap mode.
+	 */
+	if (opmode == HAL_M_HOSTAP) {
+		if (IEEE80211_IS_CHAN_2GHZ(chan))
+			AH5416(ah)->ah_ani_function =
+			    HAL_ANI_SPUR_IMMUNITY_LEVEL | HAL_ANI_FIRSTEP_LEVEL;
+		else
+			AH5416(ah)->ah_ani_function = 0;
+	}
 
 	/*
 	 * Automatic processing is done only in station mode right now.
@@ -611,7 +626,7 @@ ar5416AniReset(struct ath_hal *ah, const struct ieee80211_channel *chan,
 		ar5416AniControl(ah, HAL_ANI_NOISE_IMMUNITY_LEVEL, 0);
 		ar5416AniControl(ah, HAL_ANI_SPUR_IMMUNITY_LEVEL, 0);
 		ar5416AniControl(ah, HAL_ANI_OFDM_WEAK_SIGNAL_DETECTION,
-			AH_TRUE);
+			AH_FALSE);
 		ar5416AniControl(ah, HAL_ANI_CCK_WEAK_SIGNAL_THR, AH_FALSE);
 		ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL, 0);
 		ichan->privFlags |= CHANNEL_ANI_SETUP;
@@ -715,6 +730,19 @@ ar5416AniLowerImmunity(struct ath_hal *ah)
 
 	aniState = ahp->ah_curani;
 	params = aniState->params;
+
+	/*
+	 * In the case of AP mode operation, we cannot bucketize beacons
+	 * according to RSSI.  Instead, lower Firstep level, down to min, and
+	 * simply return.
+	 */
+	if (AH_PRIVATE(ah)->ah_opmode == HAL_M_HOSTAP) {
+		if (aniState->firstepLevel > 0) {
+			if (ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
+			    aniState->firstepLevel - 1))
+				return;
+		}
+	}
 	if (ANI_ENA_RSSI(ah)) {
 		int32_t rssi = BEACON_RSSI(ahp);
 		if (rssi > params->rssiThrHigh) {
@@ -729,41 +757,41 @@ ar5416AniLowerImmunity(struct ath_hal *ah)
 			 * detection or lower firstep level.
 			 */
 			if (aniState->ofdmWeakSigDetectOff) {
-				ar5416AniControl(ah,
+				if (ar5416AniControl(ah,
 				    HAL_ANI_OFDM_WEAK_SIGNAL_DETECTION,
-				    AH_TRUE);
-				return;
+				    AH_TRUE))
+					return;
 			}
 			if (aniState->firstepLevel > 0) {
-				ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
-						 aniState->firstepLevel - 1);
-				return;
+				if (ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
+						 aniState->firstepLevel - 1))
+					return;
 			}
 		} else {
 			/*
 			 * Beacon rssi is low, reduce firstep level.
 			 */
 			if (aniState->firstepLevel > 0) {
-				ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
-						 aniState->firstepLevel - 1);
-				return;
+				if (ar5416AniControl(ah, HAL_ANI_FIRSTEP_LEVEL,
+						 aniState->firstepLevel - 1))
+					return;
 			}
 		}
 	}
 	/* then lower spur immunity level, down to zero */
 	if (aniState->spurImmunityLevel > 0) {
-		ar5416AniControl(ah, HAL_ANI_SPUR_IMMUNITY_LEVEL,
-				 aniState->spurImmunityLevel - 1);
-		return;
+		if (ar5416AniControl(ah, HAL_ANI_SPUR_IMMUNITY_LEVEL,
+				 aniState->spurImmunityLevel - 1))
+			return;
 	}
 	/* 
 	 * if all else fails, lower noise immunity level down to a min value
 	 * zero for now
 	 */
 	if (aniState->noiseImmunityLevel > 0) {
-		ar5416AniControl(ah, HAL_ANI_NOISE_IMMUNITY_LEVEL,
-				 aniState->noiseImmunityLevel - 1);
-		return;
+		if (ar5416AniControl(ah, HAL_ANI_NOISE_IMMUNITY_LEVEL,
+				 aniState->noiseImmunityLevel - 1))
+			return;
 	}
 }
 
@@ -782,15 +810,15 @@ ar5416AniGetListenTime(struct ath_hal *ah)
 {
 	struct ath_hal_5212 *ahp = AH5212(ah);
 	struct ar5212AniState *aniState;
-	uint32_t txFrameCount, rxFrameCount, cycleCount;
+	uint32_t rxc_pct, extc_pct, rxf_pct, txf_pct;
 	int32_t listenTime;
+	int good;
 
-	txFrameCount = OS_REG_READ(ah, AR_TFCNT);
-	rxFrameCount = OS_REG_READ(ah, AR_RFCNT);
-	cycleCount = OS_REG_READ(ah, AR_CCCNT);
+	good = ar5416GetMibCycleCountsPct(ah,
+	&rxc_pct, &extc_pct, &rxf_pct, &txf_pct);
 
 	aniState = ahp->ah_curani;
-	if (aniState->cycleCount == 0 || aniState->cycleCount > cycleCount) {
+	if (good == 0) {
 		/*
 		 * Cycle counter wrap (or initial call); it's not possible
 		 * to accurately calculate a value because the registers
@@ -799,14 +827,18 @@ ar5416AniGetListenTime(struct ath_hal *ah)
 		listenTime = 0;
 		ahp->ah_stats.ast_ani_lzero++;
 	} else {
-		int32_t ccdelta = cycleCount - aniState->cycleCount;
-		int32_t rfdelta = rxFrameCount - aniState->rxFrameCount;
-		int32_t tfdelta = txFrameCount - aniState->txFrameCount;
+		int32_t ccdelta = AH5416(ah)->ah_cycleCount - aniState->cycleCount;
+		int32_t rfdelta = AH5416(ah)->ah_rxBusy - aniState->rxFrameCount;
+		int32_t tfdelta = AH5416(ah)->ah_txBusy - aniState->txFrameCount;
 		listenTime = (ccdelta - rfdelta - tfdelta) / CLOCK_RATE;
 	}
-	aniState->cycleCount = cycleCount;
-	aniState->txFrameCount = txFrameCount;
-	aniState->rxFrameCount = rxFrameCount;
+	aniState->cycleCount = AH5416(ah)->ah_cycleCount;
+	aniState->txFrameCount = AH5416(ah)->ah_rxBusy;
+	aniState->rxFrameCount = AH5416(ah)->ah_txBusy;
+
+	HALDEBUG(ah, HAL_DEBUG_ANI, "rxc=%d, extc=%d, rxf=%d, txf=%d\n",
+	    rxc_pct, extc_pct, rxf_pct, txf_pct);
+
 	return listenTime;
 }
 
@@ -873,10 +905,13 @@ ar5416AniPoll(struct ath_hal *ah, const struct ieee80211_channel *chan)
 	/* XXX can aniState be null? */
 	if (aniState == AH_NULL)
 		return;
+
+	/* Always update from the MIB, for statistics gathering */
+	listenTime = ar5416AniGetListenTime(ah);
+
 	if (!ANI_ENA(ah))
 		return;
 
-	listenTime = ar5416AniGetListenTime(ah);
 	if (listenTime < 0) {
 		ahp->ah_stats.ast_ani_lneg++;
 		/* restart ANI period if listenTime is invalid */
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
index 630efc0..37412fa 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
@@ -908,17 +908,36 @@ ar5416FillCapabilityInfo(struct ath_hal *ah)
 		pCap->halRfSilentSupport = AH_TRUE;
 	}
 
+	/*
+	 * The MAC will mark frames as RXed if there's a descriptor
+	 * to write them to. So if it hits a self-linked final descriptor,
+	 * it'll keep ACKing frames even though they're being silently
+	 * dropped. Thus, this particular feature of the driver can't
+	 * be used for 802.11n devices.
+	 */
 	ahpriv->ah_rxornIsFatal = AH_FALSE;
 
+	/*
+	 * If it's a PCI NIC, ask the HAL OS layer to serialise
+	 * register access, or SMP machines may cause the hardware
+	 * to hang. This is applicable to AR5416 and AR9220; I'm not
+	 * sure about AR9160 or AR9227.
+	 */
+	if (! AH_PRIVATE(ah)->ah_ispcie)
+		pCap->halSerialiseRegWar = 1;
+
 	return AH_TRUE;
 }
 
 static const char*
 ar5416Probe(uint16_t vendorid, uint16_t devid)
 {
-	if (vendorid == ATHEROS_VENDOR_ID &&
-	    (devid == AR5416_DEVID_PCI || devid == AR5416_DEVID_PCIE))
-		return "Atheros 5416";
+	if (vendorid == ATHEROS_VENDOR_ID) {
+		if (devid == AR5416_DEVID_PCI)
+			return "Atheros 5416";
+		if (devid == AR5416_DEVID_PCIE)
+			return "Atheros 5418";
+	}
 	return AH_NULL;
 }
 AH_CHIP(AR5416, ar5416Probe, ar5416Attach);
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c b/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c
index cb57870..d51417f4 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c
@@ -642,15 +642,7 @@ ar5416LoadNF(struct ath_hal *ah, const struct ieee80211_channel *chan)
 	OS_REG_SET_BIT(ah, AR_PHY_AGC_CONTROL, AR_PHY_AGC_CONTROL_NF);
 
 	/* Wait for load to complete, should be fast, a few 10s of us. */
-	/*
-	 * XXX For now, don't be so aggressive in waiting for the NF
-	 * XXX load to complete. A very busy 11n RX load will cause this
-	 * XXX to always fail; so just leave it.
-	 * XXX Later on we may wish to split longcal into two parts - one to do
-	 * XXX the initial longcal, and one to load in an updated NF value
-	 * XXX once it's finished - say, by checking it every 500ms.
-	 */
-	if (! ar5212WaitNFCalComplete(ah, 5)) {
+	if (! ar5212WaitNFCalComplete(ah, 1000)) {
 		/*
 		 * We timed out waiting for the noisefloor to load, probably due to an
 		 * in-progress rx. Simply return here and allow the load plenty of time
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c b/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c
index 3bc2cc5..35511ae 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_interrupts.c
@@ -70,6 +70,13 @@ ar5416GetPendingInterrupts(struct ath_hal *ah, HAL_INT *masked)
 	uint32_t isr, isr0, isr1, sync_cause = 0;
 	HAL_CAPABILITIES *pCap = &AH_PRIVATE(ah)->ah_caps;
 
+#ifdef	AH_INTERRUPT_DEBUGGING
+	/*
+	 * Blank the interrupt debugging area regardless.
+	 */
+	bzero(&ah->ah_intrstate, sizeof(ah->ah_intrstate));
+#endif
+
 	/*
 	 * Verify there's a mac interrupt and the RTC is on.
 	 */
@@ -90,6 +97,16 @@ ar5416GetPendingInterrupts(struct ath_hal *ah, HAL_INT *masked)
 			return AH_FALSE;
 	}
 
+#ifdef	AH_INTERRUPT_DEBUGGING
+	ah->ah_intrstate[0] = isr;
+	ah->ah_intrstate[1] = OS_REG_READ(ah, AR_ISR_S0);
+	ah->ah_intrstate[2] = OS_REG_READ(ah, AR_ISR_S1);
+	ah->ah_intrstate[3] = OS_REG_READ(ah, AR_ISR_S2);
+	ah->ah_intrstate[4] = OS_REG_READ(ah, AR_ISR_S3);
+	ah->ah_intrstate[5] = OS_REG_READ(ah, AR_ISR_S4);
+	ah->ah_intrstate[6] = OS_REG_READ(ah, AR_ISR_S5);
+#endif
+
 	if (isr != 0) {
 		struct ath_hal_5212 *ahp = AH5212(ah);
 		uint32_t mask2;
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c b/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c
index 880900f..b1c1b98 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_misc.c
@@ -172,6 +172,57 @@ ar5416SetCoverageClass(struct ath_hal *ah, uint8_t coverageclass, int now)
 }
 
 /*
+ * Return the busy for rx_frame, rx_clear, and tx_frame
+ */
+uint32_t
+ar5416GetMibCycleCountsPct(struct ath_hal *ah, uint32_t *rxc_pcnt,
+    uint32_t *extc_pcnt, uint32_t *rxf_pcnt, uint32_t *txf_pcnt)
+{
+	struct ath_hal_5416 *ahp = AH5416(ah);
+	u_int32_t good = 1;
+
+	/* XXX freeze/unfreeze mib counters */
+	uint32_t rc = OS_REG_READ(ah, AR_RCCNT);
+	uint32_t ec = OS_REG_READ(ah, AR_EXTRCCNT);
+	uint32_t rf = OS_REG_READ(ah, AR_RFCNT);
+	uint32_t tf = OS_REG_READ(ah, AR_TFCNT);
+	uint32_t cc = OS_REG_READ(ah, AR_CCCNT); /* read cycles last */
+
+	if (ahp->ah_cycleCount == 0 || ahp->ah_cycleCount > cc) {
+		/*
+		 * Cycle counter wrap (or initial call); it's not possible
+		 * to accurately calculate a value because the registers
+		 * right shift rather than wrap--so punt and return 0.
+		 */
+		HALDEBUG(ah, HAL_DEBUG_ANY,
+			    "%s: cycle counter wrap. ExtBusy = 0\n", __func__);
+			good = 0;
+	} else {
+		uint32_t cc_d = cc - ahp->ah_cycleCount;
+		uint32_t rc_d = rc - ahp->ah_ctlBusy;
+		uint32_t ec_d = ec - ahp->ah_extBusy;
+		uint32_t rf_d = rf - ahp->ah_rxBusy;
+		uint32_t tf_d = tf - ahp->ah_txBusy;
+
+		if (cc_d != 0) {
+			*rxc_pcnt = rc_d * 100 / cc_d;
+			*rxf_pcnt = rf_d * 100 / cc_d;
+			*txf_pcnt = tf_d * 100 / cc_d;
+			*extc_pcnt = ec_d * 100 / cc_d;
+		} else {
+			good = 0;
+		}
+	}
+	ahp->ah_cycleCount = cc;
+	ahp->ah_rxBusy = rf;
+	ahp->ah_ctlBusy = rc;
+	ahp->ah_txBusy = tf;
+	ahp->ah_extBusy = ec;
+
+	return good;
+}
+
+/*
  * Return approximation of extension channel busy over an time interval
  * 0% (clear) -> 100% (busy)
  *
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c b/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
index 97d292e..86a3f92 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
@@ -146,7 +146,9 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 
 	/* For chips on which the RTC reset is done, save TSF before it gets cleared */
 	if (AR_SREV_HOWL(ah) ||
-	    (AR_SREV_MERLIN(ah) && ath_hal_eepromGetFlag(ah, AR_EEP_OL_PWRCTRL)))
+	    (AR_SREV_MERLIN(ah) &&
+	     ath_hal_eepromGetFlag(ah, AR_EEP_OL_PWRCTRL)) ||
+	    (ah->ah_config.ah_force_full_reset))
 		tsf = ar5416GetTsf64(ah);
 
 	/* Mark PHY as inactive; marked active in ar5416InitBB() */
@@ -722,6 +724,20 @@ ar5416SetRfMode(struct ath_hal *ah, const struct ieee80211_channel *chan)
 		rfMode |= IEEE80211_IS_CHAN_5GHZ(chan) ?
 			AR_PHY_MODE_RF5GHZ : AR_PHY_MODE_RF2GHZ;
 	}
+
+	/*
+	 * Set half/quarter mode flags if required.
+	 *
+	 * This doesn't change the IFS timings at all; that needs to
+	 * be done as part of the MAC setup.  Similarly, the PLL
+	 * configuration also needs some changes for the half/quarter
+	 * rate clock.
+	 */
+	if (IEEE80211_IS_CHAN_HALF(chan))
+		rfMode |= AR_PHY_MODE_HALF;
+	else if (IEEE80211_IS_CHAN_QUARTER(chan))
+		rfMode |= AR_PHY_MODE_QUARTER;
+
 	OS_REG_WRITE(ah, AR_PHY_MODE, rfMode);
 }
 
@@ -733,12 +749,15 @@ ar5416ChipReset(struct ath_hal *ah, const struct ieee80211_channel *chan)
 {
 	OS_MARK(ah, AH_MARK_CHIPRESET, chan ? chan->ic_freq : 0);
 	/*
-	 * Warm reset is optimistic.
+	 * Warm reset is optimistic for open-loop TX power control.
 	 */
 	if (AR_SREV_MERLIN(ah) &&
 	    ath_hal_eepromGetFlag(ah, AR_EEP_OL_PWRCTRL)) {
 		if (!ar5416SetResetReg(ah, HAL_RESET_POWER_ON))
 			return AH_FALSE;
+	} else if (ah->ah_config.ah_force_full_reset) {
+		if (!ar5416SetResetReg(ah, HAL_RESET_POWER_ON))
+			return AH_FALSE;
 	} else {
 		if (!ar5416SetResetReg(ah, HAL_RESET_WARM))
 			return AH_FALSE;
@@ -1178,7 +1197,7 @@ ar5416GetRfgain(struct ath_hal *ah)
 HAL_BOOL
 ar5416Disable(struct ath_hal *ah)
 {
-	if (!ar5212SetPowerMode(ah, HAL_PM_AWAKE, AH_TRUE))
+	if (!ar5416SetPowerMode(ah, HAL_PM_AWAKE, AH_TRUE))
 		return AH_FALSE;
 	if (! ar5416SetResetReg(ah, HAL_RESET_COLD))
 		return AH_FALSE;
@@ -1209,6 +1228,12 @@ ar5416PhyDisable(struct ath_hal *ah)
 HAL_BOOL
 ar5416SetResetReg(struct ath_hal *ah, uint32_t type)
 {
+	/*
+	 * Set force wake
+	 */
+	OS_REG_WRITE(ah, AR_RTC_FORCE_WAKE,
+	    AR_RTC_FORCE_WAKE_EN | AR_RTC_FORCE_WAKE_ON_INT);
+
 	switch (type) {
 	case HAL_RESET_POWER_ON:
 		return ar5416SetResetPowerOn(ah);
@@ -1239,10 +1264,15 @@ ar5416SetResetPowerOn(struct ath_hal *ah)
             AR_RTC_FORCE_WAKE_EN | AR_RTC_FORCE_WAKE_ON_INT);    
 
     /*
-     * RTC reset and clear
+     * PowerOn reset can be used in open loop power control or failure recovery.
+     * If we do RTC reset while DMA is still running, hardware may corrupt memory.
+     * Therefore, we need to reset AHB first to stop DMA.
      */
     if (! AR_SREV_HOWL(ah))
     	OS_REG_WRITE(ah, AR_RC, AR_RC_AHB);
+    /*
+     * RTC reset and clear
+     */
     OS_REG_WRITE(ah, AR_RTC_RESET, 0);
     OS_DELAY(20);
 
@@ -1293,6 +1323,11 @@ ar5416SetReset(struct ath_hal *ah, int type)
 #endif	/* AH_SUPPORT_AR9130 */
         /*
          * Reset AHB
+         *
+         * (In case the last interrupt source was a bus timeout.)
+         * XXX TODO: this is not the way to do it! It should be recorded
+         * XXX by the interrupt handler and passed _into_ the
+         * XXX reset path routine so this occurs.
          */
         tmpReg = OS_REG_READ(ah, AR_INTR_SYNC_CAUSE);
         if (tmpReg & (AR_INTR_SYNC_LOCAL_TIMEOUT|AR_INTR_SYNC_RADM_CPL_TIMEOUT)) {
@@ -2608,7 +2643,7 @@ ar5416OverrideIni(struct ath_hal *ah, const struct ieee80211_channel *chan)
 		if (!AR_SREV_9271(ah))
 			val &= ~AR_PCU_MISC_MODE2_HWWAR1;
 
-		if (AR_SREV_KIWI_11_OR_LATER(ah))
+		if (AR_SREV_KIWI_10_OR_LATER(ah))
 			val = val & (~AR_PCU_MISC_MODE2_HWWAR2);
 
 		OS_REG_WRITE(ah, AR_PCU_MISC_MODE2, val);
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416phy.h b/sys/dev/ath/ath_hal/ar5416/ar5416phy.h
index 82e3801..a983277 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416phy.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416phy.h
@@ -121,12 +121,6 @@
 #define AR_PHY_EXT_MINCCA_PWR_S 23
 #define AR_PHY_EXT_CCA_THRESH62	0x007F0000
 #define AR_PHY_EXT_CCA_THRESH62_S	16
-/*
- * This duplicates AR_PHY_EXT_CCA_CYCPWR_THR1; it reads more like
- * an ANI register this way.
- */
-#define	AR_PHY_EXT_TIMING5_CYCPWR_THR1		0x0000FE00
-#define	AR_PHY_EXT_TIMING5_CYCPWR_THR1_S	9
 
 #define AR9280_PHY_EXT_MINCCA_PWR       0x01FF0000
 #define AR9280_PHY_EXT_MINCCA_PWR_S     16
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416reg.h b/sys/dev/ath/ath_hal/ar5416/ar5416reg.h
index 8644f0d..6f92eb7 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416reg.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416reg.h
@@ -612,10 +612,10 @@
 #define	AR_XSREV_REVISION_KITE_11	1	/* Kite 1.1 */
 #define	AR_XSREV_REVISION_KITE_12	2	/* Kite 1.2 */
 #define	AR_XSREV_VERSION_KIWI		0x180	/* Kiwi (AR9287) */
-#define	AR_XSREV_REVISION_KIWI_10	0
-#define	AR_XSREV_REVISION_KIWI_11	1
-#define	AR_XSREV_REVISION_KIWI_12	2
-#define	AR_XSREV_REVISION_KIWI_13	3
+#define	AR_XSREV_REVISION_KIWI_10	0	/* Kiwi 1.0 */
+#define	AR_XSREV_REVISION_KIWI_11	1	/* Kiwi 1.1 */
+#define	AR_XSREV_REVISION_KIWI_12	2	/* Kiwi 1.2 */
+#define	AR_XSREV_REVISION_KIWI_13	3	/* Kiwi 1.3 */
 
 /* Owl (AR5416) */
 #define	AR_SREV_OWL(_ah) \
@@ -701,6 +701,10 @@
 #define AR_SREV_KIWI(_ah) \
 	(AH_PRIVATE((_ah))->ah_macVersion == AR_XSREV_VERSION_KIWI)
 
+#define AR_SREV_KIWI_10_OR_LATER(_ah) \
+	(AH_PRIVATE((_ah))->ah_macVersion >= AR_XSREV_VERSION_KIWI)
+
+/* XXX TODO: make these handle macVersion > Kiwi */
 #define AR_SREV_KIWI_11_OR_LATER(_ah) \
 	(AR_SREV_KIWI(_ah) && \
 	 AH_PRIVATE((_ah))->ah_macRev >= AR_XSREV_REVISION_KIWI_11)
diff --git a/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c b/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c
index 62864b5..437e8ee 100644
--- a/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c
+++ b/sys/dev/ath/ath_hal/ar9002/ar9280_attach.c
@@ -892,9 +892,12 @@ ar9280SetAntennaSwitch(struct ath_hal *ah, HAL_ANT_SETTING settings)
 static const char*
 ar9280Probe(uint16_t vendorid, uint16_t devid)
 {
-	if (vendorid == ATHEROS_VENDOR_ID &&
-	    (devid == AR9280_DEVID_PCI || devid == AR9280_DEVID_PCIE))
-		return "Atheros 9280";
+	if (vendorid == ATHEROS_VENDOR_ID) {
+		if (devid == AR9280_DEVID_PCI)
+			return "Atheros 9220";
+		if (devid == AR9280_DEVID_PCIE)
+			return "Atheros 9280";
+	}
 	return AH_NULL;
 }
 AH_CHIP(AR9280, ar9280Probe, ar9280Attach);
diff --git a/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c b/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c
index 5c0de21..78f7493 100644
--- a/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c
+++ b/sys/dev/ath/ath_hal/ar9002/ar9287_attach.c
@@ -76,7 +76,7 @@ ar9287AniSetup(struct ath_hal *ah)
 	/*
 	 * These are the parameters from the AR5416 ANI code;
 	 * they likely need quite a bit of adjustment for the
-	 * AR9280.
+	 * AR9287.
 	 */
         static const struct ar5212AniParams aniparams = {
                 .maxNoiseImmunityLevel  = 4,    /* levels 0..4 */
@@ -402,13 +402,6 @@ ar9287WriteIni(struct ath_hal *ah, const struct ieee80211_channel *chan)
 	regWrites = ath_hal_ini_write(ah, &AH5212(ah)->ah_ini_common, 1, regWrites);
 }
 
-#define	AR_BASE_FREQ_2GHZ	2300
-#define	AR_BASE_FREQ_5GHZ	4900
-#define	AR_SPUR_FEEQ_BOUND_HT40	19
-#define	AR_SPUR_FEEQ_BOUND_HT20	10
-
-
-
 /*
  * Fill all software cached or static hardware state information.
  * Return failure if capabilities are to come from EEPROM and
@@ -460,7 +453,7 @@ ar9287FillCapabilityInfo(struct ath_hal *ah)
  * This has been disabled - having the HAL flip chainmasks on/off
  * when attempting to implement 11n disrupts things. For now, just
  * leave this flipped off and worry about implementing TX diversity
- * for legacy and MCS0-7 when 11n is fully functioning.
+ * for legacy and MCS0-15 when 11n is fully functioning.
  */
 HAL_BOOL
 ar9287SetAntennaSwitch(struct ath_hal *ah, HAL_ANT_SETTING settings)
@@ -471,9 +464,12 @@ ar9287SetAntennaSwitch(struct ath_hal *ah, HAL_ANT_SETTING settings)
 static const char*
 ar9287Probe(uint16_t vendorid, uint16_t devid)
 {
-	if (vendorid == ATHEROS_VENDOR_ID &&
-	    (devid == AR9287_DEVID_PCI || devid == AR9287_DEVID_PCIE))
-		return "Atheros 9287";
+	if (vendorid == ATHEROS_VENDOR_ID) {
+		if (devid == AR9287_DEVID_PCI)
+			return "Atheros 9227";
+		if (devid == AR9287_DEVID_PCIE)
+			return "Atheros 9287";
+	}
 	return AH_NULL;
 }
 AH_CHIP(AR9287, ar9287Probe, ar9287Attach);
diff --git a/sys/dev/ath/ath_rate/amrr/amrr.c b/sys/dev/ath/ath_rate/amrr/amrr.c
index 5fee76e..b10b826 100644
--- a/sys/dev/ath/ath_rate/amrr/amrr.c
+++ b/sys/dev/ath/ath_rate/amrr/amrr.c
@@ -122,19 +122,21 @@ ath_rate_findrate(struct ath_softc *sc, struct ath_node *an,
  */
 void
 ath_rate_getxtxrates(struct ath_softc *sc, struct ath_node *an,
-    uint8_t rix0, uint8_t *rix, uint8_t *try)
+    uint8_t rix0, struct ath_rc_series *rc)
 {
 	struct amrr_node *amn = ATH_NODE_AMRR(an);
 
-/*	rix[0] = amn->amn_tx_rate0; */
-	rix[1] = amn->amn_tx_rate1;
-	rix[2] = amn->amn_tx_rate2;
-	rix[3] = amn->amn_tx_rate3;
+	rc[0].flags = rc[1].flags = rc[2].flags = rc[3].flags = 0;
 
-	try[0] = amn->amn_tx_try0;
-	try[1] = amn->amn_tx_try1;
-	try[2] = amn->amn_tx_try2;
-	try[3] = amn->amn_tx_try3;
+	rc[0].rix = amn->amn_tx_rate0;
+	rc[1].rix = amn->amn_tx_rate1;
+	rc[2].rix = amn->amn_tx_rate2;
+	rc[3].rix = amn->amn_tx_rate3;
+
+	rc[0].tries = amn->amn_tx_try0;
+	rc[1].tries = amn->amn_tx_try1;
+	rc[2].tries = amn->amn_tx_try2;
+	rc[3].tries = amn->amn_tx_try3;
 }
 
 
@@ -153,10 +155,10 @@ ath_rate_setupxtxdesc(struct ath_softc *sc, struct ath_node *an,
 
 void
 ath_rate_tx_complete(struct ath_softc *sc, struct ath_node *an,
-	const struct ath_buf *bf)
+	const struct ath_rc_series *rc, const struct ath_tx_status *ts,
+	int frame_size, int nframes, int nbad)
 {
 	struct amrr_node *amn = ATH_NODE_AMRR(an);
-	const struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
 	int sr = ts->ts_shortretry;
 	int lr = ts->ts_longretry;
 	int retry_count = sr + lr;
diff --git a/sys/dev/ath/ath_rate/onoe/onoe.c b/sys/dev/ath/ath_rate/onoe/onoe.c
index 77236ad..b5e2c2d 100644
--- a/sys/dev/ath/ath_rate/onoe/onoe.c
+++ b/sys/dev/ath/ath_rate/onoe/onoe.c
@@ -130,19 +130,21 @@ ath_rate_findrate(struct ath_softc *sc, struct ath_node *an,
  */
 void
 ath_rate_getxtxrates(struct ath_softc *sc, struct ath_node *an,
-    uint8_t rix0, uint8_t *rix, uint8_t *try)
+    uint8_t rix0, struct ath_rc_series *rc)
 {
 	struct onoe_node *on = ATH_NODE_ONOE(an);
 
-/*	rix[0] = on->on_tx_rate0; */
-	rix[1] = on->on_tx_rate1;
-	rix[2] = on->on_tx_rate2;
-	rix[3] = on->on_tx_rate3;
+	rc[0].flags = rc[1].flags = rc[2].flags = rc[3].flags = 0;
 
-	try[0] = on->on_tx_try0;
-	try[1] = 2;
-	try[2] = 2;
-	try[3] = 2;
+	rc[0].rix = on->on_tx_rate0;
+	rc[1].rix = on->on_tx_rate1;
+	rc[2].rix = on->on_tx_rate2;
+	rc[3].rix = on->on_tx_rate3;
+
+	rc[0].tries = on->on_tx_try0;
+	rc[1].tries = 2;
+	rc[2].tries = 2;
+	rc[3].tries = 2;
 }
 
 void
@@ -160,10 +162,10 @@ ath_rate_setupxtxdesc(struct ath_softc *sc, struct ath_node *an,
 
 void
 ath_rate_tx_complete(struct ath_softc *sc, struct ath_node *an,
-	const struct ath_buf *bf)
+	const struct ath_rc_series *rc, const struct ath_tx_status *ts,
+	int frame_size, int nframes, int nbad)
 {
 	struct onoe_node *on = ATH_NODE_ONOE(an);
-	const struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
 
 	if (ts->ts_status == 0)
 		on->on_tx_ok++;
diff --git a/sys/dev/ath/ath_rate/sample/sample.c b/sys/dev/ath/ath_rate/sample/sample.c
index 27bb98c..47ca063 100644
--- a/sys/dev/ath/ath_rate/sample/sample.c
+++ b/sys/dev/ath/ath_rate/sample/sample.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
  */
 #include "opt_inet.h"
 #include "opt_wlan.h"
+#include "opt_ah.h"
 
 #include <sys/param.h>
 #include <sys/systm.h> 
@@ -146,6 +147,8 @@ ath_rate_node_cleanup(struct ath_softc *sc, struct ath_node *an)
 static int
 dot11rate(const HAL_RATE_TABLE *rt, int rix)
 {
+	if (rix < 0)
+		return -1;
 	return rt->info[rix].phy == IEEE80211_T_HT ?
 	    rt->info[rix].dot11Rate : (rt->info[rix].dot11Rate & IEEE80211_RATE_VAL) / 2;
 }
@@ -153,6 +156,8 @@ dot11rate(const HAL_RATE_TABLE *rt, int rix)
 static const char *
 dot11rate_label(const HAL_RATE_TABLE *rt, int rix)
 {
+	if (rix < 0)
+		return "";
 	return rt->info[rix].phy == IEEE80211_T_HT ? "MCS" : "Mb ";
 }
 
@@ -165,12 +170,13 @@ pick_best_rate(struct ath_node *an, const HAL_RATE_TABLE *rt,
     int size_bin, int require_acked_before)
 {
 	struct sample_node *sn = ATH_NODE_SAMPLE(an);
-        int best_rate_rix, best_rate_tt;
+        int best_rate_rix, best_rate_tt, best_rate_pct;
 	uint32_t mask;
-	int rix, tt;
+	int rix, tt, pct;
 
         best_rate_rix = 0;
         best_rate_tt = 0;
+	best_rate_pct = 0;
 	for (mask = sn->ratemask, rix = 0; mask != 0; mask >>= 1, rix++) {
 		if ((mask & 1) == 0)		/* not a supported rate */
 			continue;
@@ -187,13 +193,54 @@ pick_best_rate(struct ath_node *an, const HAL_RATE_TABLE *rt,
 		     !sn->stats[size_bin][rix].packets_acked))
 			continue;
 
+		/* Calculate percentage if possible */
+		if (sn->stats[size_bin][rix].total_packets > 0) {
+			pct = sn->stats[size_bin][rix].ewma_pct;
+		} else {
+			/* XXX for now, assume 95% ok */
+			pct = 95;
+		}
+
 		/* don't use a bit-rate that has been failing */
 		if (sn->stats[size_bin][rix].successive_failures > 3)
 			continue;
 
-		if (best_rate_tt == 0 || tt < best_rate_tt) {
-			best_rate_tt = tt;
-			best_rate_rix = rix;
+		/*
+		 * For HT, Don't use a bit rate that is much more
+		 * lossy than the best.
+		 *
+		 * XXX this isn't optimal; it's just designed to
+		 * eliminate rates that are going to be obviously
+		 * worse.
+		 */
+		if (an->an_node.ni_flags & IEEE80211_NODE_HT) {
+			if (best_rate_pct > (pct + 50))
+				continue;
+		}
+
+		/*
+		 * For non-MCS rates, use the current average txtime for
+		 * comparison.
+		 */
+		if (! (an->an_node.ni_flags & IEEE80211_NODE_HT)) {
+			if (best_rate_tt == 0 || tt <= best_rate_tt) {
+				best_rate_tt = tt;
+				best_rate_rix = rix;
+				best_rate_pct = pct;
+			}
+		}
+
+		/*
+		 * Since 2 stream rates have slightly higher TX times,
+		 * allow a little bit of leeway. This should later
+		 * be abstracted out and properly handled.
+		 */
+		if (an->an_node.ni_flags & IEEE80211_NODE_HT) {
+			if (best_rate_tt == 0 || (tt * 8 <= best_rate_tt * 10)) {
+				best_rate_tt = tt;
+				best_rate_rix = rix;
+				best_rate_pct = pct;
+			}
 		}
         }
         return (best_rate_tt ? best_rate_rix : -1);
@@ -252,6 +299,28 @@ pick_sample_rate(struct sample_softc *ssc , struct ath_node *an,
 			goto nextrate;
 		}
 
+		/*
+		 * When doing aggregation, successive failures don't happen
+		 * as often, as sometimes some of the sub-frames get through.
+		 *
+		 * If the sample rix average tx time is greater than the
+		 * average tx time of the current rix, don't immediately use
+		 * the rate for sampling.
+		 */
+		if (an->an_node.ni_flags & IEEE80211_NODE_HT) {
+			if ((sn->stats[size_bin][rix].average_tx_time * 10 >
+			    sn->stats[size_bin][current_rix].average_tx_time * 9) &&
+			    (ticks - sn->stats[size_bin][rix].last_tx < ssc->stale_failure_timeout)) {
+				mask &= ~(1<<rix);
+				goto nextrate;
+			}
+		}
+
+		/*
+		 * XXX TODO
+		 * For HT, limit sample somehow?
+		 */
+
 		/* Don't sample more than 2 rates higher for rates > 11M for non-HT rates */
 		if (! (an->an_node.ni_flags & IEEE80211_NODE_HT)) {
 			if (DOT11RATE(rix) > 2*11 && rix > current_rix + 2) {
@@ -315,6 +384,96 @@ ath_rate_update_static_rix(struct ath_softc *sc, struct ieee80211_node *ni)
 	}
 }
 
+/*
+ * Pick a non-HT rate to begin using.
+ */
+static int
+ath_rate_pick_seed_rate_legacy(struct ath_softc *sc, struct ath_node *an,
+    int frameLen)
+{
+#define	DOT11RATE(ix)	(rt->info[ix].dot11Rate & IEEE80211_RATE_VAL)
+#define	MCS(ix)		(rt->info[ix].dot11Rate | IEEE80211_RATE_MCS)
+#define	RATE(ix)	(DOT11RATE(ix) / 2)
+	int rix = -1;
+	const HAL_RATE_TABLE *rt = sc->sc_currates;
+	struct sample_node *sn = ATH_NODE_SAMPLE(an);
+	const int size_bin = size_to_bin(frameLen);
+
+	/* no packet has been sent successfully yet */
+	for (rix = rt->rateCount-1; rix > 0; rix--) {
+		if ((sn->ratemask & (1<<rix)) == 0)
+			continue;
+
+		/* Skip HT rates */
+		if (rt->info[rix].phy == IEEE80211_T_HT)
+			continue;
+
+		/*
+		 * Pick the highest rate <= 36 Mbps
+		 * that hasn't failed.
+		 */
+		if (DOT11RATE(rix) <= 72 &&
+		    sn->stats[size_bin][rix].successive_failures == 0) {
+			break;
+		}
+	}
+	return rix;
+#undef	RATE
+#undef	MCS
+#undef	DOT11RATE
+}
+
+/*
+ * Pick a HT rate to begin using.
+ *
+ * Don't use any non-HT rates; only consider HT rates.
+ */
+static int
+ath_rate_pick_seed_rate_ht(struct ath_softc *sc, struct ath_node *an,
+    int frameLen)
+{
+#define	DOT11RATE(ix)	(rt->info[ix].dot11Rate & IEEE80211_RATE_VAL)
+#define	MCS(ix)		(rt->info[ix].dot11Rate | IEEE80211_RATE_MCS)
+#define	RATE(ix)	(DOT11RATE(ix) / 2)
+	int rix = -1, ht_rix = -1;
+	const HAL_RATE_TABLE *rt = sc->sc_currates;
+	struct sample_node *sn = ATH_NODE_SAMPLE(an);
+	const int size_bin = size_to_bin(frameLen);
+
+	/* no packet has been sent successfully yet */
+	for (rix = rt->rateCount-1; rix > 0; rix--) {
+		/* Skip rates we can't use */
+		if ((sn->ratemask & (1<<rix)) == 0)
+			continue;
+
+		/* Keep a copy of the last seen HT rate index */
+		if (rt->info[rix].phy == IEEE80211_T_HT)
+			ht_rix = rix;
+
+		/* Skip non-HT rates */
+		if (rt->info[rix].phy != IEEE80211_T_HT)
+			continue;
+
+		/*
+		 * Pick a medium-speed rate regardless of stream count
+		 * which has not seen any failures. Higher rates may fail;
+		 * we'll try them later.
+		 */
+		if (((MCS(rix) & 0x7) <= 4) &&
+		    sn->stats[size_bin][rix].successive_failures == 0) {
+			break;
+		}
+	}
+
+	/*
+	 * If all the MCS rates have successive failures, rix should be
+	 * > 0; otherwise use the lowest MCS rix (hopefully MCS 0.)
+	 */
+	return MAX(rix, ht_rix);
+#undef	RATE
+#undef	MCS
+#undef	DOT11RATE
+}
 
 
 void
@@ -358,9 +517,14 @@ ath_rate_findrate(struct ath_softc *sc, struct ath_node *an,
 	if (sn->sample_tt[size_bin] < average_tx_time * (sn->packets_since_sample[size_bin]*ssc->sample_rate/100)) {
 		rix = pick_sample_rate(ssc, an, rt, size_bin);
 		IEEE80211_NOTE(an->an_node.ni_vap, IEEE80211_MSG_RATECTL,
-		     &an->an_node, "size %u sample rate %d current rate %d",
-		     bin_to_size(size_bin), RATE(rix),
-		     RATE(sn->current_rix[size_bin]));
+		     &an->an_node, "att %d sample_tt %d size %u sample rate %d %s current rate %d %s",
+		     average_tx_time,
+		     sn->sample_tt[size_bin],
+		     bin_to_size(size_bin),
+		     dot11rate(rt, rix),
+		     dot11rate_label(rt, rix),
+		     dot11rate(rt, sn->current_rix[size_bin]),
+		     dot11rate_label(rt, sn->current_rix[size_bin]));
 		if (rix != sn->current_rix[size_bin]) {
 			sn->current_sample_rix[size_bin] = rix;
 		} else {
@@ -371,29 +535,58 @@ ath_rate_findrate(struct ath_softc *sc, struct ath_node *an,
 		change_rates = 0;
 		if (!sn->packets_sent[size_bin] || best_rix == -1) {
 			/* no packet has been sent successfully yet */
-			for (rix = rt->rateCount-1; rix > 0; rix--) {
-				if ((sn->ratemask & (1<<rix)) == 0)
-					continue;
-				/* 
-				 * Pick the highest rate <= 36 Mbps
-				 * that hasn't failed.
-				 */
-				if (DOT11RATE(rix) <= 72 && 
-				    sn->stats[size_bin][rix].successive_failures == 0) {
-					break;
-				}
-			}
 			change_rates = 1;
-			best_rix = rix;
+			if (an->an_node.ni_flags & IEEE80211_NODE_HT)
+				best_rix =
+				    ath_rate_pick_seed_rate_ht(sc, an, frameLen);
+			else
+				best_rix =
+				    ath_rate_pick_seed_rate_legacy(sc, an, frameLen);
 		} else if (sn->packets_sent[size_bin] < 20) {
 			/* let the bit-rate switch quickly during the first few packets */
+			IEEE80211_NOTE(an->an_node.ni_vap,
+			    IEEE80211_MSG_RATECTL, &an->an_node,
+			    "%s: switching quickly..", __func__);
 			change_rates = 1;
 		} else if (ticks - ssc->min_switch > sn->ticks_since_switch[size_bin]) {
 			/* min_switch seconds have gone by */
+			IEEE80211_NOTE(an->an_node.ni_vap,
+			    IEEE80211_MSG_RATECTL, &an->an_node,
+			    "%s: min_switch %d > ticks_since_switch %d..",
+			    __func__, ticks - ssc->min_switch, sn->ticks_since_switch[size_bin]);
 			change_rates = 1;
-		} else if (2*average_tx_time < sn->stats[size_bin][sn->current_rix[size_bin]].average_tx_time) {
+		} else if ((! (an->an_node.ni_flags & IEEE80211_NODE_HT)) &&
+		    (2*average_tx_time < sn->stats[size_bin][sn->current_rix[size_bin]].average_tx_time)) {
 			/* the current bit-rate is twice as slow as the best one */
+			IEEE80211_NOTE(an->an_node.ni_vap,
+			    IEEE80211_MSG_RATECTL, &an->an_node,
+			    "%s: 2x att (= %d) < cur_rix att %d",
+			    __func__,
+			    2 * average_tx_time, sn->stats[size_bin][sn->current_rix[size_bin]].average_tx_time);
 			change_rates = 1;
+		} else if ((an->an_node.ni_flags & IEEE80211_NODE_HT)) {
+			int cur_rix = sn->current_rix[size_bin];
+			int cur_att = sn->stats[size_bin][cur_rix].average_tx_time;
+			/*
+			 * If the node is HT, upgrade it if the MCS rate is
+			 * higher and the average tx time is within 20% of
+			 * the current rate. It can fail a little.
+			 *
+			 * This is likely not optimal!
+			 */
+#if 0
+			printf("cur rix/att %x/%d, best rix/att %x/%d\n",
+			    MCS(cur_rix), cur_att, MCS(best_rix), average_tx_time);
+#endif
+			if ((MCS(best_rix) > MCS(cur_rix)) &&
+			    (average_tx_time * 8) <= (cur_att * 10)) {
+				IEEE80211_NOTE(an->an_node.ni_vap,
+				    IEEE80211_MSG_RATECTL, &an->an_node,
+				    "%s: HT: best_rix 0x%d > cur_rix 0x%x, average_tx_time %d, cur_att %d",
+				    __func__,
+				    MCS(best_rix), MCS(cur_rix), average_tx_time, cur_att);
+				change_rates = 1;
+			}
 		}
 
 		sn->packets_since_sample[size_bin]++;
@@ -445,22 +638,24 @@ done:
  */
 void
 ath_rate_getxtxrates(struct ath_softc *sc, struct ath_node *an,
-    uint8_t rix0, uint8_t *rix, uint8_t *try)
+    uint8_t rix0, struct ath_rc_series *rc)
 {
 	struct sample_node *sn = ATH_NODE_SAMPLE(an);
 	const struct txschedule *sched = &sn->sched[rix0];
 
 	KASSERT(rix0 == sched->r0, ("rix0 (%x) != sched->r0 (%x)!\n", rix0, sched->r0));
 
-/*	rix[0] = sched->r0; */
-	rix[1] = sched->r1;
-	rix[2] = sched->r2;
-	rix[3] = sched->r3;
+	rc[0].flags = rc[1].flags = rc[2].flags = rc[3].flags = 0;
+
+	rc[0].rix = sched->r0;
+	rc[1].rix = sched->r1;
+	rc[2].rix = sched->r2;
+	rc[3].rix = sched->r3;
 
-	try[0] = sched->t0;
-	try[1] = sched->t1;
-	try[2] = sched->t2;
-	try[3] = sched->t3;
+	rc[0].tries = sched->t0;
+	rc[1].tries = sched->t1;
+	rc[2].tries = sched->t2;
+	rc[3].tries = sched->t3;
 }
 
 void
@@ -488,6 +683,71 @@ ath_rate_setupxtxdesc(struct ath_softc *sc, struct ath_node *an,
 	    s3code, sched->t3);		/* series 3 */
 }
 
+/*
+ * Update the EWMA percentage.
+ *
+ * This is a simple hack to track an EWMA based on the current
+ * rate scenario. For the rate codes which failed, this will
+ * record a 0% against it. For the rate code which succeeded,
+ * EWMA will record the nbad*100/nframes percentage against it.
+ */
+static void
+update_ewma_stats(struct ath_softc *sc, struct ath_node *an,
+    int frame_size,
+    int rix0, int tries0,
+    int rix1, int tries1,
+    int rix2, int tries2,
+    int rix3, int tries3,
+    int short_tries, int tries, int status,
+    int nframes, int nbad)
+{
+	struct sample_node *sn = ATH_NODE_SAMPLE(an);
+	struct sample_softc *ssc = ATH_SOFTC_SAMPLE(sc);
+	const int size_bin = size_to_bin(frame_size);
+	int tries_so_far;
+	int pct;
+	int rix = rix0;
+
+	/* Calculate percentage based on current rate */
+	if (nframes == 0)
+		nframes = nbad = 1;
+	pct = ((nframes - nbad) * 1000) / nframes;
+
+	/* Figure out which rate index succeeded */
+	tries_so_far = tries0;
+
+	if (tries1 && tries_so_far < tries) {
+		tries_so_far += tries1;
+		rix = rix1;
+		/* XXX bump ewma pct */
+	}
+
+	if (tries2 && tries_so_far < tries) {
+		tries_so_far += tries2;
+		rix = rix2;
+		/* XXX bump ewma pct */
+	}
+
+	if (tries3 && tries_so_far < tries) {
+		rix = rix3;
+		/* XXX bump ewma pct */
+	}
+
+	/* rix is the successful rate, update EWMA for final rix */
+	if (sn->stats[size_bin][rix].total_packets <
+	    ssc->smoothing_minpackets) {
+		/* just average the first few packets */
+		int a_pct = (sn->stats[size_bin][rix].packets_acked * 1000) /
+		    (sn->stats[size_bin][rix].total_packets);
+		sn->stats[size_bin][rix].ewma_pct = a_pct;
+	} else {
+		/* use a ewma */
+		sn->stats[size_bin][rix].ewma_pct =
+			((sn->stats[size_bin][rix].ewma_pct * ssc->smoothing_rate) +
+			 (pct * (100 - ssc->smoothing_rate))) / 100;
+	}
+}
+
 static void
 update_stats(struct ath_softc *sc, struct ath_node *an, 
 		  int frame_size,
@@ -495,10 +755,14 @@ update_stats(struct ath_softc *sc, struct ath_node *an,
 		  int rix1, int tries1,
 		  int rix2, int tries2,
 		  int rix3, int tries3,
-		  int short_tries, int tries, int status)
+		  int short_tries, int tries, int status,
+		  int nframes, int nbad)
 {
 	struct sample_node *sn = ATH_NODE_SAMPLE(an);
 	struct sample_softc *ssc = ATH_SOFTC_SAMPLE(sc);
+#ifdef IEEE80211_DEBUG
+	const HAL_RATE_TABLE *rt = sc->sc_currates;
+#endif
 	const int size_bin = size_to_bin(frame_size);
 	const int size = bin_to_size(size_bin);
 	int tt, tries_so_far;
@@ -537,7 +801,7 @@ update_stats(struct ath_softc *sc, struct ath_node *an,
 		/* just average the first few packets */
 		int avg_tx = sn->stats[size_bin][rix0].average_tx_time;
 		int packets = sn->stats[size_bin][rix0].total_packets;
-		sn->stats[size_bin][rix0].average_tx_time = (tt+(avg_tx*packets))/(packets+1);
+		sn->stats[size_bin][rix0].average_tx_time = (tt+(avg_tx*packets))/(packets+nframes);
 	} else {
 		/* use a ewma */
 		sn->stats[size_bin][rix0].average_tx_time = 
@@ -545,38 +809,50 @@ update_stats(struct ath_softc *sc, struct ath_node *an,
 			 (tt * (100 - ssc->smoothing_rate))) / 100;
 	}
 	
-	if (status != 0) {
+	/*
+	 * XXX Don't mark the higher bit rates as also having failed; as this
+	 * unfortunately stops those rates from being tasted when trying to
+	 * TX. This happens with 11n aggregation.
+	 */
+	if (nframes == nbad) {
+#if 0
 		int y;
-		sn->stats[size_bin][rix0].successive_failures++;
+#endif
+		sn->stats[size_bin][rix0].successive_failures += nbad;
+#if 0
 		for (y = size_bin+1; y < NUM_PACKET_SIZE_BINS; y++) {
 			/*
 			 * Also say larger packets failed since we
 			 * assume if a small packet fails at a
 			 * bit-rate then a larger one will also.
 			 */
-			sn->stats[y][rix0].successive_failures++;
+			sn->stats[y][rix0].successive_failures += nbad;
 			sn->stats[y][rix0].last_tx = ticks;
 			sn->stats[y][rix0].tries += tries;
-			sn->stats[y][rix0].total_packets++;
+			sn->stats[y][rix0].total_packets += nframes;
 		}
+#endif
 	} else {
-		sn->stats[size_bin][rix0].packets_acked++;
+		sn->stats[size_bin][rix0].packets_acked += (nframes - nbad);
 		sn->stats[size_bin][rix0].successive_failures = 0;
 	}
 	sn->stats[size_bin][rix0].tries += tries;
 	sn->stats[size_bin][rix0].last_tx = ticks;
-	sn->stats[size_bin][rix0].total_packets++;
+	sn->stats[size_bin][rix0].total_packets += nframes;
 
 	if (rix0 == sn->current_sample_rix[size_bin]) {
 		IEEE80211_NOTE(an->an_node.ni_vap, IEEE80211_MSG_RATECTL,
 		   &an->an_node,
-"%s: size %d %s sample rate %d tries (%d/%d) tt %d avg_tt (%d/%d)", 
+"%s: size %d %s sample rate %d %s tries (%d/%d) tt %d avg_tt (%d/%d) nfrm %d nbad %d", 
 		    __func__, 
 		    size,
 		    status ? "FAIL" : "OK",
-		    rix0, short_tries, tries, tt, 
+		    dot11rate(rt, rix0),
+		    dot11rate_label(rt, rix0),
+		    short_tries, tries, tt, 
 		    sn->stats[size_bin][rix0].average_tx_time,
-		    sn->stats[size_bin][rix0].perfect_tx_time);
+		    sn->stats[size_bin][rix0].perfect_tx_time,
+		    nframes, nbad);
 		sn->sample_tt[size_bin] = tt;
 		sn->current_sample_rix[size_bin] = -1;
 	}
@@ -591,21 +867,21 @@ badrate(struct ifnet *ifp, int series, int hwrate, int tries, int status)
 
 void
 ath_rate_tx_complete(struct ath_softc *sc, struct ath_node *an,
-	const struct ath_buf *bf)
+	const struct ath_rc_series *rc, const struct ath_tx_status *ts,
+	int frame_size, int nframes, int nbad)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct sample_node *sn = ATH_NODE_SAMPLE(an);
-	const struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
-	const struct ath_desc *ds0 = &bf->bf_desc[0];
-	int final_rix, short_tries, long_tries, frame_size;
+	int final_rix, short_tries, long_tries;
 	const HAL_RATE_TABLE *rt = sc->sc_currates;
+	int status = ts->ts_status;
 	int mrr;
 
 	final_rix = rt->rateCodeToIndex[ts->ts_rate];
 	short_tries = ts->ts_shortretry;
 	long_tries = ts->ts_longretry + 1;
-	frame_size = ds0->ds_ctl0 & 0x0fff; /* low-order 12 bits of ds_ctl0 */
+
 	if (frame_size == 0)		    /* NB: should not happen */
 		frame_size = 1500;
 
@@ -615,63 +891,73 @@ ath_rate_tx_complete(struct ath_softc *sc, struct ath_node *an,
 		    "%s: size %d %s rate/try %d/%d no rates yet", 
 		    __func__,
 		    bin_to_size(size_to_bin(frame_size)),
-		    ts->ts_status ? "FAIL" : "OK",
+		    status ? "FAIL" : "OK",
 		    short_tries, long_tries);
 		return;
 	}
 	mrr = sc->sc_mrretry && !(ic->ic_flags & IEEE80211_F_USEPROT);
 	if (!mrr || ts->ts_finaltsi == 0) {
 		if (!IS_RATE_DEFINED(sn, final_rix)) {
-			badrate(ifp, 0, ts->ts_rate, long_tries, ts->ts_status);
+			badrate(ifp, 0, ts->ts_rate, long_tries, status);
 			return;
 		}
 		/*
 		 * Only one rate was used; optimize work.
 		 */
 		IEEE80211_NOTE(an->an_node.ni_vap, IEEE80211_MSG_RATECTL,
-		     &an->an_node, "%s: size %d (%d bytes) %s rate/try %d %s/%d/%d",
+		     &an->an_node, "%s: size %d (%d bytes) %s rate/try %d %s/%d/%d nframes/nbad [%d/%d]",
 		     __func__,
 		     bin_to_size(size_to_bin(frame_size)),
 		     frame_size,
-		     ts->ts_status ? "FAIL" : "OK",
-		     dot11rate(rt, final_rix), dot11rate_label(rt, final_rix), short_tries, long_tries);
+		     status ? "FAIL" : "OK",
+		     dot11rate(rt, final_rix), dot11rate_label(rt, final_rix),
+		     short_tries, long_tries, nframes, nbad);
 		update_stats(sc, an, frame_size, 
 			     final_rix, long_tries,
 			     0, 0,
 			     0, 0,
 			     0, 0,
-			     short_tries, long_tries, ts->ts_status);
+			     short_tries, long_tries, status,
+			     nframes, nbad);
+		update_ewma_stats(sc, an, frame_size, 
+			     final_rix, long_tries,
+			     0, 0,
+			     0, 0,
+			     0, 0,
+			     short_tries, long_tries, status,
+			     nframes, nbad);
+
 	} else {
-		int hwrates[4], tries[4], rix[4];
 		int finalTSIdx = ts->ts_finaltsi;
 		int i;
 
 		/*
 		 * Process intermediate rates that failed.
 		 */
-		ath_hal_gettxcompletionrates(sc->sc_ah, ds0, hwrates, tries);
-
-		for (i = 0; i < 4; i++) {
-			rix[i] = rt->rateCodeToIndex[hwrates[i]];
-		}
 
 		IEEE80211_NOTE(an->an_node.ni_vap, IEEE80211_MSG_RATECTL,
 		    &an->an_node,
-"%s: size %d (%d bytes) finaltsidx %d tries %d %s rate/try [%d %s/%d %d %s/%d %d %s/%d %d %s/%d]", 
+"%s: size %d (%d bytes) finaltsidx %d tries %d %s rate/try [%d %s/%d %d %s/%d %d %s/%d %d %s/%d] nframes/nbad [%d/%d]", 
 		     __func__,
 		     bin_to_size(size_to_bin(frame_size)),
 		     frame_size,
 		     finalTSIdx,
-		     long_tries, 
-		     ts->ts_status ? "FAIL" : "OK",
-		     dot11rate(rt, rix[0]), dot11rate_label(rt, rix[0]), tries[0],
-		     dot11rate(rt, rix[1]), dot11rate_label(rt, rix[1]), tries[1],
-		     dot11rate(rt, rix[2]), dot11rate_label(rt, rix[2]), tries[2],
-		     dot11rate(rt, rix[3]), dot11rate_label(rt, rix[3]), tries[3]);
+		     long_tries,
+		     status ? "FAIL" : "OK",
+		     dot11rate(rt, rc[0].rix),
+		      dot11rate_label(rt, rc[0].rix), rc[0].tries,
+		     dot11rate(rt, rc[1].rix),
+		      dot11rate_label(rt, rc[1].rix), rc[1].tries,
+		     dot11rate(rt, rc[2].rix),
+		      dot11rate_label(rt, rc[2].rix), rc[2].tries,
+		     dot11rate(rt, rc[3].rix),
+		      dot11rate_label(rt, rc[3].rix), rc[3].tries,
+		     nframes, nbad);
 
 		for (i = 0; i < 4; i++) {
-			if (tries[i] && !IS_RATE_DEFINED(sn, rix[i]))
-				badrate(ifp, 0, hwrates[i], tries[i], ts->ts_status);
+			if (rc[i].tries && !IS_RATE_DEFINED(sn, rc[i].rix))
+				badrate(ifp, 0, rc[i].ratecode, rc[i].tries,
+				    status);
 		}
 
 		/*
@@ -681,48 +967,62 @@ ath_rate_tx_complete(struct ath_softc *sc, struct ath_node *an,
 		 * sample higher rates 1 try at a time doing so
 		 * may unfairly penalize them.
 		 */
-		if (tries[0]) {
-			update_stats(sc, an, frame_size, 
-				     rix[0], tries[0], 
-				     rix[1], tries[1], 
-				     rix[2], tries[2], 
-				     rix[3], tries[3], 
-				     short_tries, long_tries, 
-				     long_tries > tries[0]);
-			long_tries -= tries[0];
+		if (rc[0].tries) {
+			update_stats(sc, an, frame_size,
+				     rc[0].rix, rc[0].tries,
+				     rc[1].rix, rc[1].tries,
+				     rc[2].rix, rc[2].tries,
+				     rc[3].rix, rc[3].tries,
+				     short_tries, long_tries,
+				     long_tries > rc[0].tries,
+				     nframes, nbad);
+			long_tries -= rc[0].tries;
 		}
 		
-		if (tries[1] && finalTSIdx > 0) {
-			update_stats(sc, an, frame_size, 
-				     rix[1], tries[1], 
-				     rix[2], tries[2], 
-				     rix[3], tries[3], 
-				     0, 0, 
-				     short_tries, long_tries, 
-				     ts->ts_status);
-			long_tries -= tries[1];
+		if (rc[1].tries && finalTSIdx > 0) {
+			update_stats(sc, an, frame_size,
+				     rc[1].rix, rc[1].tries,
+				     rc[2].rix, rc[2].tries,
+				     rc[3].rix, rc[3].tries,
+				     0, 0,
+				     short_tries, long_tries,
+				     status,
+				     nframes, nbad);
+			long_tries -= rc[1].tries;
 		}
 
-		if (tries[2] && finalTSIdx > 1) {
-			update_stats(sc, an, frame_size, 
-				     rix[2], tries[2], 
-				     rix[3], tries[3], 
+		if (rc[2].tries && finalTSIdx > 1) {
+			update_stats(sc, an, frame_size,
+				     rc[2].rix, rc[2].tries,
+				     rc[3].rix, rc[3].tries,
 				     0, 0,
 				     0, 0,
-				     short_tries, long_tries, 
-				     ts->ts_status);
-			long_tries -= tries[2];
+				     short_tries, long_tries,
+				     status,
+				     nframes, nbad);
+			long_tries -= rc[2].tries;
 		}
 
-		if (tries[3] && finalTSIdx > 2) {
-			update_stats(sc, an, frame_size, 
-				     rix[3], tries[3],
+		if (rc[3].tries && finalTSIdx > 2) {
+			update_stats(sc, an, frame_size,
+				     rc[3].rix, rc[3].tries,
 				     0, 0,
 				     0, 0,
 				     0, 0,
-				     short_tries, long_tries, 
-				     ts->ts_status);
+				     short_tries, long_tries,
+				     status,
+				     nframes, nbad);
 		}
+
+		update_ewma_stats(sc, an, frame_size,
+			     rc[0].rix, rc[0].tries,
+			     rc[1].rix, rc[1].tries,
+			     rc[2].rix, rc[2].tries,
+			     rc[3].rix, rc[3].tries,
+			     short_tries, long_tries,
+			     long_tries > rc[0].tries,
+			     nframes, nbad);
+
 	}
 }
 
@@ -844,6 +1144,7 @@ ath_rate_ctl_reset(struct ath_softc *sc, struct ieee80211_node *ni)
 			sn->stats[y][rix].total_packets = 0;
 			sn->stats[y][rix].packets_acked = 0;
 			sn->stats[y][rix].last_tx = 0;
+			sn->stats[y][rix].ewma_pct = 0;
 			
 			sn->stats[y][rix].perfect_tx_time =
 			    calc_usecs_unicast_packet(sc, size, rix, 0, 0,
@@ -881,18 +1182,24 @@ sample_stats(void *arg, struct ieee80211_node *ni)
 	uint32_t mask;
 	int rix, y;
 
-	printf("\n[%s] refcnt %d static_rix %d ratemask 0x%x\n",
+	printf("\n[%s] refcnt %d static_rix (%d %s) ratemask 0x%x\n",
 	    ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni),
-	    sn->static_rix, sn->ratemask);
+	    dot11rate(rt, sn->static_rix),
+	    dot11rate_label(rt, sn->static_rix),
+	    sn->ratemask);
 	for (y = 0; y < NUM_PACKET_SIZE_BINS; y++) {
 		printf("[%4u] cur rix %d (%d %s) since switch: packets %d ticks %u\n",
 		    bin_to_size(y), sn->current_rix[y],
 		    dot11rate(rt, sn->current_rix[y]),
 		    dot11rate_label(rt, sn->current_rix[y]),
 		    sn->packets_since_switch[y], sn->ticks_since_switch[y]);
-		printf("[%4u] last sample %d cur sample %d packets sent %d\n",
-		    bin_to_size(y), sn->last_sample_rix[y],
-		    sn->current_sample_rix[y], sn->packets_sent[y]);
+		printf("[%4u] last sample (%d %s) cur sample (%d %s) packets sent %d\n",
+		    bin_to_size(y),
+		    dot11rate(rt, sn->last_sample_rix[y]),
+		    dot11rate_label(rt, sn->last_sample_rix[y]),
+		    dot11rate(rt, sn->current_sample_rix[y]),
+		    dot11rate_label(rt, sn->current_sample_rix[y]),
+		    sn->packets_sent[y]);
 		printf("[%4u] packets since sample %d sample tt %u\n",
 		    bin_to_size(y), sn->packets_since_sample[y],
 		    sn->sample_tt[y]);
@@ -903,13 +1210,16 @@ sample_stats(void *arg, struct ieee80211_node *ni)
 		for (y = 0; y < NUM_PACKET_SIZE_BINS; y++) {
 			if (sn->stats[y][rix].total_packets == 0)
 				continue;
-			printf("[%2u %s:%4u] %8d:%-8d (%3d%%) T %8d F %4d avg %5u last %u\n",
+			printf("[%2u %s:%4u] %8ju:%-8ju (%3d%%) (EWMA %3d.%1d%%) T %8ju F %4d avg %5u last %u\n",
 			    dot11rate(rt, rix), dot11rate_label(rt, rix),
 			    bin_to_size(y),
-			    sn->stats[y][rix].total_packets,
-			    sn->stats[y][rix].packets_acked,
-			    (100*sn->stats[y][rix].packets_acked)/sn->stats[y][rix].total_packets,
-			    sn->stats[y][rix].tries,
+			    (uintmax_t) sn->stats[y][rix].total_packets,
+			    (uintmax_t) sn->stats[y][rix].packets_acked,
+			    (int) ((sn->stats[y][rix].packets_acked * 100ULL) /
+			     sn->stats[y][rix].total_packets),
+			    sn->stats[y][rix].ewma_pct / 10,
+			    sn->stats[y][rix].ewma_pct % 10,
+			    (uintmax_t) sn->stats[y][rix].tries,
 			    sn->stats[y][rix].successive_failures,
 			    sn->stats[y][rix].average_tx_time,
 			    ticks - sn->stats[y][rix].last_tx);
diff --git a/sys/dev/ath/ath_rate/sample/sample.h b/sys/dev/ath/ath_rate/sample/sample.h
index b39e0be..805ae46 100644
--- a/sys/dev/ath/ath_rate/sample/sample.h
+++ b/sys/dev/ath/ath_rate/sample/sample.h
@@ -51,15 +51,17 @@ struct sample_softc {
 	int	max_successive_failures;
 	int	stale_failure_timeout;	/* how long to honor max_successive_failures */
 	int	min_switch;		/* min time between rate changes */
+	int	min_good_pct;		/* min good percentage for a rate to be considered */
 };
 #define	ATH_SOFTC_SAMPLE(sc)	((struct sample_softc *)sc->sc_rc)
 
 struct rate_stats {	
 	unsigned average_tx_time;
 	int successive_failures;
-	int tries;
-	int total_packets;
-	int packets_acked;
+	uint64_t tries;
+	uint64_t total_packets;	/* pkts total since assoc */
+	uint64_t packets_acked;	/* pkts acked since assoc */
+	int ewma_pct;	/* EWMA percentage */
 	unsigned perfect_tx_time; /* transmit time for 0 retries */
 	int last_tx;
 };
diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c
index e1d2c0f..462bc3f 100644
--- a/sys/dev/ath/if_ath.c
+++ b/sys/dev/ath/if_ath.c
@@ -64,6 +64,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 #include <sys/priv.h>
 #include <sys/module.h>
+#include <sys/ktr.h>
+#include <sys/smp.h>	/* for mp_ncpus */
 
 #include <machine/bus.h>
 
@@ -106,6 +108,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/ath/ath_tx99/ath_tx99.h>
 #endif
 
+#define	ATH_KTR_INTR	KTR_SPARE4
+#define	ATH_KTR_ERR	KTR_SPARE3
 
 /*
  * ATH_BCBUF determines the number of vap's that can transmit
@@ -163,6 +167,7 @@ static int	ath_desc_alloc(struct ath_softc *);
 static void	ath_desc_free(struct ath_softc *);
 static struct ieee80211_node *ath_node_alloc(struct ieee80211vap *,
 			const uint8_t [IEEE80211_ADDR_LEN]);
+static void	ath_node_cleanup(struct ieee80211_node *);
 static void	ath_node_free(struct ieee80211_node *);
 static void	ath_node_getsignal(const struct ieee80211_node *,
 			int8_t *, int8_t *);
@@ -170,7 +175,8 @@ static int	ath_rxbuf_init(struct ath_softc *, struct ath_buf *);
 static void	ath_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m,
 			int subtype, int rssi, int nf);
 static void	ath_setdefantenna(struct ath_softc *, u_int);
-static void	ath_rx_proc(void *, int);
+static void	ath_rx_proc(struct ath_softc *sc, int);
+static void	ath_rx_tasklet(void *, int);
 static void	ath_txq_init(struct ath_softc *sc, struct ath_txq *, int);
 static struct ath_txq *ath_txq_setup(struct ath_softc*, int qtype, int subtype);
 static int	ath_tx_setup(struct ath_softc *, int, int);
@@ -180,10 +186,9 @@ static void	ath_tx_cleanup(struct ath_softc *);
 static void	ath_tx_proc_q0(void *, int);
 static void	ath_tx_proc_q0123(void *, int);
 static void	ath_tx_proc(void *, int);
-static void	ath_tx_draintxq(struct ath_softc *, struct ath_txq *);
 static int	ath_chan_set(struct ath_softc *, struct ieee80211_channel *);
-static void	ath_draintxq(struct ath_softc *);
-static void	ath_stoprecv(struct ath_softc *);
+static void	ath_draintxq(struct ath_softc *, ATH_RESET_TYPE reset_type);
+static void	ath_stoprecv(struct ath_softc *, int);
 static int	ath_startrecv(struct ath_softc *);
 static void	ath_chan_change(struct ath_softc *, struct ieee80211_channel *);
 static void	ath_scan_start(struct ieee80211com *);
@@ -382,7 +387,7 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 	taskqueue_start_threads(&sc->sc_tq, 1, PI_NET,
 		"%s taskq", ifp->if_xname);
 
-	TASK_INIT(&sc->sc_rxtask, 0, ath_rx_proc, sc);
+	TASK_INIT(&sc->sc_rxtask, 0, ath_rx_tasklet, sc);
 	TASK_INIT(&sc->sc_bmisstask, 0, ath_bmiss_proc, sc);
 	TASK_INIT(&sc->sc_bstucktask,0, ath_bstuck_proc, sc);
 
@@ -671,6 +676,17 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 #endif
 
 	/*
+	 * Check if the hardware requires PCI register serialisation.
+	 * Some of the Owl based MACs require this.
+	 */
+	if (mp_ncpus > 1 &&
+	    ath_hal_getcapability(ah, HAL_CAP_SERIALISE_WAR,
+	     0, NULL) == HAL_OK) {
+		sc->sc_ah->ah_config.ah_serialise_reg_war = 1;
+		device_printf(sc->sc_dev, "Enabling register serialisation\n");
+	}
+
+	/*
 	 * Indicate we need the 802.11 header padded to a
 	 * 32-bit boundary for 4-address and QoS frames.
 	 */
@@ -712,11 +728,26 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 	ic->ic_node_alloc = ath_node_alloc;
 	sc->sc_node_free = ic->ic_node_free;
 	ic->ic_node_free = ath_node_free;
+	sc->sc_node_cleanup = ic->ic_node_cleanup;
+	ic->ic_node_cleanup = ath_node_cleanup;
 	ic->ic_node_getsignal = ath_node_getsignal;
 	ic->ic_scan_start = ath_scan_start;
 	ic->ic_scan_end = ath_scan_end;
 	ic->ic_set_channel = ath_set_channel;
 
+	/* 802.11n specific - but just override anyway */
+	sc->sc_addba_request = ic->ic_addba_request;
+	sc->sc_addba_response = ic->ic_addba_response;
+	sc->sc_addba_stop = ic->ic_addba_stop;
+	sc->sc_bar_response = ic->ic_bar_response;
+	sc->sc_addba_response_timeout = ic->ic_addba_response_timeout;
+
+	ic->ic_addba_request = ath_addba_request;
+	ic->ic_addba_response = ath_addba_response;
+	ic->ic_addba_response_timeout = ath_addba_response_timeout;
+	ic->ic_addba_stop = ath_addba_stop;
+	ic->ic_bar_response = ath_bar_response;
+
 	ieee80211_radiotap_attach(ic,
 	    &sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th),
 		ATH_TX_RADIOTAP_PRESENT,
@@ -952,7 +983,7 @@ ath_vap_create(struct ieee80211com *ic,
 	/*
 	 * Check that a beacon buffer is available; the code below assumes it.
 	 */
-	if (needbeacon & STAILQ_EMPTY(&sc->sc_bbuf)) {
+	if (needbeacon & TAILQ_EMPTY(&sc->sc_bbuf)) {
 		device_printf(sc->sc_dev, "no beacon buffer available\n");
 		goto bad;
 	}
@@ -1014,8 +1045,8 @@ ath_vap_create(struct ieee80211com *ic,
 		 * multicast frames.  We know a beacon buffer is
 		 * available because we checked above.
 		 */
-		avp->av_bcbuf = STAILQ_FIRST(&sc->sc_bbuf);
-		STAILQ_REMOVE_HEAD(&sc->sc_bbuf, bf_list);
+		avp->av_bcbuf = TAILQ_FIRST(&sc->sc_bbuf);
+		TAILQ_REMOVE(&sc->sc_bbuf, avp->av_bcbuf, bf_list);
 		if (opmode != IEEE80211_M_IBSS || !sc->sc_hasveol) {
 			/*
 			 * Assign the vap to a beacon xmit slot.  As above
@@ -1112,6 +1143,7 @@ ath_vap_delete(struct ieee80211vap *vap)
 	struct ath_hal *ah = sc->sc_ah;
 	struct ath_vap *avp = ATH_VAP(vap);
 
+	DPRINTF(sc, ATH_DEBUG_RESET, "%s: called\n", __func__);
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 		/*
 		 * Quiesce the hardware while we remove the vap.  In
@@ -1119,11 +1151,32 @@ ath_vap_delete(struct ieee80211vap *vap)
 		 * the vap state by any frames pending on the tx queues.
 		 */
 		ath_hal_intrset(ah, 0);		/* disable interrupts */
-		ath_draintxq(sc);		/* stop xmit side */
-		ath_stoprecv(sc);		/* stop recv side */
+		ath_draintxq(sc, ATH_RESET_DEFAULT);		/* stop hw xmit side */
+		/* XXX Do all frames from all vaps/nodes need draining here? */
+		ath_stoprecv(sc, 1);		/* stop recv side */
 	}
 
 	ieee80211_vap_detach(vap);
+
+	/*
+	 * XXX Danger Will Robinson! Danger!
+	 *
+	 * Because ieee80211_vap_detach() can queue a frame (the station
+	 * diassociate message?) after we've drained the TXQ and
+	 * flushed the software TXQ, we will end up with a frame queued
+	 * to a node whose vap is about to be freed.
+	 *
+	 * To work around this, flush the hardware/software again.
+	 * This may be racy - the ath task may be running and the packet
+	 * may be being scheduled between sw->hw txq. Tsk.
+	 *
+	 * TODO: figure out why a new node gets allocated somewhere around
+	 * here (after the ath_tx_swq() call; and after an ath_stop_locked()
+	 * call!)
+	 */
+
+	ath_draintxq(sc, ATH_RESET_DEFAULT);
+
 	ATH_LOCK(sc);
 	/*
 	 * Reclaim beacon state.  Note this must be done before
@@ -1171,7 +1224,6 @@ ath_vap_delete(struct ieee80211vap *vap)
 		sc->sc_swbmiss = 0;
 	}
 #endif
-	ATH_UNLOCK(sc);
 	free(avp, M_80211_VAP);
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
@@ -1192,6 +1244,7 @@ ath_vap_delete(struct ieee80211vap *vap)
 		}
 		ath_hal_intrset(ah, sc->sc_imask);
 	}
+	ATH_UNLOCK(sc);
 }
 
 void
@@ -1304,6 +1357,23 @@ ath_intr(void *arg)
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ath_hal *ah = sc->sc_ah;
 	HAL_INT status = 0;
+	uint32_t txqs;
+
+	/*
+	 * If we're inside a reset path, just print a warning and
+	 * clear the ISR. The reset routine will finish it for us.
+	 */
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt) {
+		HAL_INT status;
+		ath_hal_getisr(ah, &status);	/* clear ISR */
+		ath_hal_intrset(ah, 0);		/* disable further intr's */
+		DPRINTF(sc, ATH_DEBUG_ANY,
+		    "%s: in reset, ignoring: status=0x%x\n",
+		    __func__, status);
+		ATH_PCU_UNLOCK(sc);
+		return;
+	}
 
 	if (sc->sc_invalid) {
 		/*
@@ -1311,10 +1381,14 @@ ath_intr(void *arg)
 		 * Note this can happen early on if the IRQ is shared.
 		 */
 		DPRINTF(sc, ATH_DEBUG_ANY, "%s: invalid; ignored\n", __func__);
+		ATH_PCU_UNLOCK(sc);
 		return;
 	}
-	if (!ath_hal_intrpend(ah))		/* shared irq, not for us */
+	if (!ath_hal_intrpend(ah)) {		/* shared irq, not for us */
+		ATH_PCU_UNLOCK(sc);
 		return;
+	}
+
 	if ((ifp->if_flags & IFF_UP) == 0 ||
 	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 		HAL_INT status;
@@ -1323,8 +1397,10 @@ ath_intr(void *arg)
 			__func__, ifp->if_flags);
 		ath_hal_getisr(ah, &status);	/* clear ISR */
 		ath_hal_intrset(ah, 0);		/* disable further intr's */
+		ATH_PCU_UNLOCK(sc);
 		return;
 	}
+
 	/*
 	 * Figure out the reason(s) for the interrupt.  Note
 	 * that the hal returns a pseudo-ISR that may include
@@ -1333,12 +1409,36 @@ ath_intr(void *arg)
 	 */
 	ath_hal_getisr(ah, &status);		/* NB: clears ISR too */
 	DPRINTF(sc, ATH_DEBUG_INTR, "%s: status 0x%x\n", __func__, status);
+	CTR1(ATH_KTR_INTR, "ath_intr: mask=0x%.8x", status);
+#ifdef	ATH_KTR_INTR_DEBUG
+	CTR5(ATH_KTR_INTR,
+	    "ath_intr: ISR=0x%.8x, ISR_S0=0x%.8x, ISR_S1=0x%.8x, ISR_S2=0x%.8x, ISR_S5=0x%.8x",
+	    ah->ah_intrstate[0],
+	    ah->ah_intrstate[1],
+	    ah->ah_intrstate[2],
+	    ah->ah_intrstate[3],
+	    ah->ah_intrstate[6]);
+#endif
 	status &= sc->sc_imask;			/* discard unasked for bits */
 
 	/* Short-circuit un-handled interrupts */
-	if (status == 0x0)
+	if (status == 0x0) {
+		ATH_PCU_UNLOCK(sc);
 		return;
+	}
+
+	/*
+	 * Take a note that we're inside the interrupt handler, so
+	 * the reset routines know to wait.
+	 */
+	sc->sc_intr_cnt++;
+	ATH_PCU_UNLOCK(sc);
 
+	/*
+	 * Handle the interrupt. We won't run concurrent with the reset
+	 * or channel change routines as they'll wait for sc_intr_cnt
+	 * to be 0 before continuing.
+	 */
 	if (status & HAL_INT_FATAL) {
 		sc->sc_stats.ast_hardware++;
 		ath_hal_intrset(ah, 0);		/* disable intr's until reset */
@@ -1377,7 +1477,9 @@ ath_intr(void *arg)
 			}
 		}
 		if (status & HAL_INT_RXEOL) {
-			int imask = sc->sc_imask;
+			int imask;
+			CTR0(ATH_KTR_ERR, "ath_intr: RXEOL");
+			ATH_PCU_LOCK(sc);
 			/*
 			 * NB: the hardware should re-read the link when
 			 *     RXE bit is written, but it doesn't work at
@@ -1393,26 +1495,54 @@ ath_intr(void *arg)
 			 * by a call to ath_reset() somehow, the
 			 * interrupt mask will be correctly reprogrammed.
 			 */
+			imask = sc->sc_imask;
 			imask &= ~(HAL_INT_RXEOL | HAL_INT_RXORN);
 			ath_hal_intrset(ah, imask);
 			/*
+			 * Only blank sc_rxlink if we've not yet kicked
+			 * the PCU.
+			 *
+			 * This isn't entirely correct - the correct solution
+			 * would be to have a PCU lock and engage that for
+			 * the duration of the PCU fiddling; which would include
+			 * running the RX process. Otherwise we could end up
+			 * messing up the RX descriptor chain and making the
+			 * RX desc list much shorter.
+			 */
+			if (! sc->sc_kickpcu)
+				sc->sc_rxlink = NULL;
+			sc->sc_kickpcu = 1;
+			/*
 			 * Enqueue an RX proc, to handled whatever
 			 * is in the RX queue.
 			 * This will then kick the PCU.
 			 */
 			taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
-			sc->sc_rxlink = NULL;
-			sc->sc_kickpcu = 1;
+			ATH_PCU_UNLOCK(sc);
 		}
 		if (status & HAL_INT_TXURN) {
 			sc->sc_stats.ast_txurn++;
 			/* bump tx trigger level */
 			ath_hal_updatetxtriglevel(ah, AH_TRUE);
 		}
-		if (status & HAL_INT_RX)
+		if (status & HAL_INT_RX) {
+			sc->sc_stats.ast_rx_intr++;
 			taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
-		if (status & HAL_INT_TX)
+		}
+		if (status & HAL_INT_TX) {
+			sc->sc_stats.ast_tx_intr++;
+			/*
+			 * Grab all the currently set bits in the HAL txq bitmap
+			 * and blank them. This is the only place we should be
+			 * doing this.
+			 */
+			ATH_PCU_LOCK(sc);
+			txqs = 0xffffffff;
+			ath_hal_gettxintrtxqs(sc->sc_ah, &txqs);
+			sc->sc_txq_active |= txqs;
 			taskqueue_enqueue(sc->sc_tq, &sc->sc_txtask);
+			ATH_PCU_UNLOCK(sc);
+		}
 		if (status & HAL_INT_BMISS) {
 			sc->sc_stats.ast_bmiss++;
 			taskqueue_enqueue(sc->sc_tq, &sc->sc_bmisstask);
@@ -1423,6 +1553,7 @@ ath_intr(void *arg)
 			sc->sc_stats.ast_tx_cst++;
 		if (status & HAL_INT_MIB) {
 			sc->sc_stats.ast_mib++;
+			ATH_PCU_LOCK(sc);
 			/*
 			 * Disable interrupts until we service the MIB
 			 * interrupt; otherwise it will continue to fire.
@@ -1433,13 +1564,25 @@ ath_intr(void *arg)
 			 * clear whatever condition caused the interrupt.
 			 */
 			ath_hal_mibevent(ah, &sc->sc_halstats);
-			ath_hal_intrset(ah, sc->sc_imask);
+			/*
+			 * Don't reset the interrupt if we've just
+			 * kicked the PCU, or we may get a nested
+			 * RXEOL before the rxproc has had a chance
+			 * to run.
+			 */
+			if (sc->sc_kickpcu == 0)
+				ath_hal_intrset(ah, sc->sc_imask);
+			ATH_PCU_UNLOCK(sc);
 		}
 		if (status & HAL_INT_RXORN) {
 			/* NB: hal marks HAL_INT_FATAL when RXORN is fatal */
+			CTR0(ATH_KTR_ERR, "ath_intr: RXORN");
 			sc->sc_stats.ast_rxorn++;
 		}
 	}
+	ATH_PCU_LOCK(sc);
+	sc->sc_intr_cnt--;
+	ATH_PCU_UNLOCK(sc);
 }
 
 static void
@@ -1464,7 +1607,7 @@ ath_fatal_proc(void *arg, int pending)
 		    state[0], state[1] , state[2], state[3],
 		    state[4], state[5]);
 	}
-	ath_reset(ifp);
+	ath_reset(ifp, ATH_RESET_NOLOSS);
 }
 
 static void
@@ -1524,7 +1667,7 @@ ath_bmiss_proc(void *arg, int pending)
 
 	if (ath_hal_gethangstate(sc->sc_ah, 0xff, &hangs) && hangs != 0) {
 		if_printf(ifp, "bb hang detected (0x%x), resetting\n", hangs);
-		ath_reset(ifp);
+		ath_reset(ifp, ATH_RESET_NOLOSS);
 	} else
 		ieee80211_beacon_miss(ifp->if_l2com);
 }
@@ -1609,6 +1752,13 @@ ath_init(void *arg)
 	sc->sc_beacons = 0;
 
 	/*
+	 * Initial aggregation settings.
+	 */
+	sc->sc_hwq_limit = ATH_AGGR_MIN_QDEPTH;
+	sc->sc_tid_hwq_lo = ATH_AGGR_SCHED_LOW;
+	sc->sc_tid_hwq_hi = ATH_AGGR_SCHED_HIGH;
+
+	/*
 	 * Setup the hardware after reset: the key cache
 	 * is filled as needed and the receive engine is
 	 * set going.  Frame transmit is handled entirely
@@ -1697,9 +1847,9 @@ ath_stop_locked(struct ifnet *ifp)
 			}
 			ath_hal_intrset(ah, 0);
 		}
-		ath_draintxq(sc);
+		ath_draintxq(sc, ATH_RESET_DEFAULT);
 		if (!sc->sc_invalid) {
-			ath_stoprecv(sc);
+			ath_stoprecv(sc, 1);
 			ath_hal_phydisable(ah);
 		} else
 			sc->sc_rxlink = NULL;
@@ -1707,6 +1857,46 @@ ath_stop_locked(struct ifnet *ifp)
 	}
 }
 
+#define	MAX_TXRX_ITERATIONS	1000
+static void
+ath_txrx_stop(struct ath_softc *sc)
+{
+	int i = MAX_TXRX_ITERATIONS;
+
+	ATH_UNLOCK_ASSERT(sc);
+	/* Stop any new TX/RX from occuring */
+	taskqueue_block(sc->sc_tq);
+
+	ATH_PCU_LOCK(sc);
+	/*
+	 * Sleep until all the pending operations have completed.
+	 *
+	 * The caller must ensure that reset has been incremented
+	 * or the pending operations may continue being queued.
+	 */
+	while (sc->sc_rxproc_cnt || sc->sc_txproc_cnt ||
+	    sc->sc_txstart_cnt || sc->sc_intr_cnt) {
+		if (i <= 0)
+			break;
+		msleep(sc, &sc->sc_mtx, 0, "ath_txrx_stop", 1);
+		i--;
+	}
+	ATH_PCU_UNLOCK(sc);
+
+	if (i <= 0)
+		device_printf(sc->sc_dev,
+		    "%s: didn't finish after %d iterations\n",
+		    __func__, MAX_TXRX_ITERATIONS);
+}
+#undef	MAX_TXRX_ITERATIONS
+
+static void
+ath_txrx_start(struct ath_softc *sc)
+{
+
+	taskqueue_unblock(sc->sc_tq);
+}
+
 static void
 ath_stop(struct ifnet *ifp)
 {
@@ -1725,16 +1915,47 @@ ath_stop(struct ifnet *ifp)
  * to reset or reload hardware state.
  */
 int
-ath_reset(struct ifnet *ifp)
+ath_reset(struct ifnet *ifp, ATH_RESET_TYPE reset_type)
 {
 	struct ath_softc *sc = ifp->if_softc;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ath_hal *ah = sc->sc_ah;
 	HAL_STATUS status;
+	int i;
+
+	DPRINTF(sc, ATH_DEBUG_RESET, "%s: called\n", __func__);
 
+	/* XXX ensure ATH_LOCK isn't held; ath_rx_proc can't be locked */
+	ATH_PCU_UNLOCK_ASSERT(sc);
+	ATH_UNLOCK_ASSERT(sc);
+
+	ATH_PCU_LOCK(sc);
+	/* XXX if we're already inside a reset, print out a big warning */
+	if (sc->sc_inreset_cnt > 0) {
+		device_printf(sc->sc_dev,
+		    "%s: concurrent ath_reset()! Danger!\n",
+		    __func__);
+	}
+	sc->sc_inreset_cnt++;
 	ath_hal_intrset(ah, 0);		/* disable interrupts */
-	ath_draintxq(sc);		/* stop xmit side */
-	ath_stoprecv(sc);		/* stop recv side */
+	ATH_PCU_UNLOCK(sc);
+
+	/*
+	 * Should now wait for pending TX/RX to complete
+	 * and block future ones from occuring. This needs to be
+	 * done before the TX queue is drained.
+	 */
+	ath_txrx_stop(sc);
+	ath_draintxq(sc, reset_type);	/* stop xmit side */
+
+	/*
+	 * Regardless of whether we're doing a no-loss flush or
+	 * not, stop the PCU and handle what's in the RX queue.
+	 * That way frames aren't dropped which shouldn't be.
+	 */
+	ath_stoprecv(sc, (reset_type != ATH_RESET_NOLOSS));
+	ath_rx_proc(sc, 0);
+
 	ath_settkipmic(sc);		/* configure TKIP MIC handling */
 	/* NB: indicate channel change so we do a full reset */
 	if (!ath_hal_reset(ah, sc->sc_opmode, ic->ic_curchan, AH_TRUE, &status))
@@ -1761,8 +1982,59 @@ ath_reset(struct ifnet *ifp)
 #endif
 			ath_beacon_config(sc, NULL);
 	}
+
+	/*
+	 * Release the reset lock and re-enable interrupts here.
+	 * If an interrupt was being processed in ath_intr(),
+	 * it would disable interrupts at this point. So we have
+	 * to atomically enable interrupts and decrement the
+	 * reset counter - this way ath_intr() doesn't end up
+	 * disabling interrupts without a corresponding enable
+	 * in the rest or channel change path.
+	 */
+	ATH_PCU_LOCK(sc);
+	sc->sc_inreset_cnt--;
+	/* XXX only do this if sc_inreset_cnt == 0? */
 	ath_hal_intrset(ah, sc->sc_imask);
+	ATH_PCU_UNLOCK(sc);
+
+	/*
+	 * TX and RX can be started here. If it were started with
+	 * sc_inreset_cnt > 0, the TX and RX path would abort.
+	 * Thus if this is a nested call through the reset or
+	 * channel change code, TX completion will occur but
+	 * RX completion and ath_start / ath_tx_start will not
+	 * run.
+	 */
+
+	/* Restart TX/RX as needed */
+	ath_txrx_start(sc);
+
+	/* XXX Restart TX completion and pending TX */
+	if (reset_type == ATH_RESET_NOLOSS) {
+		for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
+			if (ATH_TXQ_SETUP(sc, i)) {
+				ATH_TXQ_LOCK(&sc->sc_txq[i]);
+				ath_txq_restart_dma(sc, &sc->sc_txq[i]);
+				ath_txq_sched(sc, &sc->sc_txq[i]);
+				ATH_TXQ_UNLOCK(&sc->sc_txq[i]);
+			}
+		}
+	}
+
+	/*
+	 * This may have been set during an ath_start() call which
+	 * set this once it detected a concurrent TX was going on.
+	 * So, clear it.
+	 */
+	/* XXX do this inside of IF_LOCK? */
+	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
+	/* Handle any frames in the TX queue */
+	/*
+	 * XXX should this be done by the caller, rather than
+	 * ath_reset() ?
+	 */
 	ath_start(ifp);			/* restart xmit */
 	return 0;
 }
@@ -1786,7 +2058,8 @@ ath_reset_vap(struct ieee80211vap *vap, u_long cmd)
 			ath_hal_settxpowlimit(ah, ic->ic_txpowlimit);
 		return 0;
 	}
-	return ath_reset(ifp);
+	/* XXX? Full or NOLOSS? */
+	return ath_reset(ifp, ATH_RESET_FULL);
 }
 
 struct ath_buf *
@@ -1796,19 +2069,90 @@ _ath_getbuf_locked(struct ath_softc *sc)
 
 	ATH_TXBUF_LOCK_ASSERT(sc);
 
-	bf = STAILQ_FIRST(&sc->sc_txbuf);
+	bf = TAILQ_FIRST(&sc->sc_txbuf);
+	if (bf == NULL) {
+		sc->sc_stats.ast_tx_getnobuf++;
+	} else {
+		if (bf->bf_flags & ATH_BUF_BUSY) {
+			sc->sc_stats.ast_tx_getbusybuf++;
+			bf = NULL;
+		}
+	}
+
 	if (bf != NULL && (bf->bf_flags & ATH_BUF_BUSY) == 0)
-		STAILQ_REMOVE_HEAD(&sc->sc_txbuf, bf_list);
+		TAILQ_REMOVE(&sc->sc_txbuf, bf, bf_list);
 	else
 		bf = NULL;
+
 	if (bf == NULL) {
 		DPRINTF(sc, ATH_DEBUG_XMIT, "%s: %s\n", __func__,
-		    STAILQ_FIRST(&sc->sc_txbuf) == NULL ?
+		    TAILQ_FIRST(&sc->sc_txbuf) == NULL ?
 			"out of xmit buffers" : "xmit buffer busy");
+		return NULL;
 	}
+
+	/* Valid bf here; clear some basic fields */
+	bf->bf_next = NULL;	/* XXX just to be sure */
+	bf->bf_last = NULL;	/* XXX again, just to be sure */
+	bf->bf_comp = NULL;	/* XXX again, just to be sure */
+	bzero(&bf->bf_state, sizeof(bf->bf_state));
+
 	return bf;
 }
 
+/*
+ * When retrying a software frame, buffers marked ATH_BUF_BUSY
+ * can't be thrown back on the queue as they could still be
+ * in use by the hardware.
+ *
+ * This duplicates the buffer, or returns NULL.
+ *
+ * The descriptor is also copied but the link pointers and
+ * the DMA segments aren't copied; this frame should thus
+ * be again passed through the descriptor setup/chain routines
+ * so the link is correct.
+ *
+ * The caller must free the buffer using ath_freebuf().
+ *
+ * XXX TODO: this call shouldn't fail as it'll cause packet loss
+ * XXX in the TX pathway when retries are needed.
+ * XXX Figure out how to keep some buffers free, or factor the
+ * XXX number of busy buffers into the xmit path (ath_start())
+ * XXX so we don't over-commit.
+ */
+struct ath_buf *
+ath_buf_clone(struct ath_softc *sc, const struct ath_buf *bf)
+{
+	struct ath_buf *tbf;
+
+	tbf = ath_getbuf(sc);
+	if (tbf == NULL)
+		return NULL;	/* XXX failure? Why? */
+
+	/* Copy basics */
+	tbf->bf_next = NULL;
+	tbf->bf_nseg = bf->bf_nseg;
+	tbf->bf_txflags = bf->bf_txflags;
+	tbf->bf_flags = bf->bf_flags & ~ATH_BUF_BUSY;
+	tbf->bf_status = bf->bf_status;
+	tbf->bf_m = bf->bf_m;
+	tbf->bf_node = bf->bf_node;
+	/* will be setup by the chain/setup function */
+	tbf->bf_lastds = NULL;
+	/* for now, last == self */
+	tbf->bf_last = tbf;
+	tbf->bf_comp = bf->bf_comp;
+
+	/* NOTE: DMA segments will be setup by the setup/chain functions */
+
+	/* The caller has to re-init the descriptor + links */
+
+	/* Copy state */
+	memcpy(&tbf->bf_state, &bf->bf_state, sizeof(bf->bf_state));
+
+	return tbf;
+}
+
 struct ath_buf *
 ath_getbuf(struct ath_softc *sc)
 {
@@ -1821,6 +2165,7 @@ ath_getbuf(struct ath_softc *sc)
 
 		DPRINTF(sc, ATH_DEBUG_XMIT, "%s: stop queue\n", __func__);
 		sc->sc_stats.ast_tx_qstop++;
+		/* XXX do this inside of IF_LOCK? */
 		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 	}
 	ATH_TXBUF_UNLOCK(sc);
@@ -1838,6 +2183,20 @@ ath_start(struct ifnet *ifp)
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
 		return;
+
+	/* XXX is it ok to hold the ATH_LOCK here? */
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt > 0) {
+		device_printf(sc->sc_dev,
+		    "%s: sc_inreset_cnt > 0; bailing\n", __func__);
+		/* XXX do this inside of IF_LOCK? */
+		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+		ATH_PCU_UNLOCK(sc);
+		return;
+	}
+	sc->sc_txstart_cnt++;
+	ATH_PCU_UNLOCK(sc);
+
 	for (;;) {
 		/*
 		 * Grab a TX buffer and associated resources.
@@ -1849,7 +2208,7 @@ ath_start(struct ifnet *ifp)
 		IFQ_DEQUEUE(&ifp->if_snd, m);
 		if (m == NULL) {
 			ATH_TXBUF_LOCK(sc);
-			STAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
+			TAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
 			ATH_TXBUF_UNLOCK(sc);
 			break;
 		}
@@ -1860,7 +2219,7 @@ ath_start(struct ifnet *ifp)
 		 * buffers to send all the fragments so all
 		 * go out or none...
 		 */
-		STAILQ_INIT(&frags);
+		TAILQ_INIT(&frags);
 		if ((m->m_flags & M_FRAG) &&
 		    !ath_txfrag_setup(sc, &frags, m, ni)) {
 			DPRINTF(sc, ATH_DEBUG_XMIT,
@@ -1892,7 +2251,7 @@ ath_start(struct ifnet *ifp)
 			bf->bf_m = NULL;
 			bf->bf_node = NULL;
 			ATH_TXBUF_LOCK(sc);
-			STAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
+			TAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
 			ath_txfrag_cleanup(sc, &frags, ni);
 			ATH_TXBUF_UNLOCK(sc);
 			if (ni != NULL)
@@ -1913,14 +2272,18 @@ ath_start(struct ifnet *ifp)
 				goto reclaim;
 			}
 			m = next;
-			bf = STAILQ_FIRST(&frags);
+			bf = TAILQ_FIRST(&frags);
 			KASSERT(bf != NULL, ("no buf for txfrag"));
-			STAILQ_REMOVE_HEAD(&frags, bf_list);
+			TAILQ_REMOVE(&frags, bf, bf_list);
 			goto nextfrag;
 		}
 
 		sc->sc_wd_timer = 5;
 	}
+
+	ATH_PCU_LOCK(sc);
+	sc->sc_txstart_cnt--;
+	ATH_PCU_UNLOCK(sc);
 }
 
 static int
@@ -2339,6 +2702,8 @@ ath_beacon_setup(struct ath_softc *sc, struct ath_buf *bf)
 
 	/* setup descriptors */
 	ds = bf->bf_desc;
+	bf->bf_last = bf;
+	bf->bf_lastds = ds;
 
 	flags = HAL_TXDESC_NOACK;
 	if (ic->ic_opmode == IEEE80211_M_IBSS && sc->sc_hasveol) {
@@ -2414,11 +2779,13 @@ ath_beacon_update(struct ieee80211vap *vap, int item)
 static void
 ath_txqmove(struct ath_txq *dst, struct ath_txq *src)
 {
-	STAILQ_CONCAT(&dst->axq_q, &src->axq_q);
+	TAILQ_CONCAT(&dst->axq_q, &src->axq_q, bf_list);
 	dst->axq_link = src->axq_link;
 	src->axq_link = NULL;
 	dst->axq_depth += src->axq_depth;
+	dst->axq_aggr_depth += src->axq_aggr_depth;
 	src->axq_depth = 0;
+	src->axq_aggr_depth = 0;
 }
 
 /*
@@ -2609,7 +2976,7 @@ ath_beacon_generate(struct ath_softc *sc, struct ieee80211vap *vap)
 			 * Move frames from the s/w mcast q to the h/w cab q.
 			 * XXX MORE_DATA bit
 			 */
-			bfm = STAILQ_FIRST(&avp->av_mcastq.axq_q);
+			bfm = TAILQ_FIRST(&avp->av_mcastq.axq_q);
 			if (cabq->axq_link != NULL) {
 				*cabq->axq_link = bfm->bf_daddr;
 			} else
@@ -2620,7 +2987,7 @@ ath_beacon_generate(struct ath_softc *sc, struct ieee80211vap *vap)
 			sc->sc_stats.ast_cabq_xmit += nmcastq;
 		}
 		/* NB: gated by beacon so safe to start here */
-		if (! STAILQ_EMPTY(&(cabq->axq_q)))
+		if (! TAILQ_EMPTY(&(cabq->axq_q)))
 			ath_hal_txstart(ah, cabq->axq_qnum);
 		ATH_TXQ_UNLOCK(&avp->av_mcastq);
 		ATH_TXQ_UNLOCK(cabq);
@@ -2676,11 +3043,19 @@ ath_bstuck_proc(void *arg, int pending)
 {
 	struct ath_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
+	uint32_t hangs = 0;
+
+	if (ath_hal_gethangstate(sc->sc_ah, 0xff, &hangs) && hangs != 0)
+		if_printf(ifp, "bb hang detected (0x%x)\n", hangs);
 
 	if_printf(ifp, "stuck beacon; resetting (bmiss count %u)\n",
 		sc->sc_bmisscount);
 	sc->sc_stats.ast_bstuck++;
-	ath_reset(ifp);
+	/*
+	 * This assumes that there's no simultaneous channel mode change
+	 * occuring.
+	 */
+	ath_reset(ifp, ATH_RESET_NOLOSS);
 }
 
 /*
@@ -2699,7 +3074,7 @@ ath_beacon_return(struct ath_softc *sc, struct ath_buf *bf)
 		ieee80211_free_node(bf->bf_node);
 		bf->bf_node = NULL;
 	}
-	STAILQ_INSERT_TAIL(&sc->sc_bbuf, bf, bf_list);
+	TAILQ_INSERT_TAIL(&sc->sc_bbuf, bf, bf_list);
 }
 
 /*
@@ -2710,7 +3085,7 @@ ath_beacon_free(struct ath_softc *sc)
 {
 	struct ath_buf *bf;
 
-	STAILQ_FOREACH(bf, &sc->sc_bbuf, bf_list) {
+	TAILQ_FOREACH(bf, &sc->sc_bbuf, bf_list) {
 		if (bf->bf_m != NULL) {
 			bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 			m_freem(bf->bf_m);
@@ -3029,7 +3404,7 @@ ath_descdma_setup(struct ath_softc *sc,
 	}
 	dd->dd_bufptr = bf;
 
-	STAILQ_INIT(head);
+	TAILQ_INIT(head);
 	for (i = 0; i < nbuf; i++, bf++, ds += (ndesc * desc_len)) {
 		bf->bf_desc = (struct ath_desc *) ds;
 		bf->bf_daddr = DS2PHYS(dd, ds);
@@ -3055,7 +3430,8 @@ ath_descdma_setup(struct ath_softc *sc,
 			ath_descdma_cleanup(sc, dd, head);
 			return error;
 		}
-		STAILQ_INSERT_TAIL(head, bf, bf_list);
+		bf->bf_lastds = bf->bf_desc;	/* Just an initial value */
+		TAILQ_INSERT_TAIL(head, bf, bf_list);
 	}
 	return 0;
 fail3:
@@ -3084,7 +3460,7 @@ ath_descdma_cleanup(struct ath_softc *sc,
 	bus_dmamap_destroy(dd->dd_dmat, dd->dd_dmamap);
 	bus_dma_tag_destroy(dd->dd_dmat);
 
-	STAILQ_FOREACH(bf, head, bf_list) {
+	TAILQ_FOREACH(bf, head, bf_list) {
 		if (bf->bf_m) {
 			m_freem(bf->bf_m);
 			bf->bf_m = NULL;
@@ -3103,7 +3479,7 @@ ath_descdma_cleanup(struct ath_softc *sc,
 		}
 	}
 
-	STAILQ_INIT(head);
+	TAILQ_INIT(head);
 	free(dd->dd_bufptr, M_ATHDEV);
 	memset(dd, 0, sizeof(*dd));
 }
@@ -3162,19 +3538,38 @@ ath_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
 	}
 	ath_rate_node_init(sc, an);
 
+	/* Setup the mutex - there's no associd yet so set the name to NULL */
+	snprintf(an->an_name, sizeof(an->an_name), "%s: node %p",
+	    device_get_nameunit(sc->sc_dev), an);
+	mtx_init(&an->an_mtx, an->an_name, NULL, MTX_DEF);
+
+	/* XXX setup ath_tid */
+	ath_tx_tid_init(sc, an);
+
 	DPRINTF(sc, ATH_DEBUG_NODE, "%s: an %p\n", __func__, an);
 	return &an->an_node;
 }
 
 static void
+ath_node_cleanup(struct ieee80211_node *ni)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct ath_softc *sc = ic->ic_ifp->if_softc;
+
+	/* Cleanup ath_tid, free unused bufs, unlink bufs in TXQ */
+	ath_tx_node_flush(sc, ATH_NODE(ni));
+	ath_rate_node_cleanup(sc, ATH_NODE(ni));
+	sc->sc_node_cleanup(ni);
+}
+
+static void
 ath_node_free(struct ieee80211_node *ni)
 {
 	struct ieee80211com *ic = ni->ni_ic;
-        struct ath_softc *sc = ic->ic_ifp->if_softc;
+	struct ath_softc *sc = ic->ic_ifp->if_softc;
 
 	DPRINTF(sc, ATH_DEBUG_NODE, "%s: ni %p\n", __func__, ni);
-
-	ath_rate_node_cleanup(sc, ATH_NODE(ni));
+	mtx_destroy(&ATH_NODE(ni)->an_mtx);
 	sc->sc_node_free(ni);
 }
 
@@ -3454,13 +3849,35 @@ ath_handle_micerror(struct ieee80211com *ic,
 	}
 }
 
+/*
+ * Only run the RX proc if it's not already running.
+ * Since this may get run as part of the reset/flush path,
+ * the task can't clash with an existing, running tasklet.
+ */
 static void
-ath_rx_proc(void *arg, int npending)
+ath_rx_tasklet(void *arg, int npending)
+{
+	struct ath_softc *sc = arg;
+
+	CTR1(ATH_KTR_INTR, "ath_rx_proc: pending=%d", npending);
+	DPRINTF(sc, ATH_DEBUG_RX_PROC, "%s: pending %u\n", __func__, npending);
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt > 0) {
+		device_printf(sc->sc_dev,
+		    "%s: sc_inreset_cnt > 0; skipping\n", __func__);
+		ATH_PCU_UNLOCK(sc);
+		return;
+	}
+	ATH_PCU_UNLOCK(sc);
+	ath_rx_proc(sc, 1);
+}
+
+static void
+ath_rx_proc(struct ath_softc *sc, int resched)
 {
 #define	PA2DESC(_sc, _pa) \
 	((struct ath_desc *)((caddr_t)(_sc)->sc_rxdma.dd_desc + \
 		((_pa) - (_sc)->sc_rxdma.dd_desc_paddr)))
-	struct ath_softc *sc = arg;
 	struct ath_buf *bf;
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
@@ -3473,14 +3890,23 @@ ath_rx_proc(void *arg, int npending)
 	HAL_STATUS status;
 	int16_t nf;
 	u_int64_t tsf;
+	int npkts = 0;
 
-	DPRINTF(sc, ATH_DEBUG_RX_PROC, "%s: pending %u\n", __func__, npending);
+	/* XXX we must not hold the ATH_LOCK here */
+	ATH_UNLOCK_ASSERT(sc);
+	ATH_PCU_UNLOCK_ASSERT(sc);
+
+	ATH_PCU_LOCK(sc);
+	sc->sc_rxproc_cnt++;
+	ATH_PCU_UNLOCK(sc);
+
+	DPRINTF(sc, ATH_DEBUG_RX_PROC, "%s: called\n", __func__);
 	ngood = 0;
 	nf = ath_hal_getchannoise(ah, sc->sc_curchan);
 	sc->sc_stats.ast_rx_noise = nf;
 	tsf = ath_hal_gettsf64(ah);
 	do {
-		bf = STAILQ_FIRST(&sc->sc_rxbuf);
+		bf = TAILQ_FIRST(&sc->sc_rxbuf);
 		if (sc->sc_rxslink && bf == NULL) {	/* NB: shouldn't happen */
 			if_printf(ifp, "%s: no buffer!\n", __func__);
 			break;
@@ -3500,7 +3926,7 @@ ath_rx_proc(void *arg, int npending)
 			 */
 			/* XXX make debug msg */
 			if_printf(ifp, "%s: no mbuf!\n", __func__);
-			STAILQ_REMOVE_HEAD(&sc->sc_rxbuf, bf_list);
+			TAILQ_REMOVE(&sc->sc_rxbuf, bf, bf_list);
 			goto rx_next;
 		}
 		ds = bf->bf_desc;
@@ -3530,7 +3956,9 @@ ath_rx_proc(void *arg, int npending)
 #endif
 		if (status == HAL_EINPROGRESS)
 			break;
-		STAILQ_REMOVE_HEAD(&sc->sc_rxbuf, bf_list);
+
+		TAILQ_REMOVE(&sc->sc_rxbuf, bf, bf_list);
+		npkts++;
 
 		/* These aren't specifically errors */
 		if (rs->rs_flags & HAL_RX_GI)
@@ -3624,8 +4052,10 @@ rx_error:
 				/* NB: bpf needs the mbuf length setup */
 				len = rs->rs_datalen;
 				m->m_pkthdr.len = m->m_len = len;
+				bf->bf_m = NULL;
 				ath_rx_tap(ifp, m, rs, tsf, nf);
 				ieee80211_radiotap_rx_all(ic, m);
+				m_freem(m);
 			}
 			/* XXX pass MIC errors up for s/w reclaculation */
 			goto rx_next;
@@ -3797,7 +4227,7 @@ rx_accept:
 				ath_led_event(sc, 0);
 		}
 rx_next:
-		STAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
+		TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
 	} while (ath_rxbuf_init(sc, bf) == 0);
 
 	/* rx signal state monitoring */
@@ -3805,8 +4235,9 @@ rx_next:
 	if (ngood)
 		sc->sc_lastrx = tsf;
 
+	CTR2(ATH_KTR_INTR, "ath_rx_proc: npkts=%d, ngood=%d", npkts, ngood);
 	/* Queue DFS tasklet if needed */
-	if (ath_dfs_tasklet_needed(sc, sc->sc_curchan))
+	if (resched && ath_dfs_tasklet_needed(sc, sc->sc_curchan))
 		taskqueue_enqueue(sc->sc_tq, &sc->sc_dfstask);
 
 	/*
@@ -3814,21 +4245,30 @@ rx_next:
 	 * need to be handled, kick the PCU if there's
 	 * been an RXEOL condition.
 	 */
-	if (sc->sc_kickpcu) {
-		sc->sc_kickpcu = 0;
-		ath_stoprecv(sc);
-		sc->sc_imask |= (HAL_INT_RXEOL | HAL_INT_RXORN);
-		if (ath_startrecv(sc) != 0) {
-			if_printf(ifp,
-			    "%s: couldn't restart RX after RXEOL; resetting\n",
-			    __func__);
-			ath_reset(ifp);
-			return;
-		}
+	ATH_PCU_LOCK(sc);
+	if (resched && sc->sc_kickpcu) {
+		CTR0(ATH_KTR_ERR, "ath_rx_proc: kickpcu");
+		device_printf(sc->sc_dev, "%s: kickpcu; handled %d packets\n",
+		    __func__, npkts);
+
+		/* XXX rxslink? */
+		/*
+		 * XXX can we hold the PCU lock here?
+		 * Are there any net80211 buffer calls involved?
+		 */
+		bf = TAILQ_FIRST(&sc->sc_rxbuf);
+		ath_hal_putrxbuf(ah, bf->bf_daddr);
+		ath_hal_rxena(ah);		/* enable recv descriptors */
+		ath_mode_init(sc);		/* set filters, etc. */
+		ath_hal_startpcurecv(ah);	/* re-enable PCU/DMA engine */
+
 		ath_hal_intrset(ah, sc->sc_imask);
+		sc->sc_kickpcu = 0;
 	}
+	ATH_PCU_UNLOCK(sc);
 
-	if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
+	/* XXX check this inside of IF_LOCK? */
+	if (resched && (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
 #ifdef IEEE80211_SUPPORT_SUPERG
 		ieee80211_ff_age_all(ic, 100);
 #endif
@@ -3836,6 +4276,10 @@ rx_next:
 			ath_start(ifp);
 	}
 #undef PA2DESC
+
+	ATH_PCU_LOCK(sc);
+	sc->sc_rxproc_cnt--;
+	ATH_PCU_UNLOCK(sc);
 }
 
 static void
@@ -3844,9 +4288,12 @@ ath_txq_init(struct ath_softc *sc, struct ath_txq *txq, int qnum)
 	txq->axq_qnum = qnum;
 	txq->axq_ac = 0;
 	txq->axq_depth = 0;
+	txq->axq_aggr_depth = 0;
 	txq->axq_intrcnt = 0;
 	txq->axq_link = NULL;
-	STAILQ_INIT(&txq->axq_q);
+	txq->axq_softc = sc;
+	TAILQ_INIT(&txq->axq_q);
+	TAILQ_INIT(&txq->axq_tidq);
 	ATH_TXQ_LOCK_INIT(sc, txq);
 }
 
@@ -3972,10 +4419,15 @@ ath_txq_update(struct ath_softc *sc, int ac)
 		qi.tqi_burstTime = qi.tqi_readyTime;
 	} else {
 #endif
+		/*
+		 * XXX shouldn't this just use the default flags
+		 * used in the previous queue setup?
+		 */
 		qi.tqi_qflags = HAL_TXQ_TXOKINT_ENABLE
 			      | HAL_TXQ_TXERRINT_ENABLE
 			      | HAL_TXQ_TXDESCINT_ENABLE
 			      | HAL_TXQ_TXURNINT_ENABLE
+			      | HAL_TXQ_TXEOLINT_ENABLE
 			      ;
 		qi.tqi_aifs = wmep->wmep_aifsn;
 		qi.tqi_cwmin = ATH_EXPONENT_TO_VALUE(wmep->wmep_logcwmin);
@@ -4056,21 +4508,159 @@ ath_tx_findrix(const struct ath_softc *sc, uint8_t rate)
 	return (rix == 0xff ? 0 : rix);
 }
 
+static void
+ath_tx_update_stats(struct ath_softc *sc, struct ath_tx_status *ts,
+    struct ath_buf *bf)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+	int sr, lr, pri;
+
+	if (ts->ts_status == 0) {
+		u_int8_t txant = ts->ts_antenna;
+		sc->sc_stats.ast_ant_tx[txant]++;
+		sc->sc_ant_tx[txant]++;
+		if (ts->ts_finaltsi != 0)
+			sc->sc_stats.ast_tx_altrate++;
+		pri = M_WME_GETAC(bf->bf_m);
+		if (pri >= WME_AC_VO)
+			ic->ic_wme.wme_hipri_traffic++;
+		if ((bf->bf_txflags & HAL_TXDESC_NOACK) == 0)
+			ni->ni_inact = ni->ni_inact_reload;
+	} else {
+		if (ts->ts_status & HAL_TXERR_XRETRY)
+			sc->sc_stats.ast_tx_xretries++;
+		if (ts->ts_status & HAL_TXERR_FIFO)
+			sc->sc_stats.ast_tx_fifoerr++;
+		if (ts->ts_status & HAL_TXERR_FILT)
+			sc->sc_stats.ast_tx_filtered++;
+		if (ts->ts_status & HAL_TXERR_XTXOP)
+			sc->sc_stats.ast_tx_xtxop++;
+		if (ts->ts_status & HAL_TXERR_TIMER_EXPIRED)
+			sc->sc_stats.ast_tx_timerexpired++;
+
+		if (ts->ts_status & HAL_TX_DATA_UNDERRUN)
+			sc->sc_stats.ast_tx_data_underrun++;
+		if (ts->ts_status & HAL_TX_DELIM_UNDERRUN)
+			sc->sc_stats.ast_tx_delim_underrun++;
+
+		if (bf->bf_m->m_flags & M_FF)
+			sc->sc_stats.ast_ff_txerr++;
+	}
+	/* XXX when is this valid? */
+	if (ts->ts_status & HAL_TX_DESC_CFG_ERR)
+		sc->sc_stats.ast_tx_desccfgerr++;
+
+	sr = ts->ts_shortretry;
+	lr = ts->ts_longretry;
+	sc->sc_stats.ast_tx_shortretry += sr;
+	sc->sc_stats.ast_tx_longretry += lr;
+
+}
+
+/*
+ * The default completion. If fail is 1, this means
+ * "please don't retry the frame, and just return -1 status
+ * to the net80211 stack.
+ */
+void
+ath_tx_default_comp(struct ath_softc *sc, struct ath_buf *bf, int fail)
+{
+	struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
+	int st;
+
+	if (fail == 1)
+		st = -1;
+	else
+		st = ((bf->bf_txflags & HAL_TXDESC_NOACK) == 0) ?
+		    ts->ts_status : HAL_TXERR_XRETRY;
+
+	if (bf->bf_state.bfs_dobaw)
+		device_printf(sc->sc_dev,
+		    "%s: dobaw should've been cleared!\n", __func__);
+	if (bf->bf_next != NULL)
+		device_printf(sc->sc_dev,
+		    "%s: bf_next not NULL!\n", __func__);
+
+	/*
+	 * Do any tx complete callback.  Note this must
+	 * be done before releasing the node reference.
+	 * This will free the mbuf, release the net80211
+	 * node and recycle the ath_buf.
+	 */
+	ath_tx_freebuf(sc, bf, st);
+}
+
+/*
+ * Update rate control with the given completion status.
+ */
+void
+ath_tx_update_ratectrl(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_rc_series *rc, struct ath_tx_status *ts, int frmlen,
+    int nframes, int nbad)
+{
+	struct ath_node *an;
+
+	/* Only for unicast frames */
+	if (ni == NULL)
+		return;
+
+	an = ATH_NODE(ni);
+
+	if ((ts->ts_status & HAL_TXERR_FILT) == 0) {
+		ATH_NODE_LOCK(an);
+		ath_rate_tx_complete(sc, an, rc, ts, frmlen, nframes, nbad);
+		ATH_NODE_UNLOCK(an);
+	}
+}
+
+/*
+ * Update the busy status of the last frame on the free list.
+ * When doing TDMA, the busy flag tracks whether the hardware
+ * currently points to this buffer or not, and thus gated DMA
+ * may restart by re-reading the last descriptor in this
+ * buffer.
+ *
+ * This should be called in the completion function once one
+ * of the buffers has been used.
+ */
+static void
+ath_tx_update_busy(struct ath_softc *sc)
+{
+	struct ath_buf *last;
+
+	/*
+	 * Since the last frame may still be marked
+	 * as ATH_BUF_BUSY, unmark it here before
+	 * finishing the frame processing.
+	 * Since we've completed a frame (aggregate
+	 * or otherwise), the hardware has moved on
+	 * and is no longer referencing the previous
+	 * descriptor.
+	 */
+	ATH_TXBUF_LOCK_ASSERT(sc);
+	last = TAILQ_LAST(&sc->sc_txbuf, ath_bufhead_s);
+	if (last != NULL)
+		last->bf_flags &= ~ATH_BUF_BUSY;
+}
+
+
 /*
  * Process completed xmit descriptors from the specified queue.
+ * Kick the packet scheduler if needed. This can occur from this
+ * particular task.
  */
 static int
-ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
+ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq, int dosched)
 {
 	struct ath_hal *ah = sc->sc_ah;
-	struct ifnet *ifp = sc->sc_ifp;
-	struct ieee80211com *ic = ifp->if_l2com;
-	struct ath_buf *bf, *last;
-	struct ath_desc *ds, *ds0;
+	struct ath_buf *bf;
+	struct ath_desc *ds;
 	struct ath_tx_status *ts;
 	struct ieee80211_node *ni;
 	struct ath_node *an;
-	int sr, lr, pri, nacked;
+	int nacked;
 	HAL_STATUS status;
 
 	DPRINTF(sc, ATH_DEBUG_TX_PROC, "%s: tx queue %u head %p link %p\n",
@@ -4081,13 +4671,12 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
 	for (;;) {
 		ATH_TXQ_LOCK(txq);
 		txq->axq_intrcnt = 0;	/* reset periodic desc intr count */
-		bf = STAILQ_FIRST(&txq->axq_q);
+		bf = TAILQ_FIRST(&txq->axq_q);
 		if (bf == NULL) {
 			ATH_TXQ_UNLOCK(txq);
 			break;
 		}
-		ds0 = &bf->bf_desc[0];
-		ds = &bf->bf_desc[bf->bf_nseg - 1];
+		ds = bf->bf_lastds;	/* XXX must be setup correctly! */
 		ts = &bf->bf_status.ds_txstat;
 		status = ath_hal_txprocdesc(ah, ds, ts);
 #ifdef ATH_DEBUG
@@ -4099,104 +4688,72 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
 			ATH_TXQ_UNLOCK(txq);
 			break;
 		}
-		ATH_TXQ_REMOVE_HEAD(txq, bf_list);
+		ATH_TXQ_REMOVE(txq, bf, bf_list);
 #ifdef IEEE80211_SUPPORT_TDMA
 		if (txq->axq_depth > 0) {
 			/*
 			 * More frames follow.  Mark the buffer busy
 			 * so it's not re-used while the hardware may
 			 * still re-read the link field in the descriptor.
+			 *
+			 * Use the last buffer in an aggregate as that
+			 * is where the hardware may be - intermediate
+			 * descriptors won't be "busy".
 			 */
-			bf->bf_flags |= ATH_BUF_BUSY;
+			bf->bf_last->bf_flags |= ATH_BUF_BUSY;
 		} else
 #else
 		if (txq->axq_depth == 0)
 #endif
 			txq->axq_link = NULL;
-		ATH_TXQ_UNLOCK(txq);
+		if (bf->bf_state.bfs_aggr)
+			txq->axq_aggr_depth--;
 
 		ni = bf->bf_node;
+		/*
+		 * If unicast frame was ack'd update RSSI,
+		 * including the last rx time used to
+		 * workaround phantom bmiss interrupts.
+		 */
+		if (ni != NULL && ts->ts_status == 0 &&
+		    ((bf->bf_txflags & HAL_TXDESC_NOACK) == 0)) {
+			nacked++;
+			sc->sc_stats.ast_tx_rssi = ts->ts_rssi;
+			ATH_RSSI_LPF(sc->sc_halstats.ns_avgtxrssi,
+				ts->ts_rssi);
+		}
+		ATH_TXQ_UNLOCK(txq);
+
+		/* If unicast frame, update general statistics */
 		if (ni != NULL) {
 			an = ATH_NODE(ni);
-			if (ts->ts_status == 0) {
-				u_int8_t txant = ts->ts_antenna;
-				sc->sc_stats.ast_ant_tx[txant]++;
-				sc->sc_ant_tx[txant]++;
-				if (ts->ts_finaltsi != 0)
-					sc->sc_stats.ast_tx_altrate++;
-				pri = M_WME_GETAC(bf->bf_m);
-				if (pri >= WME_AC_VO)
-					ic->ic_wme.wme_hipri_traffic++;
-				if ((bf->bf_txflags & HAL_TXDESC_NOACK) == 0)
-					ni->ni_inact = ni->ni_inact_reload;
-			} else {
-				if (ts->ts_status & HAL_TXERR_XRETRY)
-					sc->sc_stats.ast_tx_xretries++;
-				if (ts->ts_status & HAL_TXERR_FIFO)
-					sc->sc_stats.ast_tx_fifoerr++;
-				if (ts->ts_status & HAL_TXERR_FILT)
-					sc->sc_stats.ast_tx_filtered++;
-				if (ts->ts_status & HAL_TXERR_XTXOP)
-					sc->sc_stats.ast_tx_xtxop++;
-				if (ts->ts_status & HAL_TXERR_TIMER_EXPIRED)
-					sc->sc_stats.ast_tx_timerexpired++;
-
-				/* XXX HAL_TX_DATA_UNDERRUN */
-				/* XXX HAL_TX_DELIM_UNDERRUN */
-
-				if (bf->bf_m->m_flags & M_FF)
-					sc->sc_stats.ast_ff_txerr++;
-			}
-			/* XXX when is this valid? */
-			if (ts->ts_status & HAL_TX_DESC_CFG_ERR)
-				sc->sc_stats.ast_tx_desccfgerr++;
-
-			sr = ts->ts_shortretry;
-			lr = ts->ts_longretry;
-			sc->sc_stats.ast_tx_shortretry += sr;
-			sc->sc_stats.ast_tx_longretry += lr;
-			/*
-			 * Hand the descriptor to the rate control algorithm.
-			 */
+			/* update statistics */
+			ath_tx_update_stats(sc, ts, bf);
+		}
+
+		/*
+		 * Call the completion handler.
+		 * The completion handler is responsible for
+		 * calling the rate control code.
+		 *
+		 * Frames with no completion handler get the
+		 * rate control code called here.
+		 */
+		if (bf->bf_comp == NULL) {
 			if ((ts->ts_status & HAL_TXERR_FILT) == 0 &&
 			    (bf->bf_txflags & HAL_TXDESC_NOACK) == 0) {
 				/*
-				 * If frame was ack'd update statistics,
-				 * including the last rx time used to
-				 * workaround phantom bmiss interrupts.
+				 * XXX assume this isn't an aggregate
+				 * frame.
 				 */
-				if (ts->ts_status == 0) {
-					nacked++;
-					sc->sc_stats.ast_tx_rssi = ts->ts_rssi;
-					ATH_RSSI_LPF(sc->sc_halstats.ns_avgtxrssi,
-						ts->ts_rssi);
-				}
-				ath_rate_tx_complete(sc, an, bf);
+				ath_tx_update_ratectrl(sc, ni,
+				     bf->bf_state.bfs_rc, ts,
+				    bf->bf_state.bfs_pktlen, 1,
+				    (ts->ts_status == 0 ? 0 : 1));
 			}
-			/*
-			 * Do any tx complete callback.  Note this must
-			 * be done before releasing the node reference.
-			 */
-			if (bf->bf_m->m_flags & M_TXCB)
-				ieee80211_process_callback(ni, bf->bf_m,
-				    (bf->bf_txflags & HAL_TXDESC_NOACK) == 0 ?
-				        ts->ts_status : HAL_TXERR_XRETRY);
-			ieee80211_free_node(ni);
-		}
-		bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
-		    BUS_DMASYNC_POSTWRITE);
-		bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
-
-		m_freem(bf->bf_m);
-		bf->bf_m = NULL;
-		bf->bf_node = NULL;
-
-		ATH_TXBUF_LOCK(sc);
-		last = STAILQ_LAST(&sc->sc_txbuf, ath_buf, bf_list);
-		if (last != NULL)
-			last->bf_flags &= ~ATH_BUF_BUSY;
-		STAILQ_INSERT_TAIL(&sc->sc_txbuf, bf, bf_list);
-		ATH_TXBUF_UNLOCK(sc);
+			ath_tx_default_comp(sc, bf, 0);
+		} else
+			bf->bf_comp(sc, bf, 0);
 	}
 #ifdef IEEE80211_SUPPORT_SUPERG
 	/*
@@ -4205,16 +4762,18 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
 	if (txq->axq_depth <= 1)
 		ieee80211_ff_flush(ic, txq->axq_ac);
 #endif
+
+	/* Kick the TXQ scheduler */
+	if (dosched) {
+		ATH_TXQ_LOCK(txq);
+		ath_txq_sched(sc, txq);
+		ATH_TXQ_UNLOCK(txq);
+	}
+
 	return nacked;
 }
 
-static __inline int
-txqactive(struct ath_hal *ah, int qnum)
-{
-	u_int32_t txqs = 1<<qnum;
-	ath_hal_gettxintrtxqs(ah, &txqs);
-	return (txqs & (1<<qnum));
-}
+#define	TXQACTIVE(t, q)		( (t) & (1 << (q)))
 
 /*
  * Deferred processing of transmit interrupt; special-cased
@@ -4225,17 +4784,30 @@ ath_tx_proc_q0(void *arg, int npending)
 {
 	struct ath_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
+	uint32_t txqs;
 
-	if (txqactive(sc->sc_ah, 0) && ath_tx_processq(sc, &sc->sc_txq[0]))
+	ATH_PCU_LOCK(sc);
+	sc->sc_txproc_cnt++;
+	txqs = sc->sc_txq_active;
+	sc->sc_txq_active &= ~txqs;
+	ATH_PCU_UNLOCK(sc);
+
+	if (TXQACTIVE(txqs, 0) && ath_tx_processq(sc, &sc->sc_txq[0], 1))
+		/* XXX why is lastrx updated in tx code? */
 		sc->sc_lastrx = ath_hal_gettsf64(sc->sc_ah);
-	if (txqactive(sc->sc_ah, sc->sc_cabq->axq_qnum))
-		ath_tx_processq(sc, sc->sc_cabq);
+	if (TXQACTIVE(txqs, sc->sc_cabq->axq_qnum))
+		ath_tx_processq(sc, sc->sc_cabq, 1);
+	/* XXX check this inside of IF_LOCK? */
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	sc->sc_wd_timer = 0;
 
 	if (sc->sc_softled)
 		ath_led_event(sc, sc->sc_txrix);
 
+	ATH_PCU_LOCK(sc);
+	sc->sc_txproc_cnt--;
+	ATH_PCU_UNLOCK(sc);
+
 	ath_start(ifp);
 }
 
@@ -4249,30 +4821,42 @@ ath_tx_proc_q0123(void *arg, int npending)
 	struct ath_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	int nacked;
+	uint32_t txqs;
+
+	ATH_PCU_LOCK(sc);
+	sc->sc_txproc_cnt++;
+	txqs = sc->sc_txq_active;
+	sc->sc_txq_active &= ~txqs;
+	ATH_PCU_UNLOCK(sc);
 
 	/*
 	 * Process each active queue.
 	 */
 	nacked = 0;
-	if (txqactive(sc->sc_ah, 0))
-		nacked += ath_tx_processq(sc, &sc->sc_txq[0]);
-	if (txqactive(sc->sc_ah, 1))
-		nacked += ath_tx_processq(sc, &sc->sc_txq[1]);
-	if (txqactive(sc->sc_ah, 2))
-		nacked += ath_tx_processq(sc, &sc->sc_txq[2]);
-	if (txqactive(sc->sc_ah, 3))
-		nacked += ath_tx_processq(sc, &sc->sc_txq[3]);
-	if (txqactive(sc->sc_ah, sc->sc_cabq->axq_qnum))
-		ath_tx_processq(sc, sc->sc_cabq);
+	if (TXQACTIVE(txqs, 0))
+		nacked += ath_tx_processq(sc, &sc->sc_txq[0], 1);
+	if (TXQACTIVE(txqs, 1))
+		nacked += ath_tx_processq(sc, &sc->sc_txq[1], 1);
+	if (TXQACTIVE(txqs, 2))
+		nacked += ath_tx_processq(sc, &sc->sc_txq[2], 1);
+	if (TXQACTIVE(txqs, 3))
+		nacked += ath_tx_processq(sc, &sc->sc_txq[3], 1);
+	if (TXQACTIVE(txqs, sc->sc_cabq->axq_qnum))
+		ath_tx_processq(sc, sc->sc_cabq, 1);
 	if (nacked)
 		sc->sc_lastrx = ath_hal_gettsf64(sc->sc_ah);
 
+	/* XXX check this inside of IF_LOCK? */
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	sc->sc_wd_timer = 0;
 
 	if (sc->sc_softled)
 		ath_led_event(sc, sc->sc_txrix);
 
+	ATH_PCU_LOCK(sc);
+	sc->sc_txproc_cnt--;
+	ATH_PCU_UNLOCK(sc);
+
 	ath_start(ifp);
 }
 
@@ -4285,33 +4869,106 @@ ath_tx_proc(void *arg, int npending)
 	struct ath_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	int i, nacked;
+	uint32_t txqs;
+
+	ATH_PCU_LOCK(sc);
+	sc->sc_txproc_cnt++;
+	txqs = sc->sc_txq_active;
+	sc->sc_txq_active &= ~txqs;
+	ATH_PCU_UNLOCK(sc);
 
 	/*
 	 * Process each active queue.
 	 */
 	nacked = 0;
 	for (i = 0; i < HAL_NUM_TX_QUEUES; i++)
-		if (ATH_TXQ_SETUP(sc, i) && txqactive(sc->sc_ah, i))
-			nacked += ath_tx_processq(sc, &sc->sc_txq[i]);
+		if (ATH_TXQ_SETUP(sc, i) && TXQACTIVE(txqs, i))
+			nacked += ath_tx_processq(sc, &sc->sc_txq[i], 1);
 	if (nacked)
 		sc->sc_lastrx = ath_hal_gettsf64(sc->sc_ah);
 
+	/* XXX check this inside of IF_LOCK? */
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	sc->sc_wd_timer = 0;
 
 	if (sc->sc_softled)
 		ath_led_event(sc, sc->sc_txrix);
 
+	ATH_PCU_LOCK(sc);
+	sc->sc_txproc_cnt--;
+	ATH_PCU_UNLOCK(sc);
+
 	ath_start(ifp);
 }
+#undef	TXQACTIVE
 
-static void
+/*
+ * Return a buffer to the pool and update the 'busy' flag on the
+ * previous 'tail' entry.
+ *
+ * This _must_ only be called when the buffer is involved in a completed
+ * TX. The logic is that if it was part of an active TX, the previous
+ * buffer on the list is now not involved in a halted TX DMA queue, waiting
+ * for restart (eg for TDMA.)
+ *
+ * The caller must free the mbuf and recycle the node reference.
+ */
+void
+ath_freebuf(struct ath_softc *sc, struct ath_buf *bf)
+{
+	bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
+	bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap, BUS_DMASYNC_POSTWRITE);
+
+	KASSERT((bf->bf_node == NULL), ("%s: bf->bf_node != NULL\n", __func__));
+	KASSERT((bf->bf_m == NULL), ("%s: bf->bf_m != NULL\n", __func__));
+
+	ATH_TXBUF_LOCK(sc);
+	ath_tx_update_busy(sc);
+	TAILQ_INSERT_TAIL(&sc->sc_txbuf, bf, bf_list);
+	ATH_TXBUF_UNLOCK(sc);
+}
+
+/*
+ * This is currently used by ath_tx_draintxq() and
+ * ath_tx_tid_free_pkts().
+ *
+ * It recycles a single ath_buf.
+ */
+void
+ath_tx_freebuf(struct ath_softc *sc, struct ath_buf *bf, int status)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct mbuf *m0 = bf->bf_m;
+
+	bf->bf_node = NULL;
+	bf->bf_m = NULL;
+
+	/* Free the buffer, it's not needed any longer */
+	ath_freebuf(sc, bf);
+
+	if (ni != NULL) {
+		/*
+		 * Do any callback and reclaim the node reference.
+		 */
+		if (m0->m_flags & M_TXCB)
+			ieee80211_process_callback(ni, m0, status);
+		ieee80211_free_node(ni);
+	}
+	m_freem(m0);
+
+	/*
+	 * XXX the buffer used to be freed -after-, but the DMA map was
+	 * freed where ath_freebuf() now is. I've no idea what this
+	 * will do.
+	 */
+}
+
+void
 ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq)
 {
 #ifdef ATH_DEBUG
 	struct ath_hal *ah = sc->sc_ah;
 #endif
-	struct ieee80211_node *ni;
 	struct ath_buf *bf;
 	u_int ix;
 
@@ -4320,50 +4977,55 @@ ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq)
 	 *     we do not need to block ath_tx_proc
 	 */
 	ATH_TXBUF_LOCK(sc);
-	bf = STAILQ_LAST(&sc->sc_txbuf, ath_buf, bf_list);
+	bf = TAILQ_LAST(&sc->sc_txbuf, ath_bufhead_s);
 	if (bf != NULL)
 		bf->bf_flags &= ~ATH_BUF_BUSY;
 	ATH_TXBUF_UNLOCK(sc);
+
 	for (ix = 0;; ix++) {
 		ATH_TXQ_LOCK(txq);
-		bf = STAILQ_FIRST(&txq->axq_q);
+		bf = TAILQ_FIRST(&txq->axq_q);
 		if (bf == NULL) {
 			txq->axq_link = NULL;
 			ATH_TXQ_UNLOCK(txq);
 			break;
 		}
-		ATH_TXQ_REMOVE_HEAD(txq, bf_list);
-		ATH_TXQ_UNLOCK(txq);
+		ATH_TXQ_REMOVE(txq, bf, bf_list);
+		if (bf->bf_state.bfs_aggr)
+			txq->axq_aggr_depth--;
 #ifdef ATH_DEBUG
 		if (sc->sc_debug & ATH_DEBUG_RESET) {
 			struct ieee80211com *ic = sc->sc_ifp->if_l2com;
 
 			ath_printtxbuf(sc, bf, txq->axq_qnum, ix,
-				ath_hal_txprocdesc(ah, bf->bf_desc,
+				ath_hal_txprocdesc(ah, bf->bf_lastds,
 				    &bf->bf_status.ds_txstat) == HAL_OK);
 			ieee80211_dump_pkt(ic, mtod(bf->bf_m, const uint8_t *),
 			    bf->bf_m->m_len, 0, -1);
 		}
 #endif /* ATH_DEBUG */
-		bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
-		ni = bf->bf_node;
-		bf->bf_node = NULL;
-		if (ni != NULL) {
-			/*
-			 * Do any callback and reclaim the node reference.
-			 */
-			if (bf->bf_m->m_flags & M_TXCB)
-				ieee80211_process_callback(ni, bf->bf_m, -1);
-			ieee80211_free_node(ni);
-		}
-		m_freem(bf->bf_m);
-		bf->bf_m = NULL;
+		/*
+		 * Since we're now doing magic in the completion
+		 * functions, we -must- call it for aggregation
+		 * destinations or BAW tracking will get upset.
+		 */
+		/*
+		 * Clear ATH_BUF_BUSY; the completion handler
+		 * will free the buffer.
+		 */
+		ATH_TXQ_UNLOCK(txq);
 		bf->bf_flags &= ~ATH_BUF_BUSY;
-
-		ATH_TXBUF_LOCK(sc);
-		STAILQ_INSERT_TAIL(&sc->sc_txbuf, bf, bf_list);
-		ATH_TXBUF_UNLOCK(sc);
+		if (bf->bf_comp)
+			bf->bf_comp(sc, bf, 1);
+		else
+			ath_tx_default_comp(sc, bf, 1);
 	}
+
+	/*
+	 * Drain software queued frames which are on
+	 * active TIDs.
+	 */
+	ath_tx_txq_drain(sc, txq);
 }
 
 static void
@@ -4378,17 +5040,16 @@ ath_tx_stopdma(struct ath_softc *sc, struct ath_txq *txq)
 	(void) ath_hal_stoptxdma(ah, txq->axq_qnum);
 }
 
-/*
- * Drain the transmit queues and reclaim resources.
- */
-static void
-ath_draintxq(struct ath_softc *sc)
+static int
+ath_stoptxdma(struct ath_softc *sc)
 {
 	struct ath_hal *ah = sc->sc_ah;
-	struct ifnet *ifp = sc->sc_ifp;
 	int i;
 
 	/* XXX return value */
+	if (sc->sc_invalid)
+		return 0;
+
 	if (!sc->sc_invalid) {
 		/* don't touch the hardware if marked invalid */
 		DPRINTF(sc, ATH_DEBUG_RESET, "%s: tx queue [%u] %p, link %p\n",
@@ -4400,15 +5061,42 @@ ath_draintxq(struct ath_softc *sc)
 			if (ATH_TXQ_SETUP(sc, i))
 				ath_tx_stopdma(sc, &sc->sc_txq[i]);
 	}
-	for (i = 0; i < HAL_NUM_TX_QUEUES; i++)
-		if (ATH_TXQ_SETUP(sc, i))
-			ath_tx_draintxq(sc, &sc->sc_txq[i]);
+
+	return 1;
+}
+
+/*
+ * Drain the transmit queues and reclaim resources.
+ */
+static void
+ath_draintxq(struct ath_softc *sc, ATH_RESET_TYPE reset_type)
+{
+#ifdef	ATH_DEBUG
+	struct ath_hal *ah = sc->sc_ah;
+#endif
+	struct ifnet *ifp = sc->sc_ifp;
+	int i;
+
+	(void) ath_stoptxdma(sc);
+
+	for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
+		/*
+		 * XXX TODO: should we just handle the completed TX frames
+		 * here, whether or not the reset is a full one or not?
+		 */
+		if (ATH_TXQ_SETUP(sc, i)) {
+			if (reset_type == ATH_RESET_NOLOSS)
+				ath_tx_processq(sc, &sc->sc_txq[i], 0);
+			else
+				ath_tx_draintxq(sc, &sc->sc_txq[i]);
+		}
+	}
 #ifdef ATH_DEBUG
 	if (sc->sc_debug & ATH_DEBUG_RESET) {
-		struct ath_buf *bf = STAILQ_FIRST(&sc->sc_bbuf);
+		struct ath_buf *bf = TAILQ_FIRST(&sc->sc_bbuf);
 		if (bf != NULL && bf->bf_m != NULL) {
 			ath_printtxbuf(sc, bf, sc->sc_bhalq, 0,
-				ath_hal_txprocdesc(ah, bf->bf_desc,
+				ath_hal_txprocdesc(ah, bf->bf_lastds,
 				    &bf->bf_status.ds_txstat) == HAL_OK);
 			ieee80211_dump_pkt(ifp->if_l2com,
 			    mtod(bf->bf_m, const uint8_t *), bf->bf_m->m_len,
@@ -4416,6 +5104,7 @@ ath_draintxq(struct ath_softc *sc)
 		}
 	}
 #endif /* ATH_DEBUG */
+	/* XXX check this inside of IF_LOCK? */
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	sc->sc_wd_timer = 0;
 }
@@ -4424,7 +5113,7 @@ ath_draintxq(struct ath_softc *sc)
  * Disable the receive h/w in preparation for a reset.
  */
 static void
-ath_stoprecv(struct ath_softc *sc)
+ath_stoprecv(struct ath_softc *sc, int dodelay)
 {
 #define	PA2DESC(_sc, _pa) \
 	((struct ath_desc *)((caddr_t)(_sc)->sc_rxdma.dd_desc + \
@@ -4434,7 +5123,8 @@ ath_stoprecv(struct ath_softc *sc)
 	ath_hal_stoppcurecv(ah);	/* disable PCU */
 	ath_hal_setrxfilter(ah, 0);	/* clear recv filter */
 	ath_hal_stopdmarecv(ah);	/* disable DMA engine */
-	DELAY(3000);			/* 3ms is long enough for 1 frame */
+	if (dodelay)
+		DELAY(3000);		/* 3ms is long enough for 1 frame */
 #ifdef ATH_DEBUG
 	if (sc->sc_debug & (ATH_DEBUG_RESET | ATH_DEBUG_FATAL)) {
 		struct ath_buf *bf;
@@ -4443,7 +5133,7 @@ ath_stoprecv(struct ath_softc *sc)
 		printf("%s: rx queue %p, link %p\n", __func__,
 			(caddr_t)(uintptr_t) ath_hal_getrxbuf(ah), sc->sc_rxlink);
 		ix = 0;
-		STAILQ_FOREACH(bf, &sc->sc_rxbuf, bf_list) {
+		TAILQ_FOREACH(bf, &sc->sc_rxbuf, bf_list) {
 			struct ath_desc *ds = bf->bf_desc;
 			struct ath_rx_status *rs = &bf->bf_status.ds_rxstat;
 			HAL_STATUS status = ath_hal_rxprocdesc(ah, ds,
@@ -4473,7 +5163,7 @@ ath_startrecv(struct ath_softc *sc)
 
 	sc->sc_rxlink = NULL;
 	sc->sc_rxpending = NULL;
-	STAILQ_FOREACH(bf, &sc->sc_rxbuf, bf_list) {
+	TAILQ_FOREACH(bf, &sc->sc_rxbuf, bf_list) {
 		int error = ath_rxbuf_init(sc, bf);
 		if (error != 0) {
 			DPRINTF(sc, ATH_DEBUG_RECV,
@@ -4483,7 +5173,7 @@ ath_startrecv(struct ath_softc *sc)
 		}
 	}
 
-	bf = STAILQ_FIRST(&sc->sc_rxbuf);
+	bf = TAILQ_FIRST(&sc->sc_rxbuf);
 	ath_hal_putrxbuf(ah, bf->bf_daddr);
 	ath_hal_rxena(ah);		/* enable recv descriptors */
 	ath_mode_init(sc);		/* set filters, etc. */
@@ -4521,6 +5211,22 @@ ath_chan_set(struct ath_softc *sc, struct ieee80211_channel *chan)
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ath_hal *ah = sc->sc_ah;
+	int ret = 0;
+	int dointr = 0;
+
+	/* Treat this as an interface reset */
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt > 0)
+		device_printf(sc->sc_dev, "%s: danger! concurrent reset!\n",
+		    __func__);
+	sc->sc_inreset_cnt++;
+	if (chan != sc->sc_curchan) {
+		dointr = 1;
+		/* XXX only do this if inreset_cnt is 1? */
+		ath_hal_intrset(ah, 0);
+	}
+	ATH_PCU_UNLOCK(sc);
+	ath_txrx_stop(sc);
 
 	DPRINTF(sc, ATH_DEBUG_RESET, "%s: %u (%u MHz, flags 0x%x)\n",
 	    __func__, ieee80211_chan2ieee(ic, chan),
@@ -4533,15 +5239,27 @@ ath_chan_set(struct ath_softc *sc, struct ieee80211_channel *chan)
 		 * hardware at the new frequency, and then re-enable
 		 * the relevant bits of the h/w.
 		 */
+#if 0
 		ath_hal_intrset(ah, 0);		/* disable interrupts */
-		ath_draintxq(sc);		/* clear pending tx frames */
-		ath_stoprecv(sc);		/* turn off frame recv */
+#endif
+		ath_stoprecv(sc, 1);		/* turn off frame recv */
+		/*
+		 * First, handle completed TX/RX frames.
+		 */
+		ath_rx_proc(sc, 0);
+		ath_draintxq(sc, ATH_RESET_NOLOSS);
+		/*
+		 * Next, flush the non-scheduled frames.
+		 */
+		ath_draintxq(sc, ATH_RESET_FULL);	/* clear pending tx frames */
+
 		if (!ath_hal_reset(ah, sc->sc_opmode, chan, AH_TRUE, &status)) {
 			if_printf(ifp, "%s: unable to reset "
 			    "channel %u (%u MHz, flags 0x%x), hal status %u\n",
 			    __func__, ieee80211_chan2ieee(ic, chan),
 			    chan->ic_freq, chan->ic_flags, status);
-			return EIO;
+			ret = EIO;
+			goto finish;
 		}
 		sc->sc_diversity = ath_hal_getdiversity(ah);
 
@@ -4554,7 +5272,8 @@ ath_chan_set(struct ath_softc *sc, struct ieee80211_channel *chan)
 		if (ath_startrecv(sc) != 0) {
 			if_printf(ifp, "%s: unable to restart recv logic\n",
 			    __func__);
-			return EIO;
+			ret = EIO;
+			goto finish;
 		}
 
 		/*
@@ -4576,12 +5295,28 @@ ath_chan_set(struct ath_softc *sc, struct ieee80211_channel *chan)
 			ath_beacon_config(sc, NULL);
 		}
 
+#if 0
 		/*
 		 * Re-enable interrupts.
 		 */
 		ath_hal_intrset(ah, sc->sc_imask);
+#endif
 	}
-	return 0;
+
+finish:
+	ATH_PCU_LOCK(sc);
+	sc->sc_inreset_cnt--;
+	/* XXX only do this if sc_inreset_cnt == 0? */
+	if (dointr)
+		ath_hal_intrset(ah, sc->sc_imask);
+	ATH_PCU_UNLOCK(sc);
+
+	/* XXX do this inside of IF_LOCK? */
+	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+	ath_txrx_start(sc);
+	/* XXX ath_start? */
+
+	return ret;
 }
 
 /*
@@ -4624,7 +5359,18 @@ ath_calibrate(void *arg)
 			DPRINTF(sc, ATH_DEBUG_CALIBRATE,
 				"%s: rfgain change\n", __func__);
 			sc->sc_stats.ast_per_rfgain++;
-			ath_reset(ifp);
+			/*
+			 * Drop lock - we can't hold it across the
+			 * ath_reset() call. Instead, we'll drop
+			 * out here, do a reset, then reschedule
+			 * the callout.
+			 */
+			callout_reset(&sc->sc_cal_ch, 1, ath_calibrate, sc);
+			sc->sc_resetcal = 0;
+			sc->sc_doresetcal = AH_TRUE;
+			ATH_UNLOCK(sc);
+			ath_reset(ifp, ATH_RESET_NOLOSS);
+			return;
 		}
 		/*
 		 * If this long cal is after an idle period, then
@@ -5004,7 +5750,7 @@ ath_setup_stationkey(struct ieee80211_node *ni)
 		ni->ni_ucastkey.wk_flags |= IEEE80211_KEY_DEVKEY;
 		IEEE80211_ADDR_COPY(ni->ni_ucastkey.wk_macaddr, ni->ni_macaddr);
 		/* NB: this will create a pass-thru key entry */
-		ath_keyset(sc, &ni->ni_ucastkey, vap->iv_bss);
+		ath_keyset(sc, vap, &ni->ni_ucastkey, vap->iv_bss);
 	}
 }
 
@@ -5279,6 +6025,7 @@ static void
 ath_watchdog(void *arg)
 {
 	struct ath_softc *sc = arg;
+	int do_reset = 0;
 
 	if (sc->sc_wd_timer != 0 && --sc->sc_wd_timer == 0) {
 		struct ifnet *ifp = sc->sc_ifp;
@@ -5290,10 +6037,20 @@ ath_watchdog(void *arg)
 			    hangs & 0xff ? "bb" : "mac", hangs);
 		} else
 			if_printf(ifp, "device timeout\n");
-		ath_reset(ifp);
+		do_reset = 1;
 		ifp->if_oerrors++;
 		sc->sc_stats.ast_watchdog++;
 	}
+
+	/*
+	 * We can't hold the lock across the ath_reset() call.
+	 */
+	if (do_reset) {
+		ATH_UNLOCK(sc);
+		ath_reset(sc->sc_ifp, ATH_RESET_NOLOSS);
+		ATH_LOCK(sc);
+	}
+
 	callout_schedule(&sc->sc_wd_ch, hz);
 }
 
diff --git a/sys/dev/ath/if_ath_ahb.c b/sys/dev/ath/if_ath_ahb.c
index a2bca05..21c3529 100644
--- a/sys/dev/ath/if_ath_ahb.c
+++ b/sys/dev/ath/if_ath_ahb.c
@@ -190,11 +190,13 @@ ath_ahb_attach(device_t dev)
 	}
 
 	ATH_LOCK_INIT(sc);
+	ATH_PCU_LOCK_INIT(sc);
 
 	error = ath_attach(AR9130_DEVID, sc);
 	if (error == 0)					/* success */
 		return 0;
 
+	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 	bus_dma_tag_destroy(sc->sc_dmat);
 bad3:
@@ -234,6 +236,7 @@ ath_ahb_detach(device_t dev)
 	if (sc->sc_eepromdata)
 		free(sc->sc_eepromdata, M_TEMP);
 
+	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 
 	return (0);
diff --git a/sys/dev/ath/if_ath_debug.c b/sys/dev/ath/if_ath_debug.c
index b5691ac..ac08f1f 100644
--- a/sys/dev/ath/if_ath_debug.c
+++ b/sys/dev/ath/if_ath_debug.c
@@ -123,33 +123,44 @@ ath_printrxbuf(struct ath_softc *sc, const struct ath_buf *bf,
 }
 
 void
-ath_printtxbuf(struct ath_softc *sc, const struct ath_buf *bf,
+ath_printtxbuf(struct ath_softc *sc, const struct ath_buf *first_bf,
 	u_int qnum, u_int ix, int done)
 {
-	const struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
+	const struct ath_tx_status *ts = &first_bf->bf_last->bf_status.ds_txstat;
+	const struct ath_buf *bf = first_bf;
 	struct ath_hal *ah = sc->sc_ah;
 	const struct ath_desc *ds;
 	int i;
 
 	printf("Q%u[%3u]", qnum, ix);
-	for (i = 0, ds = bf->bf_desc; i < bf->bf_nseg; i++, ds++) {
-		printf(" (DS.V:%p DS.P:%p) L:%08x D:%08x F:04%x%s\n"
-		       "        %08x %08x %08x %08x %08x %08x\n",
-		    ds, (const struct ath_desc *)bf->bf_daddr + i,
-		    ds->ds_link, ds->ds_data, bf->bf_txflags,
-		    !done ? "" : (ts->ts_status == 0) ? " *" : " !",
-		    ds->ds_ctl0, ds->ds_ctl1,
-		    ds->ds_hw[0], ds->ds_hw[1], ds->ds_hw[2], ds->ds_hw[3]);
-		if (ah->ah_magic == 0x20065416) {
-			printf("        %08x %08x %08x %08x %08x %08x %08x %08x\n",
-			    ds->ds_hw[4], ds->ds_hw[5], ds->ds_hw[6],
-			    ds->ds_hw[7], ds->ds_hw[8], ds->ds_hw[9],
-			    ds->ds_hw[10],ds->ds_hw[11]);
-			printf("        %08x %08x %08x %08x %08x %08x %08x %08x\n",
-			    ds->ds_hw[12],ds->ds_hw[13],ds->ds_hw[14],
-			    ds->ds_hw[15],ds->ds_hw[16],ds->ds_hw[17],
-			    ds->ds_hw[18], ds->ds_hw[19]);
+	while (bf != NULL) {
+		for (i = 0, ds = bf->bf_desc; i < bf->bf_nseg; i++, ds++) {
+			printf(" (DS.V:%p DS.P:%p) L:%08x D:%08x F:%04x%s\n"
+			       "        TXF: %04x Seq: %d swtry: %d ADDBAW?: %d DOBAW?: %d\n"
+			       "        %08x %08x %08x %08x %08x %08x\n",
+			    ds, (const struct ath_desc *)bf->bf_daddr + i,
+			    ds->ds_link, ds->ds_data, bf->bf_txflags,
+			    !done ? "" : (ts->ts_status == 0) ? " *" : " !",
+			    bf->bf_state.bfs_flags,
+			    bf->bf_state.bfs_seqno,
+			    bf->bf_state.bfs_retries,
+			    bf->bf_state.bfs_addedbaw,
+			    bf->bf_state.bfs_dobaw,
+			    ds->ds_ctl0, ds->ds_ctl1,
+			    ds->ds_hw[0], ds->ds_hw[1], ds->ds_hw[2], ds->ds_hw[3]);
+			if (ah->ah_magic == 0x20065416) {
+				printf("        %08x %08x %08x %08x %08x %08x %08x %08x\n",
+				    ds->ds_hw[4], ds->ds_hw[5], ds->ds_hw[6],
+				    ds->ds_hw[7], ds->ds_hw[8], ds->ds_hw[9],
+				    ds->ds_hw[10],ds->ds_hw[11]);
+				printf("        %08x %08x %08x %08x %08x %08x %08x %08x\n",
+				    ds->ds_hw[12],ds->ds_hw[13],ds->ds_hw[14],
+				    ds->ds_hw[15],ds->ds_hw[16],ds->ds_hw[17],
+				    ds->ds_hw[18], ds->ds_hw[19]);
+			}
 		}
+		printf("  [end]\n");
+		bf = bf->bf_next;
 	}
 }
 
diff --git a/sys/dev/ath/if_ath_debug.h b/sys/dev/ath/if_ath_debug.h
index c21914f..58199ff 100644
--- a/sys/dev/ath/if_ath_debug.h
+++ b/sys/dev/ath/if_ath_debug.h
@@ -57,6 +57,11 @@ enum {
 	ATH_DEBUG_TDMA		= 0x00800000,	/* TDMA processing */
 	ATH_DEBUG_TDMA_TIMER	= 0x01000000,	/* TDMA timer processing */
 	ATH_DEBUG_REGDOMAIN	= 0x02000000,	/* regulatory processing */
+	ATH_DEBUG_SW_TX		= 0x04000000,	/* per-packet software TX */
+	ATH_DEBUG_SW_TX_BAW	= 0x08000000,	/* BAW handling */
+	ATH_DEBUG_SW_TX_CTRL	= 0x10000000,	/* queue control */
+	ATH_DEBUG_SW_TX_AGGR	= 0x20000000,	/* aggregate TX */
+	ATH_DEBUG_SW_TX_RETRIES	= 0x40000000,	/* software TX retries */
 	ATH_DEBUG_FATAL		= 0x80000000,	/* fatal errors */
 	ATH_DEBUG_ANY		= 0xffffffff
 };
diff --git a/sys/dev/ath/if_ath_keycache.c b/sys/dev/ath/if_ath_keycache.c
index 842f766..e959c7a 100644
--- a/sys/dev/ath/if_ath_keycache.c
+++ b/sys/dev/ath/if_ath_keycache.c
@@ -178,7 +178,8 @@ ath_keyset_tkip(struct ath_softc *sc, const struct ieee80211_key *k,
  * cache slots for TKIP with hardware MIC support.
  */
 int
-ath_keyset(struct ath_softc *sc, const struct ieee80211_key *k,
+ath_keyset(struct ath_softc *sc, struct ieee80211vap *vap,
+	const struct ieee80211_key *k,
 	struct ieee80211_node *bss)
 {
 #define	N(a)	(sizeof(a)/sizeof(a[0]))
@@ -212,7 +213,23 @@ ath_keyset(struct ath_softc *sc, const struct ieee80211_key *k,
 	} else
 		hk.kv_type = HAL_CIPHER_CLR;
 
-	if ((k->wk_flags & IEEE80211_KEY_GROUP) && sc->sc_mcastkey) {
+	/*
+	 * XXX TODO: check this:
+	 * 
+	 * Group keys on hardware that supports multicast frame
+	 * key search should only be done in adhoc/hostap mode,
+	 * not STA mode.
+	 *
+	 * XXX TODO: what about mesh, tdma?
+	 */
+#if 0
+	if ((vap->iv_opmode == IEEE80211_M_HOSTAP ||
+	     vap->iv_opmode == IEEE80211_M_IBSS) &&
+#else
+	if (
+#endif
+	    (k->wk_flags & IEEE80211_KEY_GROUP) &&
+	    sc->sc_mcastkey) {
 		/*
 		 * Group keys on hardware that supports multicast frame
 		 * key search use a MAC that is the sender's address with
@@ -493,5 +510,5 @@ ath_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k,
 {
 	struct ath_softc *sc = vap->iv_ic->ic_ifp->if_softc;
 
-	return ath_keyset(sc, k, vap->iv_bss);
+	return ath_keyset(sc, vap, k, vap->iv_bss);
 }
diff --git a/sys/dev/ath/if_ath_keycache.h b/sys/dev/ath/if_ath_keycache.h
index f1696cb..0b79e6f 100644
--- a/sys/dev/ath/if_ath_keycache.h
+++ b/sys/dev/ath/if_ath_keycache.h
@@ -37,7 +37,7 @@ extern int ath_key_alloc(struct ieee80211vap *, struct ieee80211_key *,
 extern int ath_key_delete(struct ieee80211vap *, const struct ieee80211_key *);
 extern int ath_key_set(struct ieee80211vap *, const struct ieee80211_key *,
     const u_int8_t mac[IEEE80211_ADDR_LEN]);
-extern int ath_keyset(struct ath_softc *sc, const struct ieee80211_key *k,
-    struct ieee80211_node *bss);
+extern int ath_keyset(struct ath_softc *sc, struct ieee80211vap *vap,
+    const struct ieee80211_key *k, struct ieee80211_node *bss);
 
 #endif
diff --git a/sys/dev/ath/if_ath_misc.h b/sys/dev/ath/if_ath_misc.h
index 35feea2..c48590e 100644
--- a/sys/dev/ath/if_ath_misc.h
+++ b/sys/dev/ath/if_ath_misc.h
@@ -52,7 +52,19 @@ extern int ath_tx_findrix(const struct ath_softc *sc, uint8_t rate);
 
 extern struct ath_buf * ath_getbuf(struct ath_softc *sc);
 extern struct ath_buf * _ath_getbuf_locked(struct ath_softc *sc);
+extern struct ath_buf * ath_buf_clone(struct ath_softc *sc,
+	    const struct ath_buf *bf);
+extern void ath_freebuf(struct ath_softc *sc, struct ath_buf *bf);
 
-extern int ath_reset(struct ifnet *);
+extern int ath_reset(struct ifnet *, ATH_RESET_TYPE);
+extern void ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq);
+extern void ath_tx_default_comp(struct ath_softc *sc, struct ath_buf *bf,
+	    int fail);
+extern void ath_tx_update_ratectrl(struct ath_softc *sc,
+	    struct ieee80211_node *ni, struct ath_rc_series *rc,
+	    struct ath_tx_status *ts, int frmlen, int nframes, int nbad);
+
+extern void ath_tx_freebuf(struct ath_softc *sc, struct ath_buf *bf,
+    int status);
 
 #endif
diff --git a/sys/dev/ath/if_ath_pci.c b/sys/dev/ath/if_ath_pci.c
index 4824773..536930b 100644
--- a/sys/dev/ath/if_ath_pci.c
+++ b/sys/dev/ath/if_ath_pci.c
@@ -78,8 +78,10 @@ struct ath_pci_softc {
 static void
 ath_pci_setup(device_t dev)
 {
+#ifdef	ATH_PCI_LATENCY_WAR
 	/* Override the system latency timer */
 	pci_write_config(dev, PCIR_LATTIMER, 0x80, 1);
+#endif
 
 	/* If a PCI NIC, force wakeup */
 #ifdef	ATH_PCI_WAKEUP_WAR
@@ -190,11 +192,13 @@ ath_pci_attach(device_t dev)
 	}
 
 	ATH_LOCK_INIT(sc);
+	ATH_PCU_LOCK_INIT(sc);
 
 	error = ath_attach(pci_get_device(dev), sc);
 	if (error == 0)					/* success */
 		return 0;
 
+	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 	bus_dma_tag_destroy(sc->sc_dmat);
 bad3:
@@ -230,6 +234,7 @@ ath_pci_detach(device_t dev)
 	bus_dma_tag_destroy(sc->sc_dmat);
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, psc->sc_sr);
 
+	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 
 	return (0);
diff --git a/sys/dev/ath/if_ath_sysctl.c b/sys/dev/ath/if_ath_sysctl.c
index 9e52f79..5323504 100644
--- a/sys/dev/ath/if_ath_sysctl.c
+++ b/sys/dev/ath/if_ath_sysctl.c
@@ -263,7 +263,8 @@ ath_sysctl_tpscale(SYSCTL_HANDLER_ARGS)
 	if (error || !req->newptr)
 		return error;
 	return !ath_hal_settpscale(sc->sc_ah, scale) ? EINVAL :
-	    (ifp->if_drv_flags & IFF_DRV_RUNNING) ? ath_reset(ifp) : 0;
+	    (ifp->if_drv_flags & IFF_DRV_RUNNING) ?
+	      ath_reset(ifp, ATH_RESET_NOLOSS) : 0;
 }
 
 static int
@@ -295,7 +296,70 @@ ath_sysctl_rfkill(SYSCTL_HANDLER_ARGS)
 		return 0;
 	if (!ath_hal_setrfkill(ah, rfkill))
 		return EINVAL;
-	return (ifp->if_drv_flags & IFF_DRV_RUNNING) ? ath_reset(ifp) : 0;
+	return (ifp->if_drv_flags & IFF_DRV_RUNNING) ?
+	    ath_reset(ifp, ATH_RESET_FULL) : 0;
+}
+
+static int
+ath_sysctl_txagg(SYSCTL_HANDLER_ARGS)
+{
+	struct ath_softc *sc = arg1;
+	int i, t, param = 0;
+	int error;
+	struct ath_buf *bf;
+
+	error = sysctl_handle_int(oidp, &param, 0, req);
+	if (error || !req->newptr)
+		return error;
+
+	if (param != 1)
+		return 0;
+
+	printf("no tx bufs (empty list): %d\n", sc->sc_stats.ast_tx_getnobuf);
+	printf("no tx bufs (was busy): %d\n", sc->sc_stats.ast_tx_getbusybuf);
+
+	printf("aggr single packet: %d\n",
+	    sc->sc_aggr_stats.aggr_single_pkt);
+	printf("aggr single packet w/ BAW closed: %d\n",
+	    sc->sc_aggr_stats.aggr_baw_closed_single_pkt);
+	printf("aggr non-baw packet: %d\n",
+	    sc->sc_aggr_stats.aggr_nonbaw_pkt);
+	printf("aggr aggregate packet: %d\n",
+	    sc->sc_aggr_stats.aggr_aggr_pkt);
+	printf("aggr single packet low hwq: %d\n",
+	    sc->sc_aggr_stats.aggr_low_hwq_single_pkt);
+	printf("aggr sched, no work: %d\n",
+	    sc->sc_aggr_stats.aggr_sched_nopkt);
+	for (i = 0; i < 64; i++) {
+		printf("%2d: %10d ", i, sc->sc_aggr_stats.aggr_pkts[i]);
+		if (i % 4 == 3)
+			printf("\n");
+	}
+	printf("\n");
+
+	for (i = 0; i < HAL_NUM_TX_QUEUES; i++) {
+		if (ATH_TXQ_SETUP(sc, i)) {
+			printf("HW TXQ %d: axq_depth=%d, axq_aggr_depth=%d\n",
+			    i,
+			    sc->sc_txq[i].axq_depth,
+			    sc->sc_txq[i].axq_aggr_depth);
+		}
+	}
+
+	i = t = 0;
+	ATH_TXBUF_LOCK(sc);
+	TAILQ_FOREACH(bf, &sc->sc_txbuf, bf_list) {
+		if (bf->bf_flags & ATH_BUF_BUSY) {
+			printf("Busy: %d\n", t);
+			i++;
+		}
+		t++;
+	}
+	ATH_TXBUF_UNLOCK(sc);
+	printf("Total TX buffers: %d; Total TX buffers busy: %d\n",
+	    t, i);
+
+	return 0;
 }
 
 static int
@@ -366,7 +430,7 @@ ath_sysctl_intmit(SYSCTL_HANDLER_ARGS)
 	 * things in an inconsistent state.
 	 */
 	if (sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING)
-		ath_reset(sc->sc_ifp);
+		ath_reset(sc->sc_ifp, ATH_RESET_NOLOSS);
 
 	return 0;
 }
@@ -387,6 +451,24 @@ ath_sysctl_setcca(SYSCTL_HANDLER_ARGS)
 }
 #endif /* IEEE80211_SUPPORT_TDMA */
 
+static int
+ath_sysctl_forcebstuck(SYSCTL_HANDLER_ARGS)
+{
+	struct ath_softc *sc = arg1;
+	int val = 0;
+	int error;
+
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error || !req->newptr)
+		return error;
+	if (val == 0)
+		return 0;
+
+	taskqueue_enqueue_fast(sc->sc_tq, &sc->sc_bstucktask);
+	val = 0;
+	return 0;
+}
+
 void
 ath_sysctlattach(struct ath_softc *sc)
 {
@@ -465,6 +547,15 @@ ath_sysctlattach(struct ath_softc *sc)
 			"rfkill", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
 			ath_sysctl_rfkill, "I", "enable/disable RF kill switch");
 	}
+
+	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"txagg", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
+		ath_sysctl_txagg, "I", "");
+
+	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"forcebstuck", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
+		ath_sysctl_forcebstuck, "I", "");
+
 	if (ath_hal_hasintmit(ah)) {
 		SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 			"intmit", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
@@ -474,6 +565,17 @@ ath_sysctlattach(struct ath_softc *sc)
 	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 		"monpass", CTLFLAG_RW, &sc->sc_monpass, 0,
 		"mask of error frames to pass when monitoring");
+
+	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"hwq_limit", CTLFLAG_RW, &sc->sc_hwq_limit, 0,
+		"");
+	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tid_hwq_lo", CTLFLAG_RW, &sc->sc_tid_hwq_lo, 0,
+		"");
+	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"tid_hwq_hi", CTLFLAG_RW, &sc->sc_tid_hwq_hi, 0,
+		"");
+
 #ifdef IEEE80211_SUPPORT_TDMA
 	if (ath_hal_macversion(ah) > 0x78) {
 		sc->sc_tdmadbaprep = 2;
@@ -510,6 +612,8 @@ ath_sysctl_clearstats(SYSCTL_HANDLER_ARGS)
 	if (val == 0)
 		return 0;       /* Not clearing the stats is still valid */
 	memset(&sc->sc_stats, 0, sizeof(sc->sc_stats));
+	memset(&sc->sc_aggr_stats, 0, sizeof(sc->sc_aggr_stats));
+
 	val = 0;
 	return 0;
 }
@@ -789,4 +893,16 @@ ath_sysctl_hal_attach(struct ath_softc *sc)
 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "swba_backoff", CTLFLAG_RW,
 	    &sc->sc_ah->ah_config.ah_additional_swba_backoff, 0,
 	    "Atheros HAL additional SWBA backoff time");
+
+	sc->sc_ah->ah_config.ah_force_full_reset = 0;
+	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "force_full_reset", CTLFLAG_RW,
+	    &sc->sc_ah->ah_config.ah_force_full_reset, 0,
+	    "Force full chip reset rather than a warm reset");
+
+	/*
+	 * This is initialised by the driver.
+	 */
+	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "serialise_reg_war", CTLFLAG_RW,
+	    &sc->sc_ah->ah_config.ah_serialise_reg_war, 0,
+	    "Force register access serialisation");
 }
diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c
index f4cee6d..5625bf2 100644
--- a/sys/dev/ath/if_ath_tx.c
+++ b/sys/dev/ath/if_ath_tx.c
@@ -77,6 +77,7 @@ __FBSDID("$FreeBSD$");
 #ifdef IEEE80211_SUPPORT_TDMA
 #include <net80211/ieee80211_tdma.h>
 #endif
+#include <net80211/ieee80211_ht.h>
 
 #include <net/bpf.h>
 
@@ -100,6 +101,20 @@ __FBSDID("$FreeBSD$");
 #include <dev/ath/if_ath_tx_ht.h>
 
 /*
+ * How many retries to perform in software
+ */
+#define	SWMAX_RETRIES		10
+
+static int ath_tx_ampdu_pending(struct ath_softc *sc, struct ath_node *an,
+    int tid);
+static int ath_tx_ampdu_running(struct ath_softc *sc, struct ath_node *an,
+    int tid);
+static ieee80211_seq ath_tx_tid_seqno_assign(struct ath_softc *sc,
+    struct ieee80211_node *ni, struct ath_buf *bf, struct mbuf *m0);
+static int ath_tx_action_frame_override_queue(struct ath_softc *sc,
+    struct ieee80211_node *ni, struct mbuf *m0, int *tid);
+
+/*
  * Whether to use the 11n rate scenario functions or not
  */
 static inline int
@@ -108,6 +123,56 @@ ath_tx_is_11n(struct ath_softc *sc)
 	return (sc->sc_ah->ah_magic == 0x20065416);
 }
 
+/*
+ * Obtain the current TID from the given frame.
+ *
+ * Non-QoS frames need to go into TID 16 (IEEE80211_NONQOS_TID.)
+ * This has implications for which AC/priority the packet is placed
+ * in.
+ */
+static int
+ath_tx_gettid(struct ath_softc *sc, const struct mbuf *m0)
+{
+	const struct ieee80211_frame *wh;
+	int pri = M_WME_GETAC(m0);
+
+	wh = mtod(m0, const struct ieee80211_frame *);
+	if (! IEEE80211_QOS_HAS_SEQ(wh))
+		return IEEE80211_NONQOS_TID;
+	else
+		return WME_AC_TO_TID(pri);
+}
+
+/*
+ * Determine what the correct AC queue for the given frame
+ * should be.
+ *
+ * This code assumes that the TIDs map consistently to
+ * the underlying hardware (or software) ath_txq.
+ * Since the sender may try to set an AC which is
+ * arbitrary, non-QoS TIDs may end up being put on
+ * completely different ACs. There's no way to put a
+ * TID into multiple ath_txq's for scheduling, so
+ * for now we override the AC/TXQ selection and set
+ * non-QOS TID frames into the BE queue.
+ *
+ * This may be completely incorrect - specifically,
+ * some management frames may end up out of order
+ * compared to the QoS traffic they're controlling.
+ * I'll look into this later.
+ */
+static int
+ath_tx_getac(struct ath_softc *sc, const struct mbuf *m0)
+{
+	const struct ieee80211_frame *wh;
+	int pri = M_WME_GETAC(m0);
+	wh = mtod(m0, const struct ieee80211_frame *);
+	if (IEEE80211_QOS_HAS_SEQ(wh))
+		return pri;
+
+	return WME_AC_BE;
+}
+
 void
 ath_txfrag_cleanup(struct ath_softc *sc,
 	ath_bufhead *frags, struct ieee80211_node *ni)
@@ -116,10 +181,10 @@ ath_txfrag_cleanup(struct ath_softc *sc,
 
 	ATH_TXBUF_LOCK_ASSERT(sc);
 
-	STAILQ_FOREACH_SAFE(bf, frags, bf_list, next) {
+	TAILQ_FOREACH_SAFE(bf, frags, bf_list, next) {
 		/* NB: bf assumed clean */
-		STAILQ_REMOVE_HEAD(frags, bf_list);
-		STAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
+		TAILQ_REMOVE(frags, bf, bf_list);
+		TAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
 		ieee80211_node_decref(ni);
 	}
 }
@@ -144,11 +209,11 @@ ath_txfrag_setup(struct ath_softc *sc, ath_bufhead *frags,
 			break;
 		}
 		ieee80211_node_incref(ni);
-		STAILQ_INSERT_TAIL(frags, bf, bf_list);
+		TAILQ_INSERT_TAIL(frags, bf, bf_list);
 	}
 	ATH_TXBUF_UNLOCK(sc);
 
-	return !STAILQ_EMPTY(frags);
+	return !TAILQ_EMPTY(frags);
 }
 
 /*
@@ -225,8 +290,11 @@ ath_tx_dmasetup(struct ath_softc *sc, struct ath_buf *bf, struct mbuf *m0)
 	return 0;
 }
 
+/*
+ * Chain together segments+descriptors for a non-11n frame.
+ */
 static void
-ath_tx_chaindesclist(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
+ath_tx_chaindesclist(struct ath_softc *sc, struct ath_buf *bf)
 {
 	struct ath_hal *ah = sc->sc_ah;
 	struct ath_desc *ds, *ds0;
@@ -252,17 +320,170 @@ ath_tx_chaindesclist(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *
 			"%s: %d: %08x %08x %08x %08x %08x %08x\n",
 			__func__, i, ds->ds_link, ds->ds_data,
 			ds->ds_ctl0, ds->ds_ctl1, ds->ds_hw[0], ds->ds_hw[1]);
+		bf->bf_lastds = ds;
 	}
-
 }
 
+/*
+ * Fill in the descriptor list for a aggregate subframe.
+ *
+ * The subframe is returned with the ds_link field in the last subframe
+ * pointing to 0.
+ */
 static void
-ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
+ath_tx_chaindesclist_subframe(struct ath_softc *sc, struct ath_buf *bf)
 {
 	struct ath_hal *ah = sc->sc_ah;
+	struct ath_desc *ds, *ds0;
+	int i;
+
+	ds0 = ds = bf->bf_desc;
+
+	/*
+	 * There's no need to call ath_hal_setupfirsttxdesc here;
+	 * That's only going to occur for the first frame in an aggregate.
+	 */
+	for (i = 0; i < bf->bf_nseg; i++, ds++) {
+		ds->ds_data = bf->bf_segs[i].ds_addr;
+		if (i == bf->bf_nseg - 1)
+			ds->ds_link = 0;
+		else
+			ds->ds_link = bf->bf_daddr + sizeof(*ds) * (i + 1);
+
+		/*
+		 * This performs the setup for an aggregate frame.
+		 * This includes enabling the aggregate flags if needed.
+		 */
+		ath_hal_chaintxdesc(ah, ds,
+		    bf->bf_state.bfs_pktlen,
+		    bf->bf_state.bfs_hdrlen,
+		    HAL_PKT_TYPE_AMPDU,	/* forces aggregate bits to be set */
+		    bf->bf_state.bfs_keyix,
+		    0,			/* cipher, calculated from keyix */
+		    bf->bf_state.bfs_ndelim,
+		    bf->bf_segs[i].ds_len,	/* segment length */
+		    i == 0,		/* first segment */
+		    i == bf->bf_nseg - 1	/* last segment */
+		);
+
+		DPRINTF(sc, ATH_DEBUG_XMIT,
+			"%s: %d: %08x %08x %08x %08x %08x %08x\n",
+			__func__, i, ds->ds_link, ds->ds_data,
+			ds->ds_ctl0, ds->ds_ctl1, ds->ds_hw[0], ds->ds_hw[1]);
+		bf->bf_lastds = ds;
+	}
+}
+
+/*
+ * Setup segments+descriptors for an 11n aggregate.
+ * bf_first is the first buffer in the aggregate.
+ * The descriptor list must already been linked together using
+ * bf->bf_next.
+ */
+static void
+ath_tx_setds_11n(struct ath_softc *sc, struct ath_buf *bf_first)
+{
+	struct ath_buf *bf, *bf_prev = NULL;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR, "%s: nframes=%d, al=%d\n",
+	    __func__, bf_first->bf_state.bfs_nframes,
+	    bf_first->bf_state.bfs_al);
+
+	/*
+	 * Setup all descriptors of all subframes.
+	 */
+	bf = bf_first;
+	while (bf != NULL) {
+		DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+		    "%s: bf=%p, nseg=%d, pktlen=%d, seqno=%d\n",
+		    __func__, bf, bf->bf_nseg, bf->bf_state.bfs_pktlen,
+		    SEQNO(bf->bf_state.bfs_seqno));
+
+		/* Sub-frame setup */
+		ath_tx_chaindesclist_subframe(sc, bf);
+
+		/*
+		 * Link the last descriptor of the previous frame
+		 * to the beginning descriptor of this frame.
+		 */
+		if (bf_prev != NULL)
+			bf_prev->bf_lastds->ds_link = bf->bf_daddr;
+
+		/* Save a copy so we can link the next descriptor in */
+		bf_prev = bf;
+		bf = bf->bf_next;
+	}
+
+	/*
+	 * Setup first descriptor of first frame.
+	 * chaintxdesc() overwrites the descriptor entries;
+	 * setupfirsttxdesc() merges in things.
+	 * Otherwise various fields aren't set correctly (eg flags).
+	 */
+	ath_hal_setupfirsttxdesc(sc->sc_ah,
+	    bf_first->bf_desc,
+	    bf_first->bf_state.bfs_al,
+	    bf_first->bf_state.bfs_flags | HAL_TXDESC_INTREQ,
+	    bf_first->bf_state.bfs_txpower,
+	    bf_first->bf_state.bfs_txrate0,
+	    bf_first->bf_state.bfs_try0,
+	    bf_first->bf_state.bfs_txantenna,
+	    bf_first->bf_state.bfs_ctsrate,
+	    bf_first->bf_state.bfs_ctsduration);
+
+	/*
+	 * Setup the last descriptor in the list.
+	 * bf_prev points to the last; bf is NULL here.
+	 */
+	ath_hal_setuplasttxdesc(sc->sc_ah, bf_prev->bf_desc, bf_first->bf_desc);
 
-	/* Fill in the details in the descriptor list */
-	ath_tx_chaindesclist(sc, txq, bf);
+	/*
+	 * Set the first descriptor bf_lastds field to point to
+	 * the last descriptor in the last subframe, that's where
+	 * the status update will occur.
+	 */
+	bf_first->bf_lastds = bf_prev->bf_lastds;
+
+	/*
+	 * And bf_last in the first descriptor points to the end of
+	 * the aggregate list.
+	 */
+	bf_first->bf_last = bf_prev;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR, "%s: end\n", __func__);
+}
+
+static void
+ath_tx_handoff_mcast(struct ath_softc *sc, struct ath_txq *txq,
+    struct ath_buf *bf)
+{
+	ATH_TXQ_LOCK_ASSERT(txq);
+	KASSERT((bf->bf_flags & ATH_BUF_BUSY) == 0,
+	     ("%s: busy status 0x%x", __func__, bf->bf_flags));
+	if (txq->axq_link != NULL) {
+		struct ath_buf *last = ATH_TXQ_LAST(txq, axq_q_s);
+		struct ieee80211_frame *wh;
+
+		/* mark previous frame */
+		wh = mtod(last->bf_m, struct ieee80211_frame *);
+		wh->i_fc[1] |= IEEE80211_FC1_MORE_DATA;
+		bus_dmamap_sync(sc->sc_dmat, last->bf_dmamap,
+		    BUS_DMASYNC_PREWRITE);
+
+		/* link descriptor */
+		*txq->axq_link = bf->bf_daddr;
+	}
+	ATH_TXQ_INSERT_TAIL(txq, bf, bf_list);
+	txq->axq_link = &bf->bf_lastds->ds_link;
+}
+
+/*
+ * Hand-off packet to a hardware queue.
+ */
+static void
+ath_tx_handoff_hw(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
+{
+	struct ath_hal *ah = sc->sc_ah;
 
 	/*
 	 * Insert the frame on the outbound list and pass it on
@@ -272,10 +493,44 @@ ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
 	 * the SWBA handler since frames only go out on DTIM and
 	 * to avoid possible races.
 	 */
-	ATH_TXQ_LOCK(txq);
+	ATH_TXQ_LOCK_ASSERT(txq);
 	KASSERT((bf->bf_flags & ATH_BUF_BUSY) == 0,
-	     ("busy status 0x%x", bf->bf_flags));
-	if (txq->axq_qnum != ATH_TXQ_SWQ) {
+	     ("%s: busy status 0x%x", __func__, bf->bf_flags));
+	KASSERT(txq->axq_qnum != ATH_TXQ_SWQ,
+	     ("ath_tx_handoff_hw called for mcast queue"));
+
+#if 0
+	/*
+	 * This causes a LOR. Find out where the PCU lock is being
+	 * held whilst the TXQ lock is grabbed - that shouldn't
+	 * be occuring.
+	 */
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt) {
+		ATH_PCU_UNLOCK(sc);
+		DPRINTF(sc, ATH_DEBUG_RESET,
+		    "%s: called with sc_in_reset != 0\n",
+		    __func__);
+		DPRINTF(sc, ATH_DEBUG_XMIT,
+		    "%s: queued: TXDP[%u] = %p (%p) depth %d\n",
+		    __func__, txq->axq_qnum,
+		    (caddr_t)bf->bf_daddr, bf->bf_desc,
+		    txq->axq_depth);
+		ATH_TXQ_INSERT_TAIL(txq, bf, bf_list);
+		if (bf->bf_state.bfs_aggr)
+			txq->axq_aggr_depth++;
+		/*
+		 * There's no need to update axq_link; the hardware
+		 * is in reset and once the reset is complete, any
+		 * non-empty queues will simply have DMA restarted.
+		 */
+		return;
+		}
+	ATH_PCU_UNLOCK(sc);
+#endif
+
+	/* For now, so not to generate whitespace diffs */
+	if (1) {
 #ifdef IEEE80211_SUPPORT_TDMA
 		int qbusy;
 
@@ -323,7 +578,7 @@ ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
 				 * is/was empty.
 				 */
 				ath_hal_puttxbuf(ah, txq->axq_qnum,
-					STAILQ_FIRST(&txq->axq_q)->bf_daddr);
+					TAILQ_FIRST(&txq->axq_q)->bf_daddr);
 				txq->axq_flags &= ~ATH_TXQ_PUTPENDING;
 				DPRINTF(sc, ATH_DEBUG_TDMA | ATH_DEBUG_XMIT,
 				    "%s: Q%u restarted\n", __func__,
@@ -347,26 +602,52 @@ ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
 			    (caddr_t)bf->bf_daddr, bf->bf_desc, txq->axq_depth);
 		}
 #endif /* IEEE80211_SUPPORT_TDMA */
-		txq->axq_link = &bf->bf_desc[bf->bf_nseg - 1].ds_link;
+		if (bf->bf_state.bfs_aggr)
+			txq->axq_aggr_depth++;
+		txq->axq_link = &bf->bf_lastds->ds_link;
 		ath_hal_txstart(ah, txq->axq_qnum);
-	} else {
-		if (txq->axq_link != NULL) {
-			struct ath_buf *last = ATH_TXQ_LAST(txq);
-			struct ieee80211_frame *wh;
+	}
+}
 
-			/* mark previous frame */
-			wh = mtod(last->bf_m, struct ieee80211_frame *);
-			wh->i_fc[1] |= IEEE80211_FC1_MORE_DATA;
-			bus_dmamap_sync(sc->sc_dmat, last->bf_dmamap,
-			    BUS_DMASYNC_PREWRITE);
+/*
+ * Restart TX DMA for the given TXQ.
+ *
+ * This must be called whether the queue is empty or not.
+ */
+void
+ath_txq_restart_dma(struct ath_softc *sc, struct ath_txq *txq)
+{
+	struct ath_hal *ah = sc->sc_ah;
+	struct ath_buf *bf;
 
-			/* link descriptor */
-			*txq->axq_link = bf->bf_daddr;
-		}
-		ATH_TXQ_INSERT_TAIL(txq, bf, bf_list);
-		txq->axq_link = &bf->bf_desc[bf->bf_nseg - 1].ds_link;
-	}
-	ATH_TXQ_UNLOCK(txq);
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	/* This is always going to be cleared, empty or not */
+	txq->axq_flags &= ~ATH_TXQ_PUTPENDING;
+
+	bf = TAILQ_FIRST(&txq->axq_q);
+	if (bf == NULL)
+		return;
+
+	ath_hal_puttxbuf(ah, txq->axq_qnum, bf->bf_daddr);
+	txq->axq_link = &bf->bf_lastds->ds_link;
+	ath_hal_txstart(ah, txq->axq_qnum);
+}
+
+/*
+ * Hand off a packet to the hardware (or mcast queue.)
+ *
+ * The relevant hardware txq should be locked.
+ */
+static void
+ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
+{
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	if (txq->axq_qnum == ATH_TXQ_SWQ)
+		ath_tx_handoff_mcast(sc, txq, bf);
+	else
+		ath_tx_handoff_hw(sc, txq, bf);
 }
 
 static int
@@ -422,7 +703,7 @@ ath_tx_tag_crypto(struct ath_softc *sc, struct ieee80211_node *ni,
 
 static uint8_t
 ath_tx_get_rtscts_rate(struct ath_hal *ah, const HAL_RATE_TABLE *rt,
-    int rix, int cix, int shortPreamble)
+    int cix, int shortPreamble)
 {
 	uint8_t ctsrate;
 
@@ -442,7 +723,6 @@ ath_tx_get_rtscts_rate(struct ath_hal *ah, const HAL_RATE_TABLE *rt,
 	return ctsrate;
 }
 
-
 /*
  * Calculate the RTS/CTS duration for legacy frames.
  */
@@ -488,9 +768,238 @@ ath_tx_calc_ctsduration(struct ath_hal *ah, int rix, int cix,
 	return ctsduration;
 }
 
-int
-ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf,
-    struct mbuf *m0)
+/*
+ * Update the given ath_buf with updated rts/cts setup and duration
+ * values.
+ *
+ * To support rate lookups for each software retry, the rts/cts rate
+ * and cts duration must be re-calculated.
+ *
+ * This function assumes the RTS/CTS flags have been set as needed;
+ * mrr has been disabled; and the rate control lookup has been done.
+ *
+ * XXX TODO: MRR need only be disabled for the pre-11n NICs.
+ * XXX The 11n NICs support per-rate RTS/CTS configuration.
+ */
+static void
+ath_tx_set_rtscts(struct ath_softc *sc, struct ath_buf *bf)
+{
+	uint16_t ctsduration = 0;
+	uint8_t ctsrate = 0;
+	uint8_t rix = bf->bf_state.bfs_rc[0].rix;
+	uint8_t cix = 0;
+	const HAL_RATE_TABLE *rt = sc->sc_currates;
+
+	/*
+	 * No RTS/CTS enabled? Don't bother.
+	 */
+	if ((bf->bf_state.bfs_flags &
+	    (HAL_TXDESC_RTSENA | HAL_TXDESC_CTSENA)) == 0) {
+		/* XXX is this really needed? */
+		bf->bf_state.bfs_ctsrate = 0;
+		bf->bf_state.bfs_ctsduration = 0;
+		return;
+	}
+
+	/*
+	 * If protection is enabled, use the protection rix control
+	 * rate. Otherwise use the rate0 control rate.
+	 */
+	if (bf->bf_state.bfs_doprot)
+		rix = sc->sc_protrix;
+	else
+		rix = bf->bf_state.bfs_rc[0].rix;
+
+	/*
+	 * If the raw path has hard-coded ctsrate0 to something,
+	 * use it.
+	 */
+	if (bf->bf_state.bfs_ctsrate0 != 0)
+		cix = ath_tx_findrix(sc, bf->bf_state.bfs_ctsrate0);
+	else
+		/* Control rate from above */
+		cix = rt->info[rix].controlRate;
+
+	/* Calculate the rtscts rate for the given cix */
+	ctsrate = ath_tx_get_rtscts_rate(sc->sc_ah, rt, cix,
+	    bf->bf_state.bfs_shpream);
+
+	/* The 11n chipsets do ctsduration calculations for you */
+	if (! ath_tx_is_11n(sc))
+		ctsduration = ath_tx_calc_ctsduration(sc->sc_ah, rix, cix,
+		    bf->bf_state.bfs_shpream, bf->bf_state.bfs_pktlen,
+		    rt, bf->bf_state.bfs_flags);
+
+	/* Squirrel away in ath_buf */
+	bf->bf_state.bfs_ctsrate = ctsrate;
+	bf->bf_state.bfs_ctsduration = ctsduration;
+	
+	/*
+	 * Must disable multi-rate retry when using RTS/CTS.
+	 * XXX TODO: only for pre-11n NICs.
+	 */
+	bf->bf_state.bfs_ismrr = 0;
+	bf->bf_state.bfs_try0 =
+	    bf->bf_state.bfs_rc[0].tries = ATH_TXMGTTRY;	/* XXX ew */
+}
+
+/*
+ * Setup the descriptor chain for a normal or fast-frame
+ * frame.
+ */
+static void
+ath_tx_setds(struct ath_softc *sc, struct ath_buf *bf)
+{
+	struct ath_desc *ds = bf->bf_desc;
+	struct ath_hal *ah = sc->sc_ah;
+
+	ath_hal_setuptxdesc(ah, ds
+		, bf->bf_state.bfs_pktlen	/* packet length */
+		, bf->bf_state.bfs_hdrlen	/* header length */
+		, bf->bf_state.bfs_atype	/* Atheros packet type */
+		, bf->bf_state.bfs_txpower	/* txpower */
+		, bf->bf_state.bfs_txrate0
+		, bf->bf_state.bfs_try0		/* series 0 rate/tries */
+		, bf->bf_state.bfs_keyix	/* key cache index */
+		, bf->bf_state.bfs_txantenna	/* antenna mode */
+		, bf->bf_state.bfs_flags	/* flags */
+		, bf->bf_state.bfs_ctsrate	/* rts/cts rate */
+		, bf->bf_state.bfs_ctsduration	/* rts/cts duration */
+	);
+
+	/*
+	 * This will be overriden when the descriptor chain is written.
+	 */
+	bf->bf_lastds = ds;
+	bf->bf_last = bf;
+
+	/* XXX TODO: Setup descriptor chain */
+}
+
+/*
+ * Do a rate lookup.
+ *
+ * This performs a rate lookup for the given ath_buf only if it's required.
+ * Non-data frames and raw frames don't require it.
+ *
+ * This populates the primary and MRR entries; MRR values are
+ * then disabled later on if something requires it (eg RTS/CTS on
+ * pre-11n chipsets.
+ *
+ * This needs to be done before the RTS/CTS fields are calculated
+ * as they may depend upon the rate chosen.
+ */
+static void
+ath_tx_do_ratelookup(struct ath_softc *sc, struct ath_buf *bf)
+{
+	uint8_t rate, rix;
+	int try0;
+
+	if (! bf->bf_state.bfs_doratelookup)
+		return;
+
+	/* Get rid of any previous state */
+	bzero(bf->bf_state.bfs_rc, sizeof(bf->bf_state.bfs_rc));
+
+	ATH_NODE_LOCK(ATH_NODE(bf->bf_node));
+	ath_rate_findrate(sc, ATH_NODE(bf->bf_node), bf->bf_state.bfs_shpream,
+	    bf->bf_state.bfs_pktlen, &rix, &try0, &rate);
+
+	/* In case MRR is disabled, make sure rc[0] is setup correctly */
+	bf->bf_state.bfs_rc[0].rix = rix;
+	bf->bf_state.bfs_rc[0].ratecode = rate;
+	bf->bf_state.bfs_rc[0].tries = try0;
+
+	if (bf->bf_state.bfs_ismrr && try0 != ATH_TXMAXTRY)
+		ath_rate_getxtxrates(sc, ATH_NODE(bf->bf_node), rix,
+		    bf->bf_state.bfs_rc);
+	ATH_NODE_UNLOCK(ATH_NODE(bf->bf_node));
+
+	sc->sc_txrix = rix;	/* for LED blinking */
+	sc->sc_lastdatarix = rix;	/* for fast frames */
+	bf->bf_state.bfs_try0 = try0;
+	bf->bf_state.bfs_txrate0 = rate;
+}
+
+/*
+ * Set the rate control fields in the given descriptor based on
+ * the bf_state fields and node state.
+ *
+ * The bfs fields should already be set with the relevant rate
+ * control information, including whether MRR is to be enabled.
+ *
+ * Since the FreeBSD HAL currently sets up the first TX rate
+ * in ath_hal_setuptxdesc(), this will setup the MRR
+ * conditionally for the pre-11n chips, and call ath_buf_set_rate
+ * unconditionally for 11n chips. These require the 11n rate
+ * scenario to be set if MCS rates are enabled, so it's easier
+ * to just always call it. The caller can then only set rates 2, 3
+ * and 4 if multi-rate retry is needed.
+ */
+static void
+ath_tx_set_ratectrl(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_buf *bf)
+{
+	struct ath_rc_series *rc = bf->bf_state.bfs_rc;
+
+	/* If mrr is disabled, blank tries 1, 2, 3 */
+	if (! bf->bf_state.bfs_ismrr)
+		rc[1].tries = rc[2].tries = rc[3].tries = 0;
+
+	/*
+	 * Always call - that way a retried descriptor will
+	 * have the MRR fields overwritten.
+	 *
+	 * XXX TODO: see if this is really needed - setting up
+	 * the first descriptor should set the MRR fields to 0
+	 * for us anyway.
+	 */
+	if (ath_tx_is_11n(sc)) {
+		ath_buf_set_rate(sc, ni, bf);
+	} else {
+		ath_hal_setupxtxdesc(sc->sc_ah, bf->bf_desc
+			, rc[1].ratecode, rc[1].tries
+			, rc[2].ratecode, rc[2].tries
+			, rc[3].ratecode, rc[3].tries
+		);
+	}
+}
+
+/*
+ * Transmit the given frame to the hardware.
+ *
+ * The frame must already be setup; rate control must already have
+ * been done.
+ *
+ * XXX since the TXQ lock is being held here (and I dislike holding
+ * it for this long when not doing software aggregation), later on
+ * break this function into "setup_normal" and "xmit_normal". The
+ * lock only needs to be held for the ath_tx_handoff call.
+ */
+static void
+ath_tx_xmit_normal(struct ath_softc *sc, struct ath_txq *txq,
+    struct ath_buf *bf)
+{
+
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	/* Setup the descriptor before handoff */
+	ath_tx_do_ratelookup(sc, bf);
+	ath_tx_rate_fill_rcflags(sc, bf);
+	ath_tx_set_rtscts(sc, bf);
+	ath_tx_setds(sc, bf);
+	ath_tx_set_ratectrl(sc, bf->bf_node, bf);
+	ath_tx_chaindesclist(sc, bf);
+
+	/* Hand off to hardware */
+	ath_tx_handoff(sc, txq, bf);
+}
+
+
+
+static int
+ath_tx_normal_setup(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_buf *bf, struct mbuf *m0)
 {
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct ath_vap *avp = ATH_VAP(vap);
@@ -499,22 +1008,17 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 	struct ieee80211com *ic = ifp->if_l2com;
 	const struct chanAccParams *cap = &ic->ic_wme.wme_chanParams;
 	int error, iswep, ismcast, isfrag, ismrr;
-	int keyix, hdrlen, pktlen, try0;
-	u_int8_t rix, txrate, ctsrate;
-	u_int8_t cix = 0xff;		/* NB: silence compiler */
+	int keyix, hdrlen, pktlen, try0 = 0;
+	u_int8_t rix = 0, txrate = 0;
 	struct ath_desc *ds;
 	struct ath_txq *txq;
 	struct ieee80211_frame *wh;
-	u_int subtype, flags, ctsduration;
+	u_int subtype, flags;
 	HAL_PKT_TYPE atype;
 	const HAL_RATE_TABLE *rt;
 	HAL_BOOL shortPreamble;
 	struct ath_node *an;
 	u_int pri;
-	uint8_t try[4], rate[4];
-
-	bzero(try, sizeof(try));
-	bzero(rate, sizeof(rate));
 
 	wh = mtod(m0, struct ieee80211_frame *);
 	iswep = wh->i_fc[1] & IEEE80211_FC1_WEP;
@@ -528,7 +1032,8 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 	pktlen = m0->m_pkthdr.len - (hdrlen & 3);
 
 	/* Handle encryption twiddling if needed */
-	if (! ath_tx_tag_crypto(sc, ni, m0, iswep, isfrag, &hdrlen, &pktlen, &keyix)) {
+	if (! ath_tx_tag_crypto(sc, ni, m0, iswep, isfrag, &hdrlen,
+	    &pktlen, &keyix)) {
 		ath_freetx(m0);
 		return EIO;
 	}
@@ -624,12 +1129,12 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 				txrate |= rt->info[rix].shortPreamble;
 			try0 = ATH_TXMAXTRY;	/* XXX?too many? */
 		} else {
-			ath_rate_findrate(sc, an, shortPreamble, pktlen,
-				&rix, &try0, &txrate);
-			sc->sc_txrix = rix;		/* for LED blinking */
-			sc->sc_lastdatarix = rix;	/* for fast frames */
-			if (try0 != ATH_TXMAXTRY)
-				ismrr = 1;
+			/*
+			 * Do rate lookup on each TX, rather than using
+			 * the hard-coded TX information decided here.
+			 */
+			ismrr = 1;
+			bf->bf_state.bfs_doratelookup = 1;
 		}
 		if (cap->cap_wmeParams[pri].wmep_noackPolicy)
 			flags |= HAL_TXDESC_NOACK;
@@ -660,7 +1165,6 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 	} else if (pktlen > vap->iv_rtsthreshold &&
 	    (ni->ni_ath_flags & IEEE80211_NODE_FF) == 0) {
 		flags |= HAL_TXDESC_RTSENA;	/* RTS based on frame length */
-		cix = rt->info[rix].controlRate;
 		sc->sc_stats.ast_tx_rts++;
 	}
 	if (flags & HAL_TXDESC_NOACK)		/* NB: avoid double counting */
@@ -683,22 +1187,20 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 	if ((ic->ic_flags & IEEE80211_F_USEPROT) &&
 	    rt->info[rix].phy == IEEE80211_T_OFDM &&
 	    (flags & HAL_TXDESC_NOACK) == 0) {
+		bf->bf_state.bfs_doprot = 1;
 		/* XXX fragments must use CCK rates w/ protection */
-		if (ic->ic_protmode == IEEE80211_PROT_RTSCTS)
+		if (ic->ic_protmode == IEEE80211_PROT_RTSCTS) {
 			flags |= HAL_TXDESC_RTSENA;
-		else if (ic->ic_protmode == IEEE80211_PROT_CTSONLY)
+		} else if (ic->ic_protmode == IEEE80211_PROT_CTSONLY) {
 			flags |= HAL_TXDESC_CTSENA;
-		if (isfrag) {
-			/*
-			 * For frags it would be desirable to use the
-			 * highest CCK rate for RTS/CTS.  But stations
-			 * farther away may detect it at a lower CCK rate
-			 * so use the configured protection rate instead
-			 * (for now).
-			 */
-			cix = rt->info[sc->sc_protrix].controlRate;
-		} else
-			cix = rt->info[sc->sc_protrix].controlRate;
+		}
+		/*
+		 * For frags it would be desirable to use the
+		 * highest CCK rate for RTS/CTS.  But stations
+		 * farther away may detect it at a lower CCK rate
+		 * so use the configured protection rate instead
+		 * (for now).
+		 */
 		sc->sc_stats.ast_tx_protect++;
 	}
 
@@ -756,23 +1258,28 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 	}
 
 	/*
-	 * Calculate RTS/CTS rate and duration if needed.
+	 * Determine if a tx interrupt should be generated for
+	 * this descriptor.  We take a tx interrupt to reap
+	 * descriptors when the h/w hits an EOL condition or
+	 * when the descriptor is specifically marked to generate
+	 * an interrupt.  We periodically mark descriptors in this
+	 * way to insure timely replenishing of the supply needed
+	 * for sending frames.  Defering interrupts reduces system
+	 * load and potentially allows more concurrent work to be
+	 * done but if done to aggressively can cause senders to
+	 * backup.
+	 *
+	 * NB: use >= to deal with sc_txintrperiod changing
+	 *     dynamically through sysctl.
 	 */
-	ctsduration = 0;
-	if (flags & (HAL_TXDESC_RTSENA|HAL_TXDESC_CTSENA)) {
-		ctsrate = ath_tx_get_rtscts_rate(ah, rt, rix, cix, shortPreamble);
+	if (flags & HAL_TXDESC_INTREQ) {
+		txq->axq_intrcnt = 0;
+	} else if (++txq->axq_intrcnt >= sc->sc_txintrperiod) {
+		flags |= HAL_TXDESC_INTREQ;
+		txq->axq_intrcnt = 0;
+	}
 
-		/* The 11n chipsets do ctsduration calculations for you */
-		if (! ath_tx_is_11n(sc))
-			ctsduration = ath_tx_calc_ctsduration(ah, rix, cix, shortPreamble,
-			    pktlen, rt, flags);
-		/*
-		 * Must disable multi-rate retry when using RTS/CTS.
-		 */
-		ismrr = 0;
-		try0 = ATH_TXMGTTRY;		/* XXX */
-	} else
-		ctsrate = 0;
+	/* This point forward is actual TX bits */
 
 	/*
 	 * At this point we are committed to sending the frame
@@ -801,71 +1308,187 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 		ieee80211_radiotap_tx(vap, m0);
 	}
 
+	/* Blank the legacy rate array */
+	bzero(&bf->bf_state.bfs_rc, sizeof(bf->bf_state.bfs_rc));
+
 	/*
-	 * Determine if a tx interrupt should be generated for
-	 * this descriptor.  We take a tx interrupt to reap
-	 * descriptors when the h/w hits an EOL condition or
-	 * when the descriptor is specifically marked to generate
-	 * an interrupt.  We periodically mark descriptors in this
-	 * way to insure timely replenishing of the supply needed
-	 * for sending frames.  Defering interrupts reduces system
-	 * load and potentially allows more concurrent work to be
-	 * done but if done to aggressively can cause senders to
-	 * backup.
+	 * ath_buf_set_rate needs at least one rate/try to setup
+	 * the rate scenario.
+	 */
+	bf->bf_state.bfs_rc[0].rix = rix;
+	bf->bf_state.bfs_rc[0].tries = try0;
+	bf->bf_state.bfs_rc[0].ratecode = txrate;
+
+	/* Store the decided rate index values away */
+	bf->bf_state.bfs_pktlen = pktlen;
+	bf->bf_state.bfs_hdrlen = hdrlen;
+	bf->bf_state.bfs_atype = atype;
+	bf->bf_state.bfs_txpower = ni->ni_txpower;
+	bf->bf_state.bfs_txrate0 = txrate;
+	bf->bf_state.bfs_try0 = try0;
+	bf->bf_state.bfs_keyix = keyix;
+	bf->bf_state.bfs_txantenna = sc->sc_txantenna;
+	bf->bf_state.bfs_flags = flags;
+	bf->bf_txflags = flags;
+	bf->bf_state.bfs_shpream = shortPreamble;
+
+	/* XXX this should be done in ath_tx_setrate() */
+	bf->bf_state.bfs_ctsrate0 = 0;	/* ie, no hard-coded ctsrate */
+	bf->bf_state.bfs_ctsrate = 0;	/* calculated later */
+	bf->bf_state.bfs_ctsduration = 0;
+	bf->bf_state.bfs_ismrr = ismrr;
+
+	return 0;
+}
+
+/*
+ * Direct-dispatch the current frame to the hardware.
+ *
+ * This can be called by the net80211 code.
+ *
+ * XXX what about locking? Or, push the seqno assign into the
+ * XXX aggregate scheduler so its serialised?
+ */
+int
+ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_buf *bf, struct mbuf *m0)
+{
+	struct ieee80211vap *vap = ni->ni_vap;
+	struct ath_vap *avp = ATH_VAP(vap);
+	int r;
+	u_int pri;
+	int tid;
+	struct ath_txq *txq;
+	int ismcast;
+	const struct ieee80211_frame *wh;
+	int is_ampdu, is_ampdu_tx, is_ampdu_pending;
+	ieee80211_seq seqno;
+	uint8_t type, subtype;
+
+	/*
+	 * Determine the target hardware queue.
 	 *
-	 * NB: use >= to deal with sc_txintrperiod changing
-	 *     dynamically through sysctl.
+	 * For multicast frames, the txq gets overridden to be the
+	 * software TXQ and it's done via direct-dispatch.
+	 *
+	 * For any other frame, we do a TID/QoS lookup inside the frame
+	 * to see what the TID should be. If it's a non-QoS frame, the
+	 * AC and TID are overridden. The TID/TXQ code assumes the
+	 * TID is on a predictable hardware TXQ, so we don't support
+	 * having a node TID queued to multiple hardware TXQs.
+	 * This may change in the future but would require some locking
+	 * fudgery.
 	 */
-	if (flags & HAL_TXDESC_INTREQ) {
-		txq->axq_intrcnt = 0;
-	} else if (++txq->axq_intrcnt >= sc->sc_txintrperiod) {
-		flags |= HAL_TXDESC_INTREQ;
-		txq->axq_intrcnt = 0;
-	}
+	pri = ath_tx_getac(sc, m0);
+	tid = ath_tx_gettid(sc, m0);
 
-	if (ath_tx_is_11n(sc)) {
-		rate[0] = rix;
-		try[0] = try0;
+	txq = sc->sc_ac2q[pri];
+	wh = mtod(m0, struct ieee80211_frame *);
+	ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1);
+	type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
+	subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
+
+	/* A-MPDU TX */
+	is_ampdu_tx = ath_tx_ampdu_running(sc, ATH_NODE(ni), tid);
+	is_ampdu_pending = ath_tx_ampdu_pending(sc, ATH_NODE(ni), tid);
+	is_ampdu = is_ampdu_tx | is_ampdu_pending;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: tid=%d, ac=%d, is_ampdu=%d\n",
+	    __func__, tid, pri, is_ampdu);
+
+	/* Multicast frames go onto the software multicast queue */
+	if (ismcast)
+		txq = &avp->av_mcastq;
+
+	if ((! is_ampdu) && (vap->iv_ps_sta || avp->av_mcastq.axq_depth))
+		txq = &avp->av_mcastq;
+
+	/* Do the generic frame setup */
+	/* XXX should just bzero the bf_state? */
+	bf->bf_state.bfs_dobaw = 0;
+
+	/* A-MPDU TX? Manually set sequence number */
+	/* Don't do it whilst pending; the net80211 layer still assigns them */
+	/* XXX do we need locking here? */
+	if (is_ampdu_tx) {
+		ATH_TXQ_LOCK(txq);
+		/*
+		 * Always call; this function will
+		 * handle making sure that null data frames
+		 * don't get a sequence number from the current
+		 * TID and thus mess with the BAW.
+		 */
+		seqno = ath_tx_tid_seqno_assign(sc, ni, bf, m0);
+		if (IEEE80211_QOS_HAS_SEQ(wh) &&
+		    subtype != IEEE80211_FC0_SUBTYPE_QOS_NULL) {
+			bf->bf_state.bfs_dobaw = 1;
+		}
+		ATH_TXQ_UNLOCK(txq);
 	}
 
 	/*
-	 * Formulate first tx descriptor with tx controls.
+	 * If needed, the sequence number has been assigned.
+	 * Squirrel it away somewhere easy to get to.
 	 */
-	/* XXX check return value? */
-	/* XXX is this ok to call for 11n descriptors? */
-	/* XXX or should it go through the first, next, last 11n calls? */
-	ath_hal_setuptxdesc(ah, ds
-		, pktlen		/* packet length */
-		, hdrlen		/* header length */
-		, atype			/* Atheros packet type */
-		, ni->ni_txpower	/* txpower */
-		, txrate, try0		/* series 0 rate/tries */
-		, keyix			/* key cache index */
-		, sc->sc_txantenna	/* antenna mode */
-		, flags			/* flags */
-		, ctsrate		/* rts/cts rate */
-		, ctsduration		/* rts/cts duration */
-	);
-	bf->bf_txflags = flags;
+	bf->bf_state.bfs_seqno = M_SEQNO_GET(m0) << IEEE80211_SEQ_SEQ_SHIFT;
+
+	/* Is ampdu pending? fetch the seqno and print it out */
+	if (is_ampdu_pending)
+		DPRINTF(sc, ATH_DEBUG_SW_TX,
+		    "%s: tid %d: ampdu pending, seqno %d\n",
+		    __func__, tid, M_SEQNO_GET(m0));
+
+	/* This also sets up the DMA map */
+	r = ath_tx_normal_setup(sc, ni, bf, m0);
+
+	if (r != 0)
+		return r;
+
+	/* At this point m0 could have changed! */
+	m0 = bf->bf_m;
+
+#if 1
 	/*
-	 * Setup the multi-rate retry state only when we're
-	 * going to use it.  This assumes ath_hal_setuptxdesc
-	 * initializes the descriptors (so we don't have to)
-	 * when the hardware supports multi-rate retry and
-	 * we don't use it.
+	 * If it's a multicast frame, do a direct-dispatch to the
+	 * destination hardware queue. Don't bother software
+	 * queuing it.
 	 */
-        if (ismrr) {
-                if (ath_tx_is_11n(sc))
-                        ath_rate_getxtxrates(sc, an, rix, rate, try);
-                else
-                        ath_rate_setupxtxdesc(sc, an, ds, shortPreamble, rix);
-        }
-
-        if (ath_tx_is_11n(sc)) {
-                ath_buf_set_rate(sc, ni, bf, pktlen, flags, ctsrate, (atype == HAL_PKT_TYPE_PSPOLL), rate, try);
-        }
+	/*
+	 * If it's a BAR frame, do a direct dispatch to the
+	 * destination hardware queue. Don't bother software
+	 * queuing it, as the TID will now be paused.
+	 * Sending a BAR frame can occur from the net80211 txa timer
+	 * (ie, retries) or from the ath txtask (completion call.)
+	 * It queues directly to hardware because the TID is paused
+	 * at this point (and won't be unpaused until the BAR has
+	 * either been TXed successfully or max retries has been
+	 * reached.)
+	 */
+	if (txq == &avp->av_mcastq) {
+		ATH_TXQ_LOCK(txq);
+		ath_tx_xmit_normal(sc, txq, bf);
+		ATH_TXQ_UNLOCK(txq);
+	} else if (type == IEEE80211_FC0_TYPE_CTL &&
+		    subtype == IEEE80211_FC0_SUBTYPE_BAR) {
+		DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+		    "%s: BAR: TX'ing direct\n", __func__);
+		ATH_TXQ_LOCK(txq);
+		ath_tx_xmit_normal(sc, txq, bf);
+		ATH_TXQ_UNLOCK(txq);
+	} else {
+		/* add to software queue */
+		ath_tx_swq(sc, ni, txq, bf);
+	}
+#else
+	/*
+	 * For now, since there's no software queue,
+	 * direct-dispatch to the hardware.
+	 */
+	ATH_TXQ_LOCK(txq);
+	ath_tx_xmit_normal(sc, txq, bf);
+	ATH_TXQ_UNLOCK(txq);
+#endif
 
-	ath_tx_handoff(sc, txq, bf);
 	return 0;
 }
 
@@ -880,17 +1503,15 @@ ath_tx_raw_start(struct ath_softc *sc, struct ieee80211_node *ni,
 	struct ieee80211vap *vap = ni->ni_vap;
 	int error, ismcast, ismrr;
 	int keyix, hdrlen, pktlen, try0, txantenna;
-	u_int8_t rix, cix, txrate, ctsrate, rate1, rate2, rate3;
+	u_int8_t rix, txrate;
 	struct ieee80211_frame *wh;
-	u_int flags, ctsduration;
+	u_int flags;
 	HAL_PKT_TYPE atype;
 	const HAL_RATE_TABLE *rt;
 	struct ath_desc *ds;
 	u_int pri;
-	uint8_t try[4], rate[4];
-
-	bzero(try, sizeof(try));
-	bzero(rate, sizeof(rate));
+	int o_tid = -1;
+	int do_override;
 
 	wh = mtod(m0, struct ieee80211_frame *);
 	ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1);
@@ -902,14 +1523,24 @@ ath_tx_raw_start(struct ath_softc *sc, struct ieee80211_node *ni,
 	/* XXX honor IEEE80211_BPF_DATAPAD */
 	pktlen = m0->m_pkthdr.len - (hdrlen & 3) + IEEE80211_CRC_LEN;
 
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: ismcast=%d\n",
+	    __func__, ismcast);
+
 	/* Handle encryption twiddling if needed */
-	if (! ath_tx_tag_crypto(sc, ni, m0, params->ibp_flags & IEEE80211_BPF_CRYPTO, 0, &hdrlen, &pktlen, &keyix)) {
+	if (! ath_tx_tag_crypto(sc, ni,
+	    m0, params->ibp_flags & IEEE80211_BPF_CRYPTO, 0,
+	    &hdrlen, &pktlen, &keyix)) {
 		ath_freetx(m0);
 		return EIO;
 	}
 	/* packet header may have moved, reset our local pointer */
 	wh = mtod(m0, struct ieee80211_frame *);
 
+	/* Do the generic frame setup */
+	/* XXX should just bzero the bf_state? */
+	bf->bf_state.bfs_dobaw = 0;
+
 	error = ath_tx_dmasetup(sc, bf, m0);
 	if (error != 0)
 		return error;
@@ -921,8 +1552,11 @@ ath_tx_raw_start(struct ath_softc *sc, struct ieee80211_node *ni,
 	flags |= HAL_TXDESC_INTREQ;		/* force interrupt */
 	if (params->ibp_flags & IEEE80211_BPF_RTS)
 		flags |= HAL_TXDESC_RTSENA;
-	else if (params->ibp_flags & IEEE80211_BPF_CTS)
+	else if (params->ibp_flags & IEEE80211_BPF_CTS) {
+		/* XXX assume 11g/11n protection? */
+		bf->bf_state.bfs_doprot = 1;
 		flags |= HAL_TXDESC_CTSENA;
+	}
 	/* XXX leave ismcast to injector? */
 	if ((params->ibp_flags & IEEE80211_BPF_NOACK) || ismcast)
 		flags |= HAL_TXDESC_NOACK;
@@ -940,23 +1574,18 @@ ath_tx_raw_start(struct ath_softc *sc, struct ieee80211_node *ni,
 	if (txantenna == 0)			/* XXX? */
 		txantenna = sc->sc_txantenna;
 
-	ctsduration = 0;
-	if (flags & (HAL_TXDESC_RTSENA|HAL_TXDESC_CTSENA)) {
-		cix = ath_tx_findrix(sc, params->ibp_ctsrate);
-		ctsrate = ath_tx_get_rtscts_rate(ah, rt, rix, cix, params->ibp_flags & IEEE80211_BPF_SHORTPRE);
-		/* The 11n chipsets do ctsduration calculations for you */
-		if (! ath_tx_is_11n(sc))
-			ctsduration = ath_tx_calc_ctsduration(ah, rix, cix,
-			    params->ibp_flags & IEEE80211_BPF_SHORTPRE, pktlen,
-			    rt, flags);
-		/*
-		 * Must disable multi-rate retry when using RTS/CTS.
-		 */
-		ismrr = 0;			/* XXX */
-	} else
-		ctsrate = 0;
+	/*
+	 * Since ctsrate is fixed, store it away for later
+	 * use when the descriptor fields are being set.
+	 */
+	if (flags & (HAL_TXDESC_RTSENA|HAL_TXDESC_CTSENA))
+		bf->bf_state.bfs_ctsrate0 = params->ibp_ctsrate;
 
 	pri = params->ibp_pri & 3;
+	/* Override pri if the frame isn't a QoS one */
+	if (! IEEE80211_QOS_HAS_SEQ(wh))
+		pri = ath_tx_getac(sc, m0);
+
 	/*
 	 * NB: we mark all packets as type PSPOLL so the h/w won't
 	 * set the sequence number, duration, etc.
@@ -988,77 +1617,95 @@ ath_tx_raw_start(struct ath_softc *sc, struct ieee80211_node *ni,
 	 */
 	ds = bf->bf_desc;
 	/* XXX check return value? */
-	ath_hal_setuptxdesc(ah, ds
-		, pktlen		/* packet length */
-		, hdrlen		/* header length */
-		, atype			/* Atheros packet type */
-		, params->ibp_power	/* txpower */
-		, txrate, try0		/* series 0 rate/tries */
-		, keyix			/* key cache index */
-		, txantenna		/* antenna mode */
-		, flags			/* flags */
-		, ctsrate		/* rts/cts rate */
-		, ctsduration		/* rts/cts duration */
-	);
+
+	/* Store the decided rate index values away */
+	bf->bf_state.bfs_pktlen = pktlen;
+	bf->bf_state.bfs_hdrlen = hdrlen;
+	bf->bf_state.bfs_atype = atype;
+	bf->bf_state.bfs_txpower = params->ibp_power;
+	bf->bf_state.bfs_txrate0 = txrate;
+	bf->bf_state.bfs_try0 = try0;
+	bf->bf_state.bfs_keyix = keyix;
+	bf->bf_state.bfs_txantenna = txantenna;
+	bf->bf_state.bfs_flags = flags;
 	bf->bf_txflags = flags;
+	bf->bf_state.bfs_shpream =
+	    !! (params->ibp_flags & IEEE80211_BPF_SHORTPRE);
 
-	if (ath_tx_is_11n(sc)) {
-		rate[0] = ath_tx_findrix(sc, params->ibp_rate0);
-		try[0] = params->ibp_try0;
-
-		if (ismrr) {
-			/* Remember, rate[] is actually an array of rix's -adrian */
-			rate[0] = ath_tx_findrix(sc, params->ibp_rate0);
-			rate[1] = ath_tx_findrix(sc, params->ibp_rate1);
-			rate[2] = ath_tx_findrix(sc, params->ibp_rate2);
-			rate[3] = ath_tx_findrix(sc, params->ibp_rate3);
-
-			try[0] = params->ibp_try0;
-			try[1] = params->ibp_try1;
-			try[2] = params->ibp_try2;
-			try[3] = params->ibp_try3;
-		}
-	} else {
-		if (ismrr) {
-			rix = ath_tx_findrix(sc, params->ibp_rate1);
-			rate1 = rt->info[rix].rateCode;
-			if (params->ibp_flags & IEEE80211_BPF_SHORTPRE)
-				rate1 |= rt->info[rix].shortPreamble;
-			if (params->ibp_try2) {
-				rix = ath_tx_findrix(sc, params->ibp_rate2);
-				rate2 = rt->info[rix].rateCode;
-				if (params->ibp_flags & IEEE80211_BPF_SHORTPRE)
-					rate2 |= rt->info[rix].shortPreamble;
-			} else
-				rate2 = 0;
-			if (params->ibp_try3) {
-				rix = ath_tx_findrix(sc, params->ibp_rate3);
-				rate3 = rt->info[rix].rateCode;
-				if (params->ibp_flags & IEEE80211_BPF_SHORTPRE)
-					rate3 |= rt->info[rix].shortPreamble;
-			} else
-				rate3 = 0;
-			ath_hal_setupxtxdesc(ah, ds
-				, rate1, params->ibp_try1	/* series 1 */
-				, rate2, params->ibp_try2	/* series 2 */
-				, rate3, params->ibp_try3	/* series 3 */
-			);
-		}
-	}
+	/* XXX this should be done in ath_tx_setrate() */
+	bf->bf_state.bfs_ctsrate = 0;
+	bf->bf_state.bfs_ctsduration = 0;
+	bf->bf_state.bfs_ismrr = ismrr;
 
-	if (ath_tx_is_11n(sc)) {
-		/*
-		 * notice that rix doesn't include any of the "magic" flags txrate
-		 * does for communicating "other stuff" to the HAL.
-		 */
-		ath_buf_set_rate(sc, ni, bf, pktlen, flags, ctsrate, (atype == HAL_PKT_TYPE_PSPOLL), rate, try);
+	/* Blank the legacy rate array */
+	bzero(&bf->bf_state.bfs_rc, sizeof(bf->bf_state.bfs_rc));
+
+	bf->bf_state.bfs_rc[0].rix =
+	    ath_tx_findrix(sc, params->ibp_rate0);
+	bf->bf_state.bfs_rc[0].tries = try0;
+	bf->bf_state.bfs_rc[0].ratecode = txrate;
+
+	if (ismrr) {
+		int rix;
+
+		rix = ath_tx_findrix(sc, params->ibp_rate1);
+		bf->bf_state.bfs_rc[1].rix = rix;
+		bf->bf_state.bfs_rc[1].tries = params->ibp_try1;
+
+		rix = ath_tx_findrix(sc, params->ibp_rate2);
+		bf->bf_state.bfs_rc[2].rix = rix;
+		bf->bf_state.bfs_rc[2].tries = params->ibp_try2;
+
+		rix = ath_tx_findrix(sc, params->ibp_rate3);
+		bf->bf_state.bfs_rc[3].rix = rix;
+		bf->bf_state.bfs_rc[3].tries = params->ibp_try3;
 	}
+	/*
+	 * All the required rate control decisions have been made;
+	 * fill in the rc flags.
+	 */
+	ath_tx_rate_fill_rcflags(sc, bf);
 
 	/* NB: no buffered multicast in power save support */
-	ath_tx_handoff(sc, sc->sc_ac2q[pri], bf);
+
+	/* XXX If it's an ADDBA, override the correct queue */
+	do_override = ath_tx_action_frame_override_queue(sc, ni, m0, &o_tid);
+
+	/* Map ADDBA to the correct priority */
+	if (do_override) {
+#if 0
+		device_printf(sc->sc_dev,
+		    "%s: overriding tid %d pri %d -> %d\n",
+		    __func__, o_tid, pri, TID_TO_WME_AC(o_tid));
+#endif
+		pri = TID_TO_WME_AC(o_tid);
+	}
+
+	/*
+	 * If we're overiding the ADDBA destination, dump directly
+	 * into the hardware queue, right after any pending
+	 * frames to that node are.
+	 */
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: dooverride=%d\n",
+	    __func__, do_override);
+
+	if (do_override) {
+		ATH_TXQ_LOCK(sc->sc_ac2q[pri]);
+		ath_tx_xmit_normal(sc, sc->sc_ac2q[pri], bf);
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[pri]);
+	} else {
+		/* Queue to software queue */
+		ath_tx_swq(sc, ni, sc->sc_ac2q[pri], bf);
+	}
+
 	return 0;
 }
 
+/*
+ * Send a raw frame.
+ *
+ * This can be called by net80211.
+ */
 int
 ath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	const struct ieee80211_bpf_params *params)
@@ -1069,6 +1716,17 @@ ath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	struct ath_buf *bf;
 	int error;
 
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt > 0) {
+		device_printf(sc->sc_dev, "%s: sc_inreset_cnt > 0; bailing\n",
+		    __func__);
+		error = EIO;
+		ATH_PCU_UNLOCK(sc);
+		goto bad0;
+	}
+	sc->sc_txstart_cnt++;
+	ATH_PCU_UNLOCK(sc);
+
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid) {
 		DPRINTF(sc, ATH_DEBUG_XMIT, "%s: discard frame, %s", __func__,
 		    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ?
@@ -1111,14 +1769,2168 @@ ath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	ifp->if_opackets++;
 	sc->sc_stats.ast_tx_raw++;
 
+	ATH_PCU_LOCK(sc);
+	sc->sc_txstart_cnt--;
+	ATH_PCU_UNLOCK(sc);
+
 	return 0;
 bad2:
 	ATH_TXBUF_LOCK(sc);
-	STAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
+	TAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
 	ATH_TXBUF_UNLOCK(sc);
 bad:
+	ATH_PCU_LOCK(sc);
+	sc->sc_txstart_cnt--;
+	ATH_PCU_UNLOCK(sc);
+bad0:
 	ifp->if_oerrors++;
 	sc->sc_stats.ast_tx_raw_fail++;
 	ieee80211_free_node(ni);
+
 	return error;
 }
+
+/* Some helper functions */
+
+/*
+ * ADDBA (and potentially others) need to be placed in the same
+ * hardware queue as the TID/node it's relating to. This is so
+ * it goes out after any pending non-aggregate frames to the
+ * same node/TID.
+ *
+ * If this isn't done, the ADDBA can go out before the frames
+ * queued in hardware. Even though these frames have a sequence
+ * number -earlier- than the ADDBA can be transmitted (but
+ * no frames whose sequence numbers are after the ADDBA should
+ * be!) they'll arrive after the ADDBA - and the receiving end
+ * will simply drop them as being out of the BAW.
+ *
+ * The frames can't be appended to the TID software queue - it'll
+ * never be sent out. So these frames have to be directly
+ * dispatched to the hardware, rather than queued in software.
+ * So if this function returns true, the TXQ has to be
+ * overridden and it has to be directly dispatched.
+ *
+ * It's a dirty hack, but someone's gotta do it.
+ */
+
+/*
+ * XXX doesn't belong here!
+ */
+static int
+ieee80211_is_action(struct ieee80211_frame *wh)
+{
+	/* Type: Management frame? */
+	if ((wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) !=
+	    IEEE80211_FC0_TYPE_MGT)
+		return 0;
+
+	/* Subtype: Action frame? */
+	if ((wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK) !=
+	    IEEE80211_FC0_SUBTYPE_ACTION)
+		return 0;
+
+	return 1;
+}
+
+#define	MS(_v, _f)	(((_v) & _f) >> _f##_S)
+/*
+ * Return an alternate TID for ADDBA request frames.
+ *
+ * Yes, this likely should be done in the net80211 layer.
+ */
+static int
+ath_tx_action_frame_override_queue(struct ath_softc *sc,
+    struct ieee80211_node *ni,
+    struct mbuf *m0, int *tid)
+{
+	struct ieee80211_frame *wh = mtod(m0, struct ieee80211_frame *);
+	struct ieee80211_action_ba_addbarequest *ia;
+	uint8_t *frm;
+	uint16_t baparamset;
+
+	/* Not action frame? Bail */
+	if (! ieee80211_is_action(wh))
+		return 0;
+
+	/* XXX Not needed for frames we send? */
+#if 0
+	/* Correct length? */
+	if (! ieee80211_parse_action(ni, m))
+		return 0;
+#endif
+
+	/* Extract out action frame */
+	frm = (u_int8_t *)&wh[1];
+	ia = (struct ieee80211_action_ba_addbarequest *) frm;
+
+	/* Not ADDBA? Bail */
+	if (ia->rq_header.ia_category != IEEE80211_ACTION_CAT_BA)
+		return 0;
+	if (ia->rq_header.ia_action != IEEE80211_ACTION_BA_ADDBA_REQUEST)
+		return 0;
+
+	/* Extract TID, return it */
+	baparamset = le16toh(ia->rq_baparamset);
+	*tid = (int) MS(baparamset, IEEE80211_BAPS_TID);
+
+	return 1;
+}
+#undef	MS
+
+/* Per-node software queue operations */
+
+/*
+ * Add the current packet to the given BAW.
+ * It is assumed that the current packet
+ *
+ * + fits inside the BAW;
+ * + already has had a sequence number allocated.
+ *
+ * Since the BAW status may be modified by both the ath task and
+ * the net80211/ifnet contexts, the TID must be locked.
+ */
+void
+ath_tx_addto_baw(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid, struct ath_buf *bf)
+{
+	int index, cindex;
+	struct ieee80211_tx_ampdu *tap;
+
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[tid->ac]);
+
+	if (bf->bf_state.bfs_isretried)
+		return;
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+
+	if (bf->bf_state.bfs_addedbaw)
+		device_printf(sc->sc_dev,
+		    "%s: re-added? tid=%d, seqno %d; window %d:%d; baw head=%d tail=%d\n",
+		    __func__, tid->tid, SEQNO(bf->bf_state.bfs_seqno),
+		    tap->txa_start, tap->txa_wnd, tid->baw_head, tid->baw_tail);
+
+	/*
+	 * ni->ni_txseqs[] is the currently allocated seqno.
+	 * the txa state contains the current baw start.
+	 */
+	index  = ATH_BA_INDEX(tap->txa_start, SEQNO(bf->bf_state.bfs_seqno));
+	cindex = (tid->baw_head + index) & (ATH_TID_MAX_BUFS - 1);
+	DPRINTF(sc, ATH_DEBUG_SW_TX_BAW,
+	    "%s: tid=%d, seqno %d; window %d:%d; index=%d cindex=%d baw head=%d tail=%d\n",
+	    __func__, tid->tid, SEQNO(bf->bf_state.bfs_seqno),
+	    tap->txa_start, tap->txa_wnd, index, cindex, tid->baw_head, tid->baw_tail);
+
+
+#if 0
+	assert(tid->tx_buf[cindex] == NULL);
+#endif
+	if (tid->tx_buf[cindex] != NULL) {
+		device_printf(sc->sc_dev,
+		    "%s: ba packet dup (index=%d, cindex=%d, "
+		    "head=%d, tail=%d)\n",
+		    __func__, index, cindex, tid->baw_head, tid->baw_tail);
+		device_printf(sc->sc_dev,
+		    "%s: BA bf: %p; seqno=%d ; new bf: %p; seqno=%d\n",
+		    __func__,
+		    tid->tx_buf[cindex],
+		    SEQNO(tid->tx_buf[cindex]->bf_state.bfs_seqno),
+		    bf,
+		    SEQNO(bf->bf_state.bfs_seqno)
+		);
+	}
+	tid->tx_buf[cindex] = bf;
+
+	if (index >= ((tid->baw_tail - tid->baw_head) & (ATH_TID_MAX_BUFS - 1))) {
+		tid->baw_tail = cindex;
+		INCR(tid->baw_tail, ATH_TID_MAX_BUFS);
+	}
+}
+
+/*
+ * Flip the BAW buffer entry over from the existing one to the new one.
+ *
+ * When software retransmitting a (sub-)frame, it is entirely possible that
+ * the frame ath_buf is marked as BUSY and can't be immediately reused.
+ * In that instance the buffer is cloned and the new buffer is used for
+ * retransmit. We thus need to update the ath_buf slot in the BAW buf
+ * tracking array to maintain consistency.
+ */
+static void
+ath_tx_switch_baw_buf(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid, struct ath_buf *old_bf, struct ath_buf *new_bf)
+{
+	int index, cindex;
+	struct ieee80211_tx_ampdu *tap;
+	int seqno = SEQNO(old_bf->bf_state.bfs_seqno);
+
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[tid->ac]);
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+	index  = ATH_BA_INDEX(tap->txa_start, seqno);
+	cindex = (tid->baw_head + index) & (ATH_TID_MAX_BUFS - 1);
+
+	/*
+	 * Just warn for now; if it happens then we should find out
+	 * about it. It's highly likely the aggregation session will
+	 * soon hang.
+	 */
+	if (old_bf->bf_state.bfs_seqno != new_bf->bf_state.bfs_seqno) {
+		device_printf(sc->sc_dev, "%s: retransmitted buffer"
+		    " has mismatching seqno's, BA session may hang.\n",
+		    __func__);
+		device_printf(sc->sc_dev, "%s: old seqno=%d, new_seqno=%d\n",
+		    __func__,
+		    old_bf->bf_state.bfs_seqno,
+		    new_bf->bf_state.bfs_seqno);
+	}
+
+	if (tid->tx_buf[cindex] != old_bf) {
+		device_printf(sc->sc_dev, "%s: ath_buf pointer incorrect; "
+		    " has m BA session may hang.\n",
+		    __func__);
+		device_printf(sc->sc_dev, "%s: old bf=%p, new bf=%p\n",
+		    __func__,
+		    old_bf, new_bf);
+	}
+
+	tid->tx_buf[cindex] = new_bf;
+}
+
+/*
+ * seq_start - left edge of BAW
+ * seq_next - current/next sequence number to allocate
+ *
+ * Since the BAW status may be modified by both the ath task and
+ * the net80211/ifnet contexts, the TID must be locked.
+ */
+static void
+ath_tx_update_baw(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid, const struct ath_buf *bf)
+{
+	int index, cindex;
+	struct ieee80211_tx_ampdu *tap;
+	int seqno = SEQNO(bf->bf_state.bfs_seqno);
+
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[tid->ac]);
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+	index  = ATH_BA_INDEX(tap->txa_start, seqno);
+	cindex = (tid->baw_head + index) & (ATH_TID_MAX_BUFS - 1);
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_BAW,
+	    "%s: tid=%d, baw=%d:%d, seqno=%d, index=%d, cindex=%d, baw head=%d, tail=%d\n",
+	    __func__, tid->tid, tap->txa_start, tap->txa_wnd, seqno, index,
+	    cindex, tid->baw_head, tid->baw_tail);
+
+	/*
+	 * If this occurs then we have a big problem - something else
+	 * has slid tap->txa_start along without updating the BAW
+	 * tracking start/end pointers. Thus the TX BAW state is now
+	 * completely busted.
+	 *
+	 * But for now, since I haven't yet fixed TDMA and buffer cloning,
+	 * it's quite possible that a cloned buffer is making its way
+	 * here and causing it to fire off. Disable TDMA for now.
+	 */
+	if (tid->tx_buf[cindex] != bf) {
+		device_printf(sc->sc_dev,
+		    "%s: comp bf=%p, seq=%d; slot bf=%p, seqno=%d\n",
+		    __func__,
+		    bf, SEQNO(bf->bf_state.bfs_seqno),
+		    tid->tx_buf[cindex],
+		    SEQNO(tid->tx_buf[cindex]->bf_state.bfs_seqno));
+	}
+
+	tid->tx_buf[cindex] = NULL;
+
+	while (tid->baw_head != tid->baw_tail && !tid->tx_buf[tid->baw_head]) {
+		INCR(tap->txa_start, IEEE80211_SEQ_RANGE);
+		INCR(tid->baw_head, ATH_TID_MAX_BUFS);
+	}
+	DPRINTF(sc, ATH_DEBUG_SW_TX_BAW, "%s: baw is now %d:%d, baw head=%d\n",
+	    __func__, tap->txa_start, tap->txa_wnd, tid->baw_head);
+}
+
+/*
+ * Mark the current node/TID as ready to TX.
+ *
+ * This is done to make it easy for the software scheduler to
+ * find which nodes have data to send.
+ *
+ * The TXQ lock must be held.
+ */
+static void
+ath_tx_tid_sched(struct ath_softc *sc, struct ath_tid *tid)
+{
+	struct ath_txq *txq = sc->sc_ac2q[tid->ac];
+
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	if (tid->paused)
+		return;		/* paused, can't schedule yet */
+
+	if (tid->sched)
+		return;		/* already scheduled */
+
+	tid->sched = 1;
+
+	TAILQ_INSERT_TAIL(&txq->axq_tidq, tid, axq_qelem);
+}
+
+/*
+ * Mark the current node as no longer needing to be polled for
+ * TX packets.
+ *
+ * The TXQ lock must be held.
+ */
+static void
+ath_tx_tid_unsched(struct ath_softc *sc, struct ath_tid *tid)
+{
+	struct ath_txq *txq = sc->sc_ac2q[tid->ac];
+
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	if (tid->sched == 0)
+		return;
+
+	tid->sched = 0;
+	TAILQ_REMOVE(&txq->axq_tidq, tid, axq_qelem);
+}
+
+/*
+ * Assign a sequence number manually to the given frame.
+ *
+ * This should only be called for A-MPDU TX frames.
+ */
+static ieee80211_seq
+ath_tx_tid_seqno_assign(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_buf *bf, struct mbuf *m0)
+{
+	struct ieee80211_frame *wh;
+	int tid, pri;
+	ieee80211_seq seqno;
+	uint8_t subtype;
+
+	/* TID lookup */
+	wh = mtod(m0, struct ieee80211_frame *);
+	pri = M_WME_GETAC(m0);			/* honor classification */
+	tid = WME_AC_TO_TID(pri);
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: pri=%d, tid=%d, qos has seq=%d\n",
+	    __func__, pri, tid, IEEE80211_QOS_HAS_SEQ(wh));
+
+	/* XXX Is it a control frame? Ignore */
+
+	/* Does the packet require a sequence number? */
+	if (! IEEE80211_QOS_HAS_SEQ(wh))
+		return -1;
+
+	/*
+	 * Is it a QOS NULL Data frame? Give it a sequence number from
+	 * the default TID (IEEE80211_NONQOS_TID.)
+	 *
+	 * The RX path of everything I've looked at doesn't include the NULL
+	 * data frame sequence number in the aggregation state updates, so
+	 * assigning it a sequence number there will cause a BAW hole on the
+	 * RX side.
+	 */
+	subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
+	if (subtype == IEEE80211_FC0_SUBTYPE_QOS_NULL) {
+		seqno = ni->ni_txseqs[IEEE80211_NONQOS_TID];
+		INCR(ni->ni_txseqs[IEEE80211_NONQOS_TID], IEEE80211_SEQ_RANGE);
+	} else {
+		/* Manually assign sequence number */
+		seqno = ni->ni_txseqs[tid];
+		INCR(ni->ni_txseqs[tid], IEEE80211_SEQ_RANGE);
+	}
+	*(uint16_t *)&wh->i_seq[0] = htole16(seqno << IEEE80211_SEQ_SEQ_SHIFT);
+	M_SEQNO_SET(m0, seqno);
+
+	/* Return so caller can do something with it if needed */
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s:  -> seqno=%d\n", __func__, seqno);
+	return seqno;
+}
+
+/*
+ * Attempt to direct dispatch an aggregate frame to hardware.
+ * If the frame is out of BAW, queue.
+ * Otherwise, schedule it as a single frame.
+ */
+static void
+ath_tx_xmit_aggr(struct ath_softc *sc, struct ath_node *an, struct ath_buf *bf)
+{
+	struct ath_tid *tid = &an->an_tid[bf->bf_state.bfs_tid];
+	struct ath_txq *txq = bf->bf_state.bfs_txq;
+	struct ieee80211_tx_ampdu *tap;
+
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+
+	/* paused? queue */
+	if (tid->paused) {
+		ATH_TXQ_INSERT_TAIL(tid, bf, bf_list);
+		return;
+	}
+
+	/* outside baw? queue */
+	if (bf->bf_state.bfs_dobaw &&
+	    (! BAW_WITHIN(tap->txa_start, tap->txa_wnd,
+	    SEQNO(bf->bf_state.bfs_seqno)))) {
+		ATH_TXQ_INSERT_TAIL(tid, bf, bf_list);
+		ath_tx_tid_sched(sc, tid);
+		return;
+	}
+
+	/* Direct dispatch to hardware */
+	ath_tx_do_ratelookup(sc, bf);
+	ath_tx_rate_fill_rcflags(sc, bf);
+	ath_tx_set_rtscts(sc, bf);
+	ath_tx_setds(sc, bf);
+	ath_tx_set_ratectrl(sc, bf->bf_node, bf);
+	ath_tx_chaindesclist(sc, bf);
+
+	/* Statistics */
+	sc->sc_aggr_stats.aggr_low_hwq_single_pkt++;
+
+	/* Track per-TID hardware queue depth correctly */
+	tid->hwq_depth++;
+
+	/* Add to BAW */
+	if (bf->bf_state.bfs_dobaw) {
+		ath_tx_addto_baw(sc, an, tid, bf);
+		bf->bf_state.bfs_addedbaw = 1;
+	}
+
+	/* Set completion handler, multi-frame aggregate or not */
+	bf->bf_comp = ath_tx_aggr_comp;
+
+	/* Hand off to hardware */
+	ath_tx_handoff(sc, txq, bf);
+}
+
+/*
+ * Attempt to send the packet.
+ * If the queue isn't busy, direct-dispatch.
+ * If the queue is busy enough, queue the given packet on the
+ *  relevant software queue.
+ */
+void
+ath_tx_swq(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_txq *txq,
+    struct ath_buf *bf)
+{
+	struct ath_node *an = ATH_NODE(ni);
+	struct ieee80211_frame *wh;
+	struct ath_tid *atid;
+	int pri, tid;
+	struct mbuf *m0 = bf->bf_m;
+
+	/* Fetch the TID - non-QoS frames get assigned to TID 16 */
+	wh = mtod(m0, struct ieee80211_frame *);
+	pri = ath_tx_getac(sc, m0);
+	tid = ath_tx_gettid(sc, m0);
+	atid = &an->an_tid[tid];
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: bf=%p, pri=%d, tid=%d, qos=%d\n",
+	    __func__, bf, pri, tid, IEEE80211_QOS_HAS_SEQ(wh));
+
+	/* Set local packet state, used to queue packets to hardware */
+	bf->bf_state.bfs_tid = tid;
+	bf->bf_state.bfs_txq = txq;
+	bf->bf_state.bfs_pri = pri;
+
+	/*
+	 * If the hardware queue isn't busy, queue it directly.
+	 * If the hardware queue is busy, queue it.
+	 * If the TID is paused or the traffic it outside BAW, software
+	 * queue it.
+	 */
+	ATH_TXQ_LOCK(txq);
+	if (atid->paused) {
+		/* TID is paused, queue */
+		ATH_TXQ_INSERT_TAIL(atid, bf, bf_list);
+	} else if (ath_tx_ampdu_pending(sc, an, tid)) {
+		/* AMPDU pending; queue */
+		ATH_TXQ_INSERT_TAIL(atid, bf, bf_list);
+		/* XXX sched? */
+	} else if (ath_tx_ampdu_running(sc, an, tid)) {
+		/* AMPDU running, attempt direct dispatch if possible */
+		if (txq->axq_depth < sc->sc_hwq_limit)
+			ath_tx_xmit_aggr(sc, an, bf);
+		else {
+			ATH_TXQ_INSERT_TAIL(atid, bf, bf_list);
+			ath_tx_tid_sched(sc, atid);
+		}
+	} else if (txq->axq_depth < sc->sc_hwq_limit) {
+		/* AMPDU not running, attempt direct dispatch */
+		ath_tx_xmit_normal(sc, txq, bf);
+	} else {
+		/* Busy; queue */
+		ATH_TXQ_INSERT_TAIL(atid, bf, bf_list);
+		ath_tx_tid_sched(sc, atid);
+	}
+	ATH_TXQ_UNLOCK(txq);
+}
+
+/*
+ * Do the basic frame setup stuff that's required before the frame
+ * is added to a software queue.
+ *
+ * All frames get mostly the same treatment and it's done once.
+ * Retransmits fiddle with things like the rate control setup,
+ * setting the retransmit bit in the packet; doing relevant DMA/bus
+ * syncing and relinking it (back) into the hardware TX queue.
+ *
+ * Note that this may cause the mbuf to be reallocated, so
+ * m0 may not be valid.
+ */
+
+
+/*
+ * Configure the per-TID node state.
+ *
+ * This likely belongs in if_ath_node.c but I can't think of anywhere
+ * else to put it just yet.
+ *
+ * This sets up the SLISTs and the mutex as appropriate.
+ */
+void
+ath_tx_tid_init(struct ath_softc *sc, struct ath_node *an)
+{
+	int i, j;
+	struct ath_tid *atid;
+
+	for (i = 0; i < IEEE80211_TID_SIZE; i++) {
+		atid = &an->an_tid[i];
+		TAILQ_INIT(&atid->axq_q);
+		atid->tid = i;
+		atid->an = an;
+		for (j = 0; j < ATH_TID_MAX_BUFS; j++)
+			atid->tx_buf[j] = NULL;
+		atid->baw_head = atid->baw_tail = 0;
+		atid->paused = 0;
+		atid->sched = 0;
+		atid->hwq_depth = 0;
+		atid->cleanup_inprogress = 0;
+		if (i == IEEE80211_NONQOS_TID)
+			atid->ac = WME_AC_BE;
+		else
+			atid->ac = TID_TO_WME_AC(i);
+	}
+}
+
+/*
+ * Pause the current TID. This stops packets from being transmitted
+ * on it.
+ *
+ * Since this is also called from upper layers as well as the driver,
+ * it will get the TID lock.
+ */
+static void
+ath_tx_tid_pause(struct ath_softc *sc, struct ath_tid *tid)
+{
+	ATH_TXQ_LOCK(sc->sc_ac2q[tid->ac]);
+	tid->paused++;
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL, "%s: paused = %d\n",
+	    __func__, tid->paused);
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[tid->ac]);
+}
+
+/*
+ * Unpause the current TID, and schedule it if needed.
+ */
+static void
+ath_tx_tid_resume(struct ath_softc *sc, struct ath_tid *tid)
+{
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[tid->ac]);
+
+	tid->paused--;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL, "%s: unpaused = %d\n",
+	    __func__, tid->paused);
+
+	if (tid->paused || tid->axq_depth == 0) {
+		return;
+	}
+
+	ath_tx_tid_sched(sc, tid);
+	/* Punt some frames to the hardware if needed */
+	ath_txq_sched(sc, sc->sc_ac2q[tid->ac]);
+}
+
+/*
+ * Free any packets currently pending in the software TX queue.
+ *
+ * This will be called when a node is being deleted.
+ *
+ * It can also be called on an active node during an interface
+ * reset or state transition.
+ *
+ * (From Linux/reference):
+ *
+ * TODO: For frame(s) that are in the retry state, we will reuse the
+ * sequence number(s) without setting the retry bit. The
+ * alternative is to give up on these and BAR the receiver's window
+ * forward.
+ */
+static void
+ath_tx_tid_drain(struct ath_softc *sc, struct ath_node *an, struct ath_tid *tid,
+    ath_bufhead *bf_cq)
+{
+	struct ath_buf *bf;
+	struct ieee80211_tx_ampdu *tap;
+	struct ieee80211_node *ni = &an->an_node;
+	int t = 0;
+	struct ath_txq *txq = sc->sc_ac2q[tid->ac];
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[tid->ac]);
+
+	/* Walk the queue, free frames */
+	for (;;) {
+		bf = TAILQ_FIRST(&tid->axq_q);
+		if (bf == NULL) {
+			break;
+		}
+
+		if (t == 0) {
+			device_printf(sc->sc_dev,
+			    "%s: node %p: tid %d: txq_depth=%d, "
+			    "txq_aggr_depth=%d, sched=%d, paused=%d, "
+			    "hwq_depth=%d, incomp=%d, baw_head=%d, baw_tail=%d "
+			    "txa_start=%d, ni_txseqs=%d\n",
+			     __func__, ni, tid->tid, txq->axq_depth,
+			     txq->axq_aggr_depth, tid->sched, tid->paused,
+			     tid->hwq_depth, tid->incomp, tid->baw_head,
+			     tid->baw_tail, tap == NULL ? -1 : tap->txa_start,
+			     ni->ni_txseqs[tid->tid]);
+			t = 1;
+		}
+
+
+		/*
+		 * If the current TID is running AMPDU, update
+		 * the BAW.
+		 */
+		if (ath_tx_ampdu_running(sc, an, tid->tid) &&
+		    bf->bf_state.bfs_dobaw) {
+			/*
+			 * Only remove the frame from the BAW if it's
+			 * been transmitted at least once; this means
+			 * the frame was in the BAW to begin with.
+			 */
+			if (bf->bf_state.bfs_retries > 0) {
+				ath_tx_update_baw(sc, an, tid, bf);
+				bf->bf_state.bfs_dobaw = 0;
+			}
+			/*
+			 * This has become a non-fatal error now
+			 */
+			if (! bf->bf_state.bfs_addedbaw)
+				device_printf(sc->sc_dev,
+				    "%s: wasn't added: seqno %d\n",
+				    __func__, SEQNO(bf->bf_state.bfs_seqno));
+		}
+		ATH_TXQ_REMOVE(tid, bf, bf_list);
+		TAILQ_INSERT_TAIL(bf_cq, bf, bf_list);
+	}
+
+	/*
+	 * Now that it's completed, grab the TID lock and update
+	 * the sequence number and BAW window.
+	 * Because sequence numbers have been assigned to frames
+	 * that haven't been sent yet, it's entirely possible
+	 * we'll be called with some pending frames that have not
+	 * been transmitted.
+	 *
+	 * The cleaner solution is to do the sequence number allocation
+	 * when the packet is first transmitted - and thus the "retries"
+	 * check above would be enough to update the BAW/seqno.
+	 */
+
+	/* But don't do it for non-QoS TIDs */
+	if (tap) {
+#if 0
+		DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+		    "%s: node %p: TID %d: sliding BAW left edge to %d\n",
+		    __func__, an, tid->tid, tap->txa_start);
+#endif
+		ni->ni_txseqs[tid->tid] = tap->txa_start;
+		tid->baw_tail = tid->baw_head;
+	}
+}
+
+/*
+ * Flush all software queued packets for the given node.
+ *
+ * This occurs when a completion handler frees the last buffer
+ * for a node, and the node is thus freed. This causes the node
+ * to be cleaned up, which ends up calling ath_tx_node_flush.
+ */
+void
+ath_tx_node_flush(struct ath_softc *sc, struct ath_node *an)
+{
+	int tid;
+	ath_bufhead bf_cq;
+	struct ath_buf *bf;
+
+	TAILQ_INIT(&bf_cq);
+
+	for (tid = 0; tid < IEEE80211_TID_SIZE; tid++) {
+		struct ath_tid *atid = &an->an_tid[tid];
+		struct ath_txq *txq = sc->sc_ac2q[atid->ac];
+
+		/* Remove this tid from the list of active tids */
+		ATH_TXQ_LOCK(txq);
+		ath_tx_tid_unsched(sc, atid);
+
+		/* Free packets */
+		ath_tx_tid_drain(sc, an, atid, &bf_cq);
+		ATH_TXQ_UNLOCK(txq);
+	}
+
+	/* Handle completed frames */
+	while ((bf = TAILQ_FIRST(&bf_cq)) != NULL) {
+		TAILQ_REMOVE(&bf_cq, bf, bf_list);
+		ath_tx_default_comp(sc, bf, 0);
+	}
+}
+
+/*
+ * Drain all the software TXQs currently with traffic queued.
+ */
+void
+ath_tx_txq_drain(struct ath_softc *sc, struct ath_txq *txq)
+{
+	struct ath_tid *tid;
+	ath_bufhead bf_cq;
+	struct ath_buf *bf;
+
+	TAILQ_INIT(&bf_cq);
+	ATH_TXQ_LOCK(txq);
+
+	/*
+	 * Iterate over all active tids for the given txq,
+	 * flushing and unsched'ing them
+	 */
+	while (! TAILQ_EMPTY(&txq->axq_tidq)) {
+		tid = TAILQ_FIRST(&txq->axq_tidq);
+		ath_tx_tid_drain(sc, tid->an, tid, &bf_cq);
+		ath_tx_tid_unsched(sc, tid);
+	}
+
+	ATH_TXQ_UNLOCK(txq);
+
+	while ((bf = TAILQ_FIRST(&bf_cq)) != NULL) {
+		TAILQ_REMOVE(&bf_cq, bf, bf_list);
+		ath_tx_default_comp(sc, bf, 0);
+	}
+}
+
+/*
+ * Handle completion of non-aggregate session frames.
+ */
+void
+ath_tx_normal_comp(struct ath_softc *sc, struct ath_buf *bf, int fail)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+	struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
+
+	/* The TID state is protected behind the TXQ lock */
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: bf=%p: fail=%d, hwq_depth now %d\n",
+	    __func__, bf, fail, atid->hwq_depth - 1);
+
+	atid->hwq_depth--;
+	if (atid->hwq_depth < 0)
+		device_printf(sc->sc_dev, "%s: hwq_depth < 0: %d\n",
+		    __func__, atid->hwq_depth);
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	/*
+	 * punt to rate control if we're not being cleaned up
+	 * during a hw queue drain and the frame wanted an ACK.
+	 */
+	if (fail == 0 && ((bf->bf_txflags & HAL_TXDESC_NOACK) == 0))
+		ath_tx_update_ratectrl(sc, ni, bf->bf_state.bfs_rc,
+		    ts, bf->bf_state.bfs_pktlen,
+		    1, (ts->ts_status == 0) ? 0 : 1);
+
+	ath_tx_default_comp(sc, bf, fail);
+}
+
+/*
+ * Handle cleanup of aggregate session packets that aren't
+ * an A-MPDU.
+ *
+ * There's no need to update the BAW here - the session is being
+ * torn down.
+ */
+static void
+ath_tx_comp_cleanup_unaggr(struct ath_softc *sc, struct ath_buf *bf)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL, "%s: TID %d: incomp=%d\n",
+	    __func__, tid, atid->incomp);
+
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+	atid->incomp--;
+	if (atid->incomp == 0) {
+		DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+		    "%s: TID %d: cleaned up! resume!\n",
+		    __func__, tid);
+		atid->cleanup_inprogress = 0;
+		ath_tx_tid_resume(sc, atid);
+	}
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	ath_tx_default_comp(sc, bf, 0);
+}
+
+/*
+ * Performs transmit side cleanup when TID changes from aggregated to
+ * unaggregated.
+ *
+ * - Discard all retry frames from the s/w queue.
+ * - Fix the tx completion function for all buffers in s/w queue.
+ * - Count the number of unacked frames, and let transmit completion
+ *   handle it later.
+ *
+ * The caller is responsible for pausing the TID.
+ */
+static void
+ath_tx_cleanup(struct ath_softc *sc, struct ath_node *an, int tid)
+{
+	struct ath_tid *atid = &an->an_tid[tid];
+	struct ieee80211_tx_ampdu *tap;
+	struct ath_buf *bf, *bf_next;
+	ath_bufhead bf_cq;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: TID %d: called\n", __func__, tid);
+
+	TAILQ_INIT(&bf_cq);
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+
+	/*
+	 * Update the frames in the software TX queue:
+	 *
+	 * + Discard retry frames in the queue
+	 * + Fix the completion function to be non-aggregate
+	 */
+	bf = TAILQ_FIRST(&atid->axq_q);
+	while (bf) {
+		if (bf->bf_state.bfs_isretried) {
+			bf_next = TAILQ_NEXT(bf, bf_list);
+			TAILQ_REMOVE(&atid->axq_q, bf, bf_list);
+			atid->axq_depth--;
+			if (bf->bf_state.bfs_dobaw) {
+				ath_tx_update_baw(sc, an, atid, bf);
+				if (! bf->bf_state.bfs_addedbaw)
+					device_printf(sc->sc_dev,
+					    "%s: wasn't added: seqno %d\n",
+					    __func__, SEQNO(bf->bf_state.bfs_seqno));
+			}
+			bf->bf_state.bfs_dobaw = 0;
+			/*
+			 * Call the default completion handler with "fail" just
+			 * so upper levels are suitably notified about this.
+			 */
+			TAILQ_INSERT_TAIL(&bf_cq, bf, bf_list);
+			bf = bf_next;
+			continue;
+		}
+		/* Give these the default completion handler */
+		bf->bf_comp = ath_tx_normal_comp;
+		bf = TAILQ_NEXT(bf, bf_list);
+	}
+
+	/* The caller is required to pause the TID */
+#if 0
+	/* Pause the TID */
+	ath_tx_tid_pause(sc, atid);
+#endif
+
+	/*
+	 * Calculate what hardware-queued frames exist based
+	 * on the current BAW size. Ie, what frames have been
+	 * added to the TX hardware queue for this TID but
+	 * not yet ACKed.
+	 */
+	tap = ath_tx_get_tx_tid(an, tid);
+	/* Need the lock - fiddling with BAW */
+	while (atid->baw_head != atid->baw_tail) {
+		if (atid->tx_buf[atid->baw_head]) {
+			atid->incomp++;
+			atid->cleanup_inprogress = 1;
+			atid->tx_buf[atid->baw_head] = NULL;
+		}
+		INCR(atid->baw_head, ATH_TID_MAX_BUFS);
+		INCR(tap->txa_start, IEEE80211_SEQ_RANGE);
+	}
+
+	/*
+	 * If cleanup is required, defer TID scheduling
+	 * until all the HW queued packets have been
+	 * sent.
+	 */
+	if (! atid->cleanup_inprogress)
+		ath_tx_tid_resume(sc, atid);
+
+	if (atid->cleanup_inprogress)
+		DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+		    "%s: TID %d: cleanup needed: %d packets\n",
+		    __func__, tid, atid->incomp);
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	/* Handle completing frames and fail them */
+	while ((bf = TAILQ_FIRST(&bf_cq)) != NULL) {
+		TAILQ_REMOVE(&bf_cq, bf, bf_list);
+		ath_tx_default_comp(sc, bf, 1);
+	}
+}
+
+static void
+ath_tx_set_retry(struct ath_softc *sc, struct ath_buf *bf)
+{
+	struct ieee80211_frame *wh;
+
+	wh = mtod(bf->bf_m, struct ieee80211_frame *);
+	/* Only update/resync if needed */
+	if (bf->bf_state.bfs_isretried == 0) {
+		wh->i_fc[1] |= IEEE80211_FC1_RETRY;
+		bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
+		    BUS_DMASYNC_PREWRITE);
+	}
+	sc->sc_stats.ast_tx_swretries++;
+	bf->bf_state.bfs_isretried = 1;
+	bf->bf_state.bfs_retries ++;
+}
+
+static struct ath_buf *
+ath_tx_retry_clone(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid, struct ath_buf *bf)
+{
+	struct ath_buf *nbf;
+	int error;
+
+	nbf = ath_buf_clone(sc, bf);
+
+#if 0
+	device_printf(sc->sc_dev, "%s: ATH_BUF_BUSY; cloning\n",
+	    __func__);
+#endif
+
+	if (nbf == NULL) {
+		/* Failed to clone */
+		device_printf(sc->sc_dev,
+		    "%s: failed to clone a busy buffer\n",
+		    __func__);
+		return NULL;
+	}
+
+	/* Setup the dma for the new buffer */
+	error = ath_tx_dmasetup(sc, nbf, nbf->bf_m);
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "%s: failed to setup dma for clone\n",
+		    __func__);
+		/*
+		 * Put this at the head of the list, not tail;
+		 * that way it doesn't interfere with the
+		 * busy buffer logic (which uses the tail of
+		 * the list.)
+		 */
+		ATH_TXBUF_LOCK(sc);
+		TAILQ_INSERT_HEAD(&sc->sc_txbuf, nbf, bf_list);
+		ATH_TXBUF_UNLOCK(sc);
+		return NULL;
+	}
+
+	/* Update BAW if required, before we free the original buf */
+	if (bf->bf_state.bfs_dobaw)
+		ath_tx_switch_baw_buf(sc, an, tid, bf, nbf);
+
+	/* Free current buffer; return the older buffer */
+	bf->bf_m = NULL;
+	bf->bf_node = NULL;
+	ath_freebuf(sc, bf);
+	return nbf;
+}
+
+/*
+ * Handle retrying an unaggregate frame in an aggregate
+ * session.
+ *
+ * If too many retries occur, pause the TID, wait for
+ * any further retransmits (as there's no reason why
+ * non-aggregate frames in an aggregate session are
+ * transmitted in-order; they just have to be in-BAW)
+ * and then queue a BAR.
+ */
+static void
+ath_tx_aggr_retry_unaggr(struct ath_softc *sc, struct ath_buf *bf)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+	struct ieee80211_tx_ampdu *tap;
+	int txseq;
+
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+
+	tap = ath_tx_get_tx_tid(an, tid);
+
+	/*
+	 * If the buffer is marked as busy, we can't directly
+	 * reuse it. Instead, try to clone the buffer.
+	 * If the clone is successful, recycle the old buffer.
+	 * If the clone is unsuccessful, set bfs_retries to max
+	 * to force the next bit of code to free the buffer
+	 * for us.
+	 */
+	if ((bf->bf_state.bfs_retries < SWMAX_RETRIES) &&
+	    (bf->bf_flags & ATH_BUF_BUSY)) {
+		struct ath_buf *nbf;
+		nbf = ath_tx_retry_clone(sc, an, atid, bf);
+		if (nbf)
+			/* bf has been freed at this point */
+			bf = nbf;
+		else
+			bf->bf_state.bfs_retries = SWMAX_RETRIES + 1;
+	}
+
+	if (bf->bf_state.bfs_retries >= SWMAX_RETRIES) {
+		DPRINTF(sc, ATH_DEBUG_SW_TX_RETRIES,
+		    "%s: exceeded retries; seqno %d\n",
+		    __func__, SEQNO(bf->bf_state.bfs_seqno));
+		sc->sc_stats.ast_tx_swretrymax++;
+
+		/* Update BAW anyway */
+		if (bf->bf_state.bfs_dobaw) {
+			ath_tx_update_baw(sc, an, atid, bf);
+			if (! bf->bf_state.bfs_addedbaw)
+				device_printf(sc->sc_dev,
+				    "%s: wasn't added: seqno %d\n",
+				    __func__, SEQNO(bf->bf_state.bfs_seqno));
+		}
+		bf->bf_state.bfs_dobaw = 0;
+
+		/* Send BAR frame */
+		/*
+		 * This'll end up going into net80211 and back out
+		 * again, via ic->ic_raw_xmit().
+		 */
+		txseq = tap->txa_start;
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+		device_printf(sc->sc_dev,
+		    "%s: TID %d: send BAR; seq %d\n", __func__, tid, txseq);
+
+		/* XXX TODO: send BAR */
+
+		/* Free buffer, bf is free after this call */
+		ath_tx_default_comp(sc, bf, 0);
+		return;
+	}
+
+	/*
+	 * This increments the retry counter as well as
+	 * sets the retry flag in the ath_buf and packet
+	 * body.
+	 */
+	ath_tx_set_retry(sc, bf);
+
+	/*
+	 * Insert this at the head of the queue, so it's
+	 * retried before any current/subsequent frames.
+	 */
+	ATH_TXQ_INSERT_HEAD(atid, bf, bf_list);
+	ath_tx_tid_sched(sc, atid);
+
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+}
+
+/*
+ * Common code for aggregate excessive retry/subframe retry.
+ * If retrying, queues buffers to bf_q. If not, frees the
+ * buffers.
+ *
+ * XXX should unify this with ath_tx_aggr_retry_unaggr()
+ */
+static int
+ath_tx_retry_subframe(struct ath_softc *sc, struct ath_buf *bf,
+    ath_bufhead *bf_q)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[atid->ac]);
+
+	ath_hal_clr11n_aggr(sc->sc_ah, bf->bf_desc);
+	ath_hal_set11nburstduration(sc->sc_ah, bf->bf_desc, 0);
+	/* ath_hal_set11n_virtualmorefrag(sc->sc_ah, bf->bf_desc, 0); */
+
+	/*
+	 * If the buffer is marked as busy, we can't directly
+	 * reuse it. Instead, try to clone the buffer.
+	 * If the clone is successful, recycle the old buffer.
+	 * If the clone is unsuccessful, set bfs_retries to max
+	 * to force the next bit of code to free the buffer
+	 * for us.
+	 */
+	if ((bf->bf_state.bfs_retries < SWMAX_RETRIES) &&
+	    (bf->bf_flags & ATH_BUF_BUSY)) {
+		struct ath_buf *nbf;
+		nbf = ath_tx_retry_clone(sc, an, atid, bf);
+		if (nbf)
+			/* bf has been freed at this point */
+			bf = nbf;
+		else
+			bf->bf_state.bfs_retries = SWMAX_RETRIES + 1;
+	}
+
+	if (bf->bf_state.bfs_retries >= SWMAX_RETRIES) {
+		sc->sc_stats.ast_tx_swretrymax++;
+		DPRINTF(sc, ATH_DEBUG_SW_TX_RETRIES,
+		    "%s: max retries: seqno %d\n",
+		    __func__, SEQNO(bf->bf_state.bfs_seqno));
+		ath_tx_update_baw(sc, an, atid, bf);
+		if (! bf->bf_state.bfs_addedbaw)
+			device_printf(sc->sc_dev,
+			    "%s: wasn't added: seqno %d\n",
+			    __func__, SEQNO(bf->bf_state.bfs_seqno));
+		bf->bf_state.bfs_dobaw = 0;
+		return 1;
+	}
+
+	ath_tx_set_retry(sc, bf);
+	bf->bf_next = NULL;		/* Just to make sure */
+
+	TAILQ_INSERT_TAIL(bf_q, bf, bf_list);
+	return 0;
+}
+
+/*
+ * error pkt completion for an aggregate destination
+ */
+static void
+ath_tx_comp_aggr_error(struct ath_softc *sc, struct ath_buf *bf_first,
+    struct ath_tid *tid)
+{
+	struct ieee80211_node *ni = bf_first->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	struct ath_buf *bf_next, *bf;
+	ath_bufhead bf_q;
+	int drops = 0;
+	struct ieee80211_tx_ampdu *tap;
+	ath_bufhead bf_cq;
+
+	TAILQ_INIT(&bf_q);
+	TAILQ_INIT(&bf_cq);
+	sc->sc_stats.ast_tx_aggrfail++;
+
+	/*
+	 * Update rate control - all frames have failed.
+	 *
+	 * XXX use the length in the first frame in the series;
+	 * XXX just so things are consistent for now.
+	 */
+	ath_tx_update_ratectrl(sc, ni, bf_first->bf_state.bfs_rc,
+	    &bf_first->bf_status.ds_txstat,
+	    bf_first->bf_state.bfs_pktlen,
+	    bf_first->bf_state.bfs_nframes, bf_first->bf_state.bfs_nframes);
+
+	ATH_TXQ_LOCK(sc->sc_ac2q[tid->ac]);
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+
+	/* Retry all subframes */
+	bf = bf_first;
+	while (bf) {
+		bf_next = bf->bf_next;
+		bf->bf_next = NULL;	/* Remove it from the aggr list */
+		if (ath_tx_retry_subframe(sc, bf, &bf_q)) {
+			drops++;
+			bf->bf_next = NULL;
+			TAILQ_INSERT_TAIL(&bf_cq, bf, bf_list);
+		}
+		bf = bf_next;
+	}
+
+	/* Prepend all frames to the beginning of the queue */
+	while ((bf = TAILQ_LAST(&bf_q, ath_bufhead_s)) != NULL) {
+		TAILQ_REMOVE(&bf_q, bf, bf_list);
+		ATH_TXQ_INSERT_HEAD(tid, bf, bf_list);
+	}
+
+	ath_tx_tid_sched(sc, tid);
+
+	/*
+	 * send bar if we dropped any frames
+	 *
+	 * Keep the txq lock held for now, as we need to ensure
+	 * that ni_txseqs[] is consistent (as it's being updated
+	 * in the ifnet TX context or raw TX context.)
+	 */
+	if (drops) {
+		int txseq = tap->txa_start;
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[tid->ac]);
+		device_printf(sc->sc_dev,
+		    "%s: TID %d: send BAR; seq %d\n",
+		    __func__, tid->tid, txseq);
+
+		/* XXX TODO: send BAR */
+	} else {
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[tid->ac]);
+	}
+
+	/* Complete frames which errored out */
+	while ((bf = TAILQ_FIRST(&bf_cq)) != NULL) {
+		TAILQ_REMOVE(&bf_cq, bf, bf_list);
+		ath_tx_default_comp(sc, bf, 0);
+	}
+}
+
+/*
+ * Handle clean-up of packets from an aggregate list.
+ *
+ * There's no need to update the BAW here - the session is being
+ * torn down.
+ */
+static void
+ath_tx_comp_cleanup_aggr(struct ath_softc *sc, struct ath_buf *bf_first)
+{
+	struct ath_buf *bf, *bf_next;
+	struct ieee80211_node *ni = bf_first->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf_first->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+
+	bf = bf_first;
+
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+
+	/* update incomp */
+	while (bf) {
+		atid->incomp--;
+		bf = bf->bf_next;
+	}
+
+	if (atid->incomp == 0) {
+		DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+		    "%s: TID %d: cleaned up! resume!\n",
+		    __func__, tid);
+		atid->cleanup_inprogress = 0;
+		ath_tx_tid_resume(sc, atid);
+	}
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	/* Handle frame completion */
+	while (bf) {
+		bf_next = bf->bf_next;
+		ath_tx_default_comp(sc, bf, 1);
+		bf = bf_next;
+	}
+}
+
+/*
+ * Handle completion of an set of aggregate frames.
+ *
+ * XXX for now, simply complete each sub-frame.
+ *
+ * Note: the completion handler is the last descriptor in the aggregate,
+ * not the last descriptor in the first frame.
+ */
+static void
+ath_tx_aggr_comp_aggr(struct ath_softc *sc, struct ath_buf *bf_first, int fail)
+{
+	//struct ath_desc *ds = bf->bf_lastds;
+	struct ieee80211_node *ni = bf_first->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf_first->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+	struct ath_tx_status ts;
+	struct ieee80211_tx_ampdu *tap;
+	ath_bufhead bf_q;
+	ath_bufhead bf_cq;
+	int seq_st, tx_ok;
+	int hasba, isaggr;
+	uint32_t ba[2];
+	struct ath_buf *bf, *bf_next;
+	int ba_index;
+	int drops = 0;
+	int nframes = 0, nbad = 0, nf;
+	int pktlen;
+	/* XXX there's too much on the stack? */
+	struct ath_rc_series rc[4];
+	int txseq;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR, "%s: called; hwq_depth=%d\n",
+	    __func__, atid->hwq_depth);
+
+	/* The TID state is kept behind the TXQ lock */
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+
+	atid->hwq_depth--;
+	if (atid->hwq_depth < 0)
+		device_printf(sc->sc_dev, "%s: hwq_depth < 0: %d\n",
+		    __func__, atid->hwq_depth);
+
+	/*
+	 * Punt cleanup to the relevant function, not our problem now
+	 */
+	if (atid->cleanup_inprogress) {
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+		ath_tx_comp_cleanup_aggr(sc, bf_first);
+		return;
+	}
+
+	/*
+	 * Take a copy; this may be needed -after- bf_first
+	 * has been completed and freed.
+	 */
+	ts = bf_first->bf_status.ds_txstat;
+	/*
+	 * XXX for now, use the first frame in the aggregate for
+	 * XXX rate control completion; it's at least consistent.
+	 */
+	pktlen = bf_first->bf_state.bfs_pktlen;
+
+	/*
+	 * handle errors first
+	 */
+	if (ts.ts_status & HAL_TXERR_XRETRY) {
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+		ath_tx_comp_aggr_error(sc, bf_first, atid);
+		return;
+	}
+
+	TAILQ_INIT(&bf_q);
+	TAILQ_INIT(&bf_cq);
+	tap = ath_tx_get_tx_tid(an, tid);
+
+	/*
+	 * extract starting sequence and block-ack bitmap
+	 */
+	/* XXX endian-ness of seq_st, ba? */
+	seq_st = ts.ts_seqnum;
+	hasba = !! (ts.ts_flags & HAL_TX_BA);
+	tx_ok = (ts.ts_status == 0);
+	isaggr = bf_first->bf_state.bfs_aggr;
+	ba[0] = ts.ts_ba_low;
+	ba[1] = ts.ts_ba_high;
+
+	/*
+	 * Copy the TX completion status and the rate control
+	 * series from the first descriptor, as it may be freed
+	 * before the rate control code can get its grubby fingers
+	 * into things.
+	 */
+	memcpy(rc, bf_first->bf_state.bfs_rc, sizeof(rc));
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+	    "%s: txa_start=%d, tx_ok=%d, status=%.8x, flags=%.8x, isaggr=%d, seq_st=%d, hasba=%d, ba=%.8x, %.8x\n",
+	    __func__, tap->txa_start, tx_ok, ts.ts_status, ts.ts_flags,
+	    isaggr, seq_st, hasba, ba[0], ba[1]);
+
+	/* Occasionally, the MAC sends a tx status for the wrong TID. */
+	if (tid != ts.ts_tid) {
+		device_printf(sc->sc_dev, "%s: tid %d != hw tid %d\n",
+		    __func__, tid, ts.ts_tid);
+		tx_ok = 0;
+	}
+
+	/* AR5416 BA bug; this requires an interface reset */
+	if (isaggr && tx_ok && (! hasba)) {
+		device_printf(sc->sc_dev,
+		    "%s: AR5416 bug: hasba=%d; txok=%d, isaggr=%d, seq_st=%d\n",
+		    __func__, hasba, tx_ok, isaggr, seq_st);
+		/* XXX TODO: schedule an interface reset */
+	}
+
+	/*
+	 * Walk the list of frames, figure out which ones were correctly
+	 * sent and which weren't.
+	 */
+	bf = bf_first;
+	nf = bf_first->bf_state.bfs_nframes;
+
+	/* bf_first is going to be invalid once this list is walked */
+	bf_first = NULL;
+
+	/*
+	 * Walk the list of completed frames and determine
+	 * which need to be completed and which need to be
+	 * retransmitted.
+	 *
+	 * For completed frames, the completion functions need
+	 * to be called at the end of this function as the last
+	 * node reference may free the node.
+	 *
+	 * Finally, since the TXQ lock can't be held during the
+	 * completion callback (to avoid lock recursion),
+	 * the completion calls have to be done outside of the
+	 * lock.
+	 */
+	while (bf) {
+		nframes++;
+		ba_index = ATH_BA_INDEX(seq_st, SEQNO(bf->bf_state.bfs_seqno));
+		bf_next = bf->bf_next;
+		bf->bf_next = NULL;	/* Remove it from the aggr list */
+
+		DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+		    "%s: checking bf=%p seqno=%d; ack=%d\n",
+		    __func__, bf, SEQNO(bf->bf_state.bfs_seqno),
+		    ATH_BA_ISSET(ba, ba_index));
+
+		if (tx_ok && ATH_BA_ISSET(ba, ba_index)) {
+			ath_tx_update_baw(sc, an, atid, bf);
+			bf->bf_state.bfs_dobaw = 0;
+			if (! bf->bf_state.bfs_addedbaw)
+				device_printf(sc->sc_dev,
+				    "%s: wasn't added: seqno %d\n",
+				    __func__, SEQNO(bf->bf_state.bfs_seqno));
+			bf->bf_next = NULL;
+			TAILQ_INSERT_TAIL(&bf_cq, bf, bf_list);
+		} else {
+			if (ath_tx_retry_subframe(sc, bf, &bf_q)) {
+				drops++;
+				bf->bf_next = NULL;
+				TAILQ_INSERT_TAIL(&bf_cq, bf, bf_list);
+			}
+			nbad++;
+		}
+		bf = bf_next;
+	}
+
+	/*
+	 * Now that the BAW updates have been done, unlock
+	 *
+	 * txseq is grabbed before the lock is released so we
+	 * have a consistent view of what -was- in the BAW.
+	 * Anything after this point will not yet have been
+	 * TXed.
+	 */
+	txseq = tap->txa_start;
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	if (nframes != nf)
+		device_printf(sc->sc_dev,
+		    "%s: num frames seen=%d; bf nframes=%d\n",
+		    __func__, nframes, nf);
+
+	/*
+	 * Now we know how many frames were bad, call the rate
+	 * control code.
+	 */
+	if (fail == 0)
+		ath_tx_update_ratectrl(sc, ni, rc, &ts, pktlen, nframes, nbad);
+
+	/*
+	 * send bar if we dropped any frames
+	 */
+	if (drops) {
+		device_printf(sc->sc_dev,
+		    "%s: TID %d: send BAR; seq %d\n", __func__, tid, txseq);
+		/* XXX TODO: send BAR */
+	}
+
+	/* Prepend all frames to the beginning of the queue */
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+	while ((bf = TAILQ_LAST(&bf_q, ath_bufhead_s)) != NULL) {
+		TAILQ_REMOVE(&bf_q, bf, bf_list);
+		ATH_TXQ_INSERT_HEAD(atid, bf, bf_list);
+	}
+	ath_tx_tid_sched(sc, atid);
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+	    "%s: txa_start now %d\n", __func__, tap->txa_start);
+
+	/* Do deferred completion */
+	while ((bf = TAILQ_FIRST(&bf_cq)) != NULL) {
+		TAILQ_REMOVE(&bf_cq, bf, bf_list);
+		ath_tx_default_comp(sc, bf, 0);
+	}
+}
+
+/*
+ * Handle completion of unaggregated frames in an ADDBA
+ * session.
+ *
+ * Fail is set to 1 if the entry is being freed via a call to
+ * ath_tx_draintxq().
+ */
+static void
+ath_tx_aggr_comp_unaggr(struct ath_softc *sc, struct ath_buf *bf, int fail)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ath_node *an = ATH_NODE(ni);
+	int tid = bf->bf_state.bfs_tid;
+	struct ath_tid *atid = &an->an_tid[tid];
+	struct ath_tx_status *ts = &bf->bf_status.ds_txstat;
+
+	/*
+	 * Update rate control status here, before we possibly
+	 * punt to retry or cleanup.
+	 *
+	 * Do it outside of the TXQ lock.
+	 */
+	if (fail == 0 && ((bf->bf_txflags & HAL_TXDESC_NOACK) == 0))
+		ath_tx_update_ratectrl(sc, ni, bf->bf_state.bfs_rc,
+		    &bf->bf_status.ds_txstat,
+		    bf->bf_state.bfs_pktlen,
+		    1, (ts->ts_status == 0) ? 0 : 1);
+
+	/*
+	 * This is called early so atid->hwq_depth can be tracked.
+	 * This unfortunately means that it's released and regrabbed
+	 * during retry and cleanup. That's rather inefficient.
+	 */
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+
+	if (tid == IEEE80211_NONQOS_TID)
+		device_printf(sc->sc_dev, "%s: TID=16!\n", __func__);
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: bf=%p: tid=%d, hwq_depth=%d\n",
+	    __func__, bf, bf->bf_state.bfs_tid, atid->hwq_depth);
+
+	atid->hwq_depth--;
+	if (atid->hwq_depth < 0)
+		device_printf(sc->sc_dev, "%s: hwq_depth < 0: %d\n",
+		    __func__, atid->hwq_depth);
+
+	/*
+	 * If a cleanup is in progress, punt to comp_cleanup;
+	 * rather than handling it here. It's thus their
+	 * responsibility to clean up, call the completion
+	 * function in net80211, etc.
+	 */
+	if (atid->cleanup_inprogress) {
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+		ath_tx_comp_cleanup_unaggr(sc, bf);
+		return;
+	}
+
+	/*
+	 * Don't bother with the retry check if all frames
+	 * are being failed (eg during queue deletion.)
+	 */
+	if (fail == 0 && ts->ts_status & HAL_TXERR_XRETRY) {
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+		ath_tx_aggr_retry_unaggr(sc, bf);
+		return;
+	}
+
+	/* Success? Complete */
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: TID=%d, seqno %d\n",
+	    __func__, tid, SEQNO(bf->bf_state.bfs_seqno));
+	if (bf->bf_state.bfs_dobaw) {
+		ath_tx_update_baw(sc, an, atid, bf);
+		bf->bf_state.bfs_dobaw = 0;
+		if (! bf->bf_state.bfs_addedbaw)
+			device_printf(sc->sc_dev,
+			    "%s: wasn't added: seqno %d\n",
+			    __func__, SEQNO(bf->bf_state.bfs_seqno));
+	}
+
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+
+	ath_tx_default_comp(sc, bf, fail);
+	/* bf is freed at this point */
+}
+
+void
+ath_tx_aggr_comp(struct ath_softc *sc, struct ath_buf *bf, int fail)
+{
+	if (bf->bf_state.bfs_aggr)
+		ath_tx_aggr_comp_aggr(sc, bf, fail);
+	else
+		ath_tx_aggr_comp_unaggr(sc, bf, fail);
+}
+
+/*
+ * Schedule some packets from the given node/TID to the hardware.
+ *
+ * This is the aggregate version.
+ */
+void
+ath_tx_tid_hw_queue_aggr(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid)
+{
+	struct ath_buf *bf;
+	struct ath_txq *txq = sc->sc_ac2q[tid->ac];
+	struct ieee80211_tx_ampdu *tap;
+	struct ieee80211_node *ni = &an->an_node;
+	ATH_AGGR_STATUS status;
+	ath_bufhead bf_q;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: tid=%d\n", __func__, tid->tid);
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+
+	if (tid->tid == IEEE80211_NONQOS_TID)
+		device_printf(sc->sc_dev, "%s: called for TID=NONQOS_TID?\n",
+		    __func__);
+
+	for (;;) {
+		status = ATH_AGGR_DONE;
+
+		/*
+		 * If the upper layer has paused the TID, don't
+		 * queue any further packets.
+		 *
+		 * This can also occur from the completion task because
+		 * of packet loss; but as its serialised with this code,
+		 * it won't "appear" half way through queuing packets.
+		 */
+		if (tid->paused)
+			break;
+
+		bf = TAILQ_FIRST(&tid->axq_q);
+		if (bf == NULL) {
+			break;
+		}
+
+		/*
+		 * If the packet doesn't fall within the BAW (eg a NULL
+		 * data frame), schedule it directly; continue.
+		 */
+		if (! bf->bf_state.bfs_dobaw) {
+			DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR, "%s: non-baw packet\n",
+			    __func__);
+			ATH_TXQ_REMOVE(tid, bf, bf_list);
+			bf->bf_state.bfs_aggr = 0;
+			ath_tx_do_ratelookup(sc, bf);
+			ath_tx_rate_fill_rcflags(sc, bf);
+			ath_tx_set_rtscts(sc, bf);
+			ath_tx_setds(sc, bf);
+			ath_tx_chaindesclist(sc, bf);
+			ath_hal_clr11n_aggr(sc->sc_ah, bf->bf_desc);
+			ath_tx_set_ratectrl(sc, ni, bf);
+
+			sc->sc_aggr_stats.aggr_nonbaw_pkt++;
+
+			/* Queue the packet; continue */
+			goto queuepkt;
+		}
+
+		TAILQ_INIT(&bf_q);
+
+		/*
+		 * Do a rate control lookup on the first frame in the
+		 * list. The rate control code needs that to occur
+		 * before it can determine whether to TX.
+		 * It's inaccurate because the rate control code doesn't
+		 * really "do" aggregate lookups, so it only considers
+		 * the size of the first frame.
+		 */
+		ath_tx_do_ratelookup(sc, bf);
+		bf->bf_state.bfs_rc[3].rix = 0;
+		bf->bf_state.bfs_rc[3].tries = 0;
+		ath_tx_rate_fill_rcflags(sc, bf);
+
+		status = ath_tx_form_aggr(sc, an, tid, &bf_q);
+
+		DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+		    "%s: ath_tx_form_aggr() status=%d\n", __func__, status);
+
+		/*
+		 * No frames to be picked up - out of BAW
+		 */
+		if (TAILQ_EMPTY(&bf_q))
+			break;
+
+		/*
+		 * This assumes that the descriptor list in the ath_bufhead
+		 * are already linked together via bf_next pointers.
+		 */
+		bf = TAILQ_FIRST(&bf_q);
+
+		/*
+		 * If it's the only frame send as non-aggregate
+		 * assume that ath_tx_form_aggr() has checked
+		 * whether it's in the BAW and added it appropriately.
+		 */
+		if (bf->bf_state.bfs_nframes == 1) {
+			DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+			    "%s: single-frame aggregate\n", __func__);
+			bf->bf_state.bfs_aggr = 0;
+			ath_tx_set_rtscts(sc, bf);
+			ath_tx_setds(sc, bf);
+			ath_tx_chaindesclist(sc, bf);
+			ath_hal_clr11n_aggr(sc->sc_ah, bf->bf_desc);
+			ath_tx_set_ratectrl(sc, ni, bf);
+			if (status == ATH_AGGR_BAW_CLOSED)
+				sc->sc_aggr_stats.aggr_baw_closed_single_pkt++;
+			else
+				sc->sc_aggr_stats.aggr_single_pkt++;
+		} else {
+			DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+			    "%s: multi-frame aggregate: %d frames, length %d\n",
+			     __func__, bf->bf_state.bfs_nframes,
+			    bf->bf_state.bfs_al);
+			bf->bf_state.bfs_aggr = 1;
+			sc->sc_aggr_stats.aggr_pkts[bf->bf_state.bfs_nframes]++;
+			sc->sc_aggr_stats.aggr_aggr_pkt++;
+
+			/*
+			 * Update the rate and rtscts information based on the
+			 * rate decision made by the rate control code;
+			 * the first frame in the aggregate needs it.
+			 */
+			ath_tx_set_rtscts(sc, bf);
+
+			/*
+			 * Setup the relevant descriptor fields
+			 * for aggregation. The first descriptor
+			 * already points to the rest in the chain.
+			 */
+			ath_tx_setds_11n(sc, bf);
+
+			/*
+			 * setup first desc with rate and aggr info
+			 */
+			ath_tx_set_ratectrl(sc, ni, bf);
+		}
+	queuepkt:
+		//txq = bf->bf_state.bfs_txq;
+
+		/* Set completion handler, multi-frame aggregate or not */
+		bf->bf_comp = ath_tx_aggr_comp;
+
+		if (bf->bf_state.bfs_tid == IEEE80211_NONQOS_TID)
+		    device_printf(sc->sc_dev, "%s: TID=16?\n", __func__);
+
+		/* Punt to txq */
+		ath_tx_handoff(sc, txq, bf);
+
+		/* Track outstanding buffer count to hardware */
+		/* aggregates are "one" buffer */
+		tid->hwq_depth++;
+
+		/*
+		 * Break out if ath_tx_form_aggr() indicated
+		 * there can't be any further progress (eg BAW is full.)
+		 * Checking for an empty txq is done above.
+		 *
+		 * XXX locking on txq here?
+		 */
+		if (txq->axq_aggr_depth >= sc->sc_hwq_limit ||
+		    status == ATH_AGGR_BAW_CLOSED)
+			break;
+	}
+}
+
+/*
+ * Schedule some packets from the given node/TID to the hardware.
+ */
+void
+ath_tx_tid_hw_queue_norm(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid)
+{
+	struct ath_buf *bf;
+	struct ath_txq *txq = sc->sc_ac2q[tid->ac];
+	struct ieee80211_node *ni = &an->an_node;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: node %p: TID %d: called\n",
+	    __func__, an, tid->tid);
+
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	/* Check - is AMPDU pending or running? then print out something */
+	if (ath_tx_ampdu_pending(sc, an, tid->tid))
+		device_printf(sc->sc_dev, "%s: tid=%d, ampdu pending?\n",
+		    __func__, tid->tid);
+	if (ath_tx_ampdu_running(sc, an, tid->tid))
+		device_printf(sc->sc_dev, "%s: tid=%d, ampdu running?\n",
+		    __func__, tid->tid);
+
+	for (;;) {
+
+		/*
+		 * If the upper layers have paused the TID, don't
+		 * queue any further packets.
+		 */
+		if (tid->paused)
+			break;
+
+		bf = TAILQ_FIRST(&tid->axq_q);
+		if (bf == NULL) {
+			break;
+		}
+
+		ATH_TXQ_REMOVE(tid, bf, bf_list);
+
+		KASSERT(txq == bf->bf_state.bfs_txq, ("txqs not equal!\n"));
+
+		/* Sanity check! */
+		if (tid->tid != bf->bf_state.bfs_tid) {
+			device_printf(sc->sc_dev, "%s: bfs_tid %d !="
+			    " tid %d\n",
+			    __func__, bf->bf_state.bfs_tid, tid->tid);
+		}
+		/* Normal completion handler */
+		bf->bf_comp = ath_tx_normal_comp;
+
+		/* Program descriptors + rate control */
+		ath_tx_do_ratelookup(sc, bf);
+		ath_tx_rate_fill_rcflags(sc, bf);
+		ath_tx_set_rtscts(sc, bf);
+		ath_tx_setds(sc, bf);
+		ath_tx_chaindesclist(sc, bf);
+		ath_tx_set_ratectrl(sc, ni, bf);
+
+		/* Track outstanding buffer count to hardware */
+		/* aggregates are "one" buffer */
+		tid->hwq_depth++;
+
+		/* Punt to hardware or software txq */
+		ath_tx_handoff(sc, txq, bf);
+	}
+}
+
+/*
+ * Schedule some packets to the given hardware queue.
+ *
+ * This function walks the list of TIDs (ie, ath_node TIDs
+ * with queued traffic) and attempts to schedule traffic
+ * from them.
+ *
+ * TID scheduling is implemented as a FIFO, with TIDs being
+ * added to the end of the queue after some frames have been
+ * scheduled.
+ */
+void
+ath_txq_sched(struct ath_softc *sc, struct ath_txq *txq)
+{
+	struct ath_tid *tid, *next, *last;
+
+	ATH_TXQ_LOCK_ASSERT(txq);
+
+	/*
+	 * Don't schedule if the hardware queue is busy.
+	 * This (hopefully) gives some more time to aggregate
+	 * some packets in the aggregation queue.
+	 */
+	if (txq->axq_aggr_depth >= sc->sc_hwq_limit) {
+		sc->sc_aggr_stats.aggr_sched_nopkt++;
+		return;
+	}
+
+	last = TAILQ_LAST(&txq->axq_tidq, axq_t_s);
+
+	TAILQ_FOREACH_SAFE(tid, &txq->axq_tidq, axq_qelem, next) {
+		/*
+		 * Suspend paused queues here; they'll be resumed
+		 * once the addba completes or times out.
+		 */
+		DPRINTF(sc, ATH_DEBUG_SW_TX, "%s: tid=%d, paused=%d\n",
+		    __func__, tid->tid, tid->paused);
+		ath_tx_tid_unsched(sc, tid);
+		if (tid->paused) {
+			continue;
+		}
+		if (ath_tx_ampdu_running(sc, tid->an, tid->tid))
+			ath_tx_tid_hw_queue_aggr(sc, tid->an, tid);
+		else
+			ath_tx_tid_hw_queue_norm(sc, tid->an, tid);
+
+		/* Not empty? Re-schedule */
+		if (tid->axq_depth != 0)
+			ath_tx_tid_sched(sc, tid);
+
+		/* Give the software queue time to aggregate more packets */
+		if (txq->axq_aggr_depth >= sc->sc_hwq_limit) {
+			break;
+		}
+
+		/*
+		 * If this was the last entry on the original list, stop.
+		 * Otherwise nodes that have been rescheduled onto the end
+		 * of the TID FIFO list will just keep being rescheduled.
+		 */
+		if (tid == last)
+			break;
+	}
+}
+
+/*
+ * TX addba handling
+ */
+
+/*
+ * Return net80211 TID struct pointer, or NULL for none
+ */
+struct ieee80211_tx_ampdu *
+ath_tx_get_tx_tid(struct ath_node *an, int tid)
+{
+	struct ieee80211_node *ni = &an->an_node;
+	struct ieee80211_tx_ampdu *tap;
+	int ac;
+
+	if (tid == IEEE80211_NONQOS_TID)
+		return NULL;
+
+	ac = TID_TO_WME_AC(tid);
+
+	tap = &ni->ni_tx_ampdu[ac];
+	return tap;
+}
+
+/*
+ * Is AMPDU-TX running?
+ */
+static int
+ath_tx_ampdu_running(struct ath_softc *sc, struct ath_node *an, int tid)
+{
+	struct ieee80211_tx_ampdu *tap;
+
+	if (tid == IEEE80211_NONQOS_TID)
+		return 0;
+
+	tap = ath_tx_get_tx_tid(an, tid);
+	if (tap == NULL)
+		return 0;	/* Not valid; default to not running */
+
+	return !! (tap->txa_flags & IEEE80211_AGGR_RUNNING);
+}
+
+/*
+ * Is AMPDU-TX negotiation pending?
+ */
+static int
+ath_tx_ampdu_pending(struct ath_softc *sc, struct ath_node *an, int tid)
+{
+	struct ieee80211_tx_ampdu *tap;
+
+	if (tid == IEEE80211_NONQOS_TID)
+		return 0;
+
+	tap = ath_tx_get_tx_tid(an, tid);
+	if (tap == NULL)
+		return 0;	/* Not valid; default to not pending */
+
+	return !! (tap->txa_flags & IEEE80211_AGGR_XCHGPEND);
+}
+
+/*
+ * Is AMPDU-TX pending for the given TID?
+ */
+
+
+/*
+ * Method to handle sending an ADDBA request.
+ *
+ * We tap this so the relevant flags can be set to pause the TID
+ * whilst waiting for the response.
+ *
+ * XXX there's no timeout handler we can override?
+ */
+int
+ath_addba_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
+    int dialogtoken, int baparamset, int batimeout)
+{
+	struct ath_softc *sc = ni->ni_ic->ic_ifp->if_softc;
+	int tid = WME_AC_TO_TID(tap->txa_ac);
+	struct ath_node *an = ATH_NODE(ni);
+	struct ath_tid *atid = &an->an_tid[tid];
+
+	/*
+	 * XXX danger Will Robinson!
+	 *
+	 * Although the taskqueue may be running and scheduling some more
+	 * packets, these should all be _before_ the addba sequence number.
+	 * However, net80211 will keep self-assigning sequence numbers
+	 * until addba has been negotiated.
+	 *
+	 * In the past, these packets would be "paused" (which still works
+	 * fine, as they're being scheduled to the driver in the same
+	 * serialised method which is calling the addba request routine)
+	 * and when the aggregation session begins, they'll be dequeued
+	 * as aggregate packets and added to the BAW. However, now there's
+	 * a "bf->bf_state.bfs_dobaw" flag, and this isn't set for these
+	 * packets. Thus they never get included in the BAW tracking and
+	 * this can cause the initial burst of packets after the addba
+	 * negotiation to "hang", as they quickly fall outside the BAW.
+	 *
+	 * The "eventual" solution should be to tag these packets with
+	 * dobaw. Although net80211 has given us a sequence number,
+	 * it'll be "after" the left edge of the BAW and thus it'll
+	 * fall within it.
+	 */
+	ath_tx_tid_pause(sc, atid);
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: called; dialogtoken=%d, baparamset=%d, batimeout=%d\n",
+	    __func__, dialogtoken, baparamset, batimeout);
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: txa_start=%d, ni_txseqs=%d\n",
+	    __func__, tap->txa_start, ni->ni_txseqs[tid]);
+
+	return sc->sc_addba_request(ni, tap, dialogtoken, baparamset,
+	    batimeout);
+}
+
+/*
+ * Handle an ADDBA response.
+ *
+ * We unpause the queue so TX'ing can resume.
+ *
+ * Any packets TX'ed from this point should be "aggregate" (whether
+ * aggregate or not) so the BAW is updated.
+ *
+ * Note! net80211 keeps self-assigning sequence numbers until
+ * ampdu is negotiated. This means the initially-negotiated BAW left
+ * edge won't match the ni->ni_txseq.
+ *
+ * So, being very dirty, the BAW left edge is "slid" here to match
+ * ni->ni_txseq.
+ *
+ * What likely SHOULD happen is that all packets subsequent to the
+ * addba request should be tagged as aggregate and queued as non-aggregate
+ * frames; thus updating the BAW. For now though, I'll just slide the
+ * window.
+ */
+int
+ath_addba_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
+    int status, int code, int batimeout)
+{
+	struct ath_softc *sc = ni->ni_ic->ic_ifp->if_softc;
+	int tid = WME_AC_TO_TID(tap->txa_ac);
+	struct ath_node *an = ATH_NODE(ni);
+	struct ath_tid *atid = &an->an_tid[tid];
+	int r;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: called; status=%d, code=%d, batimeout=%d\n", __func__,
+	    status, code, batimeout);
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: txa_start=%d, ni_txseqs=%d\n",
+	    __func__, tap->txa_start, ni->ni_txseqs[tid]);
+
+	/*
+	 * Call this first, so the interface flags get updated
+	 * before the TID is unpaused. Otherwise a race condition
+	 * exists where the unpaused TID still doesn't yet have
+	 * IEEE80211_AGGR_RUNNING set.
+	 */
+	r = sc->sc_addba_response(ni, tap, status, code, batimeout);
+
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+	/*
+	 * XXX dirty!
+	 * Slide the BAW left edge to wherever net80211 left it for us.
+	 * Read above for more information.
+	 */
+	tap->txa_start = ni->ni_txseqs[tid];
+	ath_tx_tid_resume(sc, atid);
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+	return r;
+}
+
+
+/*
+ * Stop ADDBA on a queue.
+ */
+void
+ath_addba_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
+{
+	struct ath_softc *sc = ni->ni_ic->ic_ifp->if_softc;
+	int tid = WME_AC_TO_TID(tap->txa_ac);
+	struct ath_node *an = ATH_NODE(ni);
+	struct ath_tid *atid = &an->an_tid[tid];
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL, "%s: called\n", __func__);
+
+	/* Pause TID traffic early, so there aren't any races */
+	ath_tx_tid_pause(sc, atid);
+
+	/* There's no need to hold the TXQ lock here */
+	sc->sc_addba_stop(ni, tap);
+
+	/*
+	 * ath_tx_cleanup will resume the TID if possible, otherwise
+	 * it'll set the cleanup flag, and it'll be unpaused once
+	 * things have been cleaned up.
+	 */
+	ath_tx_cleanup(sc, an, tid);
+}
+
+/*
+ * Note: net80211 bar_timeout() doesn't call this function on BAR failure;
+ * it simply tears down the aggregation session. Ew.
+ *
+ * It however will call ieee80211_ampdu_stop() which will call
+ * ic->ic_addba_stop().
+ *
+ * XXX This uses a hard-coded max BAR count value; the whole
+ * XXX BAR TX success or failure should be better handled!
+ */
+void
+ath_bar_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
+    int status)
+{
+	struct ath_softc *sc = ni->ni_ic->ic_ifp->if_softc;
+	int tid = WME_AC_TO_TID(tap->txa_ac);
+	struct ath_node *an = ATH_NODE(ni);
+	struct ath_tid *atid = &an->an_tid[tid];
+	int attempts = tap->txa_attempts;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: called; status=%d\n", __func__, status);
+
+	/* Note: This may update the BAW details */
+	sc->sc_bar_response(ni, tap, status);
+
+	/* Unpause the TID */
+	/*
+	 * XXX if this is attempt=50, the TID will be downgraded
+	 * XXX to a non-aggregate session. So we must unpause the
+	 * XXX TID here or it'll never be done.
+	 */
+	if (status == 0 || attempts == 50) {
+		ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+		ath_tx_tid_resume(sc, atid);
+		ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+	}
+}
+
+/*
+ * This is called whenever the pending ADDBA request times out.
+ * Unpause and reschedule the TID.
+ */
+void
+ath_addba_response_timeout(struct ieee80211_node *ni,
+    struct ieee80211_tx_ampdu *tap)
+{
+	struct ath_softc *sc = ni->ni_ic->ic_ifp->if_softc;
+	int tid = WME_AC_TO_TID(tap->txa_ac);
+	struct ath_node *an = ATH_NODE(ni);
+	struct ath_tid *atid = &an->an_tid[tid];
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_CTRL,
+	    "%s: called; resuming\n", __func__);
+
+	/* Note: This updates the aggregate state to (again) pending */
+	sc->sc_addba_response_timeout(ni, tap);
+
+	/* Unpause the TID; which reschedules it */
+	ATH_TXQ_LOCK(sc->sc_ac2q[atid->ac]);
+	ath_tx_tid_resume(sc, atid);
+	ATH_TXQ_UNLOCK(sc->sc_ac2q[atid->ac]);
+}
diff --git a/sys/dev/ath/if_ath_tx.h b/sys/dev/ath/if_ath_tx.h
index e181f7a..958acf9 100644
--- a/sys/dev/ath/if_ath_tx.h
+++ b/sys/dev/ath/if_ath_tx.h
@@ -31,7 +31,58 @@
 #ifndef	__IF_ATH_TX_H__
 #define	__IF_ATH_TX_H__
 
+/*
+ * some general macros
+ */
+#define	INCR(_l, _sz)		(_l) ++; (_l) &= ((_sz) - 1)
+/*
+ * return block-ack bitmap index given sequence and starting sequence
+ */
+#define	ATH_BA_INDEX(_st, _seq)	(((_seq) - (_st)) & (IEEE80211_SEQ_RANGE - 1))
+
+#define	WME_BA_BMP_SIZE	64
+#define	WME_MAX_BA	WME_BA_BMP_SIZE
+
+/*
+ * How 'busy' to try and keep the hardware txq
+ */
+#define	ATH_AGGR_MIN_QDEPTH		2
+
+/*
+ * Watermark for scheduling TIDs in order to maximise aggregation.
+ *
+ * If hwq_depth is greater than this, don't schedule the TID
+ * for packet scheduling - the hardware is already busy servicing
+ * this TID.
+ *
+ * If hwq_depth is less than this, schedule the TID for packet
+ * scheduling in the completion handler.
+ */
+#define	ATH_AGGR_SCHED_HIGH		4
+#define	ATH_AGGR_SCHED_LOW		2
+
+/*
+ * return whether a bit at index _n in bitmap _bm is set
+ * _sz is the size of the bitmap
+ */
+#define	ATH_BA_ISSET(_bm, _n)	(((_n) < (WME_BA_BMP_SIZE)) &&		\
+	    ((_bm)[(_n) >> 5] & (1 << ((_n) & 31))))
+
+
+/* extracting the seqno from buffer seqno */
+#define	SEQNO(_a)	((_a) >> IEEE80211_SEQ_SEQ_SHIFT)
+
+/*
+ * Whether the current sequence number is within the
+ * BAW.
+ */
+#define	BAW_WITHIN(_start, _bawsz, _seqno)	\
+	    ((((_seqno) - (_start)) & 4095) < (_bawsz))
+
+extern void ath_txq_restart_dma(struct ath_softc *sc, struct ath_txq *txq);
 extern void ath_freetx(struct mbuf *m);
+extern void ath_tx_node_flush(struct ath_softc *sc, struct ath_node *an);
+extern void ath_tx_txq_drain(struct ath_softc *sc, struct ath_txq *txq);
 extern void ath_txfrag_cleanup(struct ath_softc *sc, ath_bufhead *frags,
     struct ieee80211_node *ni);
 extern int ath_txfrag_setup(struct ath_softc *sc, ath_bufhead *frags,
@@ -41,4 +92,36 @@ extern int ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni,
 extern int ath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
     const struct ieee80211_bpf_params *params);
 
+/* software queue stuff */
+extern void ath_tx_swq(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_txq *txq, struct ath_buf *bf);
+extern void ath_tx_tid_init(struct ath_softc *sc, struct ath_node *an);
+extern void ath_tx_tid_hw_queue_aggr(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid);
+extern void ath_tx_tid_hw_queue_norm(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid);
+extern void ath_txq_sched(struct ath_softc *sc, struct ath_txq *txq);
+extern void ath_tx_normal_comp(struct ath_softc *sc, struct ath_buf *bf,
+    int fail);
+extern void ath_tx_aggr_comp(struct ath_softc *sc, struct ath_buf *bf,
+    int fail);
+extern void ath_tx_addto_baw(struct ath_softc *sc, struct ath_node *an,
+    struct ath_tid *tid, struct ath_buf *bf);
+extern struct ieee80211_tx_ampdu * ath_tx_get_tx_tid(struct ath_node *an,
+    int tid);
+
+/* TX addba handling */
+extern	int ath_addba_request(struct ieee80211_node *ni,
+    struct ieee80211_tx_ampdu *tap, int dialogtoken,
+    int baparamset, int batimeout);
+extern	int ath_addba_response(struct ieee80211_node *ni,
+    struct ieee80211_tx_ampdu *tap, int dialogtoken,
+    int code, int batimeout);
+extern	void ath_addba_stop(struct ieee80211_node *ni,
+    struct ieee80211_tx_ampdu *tap);
+extern	void ath_bar_response(struct ieee80211_node *ni,
+     struct ieee80211_tx_ampdu *tap, int status);
+extern	void ath_addba_response_timeout(struct ieee80211_node *ni,
+    struct ieee80211_tx_ampdu *tap);
+
 #endif
diff --git a/sys/dev/ath/if_ath_tx_ht.c b/sys/dev/ath/if_ath_tx_ht.c
index 348a1499..bec7064 100644
--- a/sys/dev/ath/if_ath_tx_ht.c
+++ b/sys/dev/ath/if_ath_tx_ht.c
@@ -86,17 +86,357 @@ __FBSDID("$FreeBSD$");
 #include <dev/ath/ath_tx99/ath_tx99.h>
 #endif
 
+#include <dev/ath/if_ath_tx.h>		/* XXX for some support functions */
 #include <dev/ath/if_ath_tx_ht.h>
+#include <dev/ath/if_athrate.h>
+#include <dev/ath/if_ath_debug.h>
+
+/*
+ * XXX net80211?
+ */
+#define	IEEE80211_AMPDU_SUBFRAME_DEFAULT		32
+
+#define	ATH_AGGR_DELIM_SZ	4	/* delimiter size   */
+#define	ATH_AGGR_MINPLEN	256	/* in bytes, minimum packet length */
+#define	ATH_AGGR_ENCRYPTDELIM	10	/* number of delimiters for encryption padding */
+
+/*
+ * returns delimiter padding required given the packet length
+ */
+#define	ATH_AGGR_GET_NDELIM(_len)					\
+	    (((((_len) + ATH_AGGR_DELIM_SZ) < ATH_AGGR_MINPLEN) ?	\
+	    (ATH_AGGR_MINPLEN - (_len) - ATH_AGGR_DELIM_SZ) : 0) >> 2)
+
+#define	PADBYTES(_len)		((4 - ((_len) % 4)) % 4)
+
+int ath_max_4ms_framelen[4][32] = {
+	[MCS_HT20] = {
+		3212,  6432,  9648,  12864,  19300,  25736,  28952,  32172,
+		6424,  12852, 19280, 25708,  38568,  51424,  57852,  64280,
+		9628,  19260, 28896, 38528,  57792,  65532,  65532,  65532,
+		12828, 25656, 38488, 51320,  65532,  65532,  65532,  65532,
+	},
+	[MCS_HT20_SGI] = {
+		3572,  7144,  10720,  14296,  21444,  28596,  32172,  35744,
+		7140,  14284, 21428,  28568,  42856,  57144,  64288,  65532,
+		10700, 21408, 32112,  42816,  64228,  65532,  65532,  65532,
+		14256, 28516, 42780,  57040,  65532,  65532,  65532,  65532,
+	},
+	[MCS_HT40] = {
+		6680,  13360,  20044,  26724,  40092,  53456,  60140,  65532,
+		13348, 26700,  40052,  53400,  65532,  65532,  65532,  65532,
+		20004, 40008,  60016,  65532,  65532,  65532,  65532,  65532,
+		26644, 53292,  65532,  65532,  65532,  65532,  65532,  65532,
+	},
+	[MCS_HT40_SGI] = {
+		7420,  14844,  22272,  29696,  44544,  59396,  65532,  65532,
+		14832, 29668,  44504,  59340,  65532,  65532,  65532,  65532,
+		22232, 44464,  65532,  65532,  65532,  65532,  65532,  65532,
+		29616, 59232,  65532,  65532,  65532,  65532,  65532,  65532,
+	}
+};
+
+/*
+ * XXX should be in net80211
+ */
+static int ieee80211_mpdudensity_map[] = {
+	0,		/* IEEE80211_HTCAP_MPDUDENSITY_NA */
+	25,		/* IEEE80211_HTCAP_MPDUDENSITY_025 */
+	50,		/* IEEE80211_HTCAP_MPDUDENSITY_05 */
+	100,		/* IEEE80211_HTCAP_MPDUDENSITY_1 */
+	200,		/* IEEE80211_HTCAP_MPDUDENSITY_2 */
+	400,		/* IEEE80211_HTCAP_MPDUDENSITY_4 */
+	800,		/* IEEE80211_HTCAP_MPDUDENSITY_8 */
+	1600,		/* IEEE80211_HTCAP_MPDUDENSITY_16 */
+};
+
+/*
+ * XXX should be in the HAL/net80211 ?
+ */
+#define	BITS_PER_BYTE		8
+#define	OFDM_PLCP_BITS		22
+#define	HT_RC_2_MCS(_rc)	((_rc) & 0x7f)
+#define	HT_RC_2_STREAMS(_rc)	((((_rc) & 0x78) >> 3) + 1)
+#define	L_STF			8
+#define	L_LTF			8
+#define	L_SIG			4
+#define	HT_SIG			8
+#define	HT_STF			4
+#define	HT_LTF(_ns)		(4 * (_ns))
+#define	SYMBOL_TIME(_ns)	((_ns) << 2)		// ns * 4 us
+#define	SYMBOL_TIME_HALFGI(_ns)	(((_ns) * 18 + 4) / 5)	// ns * 3.6 us
+#define	NUM_SYMBOLS_PER_USEC(_usec)	(_usec >> 2)
+#define	NUM_SYMBOLS_PER_USEC_HALFGI(_usec)	(((_usec*5)-4)/18)
+#define	IS_HT_RATE(_rate)	((_rate) & 0x80)
+
+const uint32_t bits_per_symbol[][2] = {
+    /* 20MHz 40MHz */
+    {    26,   54 },     //  0: BPSK
+    {    52,  108 },     //  1: QPSK 1/2
+    {    78,  162 },     //  2: QPSK 3/4
+    {   104,  216 },     //  3: 16-QAM 1/2
+    {   156,  324 },     //  4: 16-QAM 3/4
+    {   208,  432 },     //  5: 64-QAM 2/3
+    {   234,  486 },     //  6: 64-QAM 3/4
+    {   260,  540 },     //  7: 64-QAM 5/6
+    {    52,  108 },     //  8: BPSK
+    {   104,  216 },     //  9: QPSK 1/2
+    {   156,  324 },     // 10: QPSK 3/4
+    {   208,  432 },     // 11: 16-QAM 1/2
+    {   312,  648 },     // 12: 16-QAM 3/4
+    {   416,  864 },     // 13: 64-QAM 2/3
+    {   468,  972 },     // 14: 64-QAM 3/4
+    {   520, 1080 },     // 15: 64-QAM 5/6
+    {    78,  162 },     // 16: BPSK
+    {   156,  324 },     // 17: QPSK 1/2
+    {   234,  486 },     // 18: QPSK 3/4
+    {   312,  648 },     // 19: 16-QAM 1/2
+    {   468,  972 },     // 20: 16-QAM 3/4
+    {   624, 1296 },     // 21: 64-QAM 2/3
+    {   702, 1458 },     // 22: 64-QAM 3/4
+    {   780, 1620 },     // 23: 64-QAM 5/6
+    {   104,  216 },     // 24: BPSK
+    {   208,  432 },     // 25: QPSK 1/2
+    {   312,  648 },     // 26: QPSK 3/4
+    {   416,  864 },     // 27: 16-QAM 1/2
+    {   624, 1296 },     // 28: 16-QAM 3/4
+    {   832, 1728 },     // 29: 64-QAM 2/3
+    {   936, 1944 },     // 30: 64-QAM 3/4
+    {  1040, 2160 },     // 31: 64-QAM 5/6
+};
+
+/*
+ * Fill in the rate array information based on the current
+ * node configuration and the choices made by the rate
+ * selection code and ath_buf setup code.
+ *
+ * Later on, this may end up also being made by the
+ * rate control code, but for now it can live here.
+ *
+ * This needs to be called just before the packet is
+ * queued to the software queue or hardware queue,
+ * so all of the needed fields in bf_state are setup.
+ */
+void
+ath_tx_rate_fill_rcflags(struct ath_softc *sc, struct ath_buf *bf)
+{
+	struct ieee80211_node *ni = bf->bf_node;
+	struct ieee80211com *ic = ni->ni_ic;
+	const HAL_RATE_TABLE *rt = sc->sc_currates;
+	struct ath_rc_series *rc = bf->bf_state.bfs_rc;
+	uint8_t rate;
+	int i;
+
+	for (i = 0; i < ATH_RC_NUM; i++) {
+		rc[i].flags = 0;
+		if (rc[i].tries == 0)
+			continue;
+
+		rate = rt->info[rc[i].rix].rateCode;
+
+		/*
+		 * XXX only do this for legacy rates?
+		 */
+		if (bf->bf_state.bfs_shpream)
+			rate |= rt->info[rc[i].rix].shortPreamble;
+
+		/*
+		 * Save this, used by the TX and completion code
+		 */
+		rc[i].ratecode = rate;
+
+		if (bf->bf_state.bfs_flags &
+		    (HAL_TXDESC_RTSENA | HAL_TXDESC_CTSENA))
+			rc[i].flags |= ATH_RC_RTSCTS_FLAG;
+
+		/* Only enable shortgi, 2040, dual-stream if HT is set */
+		if (IS_HT_RATE(rate)) {
+			rc[i].flags |= ATH_RC_HT_FLAG;
+
+			if (ni->ni_chw == 40)
+				rc[i].flags |= ATH_RC_CW40_FLAG;
+
+			if (ni->ni_chw == 40 &&
+			    ic->ic_htcaps & IEEE80211_HTCAP_SHORTGI40 &&
+			    ni->ni_htcap & IEEE80211_HTCAP_SHORTGI40)
+				rc[i].flags |= ATH_RC_SGI_FLAG;
+
+			if (ni->ni_chw == 20 &&
+			    ic->ic_htcaps & IEEE80211_HTCAP_SHORTGI20 &&
+			    ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20)
+				rc[i].flags |= ATH_RC_SGI_FLAG;
+
+			/* XXX dual stream? and 3-stream? */
+		}
+
+		/*
+		 * Calculate the maximum 4ms frame length based
+		 * on the MCS rate, SGI and channel width flags.
+		 */
+		if ((rc[i].flags & ATH_RC_HT_FLAG) &&
+		    (HT_RC_2_MCS(rate) < 32)) {
+			int j;
+			if (rc[i].flags & ATH_RC_CW40_FLAG) {
+				if (rc[i].flags & ATH_RC_SGI_FLAG)
+					j = MCS_HT40_SGI;
+				else
+					j = MCS_HT40;
+			} else {
+				if (rc[i].flags & ATH_RC_SGI_FLAG)
+					j = MCS_HT20_SGI;
+				else
+					j = MCS_HT20;
+			}
+			rc[i].max4msframelen =
+			    ath_max_4ms_framelen[j][HT_RC_2_MCS(rate)];
+		} else
+			rc[i].max4msframelen = 0;
+		DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+		    "%s: i=%d, rate=0x%x, flags=0x%x, max4ms=%d\n",
+		    __func__, i, rate, rc[i].flags, rc[i].max4msframelen);
+	}
+}
+
+/*
+ * Return the number of delimiters to be added to
+ * meet the minimum required mpdudensity.
+ *
+ * Caller should make sure that the rate is HT.
+ *
+ * TODO: is this delimiter calculation supposed to be the
+ * total frame length, the hdr length, the data length (including
+ * delimiters, padding, CRC, etc) or ?
+ *
+ * TODO: this should ensure that the rate control information
+ * HAS been setup for the first rate.
+ *
+ * TODO: ensure this is only called for MCS rates.
+ *
+ * TODO: enforce MCS < 31
+ */
+static int
+ath_compute_num_delims(struct ath_softc *sc, struct ath_buf *first_bf,
+    uint16_t pktlen)
+{
+	const HAL_RATE_TABLE *rt = sc->sc_currates;
+	struct ieee80211_node *ni = first_bf->bf_node;
+	struct ieee80211vap *vap = ni->ni_vap;
+	int ndelim, mindelim = 0;
+	int mpdudensity;	 /* in 1/100'th of a microsecond */
+	uint8_t rc, rix, flags;
+	int width, half_gi;
+	uint32_t nsymbits, nsymbols;
+	uint16_t minlen;
+
+	/*
+	 * vap->iv_ampdu_density is a value, rather than the actual
+	 * density.
+	 */
+	if (vap->iv_ampdu_density > IEEE80211_HTCAP_MPDUDENSITY_16)
+		mpdudensity = 1600;		/* maximum density */
+	else
+		mpdudensity = ieee80211_mpdudensity_map[vap->iv_ampdu_density];
+
+	/* Select standard number of delimiters based on frame length */
+	ndelim = ATH_AGGR_GET_NDELIM(pktlen);
+
+	/*
+	 * If encryption is enabled, add extra delimiters to let the
+	 * crypto hardware catch up. This could be tuned per-MAC and
+	 * per-rate, but for now we'll simply assume encryption is
+	 * always enabled.
+	 */
+	ndelim += ATH_AGGR_ENCRYPTDELIM;
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+	    "%s: pktlen=%d, ndelim=%d, mpdudensity=%d\n",
+	    __func__, pktlen, ndelim, mpdudensity);
+
+	/*
+	 * If the MPDU density is 0, we can return here.
+	 * Otherwise, we need to convert the desired mpdudensity
+	 * into a byte length, based on the rate in the subframe.
+	 */
+	if (mpdudensity == 0)
+		return ndelim;
+
+	/*
+	 * Convert desired mpdu density from microeconds to bytes based
+	 * on highest rate in rate series (i.e. first rate) to determine
+	 * required minimum length for subframe. Take into account
+	 * whether high rate is 20 or 40Mhz and half or full GI.
+	 */
+	rix = first_bf->bf_state.bfs_rc[0].rix;
+	rc = rt->info[rix].rateCode;
+	flags = first_bf->bf_state.bfs_rc[0].flags;
+	width = !! (flags & ATH_RC_CW40_FLAG);
+	half_gi = !! (flags & ATH_RC_SGI_FLAG);
+
+	/*
+	 * mpdudensity is in 1/100th of a usec, so divide by 100
+	 */
+	if (half_gi)
+		nsymbols = NUM_SYMBOLS_PER_USEC_HALFGI(mpdudensity);
+	else
+		nsymbols = NUM_SYMBOLS_PER_USEC(mpdudensity);
+	nsymbols /= 100;
+
+	if (nsymbols == 0)
+		nsymbols = 1;
+
+	nsymbits = bits_per_symbol[HT_RC_2_MCS(rc)][width];
+	minlen = (nsymbols * nsymbits) / BITS_PER_BYTE;
+
+	/*
+	 * Min length is the minimum frame length for the
+	 * required MPDU density.
+	 */
+	if (pktlen < minlen) {
+		mindelim = (minlen - pktlen) / ATH_AGGR_DELIM_SZ;
+		ndelim = MAX(mindelim, ndelim);
+	}
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR,
+	    "%s: pktlen=%d, minlen=%d, rix=%x, rc=%x, width=%d, hgi=%d, ndelim=%d\n",
+	    __func__, pktlen, minlen, rix, rc, width, half_gi, ndelim);
+
+	return ndelim;
+}
+
+/*
+ * Fetch the aggregation limit.
+ *
+ * It's the lowest of the four rate series 4ms frame length.
+ */
+static int
+ath_get_aggr_limit(struct ath_softc *sc, struct ath_buf *bf)
+{
+	int amin = 65530;
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		if (bf->bf_state.bfs_rc[i].tries == 0)
+			continue;
+		amin = MIN(amin, bf->bf_state.bfs_rc[i].max4msframelen);
+	}
+
+	DPRINTF(sc, ATH_DEBUG_SW_TX_AGGR, "%s: max frame len= %d\n",
+	    __func__, amin);
+
+	return amin;
+}
 
 /*
  * Setup a 11n rate series structure
  *
  * This should be called for both legacy and MCS rates.
+ *
+ * It, along with ath_buf_set_rate, must be called -after- a burst
+ * or aggregate is setup.
  */
 static void
 ath_rateseries_setup(struct ath_softc *sc, struct ieee80211_node *ni,
-    HAL_11N_RATE_SERIES *series, unsigned int pktlen, uint8_t *rix,
-    uint8_t *try, int flags)
+    struct ath_buf *bf, HAL_11N_RATE_SERIES *series)
 {
 #define	HT_RC_2_STREAMS(_rc)	((((_rc) & 0x78) >> 3) + 1)
 	struct ieee80211com *ic = ni->ni_ic;
@@ -104,18 +444,34 @@ ath_rateseries_setup(struct ath_softc *sc, struct ieee80211_node *ni,
 	HAL_BOOL shortPreamble = AH_FALSE;
 	const HAL_RATE_TABLE *rt = sc->sc_currates;
 	int i;
+	int pktlen;
+	int flags = bf->bf_state.bfs_flags;
+	struct ath_rc_series *rc = bf->bf_state.bfs_rc;
 
 	if ((ic->ic_flags & IEEE80211_F_SHPREAMBLE) &&
 	    (ni->ni_capinfo & IEEE80211_CAPINFO_SHORT_PREAMBLE))
 		shortPreamble = AH_TRUE;
 
+	/*
+	 * If this is the first frame in an aggregate series,
+	 * use the aggregate length.
+	 */
+	if (bf->bf_state.bfs_aggr)
+		pktlen = bf->bf_state.bfs_al;
+	else
+		pktlen = bf->bf_state.bfs_pktlen;
+
+	/*
+	 * XXX TODO: modify this routine to use the bfs_rc[x].flags
+	 * XXX fields.
+	 */
 	memset(series, 0, sizeof(HAL_11N_RATE_SERIES) * 4);
 	for (i = 0; i < 4;  i++) {
 		/* Only set flags for actual TX attempts */
-		if (try[i] == 0)
+		if (rc[i].tries == 0)
 			continue;
 
-		series[i].Tries = try[i];
+		series[i].Tries = rc[i].tries;
 
 		/*
 		 * XXX this isn't strictly correct - sc_txchainmask
@@ -154,7 +510,7 @@ ath_rateseries_setup(struct ath_softc *sc, struct ieee80211_node *ni,
 		    ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20)
 			series[i].RateFlags |= HAL_RATESERIES_HALFGI;
 
-		series[i].Rate = rt->info[rix[i]].rateCode;
+		series[i].Rate = rt->info[rc[i].rix].rateCode;
 
 		/* PktDuration doesn't include slot, ACK, RTS, etc timing - it's just the packet duration */
 		if (series[i].Rate & IEEE80211_RATE_MCS) {
@@ -166,9 +522,10 @@ ath_rateseries_setup(struct ath_softc *sc, struct ieee80211_node *ni,
 				, series[i].RateFlags & HAL_RATESERIES_HALFGI);
 		} else {
 			if (shortPreamble)
-				series[i].Rate |= rt->info[rix[i]].shortPreamble;
+				series[i].Rate |=
+				    rt->info[rc[i].rix].shortPreamble;
 			series[i].PktDuration = ath_hal_computetxtime(ah,
-			    rt, pktlen, rix[i], shortPreamble);
+			    rt, pktlen, rc[i].rix, shortPreamble);
 		}
 	}
 #undef	HT_RC_2_STREAMS
@@ -200,25 +557,28 @@ ath_rateseries_print(HAL_11N_RATE_SERIES *series)
  */
 
 void
-ath_buf_set_rate(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf,
-    int pktlen, int flags, uint8_t ctsrate, int is_pspoll, uint8_t *rix, uint8_t *try)
+ath_buf_set_rate(struct ath_softc *sc, struct ieee80211_node *ni,
+    struct ath_buf *bf)
 {
 	HAL_11N_RATE_SERIES series[4];
 	struct ath_desc *ds = bf->bf_desc;
 	struct ath_desc *lastds = NULL;
 	struct ath_hal *ah = sc->sc_ah;
+	int is_pspoll = (bf->bf_state.bfs_atype == HAL_PKT_TYPE_PSPOLL);
+	int ctsrate = bf->bf_state.bfs_ctsrate;
+	int flags = bf->bf_state.bfs_flags;
 
 	/* Setup rate scenario */
 	memset(&series, 0, sizeof(series));
 
-	ath_rateseries_setup(sc, ni, series, pktlen, rix, try, flags);
+	ath_rateseries_setup(sc, ni, bf, series);
 
 	/* Enforce AR5416 aggregate limit - can't do RTS w/ an agg frame > 8k */
 
 	/* Enforce RTS and CTS are mutually exclusive */
 
 	/* Get a pointer to the last tx descriptor in the list */
-	lastds = &bf->bf_desc[bf->bf_nseg - 1];
+	lastds = bf->bf_lastds;
 
 #if 0
 	printf("pktlen: %d; flags 0x%x\n", pktlen, flags);
@@ -238,6 +598,238 @@ ath_buf_set_rate(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf
 	ath_hal_setuplasttxdesc(ah, lastds, ds);
 
 	/* Set burst duration */
-	/* This should only be done if aggregate protection is enabled */
+	/*
+	 * This is only required when doing 11n burst, not aggregation
+	 * ie, if there's a second frame in a RIFS or A-MPDU burst
+	 * w/ >1 A-MPDU frame bursting back to back.
+	 * Normal A-MPDU doesn't do bursting -between- aggregates.
+	 *
+	 * .. and it's highly likely this won't ever be implemented
+	 */
 	//ath_hal_set11nburstduration(ah, ds, 8192);
 }
+
+/*
+ * Form an aggregate packet list.
+ *
+ * This function enforces the aggregate restrictions/requirements.
+ *
+ * These are:
+ *
+ * + The aggregate size maximum (64k for AR9160 and later, 8K for
+ *   AR5416 when doing RTS frame protection.)
+ * + Maximum number of sub-frames for an aggregate
+ * + The aggregate delimiter size, giving MACs time to do whatever is
+ *   needed before each frame
+ * + Enforce the BAW limit
+ *
+ * Each descriptor queued should have the DMA setup.
+ * The rate series, descriptor setup, linking, etc is all done
+ * externally. This routine simply chains them together.
+ * ath_tx_setds_11n() will take care of configuring the per-
+ * descriptor setup, and ath_buf_set_rate() will configure the
+ * rate control.
+ *
+ * Note that the TID lock is only grabbed when dequeuing packets from
+ * the TID queue. If some code in another thread adds to the head of this
+ * list, very strange behaviour will occur. Since retransmission is the
+ * only reason this will occur, and this routine is designed to be called
+ * from within the scheduler task, it won't ever clash with the completion
+ * task.
+ *
+ * So if you want to call this from an upper layer context (eg, to direct-
+ * dispatch aggregate frames to the hardware), please keep this in mind.
+ */
+ATH_AGGR_STATUS
+ath_tx_form_aggr(struct ath_softc *sc, struct ath_node *an, struct ath_tid *tid,
+    ath_bufhead *bf_q)
+{
+	//struct ieee80211_node *ni = &an->an_node;
+	struct ath_buf *bf, *bf_first = NULL, *bf_prev = NULL;
+	int nframes = 0;
+	uint16_t aggr_limit = 0, al = 0, bpad = 0, al_delta, h_baw;
+	struct ieee80211_tx_ampdu *tap;
+	int status = ATH_AGGR_DONE;
+	int prev_frames = 0;	/* XXX for AR5416 burst, not done here */
+	int prev_al = 0;	/* XXX also for AR5416 burst */
+
+	ATH_TXQ_LOCK_ASSERT(sc->sc_ac2q[tid->ac]);
+
+	tap = ath_tx_get_tx_tid(an, tid->tid);
+	if (tap == NULL) {
+		status = ATH_AGGR_ERROR;
+		goto finish;
+	}
+
+	h_baw = tap->txa_wnd / 2;
+
+	for (;;) {
+		bf = TAILQ_FIRST(&tid->axq_q);
+		if (bf_first == NULL)
+			bf_first = bf;
+		if (bf == NULL) {
+			status = ATH_AGGR_DONE;
+			break;
+		} else {
+			/*
+			 * It's the first frame;
+			 * set the aggregation limit based on the
+			 * rate control decision that has been made.
+			 */
+			aggr_limit = ath_get_aggr_limit(sc, bf_first);
+		}
+
+		/* Set this early just so things don't get confused */
+		bf->bf_next = NULL;
+
+		/*
+		 * Don't unlock the tid lock until we're sure we are going
+		 * to queue this frame.
+		 */
+
+		/*
+		 * If the frame doesn't have a sequence number that we're
+		 * tracking in the BAW (eg NULL QOS data frame), we can't
+		 * aggregate it. Stop the aggregation process; the sender
+		 * can then TX what's in the list thus far and then
+		 * TX the frame individually.
+		 */
+		if (! bf->bf_state.bfs_dobaw) {
+			status = ATH_AGGR_NONAGGR;
+			break;
+		}
+
+		/*
+		 * If any of the rates are non-HT, this packet
+		 * can't be aggregated.
+		 * XXX TODO: add a bf_state flag which gets marked
+		 * if any active rate is non-HT.
+		 */
+
+		/*
+		 * If the packet has a sequence number, do not
+		 * step outside of the block-ack window.
+		 */
+		if (! BAW_WITHIN(tap->txa_start, tap->txa_wnd,
+		    SEQNO(bf->bf_state.bfs_seqno))) {
+		    status = ATH_AGGR_BAW_CLOSED;
+		    break;
+		}
+
+		/*
+		 * XXX TODO: AR5416 has an 8K aggregation size limit
+		 * when RTS is enabled, and RTS is required for dual-stream
+		 * rates.
+		 *
+		 * For now, limit all aggregates for the AR5416 to be 8K.
+		 */
+
+		/*
+		 * do not exceed aggregation limit
+		 */
+		al_delta = ATH_AGGR_DELIM_SZ + bf->bf_state.bfs_pktlen;
+		if (nframes &&
+		    (aggr_limit < (al + bpad + al_delta + prev_al))) {
+			status = ATH_AGGR_LIMITED;
+			break;
+		}
+
+		/*
+		 * Do not exceed subframe limit.
+		 */
+		if ((nframes + prev_frames) >= MIN((h_baw),
+		    IEEE80211_AMPDU_SUBFRAME_DEFAULT)) {
+			status = ATH_AGGR_LIMITED;
+			break;
+		}
+
+		/*
+		 * this packet is part of an aggregate.
+		 */
+		ATH_TXQ_REMOVE(tid, bf, bf_list);
+
+		/* The TID lock is required for the BAW update */
+		ath_tx_addto_baw(sc, an, tid, bf);
+		bf->bf_state.bfs_addedbaw = 1;
+
+		/*
+		 * XXX TODO: If any frame in the aggregate requires RTS/CTS,
+		 * set the first frame.
+		 */
+
+		/*
+		 * XXX enforce ACK for aggregate frames (this needs to be
+		 * XXX handled more gracefully?
+		 */
+		if (bf->bf_state.bfs_flags & HAL_TXDESC_NOACK) {
+			device_printf(sc->sc_dev,
+			    "%s: HAL_TXDESC_NOACK set for an aggregate frame?\n",
+			    __func__);
+			bf->bf_state.bfs_flags &= (~HAL_TXDESC_NOACK);
+		}
+
+		/*
+		 * Add the now owned buffer (which isn't
+		 * on the software TXQ any longer) to our
+		 * aggregate frame list.
+		 */
+		TAILQ_INSERT_TAIL(bf_q, bf, bf_list);
+		nframes ++;
+
+		/* Completion handler */
+		bf->bf_comp = ath_tx_aggr_comp;
+
+		/*
+		 * add padding for previous frame to aggregation length
+		 */
+		al += bpad + al_delta;
+
+		/*
+		 * Calculate delimiters needed for the current frame
+		 */
+		bf->bf_state.bfs_ndelim =
+		    ath_compute_num_delims(sc, bf_first,
+		    bf->bf_state.bfs_pktlen);
+
+		/*
+		 * Calculate the padding needed from this set of delimiters,
+		 * used when calculating if the next frame will fit in
+		 * the aggregate.
+		 */
+		bpad = PADBYTES(al_delta) + (bf->bf_state.bfs_ndelim << 2);
+
+		/*
+		 * Chain the buffers together
+		 */
+		if (bf_prev)
+			bf_prev->bf_next = bf;
+		bf_prev = bf;
+
+		/*
+		 * XXX TODO: if any sub-frames have RTS/CTS enabled;
+		 * enable it for the entire aggregate.
+		 */
+
+#if 0
+		/*
+		 * terminate aggregation on a small packet boundary
+		 */
+		if (bf->bf_state.bfs_pktlen < ATH_AGGR_MINPLEN) {
+			status = ATH_AGGR_SHORTPKT;
+			break;
+		}
+#endif
+
+	}
+
+finish:
+	/*
+	 * Just in case the list was empty when we tried to
+	 * dequeue a packet ..
+	 */
+	if (bf_first) {
+		bf_first->bf_state.bfs_al = al;
+		bf_first->bf_state.bfs_nframes = nframes;
+	}
+	return status;
+}
diff --git a/sys/dev/ath/if_ath_tx_ht.h b/sys/dev/ath/if_ath_tx_ht.h
index cf16c46..543c56b 100644
--- a/sys/dev/ath/if_ath_tx_ht.h
+++ b/sys/dev/ath/if_ath_tx_ht.h
@@ -31,9 +31,32 @@
 #ifndef	__IF_ATH_TX_HT_H__
 #define	__IF_ATH_TX_HT_H__
 
+enum {
+	MCS_HT20,
+	MCS_HT20_SGI,
+	MCS_HT40,
+	MCS_HT40_SGI,
+};
+
+typedef enum {
+	ATH_AGGR_DONE,
+	ATH_AGGR_BAW_CLOSED,
+	ATH_AGGR_LIMITED,
+	ATH_AGGR_SHORTPKT,
+	ATH_AGGR_8K_LIMITED,
+	ATH_AGGR_ERROR,
+	ATH_AGGR_NONAGGR,
+} ATH_AGGR_STATUS;
+
+extern int	ath_max_4ms_framelen[4][32];
+
+extern void ath_tx_rate_fill_rcflags(struct ath_softc *sc, struct ath_buf *bf);
+
 extern void	ath_buf_set_rate(struct ath_softc *sc,
-		struct ieee80211_node *ni, struct ath_buf *bf,
-		int pktlen, int flags, uint8_t ctsrate, int is_pspoll,
-		uint8_t *rix, uint8_t *try);
+		struct ieee80211_node *ni, struct ath_buf *bf);
+
+extern ATH_AGGR_STATUS
+	    ath_tx_form_aggr(struct ath_softc *sc, struct ath_node *an,
+	    struct ath_tid *tid, ath_bufhead *bf_q);
 
 #endif
diff --git a/sys/dev/ath/if_athioctl.h b/sys/dev/ath/if_athioctl.h
index 065a49c..680694f 100644
--- a/sys/dev/ath/if_athioctl.h
+++ b/sys/dev/ath/if_athioctl.h
@@ -35,6 +35,16 @@
 #ifndef _DEV_ATH_ATHIOCTL_H
 #define _DEV_ATH_ATHIOCTL_H
 
+struct ath_tx_aggr_stats {
+	u_int32_t	aggr_pkts[64];
+	u_int32_t	aggr_single_pkt;
+	u_int32_t	aggr_nonbaw_pkt;
+	u_int32_t	aggr_aggr_pkt;
+	u_int32_t	aggr_baw_closed_single_pkt;
+	u_int32_t	aggr_low_hwq_single_pkt;
+	u_int32_t	aggr_sched_nopkt;
+};
+
 struct ath_stats {
 	u_int32_t	ast_watchdog;	/* device reset by watchdog */
 	u_int32_t	ast_hardware;	/* fatal hardware error interrupts */
diff --git a/sys/dev/ath/if_athrate.h b/sys/dev/ath/if_athrate.h
index 55b99a2..10f6040 100644
--- a/sys/dev/ath/if_athrate.h
+++ b/sys/dev/ath/if_athrate.h
@@ -77,6 +77,21 @@ struct ath_ratectrl {
 struct ath_ratectrl *ath_rate_attach(struct ath_softc *);
 void	ath_rate_detach(struct ath_ratectrl *);
 
+#define	ATH_RC_NUM		4
+
+#define	ATH_RC_DS_FLAG		0x01	/* dual-stream rate */
+#define	ATH_RC_CW40_FLAG	0x02	/* use HT40 */
+#define	ATH_RC_SGI_FLAG		0x04	/* use short-GI */
+#define	ATH_RC_HT_FLAG		0x08	/* use HT */
+#define	ATH_RC_RTSCTS_FLAG	0x10	/* enable RTS/CTS protection */
+
+struct ath_rc_series {
+	uint8_t rix;		/* ratetable index, not rate code */
+	uint8_t ratecode;	/* hardware rate code */
+	uint8_t tries;
+	uint8_t flags;
+	uint32_t max4msframelen;
+};
 
 /*
  * State storage handling.
@@ -105,7 +120,7 @@ void	ath_rate_newassoc(struct ath_softc *, struct ath_node *,
  * Return the four TX rate index and try counts for the current data packet.
  */
 void	ath_rate_getxtxrates(struct ath_softc *sc, struct ath_node *an,
-		uint8_t rix0, uint8_t *rix, uint8_t *try);
+		uint8_t rix0, struct ath_rc_series *rc);
 
 /*
  * Return the transmit info for a data packet.  If multi-rate state
@@ -127,8 +142,12 @@ void	ath_rate_setupxtxdesc(struct ath_softc *, struct ath_node *,
  * supplied transmit descriptor.  The routine is invoked both
  * for packets that were successfully sent and for those that
  * failed (consult the descriptor for details).
+ *
+ * For A-MPDU frames, nframes and nbad indicate how many frames
+ * were in the aggregate, and how many failed.
  */
 struct ath_buf;
 void	ath_rate_tx_complete(struct ath_softc *, struct ath_node *,
-		const struct ath_buf *);
+		const struct ath_rc_series *, const struct ath_tx_status *,
+		int pktlen, int nframes, int nbad);
 #endif /* _ATH_RATECTRL_H_ */
diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h
index 1eaf709..331bea4 100644
--- a/sys/dev/ath/if_athvar.h
+++ b/sys/dev/ath/if_athvar.h
@@ -83,12 +83,73 @@ struct taskqueue;
 struct kthread;
 struct ath_buf;
 
+#define	ATH_TID_MAX_BUFS	(2 * IEEE80211_AGGR_BAWMAX)
+
+/*
+ * Per-TID state
+ *
+ * Note that TID 16 (WME_NUM_TID+1) is for handling non-QoS frames.
+ */
+struct ath_tid {
+	TAILQ_HEAD(,ath_buf) axq_q;		/* pending buffers */
+	u_int			axq_depth;	/* SW queue depth */
+	char			axq_name[48];	/* lock name */
+	struct ath_node		*an;		/* pointer to parent */
+	int			tid;		/* tid */
+	int			ac;		/* which AC gets this trafic */
+	int			hwq_depth;	/* how many buffers are on HW */
+
+	/*
+	 * Entry on the ath_txq; when there's traffic
+	 * to send
+	 */
+	TAILQ_ENTRY(ath_tid)	axq_qelem;
+	int			sched;
+	int			paused;	/* >0 if the TID has been paused */
+
+	/*
+	 * Is the TID being cleaned up after a transition
+	 * from aggregation to non-aggregation?
+	 * When this is set to 1, this TID will be paused
+	 * and no further traffic will be queued until all
+	 * the hardware packets pending for this TID have been
+	 * TXed/completed; at which point (non-aggregation)
+	 * traffic will resume being TXed.
+	 */
+	int			cleanup_inprogress;
+	/*
+	 * How many hardware-queued packets are
+	 * waiting to be cleaned up.
+	 * This is only valid if cleanup_inprogress is 1.
+	 */
+	int			incomp;
+
+	/*
+	 * The following implements a ring representing
+	 * the frames in the current BAW.
+	 * To avoid copying the array content each time
+	 * the BAW is moved, the baw_head/baw_tail point
+	 * to the current BAW begin/end; when the BAW is
+	 * shifted the head/tail of the array are also
+	 * appropriately shifted.
+	 */
+	/* active tx buffers, beginning at current BAW */
+	struct ath_buf		*tx_buf[ATH_TID_MAX_BUFS];
+	/* where the baw head is in the array */
+	int			baw_head;
+	/* where the BAW tail is in the array */
+	int			baw_tail;
+};
+
 /* driver-specific node state */
 struct ath_node {
 	struct ieee80211_node an_node;	/* base class */
 	u_int8_t	an_mgmtrix;	/* min h/w rate index */
 	u_int8_t	an_mcastrix;	/* mcast h/w rate index */
 	struct ath_buf	*an_ff_buf[WME_NUM_AC]; /* ff staging area */
+	struct ath_tid	an_tid[IEEE80211_TID_SIZE];	/* per-TID state */
+	char		an_name[32];	/* eg "wlan0_a1" */
+	struct mtx	an_mtx;		/* protecting the ath_node state */
 	/* variable-length rate control state follows */
 };
 #define	ATH_NODE(ni)	((struct ath_node *)(ni))
@@ -109,7 +170,8 @@ struct ath_node {
 #define	ATH_RSSI(x)		ATH_EP_RND(x, HAL_RSSI_EP_MULTIPLIER)
 
 struct ath_buf {
-	STAILQ_ENTRY(ath_buf)	bf_list;
+	TAILQ_ENTRY(ath_buf)	bf_list;
+	struct ath_buf *	bf_next;	/* next buffer in the aggregate */
 	int			bf_nseg;
 	uint16_t		bf_txflags;	/* tx descriptor flags */
 	uint16_t		bf_flags;	/* status flags (below) */
@@ -119,11 +181,65 @@ struct ath_buf {
 	bus_dmamap_t		bf_dmamap;	/* DMA map for mbuf chain */
 	struct mbuf		*bf_m;		/* mbuf for buf */
 	struct ieee80211_node	*bf_node;	/* pointer to the node */
+	struct ath_desc		*bf_lastds;	/* last descriptor for comp status */
+	struct ath_buf		*bf_last;	/* last buffer in aggregate, or self for non-aggregate */
 	bus_size_t		bf_mapsize;
 #define	ATH_MAX_SCATTER		ATH_TXDESC	/* max(tx,rx,beacon) desc's */
 	bus_dma_segment_t	bf_segs[ATH_MAX_SCATTER];
+
+	/* Completion function to call on TX complete (fail or not) */
+	/*
+	 * "fail" here is set to 1 if the queue entries were removed
+	 * through a call to ath_tx_draintxq().
+	 */
+	void(* bf_comp) (struct ath_softc *sc, struct ath_buf *bf, int fail);
+
+	/* This state is kept to support software retries and aggregation */
+	struct {
+		int bfs_seqno;		/* sequence number of this packet */
+		int bfs_retries;	/* retry count */
+		uint16_t bfs_tid;	/* packet TID (or TID_MAX for no QoS) */
+		uint16_t bfs_pri;	/* packet AC priority */
+		struct ath_txq *bfs_txq;	/* eventual dest hardware TXQ */
+		uint16_t bfs_pktdur;	/* packet duration (at current rate?) */
+		uint16_t bfs_nframes;	/* number of frames in aggregate */
+		uint16_t bfs_ndelim;	/* number of delims for padding */
+
+		int bfs_aggr:1;		/* part of aggregate? */
+		int bfs_aggrburst:1;	/* part of aggregate burst? */
+		int bfs_isretried:1;	/* retried frame? */
+		int bfs_dobaw:1;	/* actually check against BAW? */
+		int bfs_addedbaw:1;	/* has been added to the BAW */
+		int bfs_shpream:1;	/* use short preamble */
+		int bfs_istxfrag:1;	/* is fragmented */
+		int bfs_ismrr:1;	/* do multi-rate TX retry */
+		int bfs_doprot:1;	/* do RTS/CTS based protection */
+		int bfs_doratelookup:1;	/* do rate lookup before each TX */
+		int bfs_nfl;		/* next fragment length */
+
+		/*
+		 * These fields are passed into the
+		 * descriptor setup functions.
+		 */
+		HAL_PKT_TYPE bfs_atype;	/* packet type */
+		int bfs_pktlen;		/* length of this packet */
+		int bfs_hdrlen;		/* length of this packet header */
+		uint16_t bfs_al;	/* length of aggregate */
+		int bfs_flags;		/* HAL descriptor flags */
+		int bfs_txrate0;	/* first TX rate */
+		int bfs_try0;		/* first try count */
+		uint8_t bfs_ctsrate0;	/* Non-zero - use this as ctsrate */
+		int bfs_keyix;		/* crypto key index */
+		int bfs_txpower;	/* tx power */
+		int bfs_txantenna;	/* TX antenna config */
+		enum ieee80211_protmode bfs_protmode;
+		HAL_11N_RATE_SERIES bfs_rc11n[ATH_RC_NUM];	/* 11n TX series */
+		int bfs_ctsrate;	/* CTS rate */
+		int bfs_ctsduration;	/* CTS duration (pre-11n NICs) */
+		struct ath_rc_series bfs_rc[ATH_RC_NUM];	/* non-11n TX series */
+	} bf_state;
 };
-typedef STAILQ_HEAD(, ath_buf) ath_bufhead;
+typedef TAILQ_HEAD(ath_bufhead_s, ath_buf) ath_bufhead;
 
 #define	ATH_BUF_BUSY	0x00000002	/* (tx) desc owned by h/w */
 
@@ -151,19 +267,28 @@ struct ath_descdma {
  * hardware queue).
  */
 struct ath_txq {
+	struct ath_softc	*axq_softc;	/* Needed for scheduling */
 	u_int			axq_qnum;	/* hardware q number */
 #define	ATH_TXQ_SWQ	(HAL_NUM_TX_QUEUES+1)	/* qnum for s/w only queue */
 	u_int			axq_ac;		/* WME AC */
 	u_int			axq_flags;
 #define	ATH_TXQ_PUTPENDING	0x0001		/* ath_hal_puttxbuf pending */
 	u_int			axq_depth;	/* queue depth (stat only) */
+	u_int			axq_aggr_depth;	/* how many aggregates are queued */
 	u_int			axq_intrcnt;	/* interrupt count */
 	u_int32_t		*axq_link;	/* link ptr in last TX desc */
-	STAILQ_HEAD(, ath_buf)	axq_q;		/* transmit queue */
+	TAILQ_HEAD(axq_q_s, ath_buf)	axq_q;		/* transmit queue */
 	struct mtx		axq_lock;	/* lock on q and link */
 	char			axq_name[12];	/* e.g. "ath0_txq4" */
+
+	/* Per-TID traffic queue for software -> hardware TX */
+	TAILQ_HEAD(axq_t_s,ath_tid)	axq_tidq;
 };
 
+#define	ATH_NODE_LOCK(_an)		mtx_lock(&(_an)->an_mtx)
+#define	ATH_NODE_UNLOCK(_an)		mtx_unlock(&(_an)->an_mtx)
+#define	ATH_NODE_LOCK_ASSERT(_an)	mtx_assert(&(_an)->an_mtx, MA_OWNED)
+
 #define	ATH_TXQ_LOCK_INIT(_sc, _tq) do { \
 	snprintf((_tq)->axq_name, sizeof((_tq)->axq_name), "%s_txq%u", \
 		device_get_nameunit((_sc)->sc_dev), (_tq)->axq_qnum); \
@@ -173,19 +298,21 @@ struct ath_txq {
 #define	ATH_TXQ_LOCK(_tq)		mtx_lock(&(_tq)->axq_lock)
 #define	ATH_TXQ_UNLOCK(_tq)		mtx_unlock(&(_tq)->axq_lock)
 #define	ATH_TXQ_LOCK_ASSERT(_tq)	mtx_assert(&(_tq)->axq_lock, MA_OWNED)
+#define	ATH_TXQ_IS_LOCKED(_tq)		mtx_owned(&(_tq)->axq_lock)
 
+#define ATH_TXQ_INSERT_HEAD(_tq, _elm, _field) do { \
+	TAILQ_INSERT_HEAD(&(_tq)->axq_q, (_elm), _field); \
+	(_tq)->axq_depth++; \
+} while (0)
 #define ATH_TXQ_INSERT_TAIL(_tq, _elm, _field) do { \
-	STAILQ_INSERT_TAIL(&(_tq)->axq_q, (_elm), _field); \
+	TAILQ_INSERT_TAIL(&(_tq)->axq_q, (_elm), _field); \
 	(_tq)->axq_depth++; \
 } while (0)
-#define ATH_TXQ_REMOVE_HEAD(_tq, _field) do { \
-	STAILQ_REMOVE_HEAD(&(_tq)->axq_q, _field); \
+#define ATH_TXQ_REMOVE(_tq, _elm, _field) do { \
+	TAILQ_REMOVE(&(_tq)->axq_q, _elm, _field); \
 	(_tq)->axq_depth--; \
 } while (0)
-/* NB: this does not do the "head empty check" that STAILQ_LAST does */
-#define	ATH_TXQ_LAST(_tq) \
-	((struct ath_buf *)(void *) \
-	 ((char *)((_tq)->axq_q.stqh_last) - __offsetof(struct ath_buf, bf_list)))
+#define	ATH_TXQ_LAST(_tq, _field)	TAILQ_LAST(&(_tq)->axq_q, _field)
 
 struct ath_vap {
 	struct ieee80211vap av_vap;	/* base class */
@@ -205,9 +332,20 @@ struct ath_vap {
 struct taskqueue;
 struct ath_tx99;
 
+/*
+ * Whether to reset the TX/RX queue with or without
+ * a queue flush.
+ */
+typedef enum {
+	ATH_RESET_DEFAULT = 0,
+	ATH_RESET_NOLOSS = 1,
+	ATH_RESET_FULL = 2,
+} ATH_RESET_TYPE;
+
 struct ath_softc {
 	struct ifnet		*sc_ifp;	/* interface common */
 	struct ath_stats	sc_stats;	/* interface statistics */
+	struct ath_tx_aggr_stats	sc_aggr_stats;
 	int			sc_debug;
 	int			sc_nvaps;	/* # vaps */
 	int			sc_nstavaps;	/* # station vaps */
@@ -216,12 +354,15 @@ struct ath_softc {
 	u_int8_t		sc_nbssid0;	/* # vap's using base mac */
 	uint32_t		sc_bssidmask;	/* bssid mask */
 
+	void 			(*sc_node_cleanup)(struct ieee80211_node *);
 	void 			(*sc_node_free)(struct ieee80211_node *);
 	device_t		sc_dev;
 	HAL_BUS_TAG		sc_st;		/* bus space tag */
 	HAL_BUS_HANDLE		sc_sh;		/* bus space handle */
 	bus_dma_tag_t		sc_dmat;	/* bus DMA tag */
 	struct mtx		sc_mtx;		/* master lock (recursive) */
+	struct mtx		sc_pcu_mtx;	/* PCU access mutex */
+	char			sc_pcu_mtx_name[32];
 	struct taskqueue	*sc_tq;		/* private task queue */
 	struct ath_hal		*sc_ah;		/* Atheros HAL */
 	struct ath_ratectrl	*sc_rc;		/* tx rate control support */
@@ -255,7 +396,6 @@ struct ath_softc {
 				sc_setcca   : 1,/* set/clr CCA with TDMA */
 				sc_resetcal : 1,/* reset cal state next trip */
 				sc_rxslink  : 1,/* do self-linked final descriptor */
-				sc_kickpcu  : 1,/* kick PCU RX on next RX proc */
 				sc_rxtsf32  : 1;/* RX dec TSF is 32 bits */
 	uint32_t		sc_eerd;	/* regdomain from EEPROM */
 	uint32_t		sc_eecc;	/* country code from EEPROM */
@@ -282,7 +422,26 @@ struct ath_softc {
 	u_int			sc_fftxqmin;	/* min frames before staging */
 	u_int			sc_fftxqmax;	/* max frames before drop */
 	u_int			sc_txantenna;	/* tx antenna (fixed or auto) */
+
 	HAL_INT			sc_imask;	/* interrupt mask copy */
+
+	/*
+	 * These are modified in the interrupt handler as well as
+	 * the task queues and other contexts. Thus these must be
+	 * protected by a mutex, or they could clash.
+	 *
+	 * For now, access to these is behind the ATH_LOCK,
+	 * just to save time.
+	 */
+	uint32_t		sc_txq_active;	/* bitmap of active TXQs */
+	uint32_t		sc_kickpcu;	/* whether to kick the PCU */
+	uint32_t		sc_rxproc_cnt;	/* In RX processing */
+	uint32_t		sc_txproc_cnt;	/* In TX processing */
+	uint32_t		sc_txstart_cnt;	/* In TX output (raw/start) */
+	uint32_t		sc_inreset_cnt;	/* In active reset/chanchange */
+	uint32_t		sc_txrx_cnt;	/* refcount on stop/start'ing TX */
+	uint32_t		sc_intr_cnt;	/* refcount on interrupt handling */
+
 	u_int			sc_keymax;	/* size of key cache */
 	u_int8_t		sc_keymap[ATH_KEYBYTES];/* key use bit map */
 
@@ -360,10 +519,38 @@ struct ath_softc {
 	int			sc_txchainmask;	/* currently configured TX chainmask */
 	int			sc_rxchainmask;	/* currently configured RX chainmask */
 
+	/*
+	 * Aggregation twiddles
+	 *
+	 * hwq_limit:	how busy to keep the hardware queue - don't schedule
+	 *		further packets to the hardware, regardless of the TID
+	 * tid_hwq_lo:	how low the per-TID hwq count has to be before the
+	 *		TID will be scheduled again
+	 * tid_hwq_hi:	how many frames to queue to the HWQ before the TID
+	 *		stops being scheduled.
+	 */
+	int			sc_hwq_limit;
+	int			sc_tid_hwq_lo;
+	int			sc_tid_hwq_hi;
+
 	/* DFS related state */
 	void			*sc_dfs;	/* Used by an optional DFS module */
 	int			sc_dodfs;	/* Whether to enable DFS rx filter bits */
 	struct task		sc_dfstask;	/* DFS processing task */
+
+	/* TX AMPDU handling */
+	int			(*sc_addba_request)(struct ieee80211_node *,
+				    struct ieee80211_tx_ampdu *, int, int, int);
+	int			(*sc_addba_response)(struct ieee80211_node *,
+				    struct ieee80211_tx_ampdu *, int, int, int);
+	void			(*sc_addba_stop)(struct ieee80211_node *,
+				    struct ieee80211_tx_ampdu *);
+	void			(*sc_addba_response_timeout)
+				    (struct ieee80211_node *,
+				    struct ieee80211_tx_ampdu *);
+	void			(*sc_bar_response)(struct ieee80211_node *ni,
+				    struct ieee80211_tx_ampdu *tap,
+				    int status);
 };
 
 #define	ATH_LOCK_INIT(_sc) \
@@ -373,6 +560,40 @@ struct ath_softc {
 #define	ATH_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
 #define	ATH_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
 #define	ATH_LOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_mtx, MA_OWNED)
+#define	ATH_UNLOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED)
+
+/*
+ * The PCU lock is non-recursive and should be treated as a spinlock.
+ * Although currently the interrupt code is run in netisr context and
+ * doesn't require this, this may change in the future.
+ * Please keep this in mind when protecting certain code paths
+ * with the PCU lock.
+ *
+ * The PCU lock is used to serialise access to the PCU so things such
+ * as TX, RX, state change (eg channel change), channel reset and updates
+ * from interrupt context (eg kickpcu, txqactive bits) do not clash.
+ *
+ * Although the current single-thread taskqueue mechanism protects the
+ * majority of these situations by simply serialising them, there are
+ * a few others which occur at the same time. These include the TX path
+ * (which only acquires ATH_LOCK when recycling buffers to the free list),
+ * ath_set_channel, the channel scanning API and perhaps quite a bit more.
+ */
+#define	ATH_PCU_LOCK_INIT(_sc) do {\
+	snprintf((_sc)->sc_pcu_mtx_name,				\
+	    sizeof((_sc)->sc_pcu_mtx_name),				\
+	    "%s PCU lock",						\
+	    device_get_nameunit((_sc)->sc_dev));			\
+	mtx_init(&(_sc)->sc_pcu_mtx, (_sc)->sc_pcu_mtx_name,		\
+		 NULL, MTX_DEF);					\
+	} while (0)
+#define	ATH_PCU_LOCK_DESTROY(_sc)	mtx_destroy(&(_sc)->sc_pcu_mtx)
+#define	ATH_PCU_LOCK(_sc)		mtx_lock(&(_sc)->sc_pcu_mtx)
+#define	ATH_PCU_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_pcu_mtx)
+#define	ATH_PCU_LOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_pcu_mtx,	\
+		MA_OWNED)
+#define	ATH_PCU_UNLOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_pcu_mtx,	\
+		MA_NOTOWNED)
 
 #define	ATH_TXQ_SETUP(sc, i)	((sc)->sc_txqsetup & (1<<i))
 
@@ -686,24 +907,33 @@ void	ath_intr(void *);
 #define ath_hal_gettxcompletionrates(_ah, _ds, _rates, _tries) \
 	((*(_ah)->ah_getTxCompletionRates)((_ah), (_ds), (_rates), (_tries)))
 
-#define	ath_hal_chaintxdesc(_ah, _ds, _pktlen, _hdrlen, _type, _keyix, \
-	_cipher, _delims, _seglen, _first, _last) \
-	((*(_ah)->ah_chainTxDesc((_ah), (_ds), (_pktlen), (_hdrlen), \
-	(_type), (_keyix), (_cipher), (_delims), (_seglen), \
-	(_first), (_last)))) 
 #define	ath_hal_setupfirsttxdesc(_ah, _ds, _aggrlen, _flags, _txpower, \
 		_txr0, _txtr0, _antm, _rcr, _rcd) \
 	((*(_ah)->ah_setupFirstTxDesc)((_ah), (_ds), (_aggrlen), (_flags), \
 	(_txpower), (_txr0), (_txtr0), (_antm), (_rcr), (_rcd)))
+#define	ath_hal_chaintxdesc(_ah, _ds, _pktlen, _hdrlen, _type, _keyix, \
+	_cipher, _delims, _seglen, _first, _last) \
+	((*(_ah)->ah_chainTxDesc)((_ah), (_ds), (_pktlen), (_hdrlen), \
+	(_type), (_keyix), (_cipher), (_delims), (_seglen), \
+	(_first), (_last)))
 #define	ath_hal_setuplasttxdesc(_ah, _ds, _ds0) \
 	((*(_ah)->ah_setupLastTxDesc)((_ah), (_ds), (_ds0)))
+
 #define	ath_hal_set11nratescenario(_ah, _ds, _dur, _rt, _series, _ns, _flags) \
 	((*(_ah)->ah_set11nRateScenario)((_ah), (_ds), (_dur), (_rt), \
 	(_series), (_ns), (_flags)))
+
+#define	ath_hal_set11n_aggr_first(_ah, _ds, _len, _num) \
+	((*(_ah)->ah_set11nAggrFirst)((_ah), (_ds), (_len), (_num)))
 #define	ath_hal_set11naggrmiddle(_ah, _ds, _num) \
-	((*(_ah)->ah_set11nAggrMiddle((_ah), (_ds), (_num))))
+	((*(_ah)->ah_set11nAggrMiddle)((_ah), (_ds), (_num)))
+#define	ath_hal_set11n_aggr_last(_ah, _ds) \
+	((*(_ah)->ah_set11nAggrLast)((_ah), (_ds)))
+
 #define	ath_hal_set11nburstduration(_ah, _ds, _dur) \
 	((*(_ah)->ah_set11nBurstDuration)((_ah), (_ds), (_dur)))
+#define	ath_hal_clr11n_aggr(_ah, _ds) \
+	((*(_ah)->ah_clr11nAggr)((_ah), (_ds)))
 
 /*
  * This is badly-named; you need to set the correct parameters
diff --git a/sys/dev/atkbdc/psm.c b/sys/dev/atkbdc/psm.c
index 0cd17ae..659c780 100644
--- a/sys/dev/atkbdc/psm.c
+++ b/sys/dev/atkbdc/psm.c
@@ -2262,8 +2262,8 @@ psmtimeout(void *arg)
 }
 
 /* Add all sysctls under the debug.psm and hw.psm nodes */
-SYSCTL_NODE(_debug, OID_AUTO, psm, CTLFLAG_RD, 0, "ps/2 mouse");
-SYSCTL_NODE(_hw, OID_AUTO, psm, CTLFLAG_RD, 0, "ps/2 mouse");
+static SYSCTL_NODE(_debug, OID_AUTO, psm, CTLFLAG_RD, 0, "ps/2 mouse");
+static SYSCTL_NODE(_hw, OID_AUTO, psm, CTLFLAG_RD, 0, "ps/2 mouse");
 
 SYSCTL_INT(_debug_psm, OID_AUTO, loglevel, CTLFLAG_RW, &verbose, 0,
     "Verbosity level");
diff --git a/sys/dev/bce/if_bce.c b/sys/dev/bce/if_bce.c
index dae8d5f..39a93aa 100644
--- a/sys/dev/bce/if_bce.c
+++ b/sys/dev/bce/if_bce.c
@@ -489,7 +489,7 @@ DRIVER_MODULE(miibus, bce, miibus_driver, miibus_devclass, 0, 0);
 /****************************************************************************/
 /* Tunable device values                                                    */
 /****************************************************************************/
-SYSCTL_NODE(_hw, OID_AUTO, bce, CTLFLAG_RD, 0, "bce driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, bce, CTLFLAG_RD, 0, "bce driver parameters");
 
 /* Allowable values are TRUE or FALSE */
 static int bce_verbose = TRUE;
diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c
index 4deecc8..f7d1730 100644
--- a/sys/dev/bge/if_bge.c
+++ b/sys/dev/bge/if_bge.c
@@ -301,6 +301,7 @@ static const struct bge_revision {
 	{ BGE_CHIPID_BCM5717_A0,	"BCM5717 A0" },
 	{ BGE_CHIPID_BCM5717_B0,	"BCM5717 B0" },
 	{ BGE_CHIPID_BCM5719_A0,	"BCM5719 A0" },
+	{ BGE_CHIPID_BCM5720_A0,	"BCM5720 A0" },
 	{ BGE_CHIPID_BCM5755_A0,	"BCM5755 A0" },
 	{ BGE_CHIPID_BCM5755_A1,	"BCM5755 A1" },
 	{ BGE_CHIPID_BCM5755_A2,	"BCM5755 A2" },
@@ -349,6 +350,7 @@ static const struct bge_revision const bge_majorrevs[] = {
 	{ BGE_ASICREV_BCM57780,		"unknown BCM57780" },
 	{ BGE_ASICREV_BCM5717,		"unknown BCM5717" },
 	{ BGE_ASICREV_BCM5719,		"unknown BCM5719" },
+	{ BGE_ASICREV_BCM5720,		"unknown BCM5720" },
 
 	{ 0, NULL }
 };
@@ -437,6 +439,7 @@ static int bge_init_tx_ring(struct bge_softc *);
 
 static int bge_chipinit(struct bge_softc *);
 static int bge_blockinit(struct bge_softc *);
+static uint32_t bge_dma_swap_options(struct bge_softc *);
 
 static int bge_has_eaddr(struct bge_softc *);
 static uint32_t bge_readmem_ind(struct bge_softc *, int);
@@ -517,7 +520,7 @@ static int bge_allow_asf = 1;
 
 TUNABLE_INT("hw.bge.allow_asf", &bge_allow_asf);
 
-SYSCTL_NODE(_hw, OID_AUTO, bge, CTLFLAG_RD, 0, "BGE driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, bge, CTLFLAG_RD, 0, "BGE driver parameters");
 SYSCTL_INT(_hw_bge, OID_AUTO, allow_asf, CTLFLAG_RD, &bge_allow_asf, 0,
 	"Allow ASF mode if available");
 
@@ -1368,25 +1371,44 @@ bge_stop_fw(struct bge_softc *sc)
 	int i;
 
 	if (sc->bge_asf_mode) {
-		bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB, BGE_FW_PAUSE);
+		bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB, BGE_FW_CMD_PAUSE);
 		CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
-		    CSR_READ_4(sc, BGE_RX_CPU_EVENT) | (1 << 14));
+		    CSR_READ_4(sc, BGE_RX_CPU_EVENT) | BGE_RX_CPU_DRV_EVENT);
 
 		for (i = 0; i < 100; i++ ) {
-			if (!(CSR_READ_4(sc, BGE_RX_CPU_EVENT) & (1 << 14)))
+			if (!(CSR_READ_4(sc, BGE_RX_CPU_EVENT) &
+			    BGE_RX_CPU_DRV_EVENT))
 				break;
 			DELAY(10);
 		}
 	}
 }
 
+static uint32_t
+bge_dma_swap_options(struct bge_softc *sc)
+{
+	uint32_t dma_options;
+
+	dma_options = BGE_MODECTL_WORDSWAP_NONFRAME |
+	    BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA;
+#if BYTE_ORDER == BIG_ENDIAN
+	dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME;
+#endif
+	if ((sc)->bge_asicrev == BGE_ASICREV_BCM5720)
+		dma_options |= BGE_MODECTL_BYTESWAP_B2HRX_DATA |
+		    BGE_MODECTL_WORDSWAP_B2HRX_DATA | BGE_MODECTL_B2HRX_ENABLE |
+		    BGE_MODECTL_HTX2B_ENABLE;
+
+	return (dma_options);
+}
+
 /*
  * Do endian, PCI and DMA initialization.
  */
 static int
 bge_chipinit(struct bge_softc *sc)
 {
-	uint32_t dma_rw_ctl, misc_ctl;
+	uint32_t dma_rw_ctl, misc_ctl, mode_ctl;
 	uint16_t val;
 	int i;
 
@@ -1504,9 +1526,8 @@ bge_chipinit(struct bge_softc *sc)
 	/*
 	 * Set up general mode register.
 	 */
-	CSR_WRITE_4(sc, BGE_MODE_CTL, BGE_DMA_SWAP_OPTIONS |
-	    BGE_MODECTL_MAC_ATTN_INTR | BGE_MODECTL_HOST_SEND_BDS |
-	    BGE_MODECTL_TX_NO_PHDR_CSUM);
+	mode_ctl = bge_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR |
+	    BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
 
 	/*
 	 * BCM5701 B5 have a bug causing data corruption when using
@@ -1516,13 +1537,15 @@ bge_chipinit(struct bge_softc *sc)
 	 */
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5701 &&
 	    sc->bge_chipid == BGE_CHIPID_BCM5701_B5)
-		BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_FORCE_PCI32);
+		mode_ctl |= BGE_MODECTL_FORCE_PCI32;
 
 	/*
 	 * Tell the firmware the driver is running
 	 */
 	if (sc->bge_asf_mode & ASF_STACKUP)
-		BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
+		mode_ctl |= BGE_MODECTL_STACKUP;
+
+	CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
 
 	/*
 	 * Disable memory write invalidate.  Apparently it is not supported
@@ -1582,8 +1605,7 @@ bge_blockinit(struct bge_softc *sc)
 	}
 
 	/* Configure mbuf pool watermarks */
-	if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
-	    sc->bge_asicrev == BGE_ASICREV_BCM57765) {
+	if (BGE_IS_5717_PLUS(sc)) {
 		CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
 		if (sc->bge_ifp->if_mtu > ETHERMTU) {
 			CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
@@ -1718,7 +1740,8 @@ bge_blockinit(struct bge_softc *sc)
 		    BGE_RCB_MAXLEN_FLAGS(BGE_MAX_FRAMELEN, 0);
 	}
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
-	    sc->bge_asicrev == BGE_ASICREV_BCM5719)
+	    sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+	    sc->bge_asicrev == BGE_ASICREV_BCM5720)
 		rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
 	else
 		rcb->bge_nicaddr = BGE_STD_RX_RINGS;
@@ -1751,7 +1774,8 @@ bge_blockinit(struct bge_softc *sc)
 		rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0,
 		    BGE_RCB_FLAG_USE_EXT_RX_BD | BGE_RCB_FLAG_RING_DISABLED);
 		if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
-		    sc->bge_asicrev == BGE_ASICREV_BCM5719)
+		    sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+		    sc->bge_asicrev == BGE_ASICREV_BCM5720)
 			rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
 		else
 			rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
@@ -1840,7 +1864,8 @@ bge_blockinit(struct bge_softc *sc)
 	RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
 	RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
-	    sc->bge_asicrev == BGE_ASICREV_BCM5719)
+	    sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+	    sc->bge_asicrev == BGE_ASICREV_BCM5720)
 		RCB_WRITE_4(sc, vrcb, bge_nicaddr, BGE_SEND_RING_5717);
 	else
 		RCB_WRITE_4(sc, vrcb, bge_nicaddr,
@@ -1854,7 +1879,8 @@ bge_blockinit(struct bge_softc *sc)
 	 * return ring control blocks, located in NIC memory.
 	 */
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
-	    sc->bge_asicrev == BGE_ASICREV_BCM5719) {
+	    sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+	    sc->bge_asicrev == BGE_ASICREV_BCM5720) {
 		/* Should be 17, use 16 until we get an SRAM map. */
 		limit = 16;
 	} else if (!BGE_IS_5705_PLUS(sc))
@@ -1898,7 +1924,11 @@ bge_blockinit(struct bge_softc *sc)
 	    BGE_TX_BACKOFF_SEED_MASK);
 
 	/* Set inter-packet gap */
-	CSR_WRITE_4(sc, BGE_TX_LENGTHS, 0x2620);
+	val = 0x2620;
+	if (sc->bge_asicrev == BGE_ASICREV_BCM5720)
+		val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
+		    (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
+	CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
 
 	/*
 	 * Specify which ring to use for packets that don't match
@@ -2053,6 +2083,11 @@ bge_blockinit(struct bge_softc *sc)
 		    sc->bge_asicrev == BGE_ASICREV_BCM57780)
 			val |= BGE_RDMAMODE_TSO6_ENABLE;
 	}
+
+	if (sc->bge_asicrev == BGE_ASICREV_BCM5720)
+		val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
+			BGE_RDMAMODE_H2BNC_VLAN_DET;
+
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5761 ||
 	    sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
 	    sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
@@ -2063,7 +2098,8 @@ bge_blockinit(struct bge_softc *sc)
 		 * Adjust tx margin to prevent TX data corruption and
 		 * fix internal FIFO overflow.
 		 */
-		if (sc->bge_asicrev == BGE_ASICREV_BCM5719) {
+		if (sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+		    sc->bge_asicrev == BGE_ASICREV_BCM5720) {
 			dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
 			    BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
 			    BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
@@ -2080,7 +2116,8 @@ bge_blockinit(struct bge_softc *sc)
 		    BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
 	}
 
-	if (sc->bge_asicrev == BGE_ASICREV_BCM5719) {
+	if (sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+	    sc->bge_asicrev == BGE_ASICREV_BCM5720) {
 		CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
 		    CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
 		    BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
@@ -2239,6 +2276,7 @@ bge_probe(device_t dev)
 				case BCOM_DEVICEID_BCM5717:
 				case BCOM_DEVICEID_BCM5718:
 				case BCOM_DEVICEID_BCM5719:
+				case BCOM_DEVICEID_BCM5720:
 					id = pci_read_config(dev,
 					    BGE_PCI_GEN2_PRODID_ASICREV, 4);
 					break;
@@ -2763,6 +2801,7 @@ bge_attach(device_t dev)
 		case BCOM_DEVICEID_BCM5717:
 		case BCOM_DEVICEID_BCM5718:
 		case BCOM_DEVICEID_BCM5719:
+		case BCOM_DEVICEID_BCM5720:
 			sc->bge_chipid = pci_read_config(dev,
 			    BGE_PCI_GEN2_PRODID_ASICREV, 4);
 			break;
@@ -2794,12 +2833,14 @@ bge_attach(device_t dev)
 	  * BCM5704  |   1   |   X   |   1   |   X   |
 	  * BCM5717  |   1   |   8   |   2   |   9   |
 	  * BCM5719  |   1   |   8   |   2   |   9   |
+	  * BCM5720  |   1   |   8   |   2   |   9   |
 	  *
 	  * Other addresses may respond but they are not
 	  * IEEE compliant PHYs and should be ignored.
 	  */
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
-	    sc->bge_asicrev == BGE_ASICREV_BCM5719) {
+	    sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+	    sc->bge_asicrev == BGE_ASICREV_BCM5720) {
 		f = pci_get_function(dev);
 		if (sc->bge_chipid == BGE_CHIPID_BCM5717_A0) {
 			if (CSR_READ_4(sc, BGE_SGDIG_STS) &
@@ -2834,6 +2875,7 @@ bge_attach(device_t dev)
 	switch (sc->bge_asicrev) {
 	case BGE_ASICREV_BCM5717:
 	case BGE_ASICREV_BCM5719:
+	case BGE_ASICREV_BCM5720:
 	case BGE_ASICREV_BCM57765:
 		sc->bge_flags |= BGE_FLAG_5717_PLUS | BGE_FLAG_5755_PLUS |
 		    BGE_FLAG_575X_PLUS | BGE_FLAG_5705_PLUS | BGE_FLAG_JUMBO |
@@ -2889,6 +2931,7 @@ bge_attach(device_t dev)
 	    sc->bge_asicrev != BGE_ASICREV_BCM5906 &&
 	    sc->bge_asicrev != BGE_ASICREV_BCM5717 &&
 	    sc->bge_asicrev != BGE_ASICREV_BCM5719 &&
+	    sc->bge_asicrev != BGE_ASICREV_BCM5720 &&
 	    sc->bge_asicrev != BGE_ASICREV_BCM5785 &&
 	    sc->bge_asicrev != BGE_ASICREV_BCM57765 &&
 	    sc->bge_asicrev != BGE_ASICREV_BCM57780) {
@@ -3009,7 +3052,8 @@ bge_attach(device_t dev)
 		 */
 		sc->bge_flags |= BGE_FLAG_PCIE;
 		sc->bge_expcap = reg;
-		if (sc->bge_asicrev == BGE_ASICREV_BCM5719)
+		if (sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
+		    sc->bge_asicrev == BGE_ASICREV_BCM5720)
 			pci_set_max_read_req(dev, 2048);
 		else if (pci_get_max_read_req(dev) != 4096)
 			pci_set_max_read_req(dev, 4096);
@@ -3595,8 +3639,7 @@ bge_reset(struct bge_softc *sc)
 	}
 
 	/* Fix up byte swapping. */
-	CSR_WRITE_4(sc, BGE_MODE_CTL, BGE_DMA_SWAP_OPTIONS |
-	    BGE_MODECTL_BYTESWAP_DATA);
+	CSR_WRITE_4(sc, BGE_MODE_CTL, bge_dma_swap_options(sc));
 
 	/* Tell the ASF firmware we are up */
 	if (sc->bge_asf_mode & ASF_STACKUP)
@@ -3627,6 +3670,10 @@ bge_reset(struct bge_softc *sc)
 	}
 	DELAY(10000);
 
+	if (sc->bge_asicrev == BGE_ASICREV_BCM5720)
+		BGE_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
+		    CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
+
 	return (0);
 }
 
@@ -4107,11 +4154,13 @@ bge_asf_driver_up(struct bge_softc *sc)
 		else {
 			sc->bge_asf_count = 2;
 			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB,
-			    BGE_FW_DRV_ALIVE);
+			    BGE_FW_CMD_DRV_ALIVE);
 			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_LEN_MB, 4);
-			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_DATA_MB, 3);
+			bge_writemem_ind(sc, BGE_SRAM_FW_CMD_DATA_MB,
+			    BGE_FW_HB_TIMEOUT_SEC);
 			CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
-			    CSR_READ_4(sc, BGE_RX_CPU_EVENT) | (1 << 14));
+			    CSR_READ_4(sc, BGE_RX_CPU_EVENT) |
+			    BGE_RX_CPU_DRV_EVENT);
 		}
 	}
 }
@@ -4242,8 +4291,30 @@ bge_stats_update_regs(struct bge_softc *sc)
 	    CSR_READ_4(sc, BGE_RXLP_LOCSTAT_DMA_HPWRQ_FULL);
 	stats->NoMoreRxBDs +=
 	    CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS);
-	stats->InputDiscards +=
-	    CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_DROPS);
+	/*
+	 * XXX
+	 * Unlike other controllers, BGE_RXLP_LOCSTAT_IFIN_DROPS
+	 * counter of BCM5717, BCM5718, BCM5719 A0 and BCM5720 A0
+	 * includes number of unwanted multicast frames.  This comes
+	 * from silicon bug and known workaround to get rough(not
+	 * exact) counter is to enable interrupt on MBUF low water
+	 * attention.  This can be accomplished by setting
+	 * BGE_HCCMODE_ATTN bit of BGE_HCC_MODE,
+	 * BGE_BMANMODE_LOMBUF_ATTN bit of BGE_BMAN_MODE and
+	 * BGE_MODECTL_FLOWCTL_ATTN_INTR bit of BGE_MODE_CTL.
+	 * However that change would generate more interrupts and
+	 * there are still possibilities of losing multiple frames
+	 * during BGE_MODECTL_FLOWCTL_ATTN_INTR interrupt handling.
+	 * Given that the workaround still would not get correct
+	 * counter I don't think it's worth to implement it.  So
+	 * ignore reading the counter on controllers that have the
+	 * silicon bug.
+	 */
+	if (sc->bge_asicrev != BGE_ASICREV_BCM5717 &&
+	    sc->bge_chipid != BGE_CHIPID_BCM5719_A0 &&
+	    sc->bge_chipid != BGE_CHIPID_BCM5720_A0)
+		stats->InputDiscards +=
+		    CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_DROPS);
 	stats->InputErrors +=
 	    CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_ERRORS);
 	stats->RecvThresholdHit +=
@@ -4839,6 +4910,11 @@ bge_init_locked(struct bge_softc *sc)
 	mode = CSR_READ_4(sc, BGE_TX_MODE);
 	if (BGE_IS_5755_PLUS(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5906)
 		mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
+	if (sc->bge_asicrev == BGE_ASICREV_BCM5720) {
+		mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
+		mode |= CSR_READ_4(sc, BGE_TX_MODE) &
+		    (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
+	}
 	/* Turn on transmitter. */
 	CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
 
diff --git a/sys/dev/bge/if_bgereg.h b/sys/dev/bge/if_bgereg.h
index 7973c86..3ba4176 100644
--- a/sys/dev/bge/if_bgereg.h
+++ b/sys/dev/bge/if_bgereg.h
@@ -91,8 +91,16 @@
 
 /* Firmware interface */
 #define	BGE_SRAM_DATA_SIG_MAGIC		0x4B657654	/* 'KevT' */
-#define	BGE_FW_DRV_ALIVE		0x00000001
-#define	BGE_FW_PAUSE			0x00000002
+
+#define	BGE_FW_CMD_DRV_ALIVE		0x00000001
+#define	BGE_FW_CMD_PAUSE		0x00000002
+#define	BGE_FW_CMD_IPV4_ADDR_CHANGE	0x00000003
+#define	BGE_FW_CMD_IPV6_ADDR_CHANGE	0x00000004
+#define	BGE_FW_CMD_LINK_UPDATE		0x0000000C
+#define	BGE_FW_CMD_DRV_ALIVE2		0x0000000D
+#define	BGE_FW_CMD_DRV_ALIVE3		0x0000000E
+
+#define	BGE_FW_HB_TIMEOUT_SEC		3
 
 #define	BGE_FW_DRV_STATE_START		0x00000001
 #define	BGE_FW_DRV_STATE_START_DONE	0x80000001
@@ -250,15 +258,6 @@
 #define	BGE_PCIMISCCTL_ASICREV_SHIFT	16
 
 #define	BGE_HIF_SWAP_OPTIONS	(BGE_PCIMISCCTL_ENDIAN_WORDSWAP)
-#if BYTE_ORDER == LITTLE_ENDIAN
-#define	BGE_DMA_SWAP_OPTIONS \
-	BGE_MODECTL_WORDSWAP_NONFRAME| \
-	BGE_MODECTL_BYTESWAP_DATA|BGE_MODECTL_WORDSWAP_DATA
-#else
-#define	BGE_DMA_SWAP_OPTIONS \
-	BGE_MODECTL_WORDSWAP_NONFRAME|BGE_MODECTL_BYTESWAP_NONFRAME| \
-	BGE_MODECTL_BYTESWAP_DATA|BGE_MODECTL_WORDSWAP_DATA
-#endif
 
 #define	BGE_INIT \
 	(BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_CLEAR_INTA| \
@@ -331,6 +330,7 @@
 #define	BGE_CHIPID_BCM5717_A0		0x05717000
 #define	BGE_CHIPID_BCM5717_B0		0x05717100
 #define	BGE_CHIPID_BCM5719_A0		0x05719000
+#define	BGE_CHIPID_BCM5720_A0		0x05720000
 #define	BGE_CHIPID_BCM57765_A0		0x57785000
 #define	BGE_CHIPID_BCM57765_B0		0x57785100
 
@@ -355,6 +355,7 @@
 /* BGE_PCI_PRODID_ASICREV ASIC rev. identifiers. */
 #define	BGE_ASICREV_BCM5717		0x5717
 #define	BGE_ASICREV_BCM5719		0x5719
+#define	BGE_ASICREV_BCM5720		0x5720
 #define	BGE_ASICREV_BCM5761		0x5761
 #define	BGE_ASICREV_BCM5784		0x5784
 #define	BGE_ASICREV_BCM5785		0x5785
@@ -799,6 +800,8 @@
 #define	BGE_TXMODE_BIGBACKOFF_ENABLE	0x00000020
 #define	BGE_TXMODE_LONGPAUSE_ENABLE	0x00000040
 #define	BGE_TXMODE_MBUF_LOCKUP_FIX	0x00000100
+#define	BGE_TXMODE_JMB_FRM_LEN		0x00400000
+#define	BGE_TXMODE_CNT_DN_MODE		0x00800000
 
 /* Transmit MAC status register */
 #define	BGE_TXSTAT_RX_XOFFED		0x00000001
@@ -812,6 +815,8 @@
 #define	BGE_TXLEN_SLOTTIME		0x000000FF
 #define	BGE_TXLEN_IPG			0x00000F00
 #define	BGE_TXLEN_CRS			0x00003000
+#define	BGE_TXLEN_JMB_FRM_LEN_MSK	0x00FF0000
+#define	BGE_TXLEN_CNT_DN_VAL_MSK	0xFF000000
 
 /* Receive MAC mode register */
 #define	BGE_RXMODE_RESET		0x00000001
@@ -1269,6 +1274,7 @@
 #define	BGE_CPMU_LSPD_1000MB_CLK	0x360C
 #define	BGE_CPMU_LNK_AWARE_PWRMD	0x3610
 #define	BGE_CPMU_HST_ACC		0x361C
+#define	BGE_CPMU_CLCK_ORIDE		0x3624
 #define	BGE_CPMU_CLCK_STAT		0x3630
 #define	BGE_CPMU_MUTEX_REQ		0x365C
 #define	BGE_CPMU_MUTEX_GNT		0x3660
@@ -1296,6 +1302,9 @@
 #define	BGE_CPMU_HST_ACC_MACCLK_MASK	0x001F0000
 #define	BGE_CPMU_HST_ACC_MACCLK_6_25	0x00130000
 
+/* Clock Speed Override Policy register */
+#define	CPMU_CLCK_ORIDE_MAC_ORIDE_EN	0x80000000
+
 /* CPMU Clock Status register */
 #define	BGE_CPMU_CLCK_STAT_MAC_CLCK_MASK	0x001F0000
 #define	BGE_CPMU_CLCK_STAT_MAC_CLCK_62_5	0x00000000
@@ -1543,6 +1552,7 @@
 #define	BGE_RDMAMODE_MULT_DMA_RD_DIS	0x01000000
 #define	BGE_RDMAMODE_TSO4_ENABLE	0x08000000
 #define	BGE_RDMAMODE_TSO6_ENABLE	0x10000000
+#define	BGE_RDMAMODE_H2BNC_VLAN_DET	0x20000000
 
 /* Read DMA status register */
 #define	BGE_RDMASTAT_PCI_TGT_ABRT_ATTN	0x00000004
@@ -1895,6 +1905,8 @@
 #define	BGE_EE_DELAY			0x6848
 #define	BGE_FASTBOOT_PC			0x6894
 
+#define	BGE_RX_CPU_DRV_EVENT		0x00004000
+
 /*
  * NVRAM Control registers
  */
@@ -1951,14 +1963,18 @@
 #define	BGE_MODECTL_WORDSWAP_NONFRAME	0x00000004
 #define	BGE_MODECTL_BYTESWAP_DATA	0x00000010
 #define	BGE_MODECTL_WORDSWAP_DATA	0x00000020
+#define	BGE_MODECTL_BYTESWAP_B2HRX_DATA	0x00000040
+#define	BGE_MODECTL_WORDSWAP_B2HRX_DATA	0x00000080
 #define	BGE_MODECTL_NO_FRAME_CRACKING	0x00000200
 #define	BGE_MODECTL_NO_RX_CRC		0x00000400
 #define	BGE_MODECTL_RX_BADFRAMES	0x00000800
 #define	BGE_MODECTL_NO_TX_INTR		0x00002000
 #define	BGE_MODECTL_NO_RX_INTR		0x00004000
 #define	BGE_MODECTL_FORCE_PCI32		0x00008000
+#define	BGE_MODECTL_B2HRX_ENABLE	0x00008000
 #define	BGE_MODECTL_STACKUP		0x00010000
 #define	BGE_MODECTL_HOST_SEND_BDS	0x00020000
+#define	BGE_MODECTL_HTX2B_ENABLE	0x00040000
 #define	BGE_MODECTL_TX_NO_PHDR_CSUM	0x00100000
 #define	BGE_MODECTL_RX_NO_PHDR_CSUM	0x00800000
 #define	BGE_MODECTL_TX_ATTN_INTR	0x01000000
@@ -2288,7 +2304,8 @@ struct bge_status_block {
 #define	BCOM_DEVICEID_BCM5717		0x1655
 #define	BCOM_DEVICEID_BCM5718		0x1656
 #define	BCOM_DEVICEID_BCM5719		0x1657
-#define	BCOM_DEVICEID_BCM5720		0x1658
+#define	BCOM_DEVICEID_BCM5720_PP	0x1658	/* Not released to public. */
+#define	BCOM_DEVICEID_BCM5720		0x165F
 #define	BCOM_DEVICEID_BCM5721		0x1659
 #define	BCOM_DEVICEID_BCM5722		0x165A
 #define	BCOM_DEVICEID_BCM5723		0x165B
diff --git a/sys/dev/bktr/bktr_os.c b/sys/dev/bktr/bktr_os.c
index b667700..4eb39ae 100644
--- a/sys/dev/bktr/bktr_os.c
+++ b/sys/dev/bktr/bktr_os.c
@@ -108,7 +108,7 @@ int bt848_amsound = 0;		/* hard-wire AM sound at 6.5 Hz (france),
 int bt848_dolby = 0;
 #endif
 
-SYSCTL_NODE(_hw, OID_AUTO, bt848, CTLFLAG_RW, 0, "Bt848 Driver mgmt");
+static SYSCTL_NODE(_hw, OID_AUTO, bt848, CTLFLAG_RW, 0, "Bt848 Driver mgmt");
 SYSCTL_INT(_hw_bt848, OID_AUTO, card, CTLFLAG_RW, &bt848_card, -1, "");
 SYSCTL_INT(_hw_bt848, OID_AUTO, tuner, CTLFLAG_RW, &bt848_tuner, -1, "");
 SYSCTL_INT(_hw_bt848, OID_AUTO, reverse_mute, CTLFLAG_RW, &bt848_reverse_mute, -1, "");
diff --git a/sys/dev/bktr/bktr_reg.h b/sys/dev/bktr/bktr_reg.h
index 982fe2c..4a3af1a 100644
--- a/sys/dev/bktr/bktr_reg.h
+++ b/sys/dev/bktr/bktr_reg.h
@@ -717,10 +717,6 @@ struct bt848_card_sig {
 /* ioctl_cmd_t int on old versions, u_long on new versions */
 /***********************************************************/
 
-#if (__FreeBSD__ == 2)
-typedef int ioctl_cmd_t;
-#endif
-
 #if defined(__FreeBSD__)
 typedef u_long ioctl_cmd_t;
 #endif
diff --git a/sys/dev/bm/if_bm.c b/sys/dev/bm/if_bm.c
index 1977f92..c478e7b 100644
--- a/sys/dev/bm/if_bm.c
+++ b/sys/dev/bm/if_bm.c
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/ofw/ofw_bus.h>
@@ -105,17 +106,28 @@ static void bm_tick		(void *xsc);
 static int bm_ifmedia_upd	(struct ifnet *);
 static void bm_ifmedia_sts	(struct ifnet *, struct ifmediareq *);
 
-static void bm_miicsr_dwrite	(struct bm_softc *, u_int16_t);
-static void bm_mii_writebit	(struct bm_softc *, int);
-static int bm_mii_readbit	(struct bm_softc *);
-static void bm_mii_sync		(struct bm_softc *);
-static void bm_mii_send		(struct bm_softc *, u_int32_t, int);
-static int bm_mii_readreg	(struct bm_softc *, struct bm_mii_frame *);
-static int bm_mii_writereg	(struct bm_softc *, struct bm_mii_frame *);
 static int bm_miibus_readreg	(device_t, int, int);
 static int bm_miibus_writereg	(device_t, int, int, int);
 static void bm_miibus_statchg	(device_t);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t bm_mii_bitbang_read(device_t);
+static void bm_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops bm_mii_bitbang_ops = {
+	bm_mii_bitbang_read,
+	bm_mii_bitbang_write,
+	{
+		BM_MII_DATAOUT,	/* MII_BIT_MDO */
+		BM_MII_DATAIN,	/* MII_BIT_MDI */
+		BM_MII_CLK,	/* MII_BIT_MDC */
+		BM_MII_OENABLE,	/* MII_BIT_DIR_HOST_PHY */
+		0,		/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static device_method_t bm_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		bm_probe),
@@ -150,171 +162,36 @@ DRIVER_MODULE(miibus, bm, miibus_driver, miibus_devclass, 0, 0);
  */
 
 /*
- * Write to the MII csr, introducing a delay to allow valid
- * MII clock pulses to be formed
+ * Write the MII serial port for the MII bit-bang module.
  */
 static void
-bm_miicsr_dwrite(struct bm_softc *sc, u_int16_t val)
-{
-	CSR_WRITE_2(sc, BM_MII_CSR, val);
-	/*
-	 * Assume this is a clock toggle and generate a 1us delay
-	 * to cover both MII's 160ns high/low minimum and 400ns
-	 * cycle miniumum
-	 */
-	DELAY(1);
-}
-
-/*
- * Write a bit to the MII bus.
- */
-static void
-bm_mii_writebit(struct bm_softc *sc, int bit)
-{
-	u_int16_t regval;
-
-	regval = BM_MII_OENABLE;
-	if (bit)
-		regval |= BM_MII_DATAOUT;
-
-	bm_miicsr_dwrite(sc, regval);
-	bm_miicsr_dwrite(sc, regval | BM_MII_CLK);
-	bm_miicsr_dwrite(sc, regval);
-}
-
-/*
- * Read a bit from the MII bus.
- */
-static int
-bm_mii_readbit(struct bm_softc *sc)
-{
-	u_int16_t regval, bitin;
-
-	/* ~BM_MII_OENABLE */
-	regval = 0;
-
-	bm_miicsr_dwrite(sc, regval);
-	bm_miicsr_dwrite(sc, regval | BM_MII_CLK);
-	bm_miicsr_dwrite(sc, regval);
-	bitin = CSR_READ_2(sc, BM_MII_CSR) & BM_MII_DATAIN;
-
-	return (bitin == BM_MII_DATAIN);
-}
-
-/*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
- */
-static void
-bm_mii_sync(struct bm_softc *sc)
-{
-	int i;
-	u_int16_t regval;
-
-	regval = BM_MII_OENABLE | BM_MII_DATAOUT;
-
-	bm_miicsr_dwrite(sc, regval);
-	for (i = 0; i < 32; i++) {
-		bm_miicsr_dwrite(sc, regval | BM_MII_CLK);
-		bm_miicsr_dwrite(sc, regval);
-	}
-}
-
-/*
- * Clock a series of bits through the MII.
- */
-static void
-bm_mii_send(struct bm_softc *sc, u_int32_t bits, int cnt)
-{
-	int i;
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1)
-		bm_mii_writebit(sc, bits & i);
-}
-
-/*
- * Read a PHY register through the MII.
- */
-static int
-bm_mii_readreg(struct bm_softc *sc, struct bm_mii_frame *frame)
+bm_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	int i, ack, bit;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = BM_MII_STARTDELIM;
-	frame->mii_opcode = BM_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	/*
-	 * Sync the PHYs
-	 */
-	bm_mii_sync(sc);
-
-	/*
-	 * Send command/address info
-	 */
-	bm_mii_send(sc, frame->mii_stdelim, 2);
-	bm_mii_send(sc, frame->mii_opcode, 2);
-	bm_mii_send(sc, frame->mii_phyaddr, 5);
-	bm_mii_send(sc, frame->mii_regaddr, 5);
-
-	/*
-	 * Check for ack.
-	 */
-	ack = bm_mii_readbit(sc);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	for (i = 0x8000; i; i >>= 1) {
-		bit = bm_mii_readbit(sc);
-		if (!ack && bit)
-			frame->mii_data |= i;
-	}
+	struct bm_softc *sc;
 
-	/*
-	 * Skip through idle bit-times
-	 */
-	bm_mii_writebit(sc, 0);
-	bm_mii_writebit(sc, 0);
+	sc = device_get_softc(dev);
 
-	return ((ack) ? 1 : 0);
+	CSR_WRITE_2(sc, BM_MII_CSR, val);
+	CSR_BARRIER(sc, BM_MII_CSR, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 /*
- * Write to a PHY register through the MII.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static int
-bm_mii_writereg(struct bm_softc *sc, struct bm_mii_frame *frame)
+static uint32_t
+bm_mii_bitbang_read(device_t dev)
 {
-	/*
-	 * Set up frame for tx
-	 */
-	frame->mii_stdelim = BM_MII_STARTDELIM;
-	frame->mii_opcode = BM_MII_WRITEOP;
-	frame->mii_turnaround = BM_MII_TURNAROUND;
-
-	/*
-	 * Sync the phy and start the bitbang write sequence
-	 */
-	bm_mii_sync(sc);
+	struct bm_softc *sc;
+	uint32_t reg;
 
-	bm_mii_send(sc, frame->mii_stdelim, 2);
-	bm_mii_send(sc, frame->mii_opcode, 2);
-	bm_mii_send(sc, frame->mii_phyaddr, 5);
-	bm_mii_send(sc, frame->mii_regaddr, 5);
-	bm_mii_send(sc, frame->mii_turnaround, 2);
-	bm_mii_send(sc, frame->mii_data, 16);
+	sc = device_get_softc(dev);
 
-	/*
-	 * Idle bit.
-	 */
-	bm_mii_writebit(sc, 0);
+	reg = CSR_READ_2(sc, BM_MII_CSR);
+	CSR_BARRIER(sc, BM_MII_CSR, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	return (0);
+	return (reg);
 }
 
 /*
@@ -323,34 +200,15 @@ bm_mii_writereg(struct bm_softc *sc, struct bm_mii_frame *frame)
 static int
 bm_miibus_readreg(device_t dev, int phy, int reg)
 {
-	struct bm_softc *sc;
-	struct bm_mii_frame frame;
-
-	sc = device_get_softc(dev);
-	bzero(&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
 
-	bm_mii_readreg(sc, &frame);
-
-	return (frame.mii_data);
+	return (mii_bitbang_readreg(dev, &bm_mii_bitbang_ops, phy, reg));
 }
 
 static int
 bm_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
-	struct bm_softc *sc;
-	struct bm_mii_frame frame;
-
-	sc = device_get_softc(dev);
-	bzero(&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
 
-	bm_mii_writereg(sc, &frame);
+	mii_bitbang_readreg(dev, &bm_mii_bitbang_ops, phy, reg);
 
 	return (0);
 }
diff --git a/sys/dev/bm/if_bmreg.h b/sys/dev/bm/if_bmreg.h
index 15ee1fe..39d9b48 100644
--- a/sys/dev/bm/if_bmreg.h
+++ b/sys/dev/bm/if_bmreg.h
@@ -129,14 +129,6 @@
 #define BM_MII_DATAIN	0x0008		/* MDIO data in */
 
 /*
- * MII constants
- */
-#define BM_MII_STARTDELIM	0x01
-#define BM_MII_READOP		0x02
-#define BM_MII_WRITEOP		0x01
-#define BM_MII_TURNAROUND	0x02
-
-/*
  * Various flags
  */
 
@@ -174,3 +166,5 @@
 #define	CSR_READ_1(sc, reg)		\
 	bus_read_1(sc->sc_memr, reg)
 
+#define CSR_BARRIER(sc, reg, length, flags)				\
+	bus_barrier(sc->sc_memr, reg, length, flags)
diff --git a/sys/dev/bm/if_bmvar.h b/sys/dev/bm/if_bmvar.h
index 7d31489..b50d65b 100644
--- a/sys/dev/bm/if_bmvar.h
+++ b/sys/dev/bm/if_bmvar.h
@@ -46,7 +46,6 @@
 /*
  * software state for transmit job mbufs (may be elements of mbuf chains)
  */
-
 struct bm_txsoft {
 	struct mbuf *txs_mbuf;		/* head of our mbuf chain */
 	bus_dmamap_t txs_dmamap;	/* our DMA map */
@@ -71,7 +70,6 @@ struct bm_rxsoft {
 	bus_dma_segment_t segment;
 };
 
-
 struct bm_softc {
 	struct ifnet    	*sc_ifp;
 	struct mtx		sc_mtx;	
@@ -113,13 +111,3 @@ struct bm_softc {
 
 	dbdma_channel_t		*sc_txdma, *sc_rxdma;
 };
-
-struct bm_mii_frame {
-	u_int8_t		mii_stdelim;
-	u_int8_t		mii_opcode;
-	u_int8_t		mii_phyaddr;
-	u_int8_t		mii_regaddr;
-	u_int8_t		mii_turnaround;
-	u_int16_t		mii_data;
-};
-
diff --git a/sys/dev/bwn/if_bwn.c b/sys/dev/bwn/if_bwn.c
index c924a34..af10ace 100644
--- a/sys/dev/bwn/if_bwn.c
+++ b/sys/dev/bwn/if_bwn.c
@@ -73,7 +73,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/bwn/if_bwnreg.h>
 #include <dev/bwn/if_bwnvar.h>
 
-SYSCTL_NODE(_hw, OID_AUTO, bwn, CTLFLAG_RD, 0, "Broadcom driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, bwn, CTLFLAG_RD, 0,
+    "Broadcom driver parameters");
 
 /*
  * Tunable & sysctl variables.
diff --git a/sys/dev/bxe/if_bxe.c b/sys/dev/bxe/if_bxe.c
index 930a4f0..9957359 100644
--- a/sys/dev/bxe/if_bxe.c
+++ b/sys/dev/bxe/if_bxe.c
@@ -423,7 +423,7 @@ DRIVER_MODULE(bxe, pci, bxe_driver, bxe_devclass, 0, 0);
 /*
  * Tunable device values
  */
-SYSCTL_NODE(_hw, OID_AUTO, bxe, CTLFLAG_RD, 0, "bxe driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, bxe, CTLFLAG_RD, 0, "bxe driver parameters");
 /* Allowable values are TRUE (1) or FALSE (0). */
 
 static int bxe_dcc_enable = FALSE;
diff --git a/sys/dev/cardbus/cardbus.c b/sys/dev/cardbus/cardbus.c
index 39d20b5..ea75770 100644
--- a/sys/dev/cardbus/cardbus.c
+++ b/sys/dev/cardbus/cardbus.c
@@ -54,7 +54,7 @@ __FBSDID("$FreeBSD$");
 #include "pcib_if.h"
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, cardbus, CTLFLAG_RD, 0, "CardBus parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, cardbus, CTLFLAG_RD, 0, "CardBus parameters");
 
 int    cardbus_debug = 0;
 TUNABLE_INT("hw.cardbus.debug", &cardbus_debug);
diff --git a/sys/dev/cesa/cesa.c b/sys/dev/cesa/cesa.c
new file mode 100644
index 0000000..54bb8e3
--- /dev/null
+++ b/sys/dev/cesa/cesa.c
@@ -0,0 +1,1614 @@
+/*-
+ * Copyright (C) 2009-2011 Semihalf.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * CESA SRAM Memory Map:
+ *
+ * +------------------------+ <= sc->sc_sram_base + CESA_SRAM_SIZE
+ * |                        |
+ * |          DATA          |
+ * |                        |
+ * +------------------------+ <= sc->sc_sram_base + CESA_DATA(0)
+ * |  struct cesa_sa_data   |
+ * +------------------------+
+ * |  struct cesa_sa_hdesc  |
+ * +------------------------+ <= sc->sc_sram_base
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+
+#include <machine/bus.h>
+#include <machine/intr.h>
+#include <machine/resource.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <sys/md5.h>
+#include <crypto/sha1.h>
+#include <crypto/rijndael/rijndael.h>
+#include <opencrypto/cryptodev.h>
+#include "cryptodev_if.h"
+
+#include <arm/mv/mvreg.h>
+#include <arm/mv/mvwin.h>
+#include <arm/mv/mvvar.h>
+#include "cesa.h"
+
+#undef DEBUG
+
+static int	cesa_probe(device_t);
+static int	cesa_attach(device_t);
+static int	cesa_detach(device_t);
+static void	cesa_intr(void *);
+static int	cesa_newsession(device_t, u_int32_t *, struct cryptoini *);
+static int	cesa_freesession(device_t, u_int64_t);
+static int	cesa_process(device_t, struct cryptop *, int);
+
+static struct resource_spec cesa_res_spec[] = {
+	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
+	{ SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE },
+	{ -1, 0 }
+};
+
+static device_method_t cesa_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cesa_probe),
+	DEVMETHOD(device_attach,	cesa_attach),
+	DEVMETHOD(device_detach,	cesa_detach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
+
+	/* Crypto device methods */
+	DEVMETHOD(cryptodev_newsession,	cesa_newsession),
+	DEVMETHOD(cryptodev_freesession,cesa_freesession),
+	DEVMETHOD(cryptodev_process,	cesa_process),
+
+	{ 0, 0 }
+};
+
+static driver_t cesa_driver = {
+	"cesa",
+	cesa_methods,
+	sizeof (struct cesa_softc)
+};
+static devclass_t cesa_devclass;
+
+DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0);
+MODULE_DEPEND(cesa, crypto, 1, 1, 1);
+
+static void
+cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd)
+{
+#ifdef DEBUG
+	device_t dev;
+
+	dev = sc->sc_dev;
+	device_printf(dev, "CESA SA Hardware Descriptor:\n");
+	device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config);
+	device_printf(dev, "\t\te_src:  0x%08X\n", cshd->cshd_enc_src);
+	device_printf(dev, "\t\te_dst:  0x%08X\n", cshd->cshd_enc_dst);
+	device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen);
+	device_printf(dev, "\t\te_key:  0x%08X\n", cshd->cshd_enc_key);
+	device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv);
+	device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf);
+	device_printf(dev, "\t\tm_src:  0x%08X\n", cshd->cshd_mac_src);
+	device_printf(dev, "\t\tm_dst:  0x%08X\n", cshd->cshd_mac_dst);
+	device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen);
+	device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen);
+	device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in);
+	device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out);
+#endif
+}
+
+static void
+cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	struct cesa_dma_mem *cdm;
+
+	if (error)
+		return;
+
+	KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1."));
+	cdm = arg;
+	cdm->cdm_paddr = segs->ds_addr;
+}
+
+static int
+cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm,
+    bus_size_t size)
+{
+	int error;
+
+	KASSERT(cdm->cdm_vaddr == NULL,
+	    ("%s(): DMA memory descriptor in use.", __func__));
+
+	error = bus_dma_tag_create(NULL,	/* parent */
+	    PAGE_SIZE, 0,			/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,		/* lowaddr */
+	    BUS_SPACE_MAXADDR,			/* highaddr */
+	    NULL, NULL,				/* filtfunc, filtfuncarg */
+	    size, 1,				/* maxsize, nsegments */
+	    size, 0,				/* maxsegsz, flags */
+	    NULL, NULL,				/* lockfunc, lockfuncarg */
+	    &cdm->cdm_tag);			/* dmat */
+	if (error) {
+		device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
+		    " %i!\n", error);
+
+		goto err1;
+	}
+
+	error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr,
+	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map);
+	if (error) {
+		device_printf(sc->sc_dev, "failed to allocate DMA safe"
+		    " memory, error %i!\n", error);
+
+		goto err2;
+	}
+
+	error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr,
+	    size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT);
+	if (error) {
+		device_printf(sc->sc_dev, "cannot get address of the DMA"
+		    " memory, error %i\n", error);
+
+		goto err3;
+	}
+
+	return (0);
+err3:
+	bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
+err2:
+	bus_dma_tag_destroy(cdm->cdm_tag);
+err1:
+	cdm->cdm_vaddr = NULL;
+	return (error);
+}
+
+static void
+cesa_free_dma_mem(struct cesa_dma_mem *cdm)
+{
+
+	bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map);
+	bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
+	bus_dma_tag_destroy(cdm->cdm_tag);
+	cdm->cdm_vaddr = NULL;
+}
+
+static void
+cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op)
+{
+
+	/* Sync only if dma memory is valid */
+        if (cdm->cdm_vaddr != NULL)
+		bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op);
+}
+
+static void
+cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op)
+{
+
+	cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op);
+	cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op);
+	cesa_sync_dma_mem(&sc->sc_requests_cdm, op);
+}
+
+static struct cesa_session *
+cesa_alloc_session(struct cesa_softc *sc)
+{
+	struct cesa_session *cs;
+
+	CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions);
+
+	return (cs);
+}
+
+static struct cesa_session *
+cesa_get_session(struct cesa_softc *sc, uint32_t sid)
+{
+
+	if (sid >= CESA_SESSIONS)
+		return (NULL);
+
+	return (&sc->sc_sessions[sid]);
+}
+
+static void
+cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs)
+{
+
+	CESA_GENERIC_FREE_LOCKED(sc, cs, sessions);
+}
+
+static struct cesa_request *
+cesa_alloc_request(struct cesa_softc *sc)
+{
+	struct cesa_request *cr;
+
+	CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests);
+	if (!cr)
+		return (NULL);
+
+	STAILQ_INIT(&cr->cr_tdesc);
+	STAILQ_INIT(&cr->cr_sdesc);
+
+	return (cr);
+}
+
+static void
+cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr)
+{
+
+	/* Free TDMA descriptors assigned to this request */
+	CESA_LOCK(sc, tdesc);
+	STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc);
+	CESA_UNLOCK(sc, tdesc);
+
+	/* Free SA descriptors assigned to this request */
+	CESA_LOCK(sc, sdesc);
+	STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
+	CESA_UNLOCK(sc, sdesc);
+
+	/* Unload DMA memory asociated with request */
+	if (cr->cr_dmap_loaded) {
+		bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap);
+		cr->cr_dmap_loaded = 0;
+	}
+
+	CESA_GENERIC_FREE_LOCKED(sc, cr, requests);
+}
+
+static void
+cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr)
+{
+
+	CESA_LOCK(sc, requests);
+	STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq);
+	CESA_UNLOCK(sc, requests);
+}
+
+static struct cesa_tdma_desc *
+cesa_alloc_tdesc(struct cesa_softc *sc)
+{
+	struct cesa_tdma_desc *ctd;
+
+	CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc);
+
+	if (!ctd)
+		device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. "
+		    "Consider increasing CESA_TDMA_DESCRIPTORS.\n");
+
+	return (ctd);
+}
+
+static struct cesa_sa_desc *
+cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr)
+{
+	struct cesa_sa_desc *csd;
+
+	CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
+	if (!csd) {
+		device_printf(sc->sc_dev, "SA descriptors pool exhaused. "
+		    "Consider increasing CESA_SA_DESCRIPTORS.\n");
+		return (NULL);
+	}
+
+	STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
+
+	/* Fill-in SA descriptor with default values */
+	csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
+	csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
+	csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
+	csd->csd_cshd->cshd_enc_src = 0;
+	csd->csd_cshd->cshd_enc_dst = 0;
+	csd->csd_cshd->cshd_enc_dlen = 0;
+	csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
+	csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
+	csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
+	csd->csd_cshd->cshd_mac_src = 0;
+	csd->csd_cshd->cshd_mac_dlen = 0;
+
+	return (csd);
+}
+
+static struct cesa_tdma_desc *
+cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src,
+    bus_size_t size)
+{
+	struct cesa_tdma_desc *ctd;
+
+	ctd = cesa_alloc_tdesc(sc);
+	if (!ctd)
+		return (NULL);
+
+	ctd->ctd_cthd->cthd_dst = dst;
+	ctd->ctd_cthd->cthd_src = src;
+	ctd->ctd_cthd->cthd_byte_count = size;
+
+	/* Handle special control packet */
+	if (size != 0)
+		ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED;
+	else
+		ctd->ctd_cthd->cthd_flags = 0;
+
+	return (ctd);
+}
+
+static struct cesa_tdma_desc *
+cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
+{
+
+	return (cesa_tdma_copy(sc, sc->sc_sram_base +
+	    sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr,
+	    sizeof(struct cesa_sa_data)));
+}
+
+static struct cesa_tdma_desc *
+cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
+{
+
+	return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base +
+	    sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data)));
+}
+
+static struct cesa_tdma_desc *
+cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd)
+{
+
+	return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr,
+	    sizeof(struct cesa_sa_hdesc)));
+}
+
+static void
+cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd)
+{
+	struct cesa_tdma_desc *ctd_prev;
+
+	if (!STAILQ_EMPTY(&cr->cr_tdesc)) {
+		ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq);
+		ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr;
+	}
+
+	ctd->ctd_cthd->cthd_next = 0;
+	STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq);
+}
+
+static int
+cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
+    struct cesa_packet *cp, struct cesa_sa_desc *csd)
+{
+	struct cesa_tdma_desc *ctd, *tmp;
+
+	/* Copy SA descriptor for this packet */
+	ctd = cesa_tdma_copy_sdesc(sc, csd);
+	if (!ctd)
+		return (ENOMEM);
+
+	cesa_append_tdesc(cr, ctd);
+
+	/* Copy data to be processed */
+	STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp)
+		cesa_append_tdesc(cr, ctd);
+	STAILQ_INIT(&cp->cp_copyin);
+
+	/* Insert control descriptor */
+	ctd = cesa_tdma_copy(sc, 0, 0, 0);
+	if (!ctd)
+		return (ENOMEM);
+
+	cesa_append_tdesc(cr, ctd);
+
+	/* Copy back results */
+	STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp)
+		cesa_append_tdesc(cr, ctd);
+	STAILQ_INIT(&cp->cp_copyout);
+
+	return (0);
+}
+
+static int
+cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
+{
+	uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
+	uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN];
+	SHA1_CTX sha1ctx;
+	MD5_CTX md5ctx;
+	uint32_t *hout;
+	uint32_t *hin;
+	int i;
+
+	memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
+	memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
+	for (i = 0; i < mklen; i++) {
+		ipad[i] ^= mkey[i];
+		opad[i] ^= mkey[i];
+	}
+
+	hin = (uint32_t *)cs->cs_hiv_in;
+	hout = (uint32_t *)cs->cs_hiv_out;
+
+	switch (alg) {
+	case CRYPTO_MD5_HMAC:
+		MD5Init(&md5ctx);
+		MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN);
+		memcpy(hin, md5ctx.state, sizeof(md5ctx.state));
+		MD5Init(&md5ctx);
+		MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN);
+		memcpy(hout, md5ctx.state, sizeof(md5ctx.state));
+		break;
+	case CRYPTO_SHA1_HMAC:
+		SHA1Init(&sha1ctx);
+		SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN);
+		memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
+		SHA1Init(&sha1ctx);
+		SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN);
+		memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
+		hin[i] = htobe32(hin[i]);
+		hout[i] = htobe32(hout[i]);
+	}
+
+	return (0);
+}
+
+static int
+cesa_prep_aes_key(struct cesa_session *cs)
+{
+	uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
+	uint32_t *dkey;
+	int i;
+
+	rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8);
+
+	cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
+	dkey = (uint32_t *)cs->cs_aes_dkey;
+
+	switch (cs->cs_klen) {
+	case 16:
+		cs->cs_config |= CESA_CSH_AES_KLEN_128;
+		for (i = 0; i < 4; i++)
+			*dkey++ = htobe32(ek[4 * 10 + i]);
+		break;
+	case 24:
+		cs->cs_config |= CESA_CSH_AES_KLEN_192;
+		for (i = 0; i < 4; i++)
+			*dkey++ = htobe32(ek[4 * 12 + i]);
+		for (i = 0; i < 2; i++)
+			*dkey++ = htobe32(ek[4 * 11 + 2 + i]);
+		break;
+	case 32:
+		cs->cs_config |= CESA_CSH_AES_KLEN_256;
+		for (i = 0; i < 4; i++)
+			*dkey++ = htobe32(ek[4 * 14 + i]);
+		for (i = 0; i < 4; i++)
+			*dkey++ = htobe32(ek[4 * 13 + i]);
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	return (0);
+}
+
+static int
+cesa_is_hash(int alg)
+{
+
+	switch (alg) {
+	case CRYPTO_MD5:
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA1_HMAC:
+		return (1);
+	default:
+		return (0);
+	}
+}
+
+static void
+cesa_start_packet(struct cesa_packet *cp, unsigned int size)
+{
+
+	cp->cp_size = size;
+	cp->cp_offset = 0;
+	STAILQ_INIT(&cp->cp_copyin);
+	STAILQ_INIT(&cp->cp_copyout);
+}
+
+static int
+cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp,
+    bus_dma_segment_t *seg)
+{
+	struct cesa_tdma_desc *ctd;
+	unsigned int bsize;
+
+	/* Calculate size of block copy */
+	bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset);
+
+	if (bsize > 0) {
+		ctd = cesa_tdma_copy(sc, sc->sc_sram_base +
+		    CESA_DATA(cp->cp_offset), seg->ds_addr, bsize);
+		if (!ctd)
+			return (-ENOMEM);
+
+		STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq);
+
+		ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base +
+		    CESA_DATA(cp->cp_offset), bsize);
+		if (!ctd)
+			return (-ENOMEM);
+
+		STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq);
+
+		seg->ds_len -= bsize;
+		seg->ds_addr += bsize;
+		cp->cp_offset += bsize;
+	}
+
+	return (bsize);
+}
+
+static void
+cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	unsigned int mpsize, fragmented;
+	unsigned int mlen, mskip, tmlen;
+	struct cesa_chain_info *cci;
+	unsigned int elen, eskip;
+	unsigned int skip, len;
+	struct cesa_sa_desc *csd;
+	struct cesa_request *cr;
+	struct cesa_softc *sc;
+	struct cesa_packet cp;
+	bus_dma_segment_t seg;
+	uint32_t config;
+	int size;
+
+	cci = arg;
+	sc = cci->cci_sc;
+	cr = cci->cci_cr;
+
+	if (error) {
+		cci->cci_error = error;
+		return;
+	}
+
+	elen = cci->cci_enc ? cci->cci_enc->crd_len : 0;
+	eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0;
+	mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0;
+	mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
+
+	if (elen && mlen &&
+	    ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
+	    (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
+	    (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
+		/*
+		 * Data alignment in the request does not meet CESA requiremnts
+		 * for combined encryption/decryption and hashing. We have to
+		 * split the request to separate operations and process them
+		 * one by one.
+		 */
+		config = cci->cci_config;
+		if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
+			config &= ~CESA_CSHD_OP_MASK;
+
+			cci->cci_config = config | CESA_CSHD_MAC;
+			cci->cci_enc = NULL;
+			cci->cci_mac = cr->cr_mac;
+			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+
+			cci->cci_config = config | CESA_CSHD_ENC;
+			cci->cci_enc = cr->cr_enc;
+			cci->cci_mac = NULL;
+			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+		} else {
+			config &= ~CESA_CSHD_OP_MASK;
+
+			cci->cci_config = config | CESA_CSHD_ENC;
+			cci->cci_enc = cr->cr_enc;
+			cci->cci_mac = NULL;
+			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+
+			cci->cci_config = config | CESA_CSHD_MAC;
+			cci->cci_enc = NULL;
+			cci->cci_mac = cr->cr_mac;
+			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+		}
+
+		return;
+	}
+
+	tmlen = mlen;
+	fragmented = 0;
+	mpsize = CESA_MAX_PACKET_SIZE;
+	mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
+
+	if (elen && mlen) {
+		skip = MIN(eskip, mskip);
+		len = MAX(elen + eskip, mlen + mskip) - skip;
+	} else if (elen) {
+		skip = eskip;
+		len = elen;
+	} else {
+		skip = mskip;
+		len = mlen;
+	}
+
+	/* Start first packet in chain */
+	cesa_start_packet(&cp, MIN(mpsize, len));
+
+	while (nseg-- && len > 0) {
+		seg = *(segs++);
+
+		/*
+		 * Skip data in buffer on which neither ENC nor MAC operation
+		 * is requested.
+		 */
+		if (skip > 0) {
+			size = MIN(skip, seg.ds_len);
+			skip -= size;
+
+			seg.ds_addr += size;
+			seg.ds_len -= size;
+
+			if (eskip > 0)
+				eskip -= size;
+
+			if (mskip > 0)
+				mskip -= size;
+
+			if (seg.ds_len == 0)
+				continue;
+		}
+
+		while (1) {
+			/*
+			 * Fill in current packet with data. Break if there is
+			 * no more data in current DMA segment or an error
+			 * occured.
+			 */
+			size = cesa_fill_packet(sc, &cp, &seg);
+			if (size <= 0) {
+				error = -size;
+				break;
+			}
+
+			len -= size;
+
+			/* If packet is full, append it to the chain */
+			if (cp.cp_size == cp.cp_offset) {
+				csd = cesa_alloc_sdesc(sc, cr);
+				if (!csd) {
+					error = ENOMEM;
+					break;
+				}
+
+				/* Create SA descriptor for this packet */
+				csd->csd_cshd->cshd_config = cci->cci_config;
+				csd->csd_cshd->cshd_mac_total_dlen = tmlen;
+
+				/*
+				 * Enable fragmentation if request will not fit
+				 * into one packet.
+				 */
+				if (len > 0) {
+					if (!fragmented) {
+						fragmented = 1;
+						csd->csd_cshd->cshd_config |=
+						    CESA_CSHD_FRAG_FIRST;
+					} else
+						csd->csd_cshd->cshd_config |=
+						    CESA_CSHD_FRAG_MIDDLE;
+				} else if (fragmented)
+					csd->csd_cshd->cshd_config |=
+					    CESA_CSHD_FRAG_LAST;
+
+				if (eskip < cp.cp_size && elen > 0) {
+					csd->csd_cshd->cshd_enc_src =
+					    CESA_DATA(eskip);
+					csd->csd_cshd->cshd_enc_dst =
+					    CESA_DATA(eskip);
+					csd->csd_cshd->cshd_enc_dlen =
+					    MIN(elen, cp.cp_size - eskip);
+				}
+
+				if (mskip < cp.cp_size && mlen > 0) {
+					csd->csd_cshd->cshd_mac_src =
+					    CESA_DATA(mskip);
+					csd->csd_cshd->cshd_mac_dlen =
+					    MIN(mlen, cp.cp_size - mskip);
+				}
+
+				elen -= csd->csd_cshd->cshd_enc_dlen;
+				eskip -= MIN(eskip, cp.cp_size);
+				mlen -= csd->csd_cshd->cshd_mac_dlen;
+				mskip -= MIN(mskip, cp.cp_size);
+
+				cesa_dump_cshd(sc, csd->csd_cshd);
+
+				/* Append packet to the request */
+				error = cesa_append_packet(sc, cr, &cp, csd);
+				if (error)
+					break;
+
+				/* Start a new packet, as current is full */
+				cesa_start_packet(&cp, MIN(mpsize, len));
+			}
+		}
+
+		if (error)
+			break;
+	}
+
+	if (error) {
+		/*
+		 * Move all allocated resources to the request. They will be
+		 * freed later.
+		 */
+		STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin);
+		STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout);
+		cci->cci_error = error;
+	}
+}
+
+static void
+cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
+    bus_size_t size, int error)
+{
+
+	cesa_create_chain_cb(arg, segs, nseg, error);
+}
+
+static int
+cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
+{
+	struct cesa_chain_info cci;
+	struct cesa_tdma_desc *ctd;
+	uint32_t config;
+	int error;
+
+	error = 0;
+	CESA_LOCK_ASSERT(sc, sessions);
+
+	/* Create request metadata */
+	if (cr->cr_enc) {
+		if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC &&
+		    (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+			memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
+			    cr->cr_cs->cs_klen);
+		else
+			memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
+			    cr->cr_cs->cs_klen);
+	}
+
+	if (cr->cr_mac) {
+		memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
+		    CESA_MAX_HASH_LEN);
+		memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
+		    CESA_MAX_HASH_LEN);
+	}
+
+	ctd = cesa_tdma_copyin_sa_data(sc, cr);
+	if (!ctd)
+		return (ENOMEM);
+
+	cesa_append_tdesc(cr, ctd);
+
+	/* Prepare SA configuration */
+	config = cr->cr_cs->cs_config;
+
+	if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+		config |= CESA_CSHD_DECRYPT;
+	if (cr->cr_enc && !cr->cr_mac)
+		config |= CESA_CSHD_ENC;
+	if (!cr->cr_enc && cr->cr_mac)
+		config |= CESA_CSHD_MAC;
+	if (cr->cr_enc && cr->cr_mac)
+		config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
+		    CESA_CSHD_ENC_AND_MAC;
+
+	/* Create data packets */
+	cci.cci_sc = sc;
+	cci.cci_cr = cr;
+	cci.cci_enc = cr->cr_enc;
+	cci.cci_mac = cr->cr_mac;
+	cci.cci_config = config;
+	cci.cci_error = 0;
+
+	if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
+		error = bus_dmamap_load_uio(sc->sc_data_dtag,
+		    cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf,
+		    cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
+	else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF)
+		error = bus_dmamap_load_mbuf(sc->sc_data_dtag,
+		    cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf,
+		    cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
+	else
+		error = bus_dmamap_load(sc->sc_data_dtag,
+		    cr->cr_dmap, cr->cr_crp->crp_buf,
+		    cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci,
+		    BUS_DMA_NOWAIT);
+
+	if (!error)
+		cr->cr_dmap_loaded = 1;
+
+	if (cci.cci_error)
+		error = cci.cci_error;
+
+	if (error)
+		return (error);
+
+	/* Read back request metadata */
+	ctd = cesa_tdma_copyout_sa_data(sc, cr);
+	if (!ctd)
+		return (ENOMEM);
+
+	cesa_append_tdesc(cr, ctd);
+
+	return (0);
+}
+
+static void
+cesa_execute(struct cesa_softc *sc)
+{
+	struct cesa_tdma_desc *prev_ctd, *ctd;
+	struct cesa_request *prev_cr, *cr;
+
+	CESA_LOCK(sc, requests);
+
+	/*
+	 * If ready list is empty, there is nothing to execute. If queued list
+	 * is not empty, the hardware is busy and we cannot start another
+	 * execution.
+	 */
+	if (STAILQ_EMPTY(&sc->sc_ready_requests) ||
+	    !STAILQ_EMPTY(&sc->sc_queued_requests)) {
+		CESA_UNLOCK(sc, requests);
+		return;
+	}
+
+	/* Move all ready requests to queued list */
+	STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests);
+	STAILQ_INIT(&sc->sc_ready_requests);
+
+	/* Create one execution chain from all requests on the list */
+	if (STAILQ_FIRST(&sc->sc_queued_requests) !=
+	    STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) {
+		prev_cr = NULL;
+		cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD |
+		    BUS_DMASYNC_POSTWRITE);
+
+		STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
+			if (prev_cr) {
+				ctd = STAILQ_FIRST(&cr->cr_tdesc);
+				prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc,
+				    cesa_tdma_desc, ctd_stq);
+
+				prev_ctd->ctd_cthd->cthd_next =
+				    ctd->ctd_cthd_paddr;
+			}
+
+			prev_cr = cr;
+		}
+
+		cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD |
+		    BUS_DMASYNC_PREWRITE);
+	}
+
+	/* Start chain execution in hardware */
+	cr = STAILQ_FIRST(&sc->sc_queued_requests);
+	ctd = STAILQ_FIRST(&cr->cr_tdesc);
+
+	CESA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr);
+	CESA_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE);
+
+	CESA_UNLOCK(sc, requests);
+}
+
+static int
+cesa_setup_sram(struct cesa_softc *sc)
+{
+	phandle_t sram_node;
+	ihandle_t sram_ihandle;
+	pcell_t sram_handle, sram_reg;
+
+	if (OF_getprop(ofw_bus_get_node(sc->sc_dev), "sram-handle",
+	    (void *)&sram_handle, sizeof(sram_handle)) <= 0)
+		return (ENXIO);
+
+	sram_ihandle = (ihandle_t)sram_handle;
+	sram_ihandle = fdt32_to_cpu(sram_ihandle);
+	sram_node = OF_instance_to_package(sram_ihandle);
+
+	if (OF_getprop(sram_node, "reg", (void *)&sram_reg,
+	    sizeof(sram_reg)) <= 0)
+		return (ENXIO);
+
+	sc->sc_sram_base = fdt32_to_cpu(sram_reg);
+
+	return (0);
+}
+
+static int
+cesa_probe(device_t dev)
+{
+	if (!ofw_bus_is_compatible(dev, "mrvl,cesa"))
+		return (ENXIO);
+
+	device_set_desc(dev, "Marvell Cryptographic Engine and Security "
+	    "Accelerator");
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+cesa_attach(device_t dev)
+{
+	struct cesa_softc *sc;
+	uint32_t d, r;
+	int error;
+	int i;
+
+	sc = device_get_softc(dev);
+	sc->sc_blocked = 0;
+	sc->sc_error = 0;
+	sc->sc_dev = dev;
+
+	error = cesa_setup_sram(sc);
+	if (error) {
+		device_printf(dev, "could not setup SRAM\n");
+		return (error);
+	}
+
+	soc_id(&d, &r);
+
+	switch (d) {
+	case MV_DEV_88F6281:
+		sc->sc_tperr = 0;
+		break;
+	case MV_DEV_MV78100:
+	case MV_DEV_MV78100_Z0:
+		sc->sc_tperr = CESA_ICR_TPERR;
+		break;
+	default:
+		return (ENXIO);
+	}
+
+	/* Initialize mutexes */
+	mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev),
+	    "CESA Shared Data", MTX_DEF);
+	mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev),
+	    "CESA TDMA Descriptors Pool", MTX_DEF);
+	mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev),
+	    "CESA SA Descriptors Pool", MTX_DEF);
+	mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev),
+	    "CESA Requests Pool", MTX_DEF);
+	mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
+	    "CESA Sessions Pool", MTX_DEF);
+
+	/* Allocate I/O and IRQ resources */
+	error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res);
+	if (error) {
+		device_printf(dev, "could not allocate resources\n");
+		goto err0;
+	}
+
+	sc->sc_bsh = rman_get_bushandle(*(sc->sc_res));
+	sc->sc_bst = rman_get_bustag(*(sc->sc_res));
+
+	/* Setup interrupt handler */
+	error = bus_setup_intr(dev, sc->sc_res[1], INTR_TYPE_NET | INTR_MPSAFE,
+	    NULL, cesa_intr, sc, &(sc->sc_icookie));
+	if (error) {
+		device_printf(dev, "could not setup engine completion irq\n");
+		goto err1;
+	}
+
+	/* Create DMA tag for processed data */
+	error = bus_dma_tag_create(NULL,	/* parent */
+	    1, 0,				/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,		/* lowaddr */
+	    BUS_SPACE_MAXADDR,			/* highaddr */
+	    NULL, NULL,				/* filtfunc, filtfuncarg */
+	    CESA_MAX_REQUEST_SIZE,		/* maxsize */
+	    CESA_MAX_FRAGMENTS,			/* nsegments */
+	    CESA_MAX_REQUEST_SIZE, 0,		/* maxsegsz, flags */
+	    NULL, NULL,				/* lockfunc, lockfuncarg */
+	    &sc->sc_data_dtag);			/* dmat */
+	if (error)
+		goto err2;
+
+	/* Initialize data structures: TDMA Descriptors Pool */
+	error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm,
+	    CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc));
+	if (error)
+		goto err3;
+
+	STAILQ_INIT(&sc->sc_free_tdesc);
+	for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) {
+		sc->sc_tdesc[i].ctd_cthd =
+		    (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i;
+		sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr +
+		    (i * sizeof(struct cesa_tdma_hdesc));
+		STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i],
+		    ctd_stq);
+	}
+
+	/* Initialize data structures: SA Descriptors Pool */
+	error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm,
+	    CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc));
+	if (error)
+		goto err4;
+
+	STAILQ_INIT(&sc->sc_free_sdesc);
+	for (i = 0; i < CESA_SA_DESCRIPTORS; i++) {
+		sc->sc_sdesc[i].csd_cshd =
+		    (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i;
+		sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr +
+		    (i * sizeof(struct cesa_sa_hdesc));
+		STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i],
+		    csd_stq);
+	}
+
+	/* Initialize data structures: Requests Pool */
+	error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm,
+	    CESA_REQUESTS * sizeof(struct cesa_sa_data));
+	if (error)
+		goto err5;
+
+	STAILQ_INIT(&sc->sc_free_requests);
+	STAILQ_INIT(&sc->sc_ready_requests);
+	STAILQ_INIT(&sc->sc_queued_requests);
+	for (i = 0; i < CESA_REQUESTS; i++) {
+		sc->sc_requests[i].cr_csd =
+		    (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i;
+		sc->sc_requests[i].cr_csd_paddr =
+		    sc->sc_requests_cdm.cdm_paddr +
+		    (i * sizeof(struct cesa_sa_data));
+
+		/* Preallocate DMA maps */
+		error = bus_dmamap_create(sc->sc_data_dtag, 0,
+		    &sc->sc_requests[i].cr_dmap);
+		if (error && i > 0) {
+			i--;
+			do {
+				bus_dmamap_destroy(sc->sc_data_dtag,
+				    sc->sc_requests[i].cr_dmap);
+			} while (i--);
+
+			goto err6;
+		}
+
+		STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i],
+		    cr_stq);
+	}
+
+	/* Initialize data structures: Sessions Pool */
+	STAILQ_INIT(&sc->sc_free_sessions);
+	for (i = 0; i < CESA_SESSIONS; i++) {
+		sc->sc_sessions[i].cs_sid = i;
+		STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i],
+		    cs_stq);
+	}
+
+	/*
+	 * Initialize TDMA:
+	 * - Burst limit: 128 bytes,
+	 * - Outstanding reads enabled,
+	 * - No byte-swap.
+	 */
+	CESA_WRITE(sc, CESA_TDMA_CR, CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 |
+	    CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE);
+
+	/*
+	 * Initialize SA:
+	 * - SA descriptor is present at beginning of CESA SRAM,
+	 * - Multi-packet chain mode,
+	 * - Cooperation with TDMA enabled.
+	 */
+	CESA_WRITE(sc, CESA_SA_DPR, 0);
+	CESA_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA |
+	    CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE);
+
+	/* Unmask interrupts */
+	CESA_WRITE(sc, CESA_ICR, 0);
+	CESA_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr);
+	CESA_WRITE(sc, CESA_TDMA_ECR, 0);
+	CESA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS |
+	    CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT |
+	    CESA_TDMA_EMR_DATA_ERROR);
+
+	/* Register in OCF */
+	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
+	if (sc->sc_cid) {
+		device_printf(dev, "could not get crypto driver id\n");
+		goto err7;
+	}
+
+	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
+
+	return (0);
+err7:
+	for (i = 0; i < CESA_REQUESTS; i++)
+		bus_dmamap_destroy(sc->sc_data_dtag,
+		    sc->sc_requests[i].cr_dmap);
+err6:
+	cesa_free_dma_mem(&sc->sc_requests_cdm);
+err5:
+	cesa_free_dma_mem(&sc->sc_sdesc_cdm);
+err4:
+	cesa_free_dma_mem(&sc->sc_tdesc_cdm);
+err3:
+	bus_dma_tag_destroy(sc->sc_data_dtag);
+err2:
+	bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie);
+err1:
+	bus_release_resources(dev, cesa_res_spec, sc->sc_res);
+err0:
+	mtx_destroy(&sc->sc_sessions_lock);
+	mtx_destroy(&sc->sc_requests_lock);
+	mtx_destroy(&sc->sc_sdesc_lock);
+	mtx_destroy(&sc->sc_tdesc_lock);
+	mtx_destroy(&sc->sc_sc_lock);
+	return (ENXIO);
+}
+
+static int
+cesa_detach(device_t dev)
+{
+	struct cesa_softc *sc;
+	int i;
+ 
+	sc = device_get_softc(dev);
+
+	/* TODO: Wait for queued requests completion before shutdown. */
+
+	/* Mask interrupts */
+	CESA_WRITE(sc, CESA_ICM, 0);
+	CESA_WRITE(sc, CESA_TDMA_EMR, 0);
+
+	/* Unregister from OCF */
+	crypto_unregister_all(sc->sc_cid);
+
+	/* Free DMA Maps */
+	for (i = 0; i < CESA_REQUESTS; i++)
+		bus_dmamap_destroy(sc->sc_data_dtag,
+		    sc->sc_requests[i].cr_dmap);
+
+	/* Free DMA Memory */
+	cesa_free_dma_mem(&sc->sc_requests_cdm);
+	cesa_free_dma_mem(&sc->sc_sdesc_cdm);
+	cesa_free_dma_mem(&sc->sc_tdesc_cdm);
+
+	/* Free DMA Tag */
+	bus_dma_tag_destroy(sc->sc_data_dtag);
+
+	/* Stop interrupt */
+	bus_teardown_intr(dev, sc->sc_res[1], sc->sc_icookie);
+
+	/* Relase I/O and IRQ resources */
+	bus_release_resources(dev, cesa_res_spec, sc->sc_res);
+
+	/* Destory mutexes */
+	mtx_destroy(&sc->sc_sessions_lock);
+	mtx_destroy(&sc->sc_requests_lock);
+	mtx_destroy(&sc->sc_sdesc_lock);
+	mtx_destroy(&sc->sc_tdesc_lock);
+	mtx_destroy(&sc->sc_sc_lock);
+
+	return (0);
+}
+
+static void
+cesa_intr(void *arg)
+{
+	STAILQ_HEAD(, cesa_request) requests;
+	struct cesa_request *cr, *tmp;
+	struct cesa_softc *sc;
+	uint32_t ecr, icr;
+	int blocked;
+
+	sc = arg;
+
+	/* Ack interrupt */
+	ecr = CESA_READ(sc, CESA_TDMA_ECR);
+	CESA_WRITE(sc, CESA_TDMA_ECR, 0);
+	icr = CESA_READ(sc, CESA_ICR);
+	CESA_WRITE(sc, CESA_ICR, 0);
+
+	/* Check for TDMA errors */
+	if (ecr & CESA_TDMA_ECR_MISS) {
+		device_printf(sc->sc_dev, "TDMA Miss error detected!\n");
+		sc->sc_error = EIO;
+	}
+
+	if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) {
+		device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n");
+		sc->sc_error = EIO;
+	}
+
+	if (ecr & CESA_TDMA_ECR_BOTH_HIT) {
+		device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n");
+		sc->sc_error = EIO;
+	}
+
+	if (ecr & CESA_TDMA_ECR_DATA_ERROR) {
+		device_printf(sc->sc_dev, "TDMA Data error detected!\n");
+		sc->sc_error = EIO;
+	}
+
+	/* Check for CESA errors */
+	if (icr & sc->sc_tperr) {
+		device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n");
+		sc->sc_error = EIO;
+	}
+
+	/* If there is nothing more to do, return */
+	if ((icr & CESA_ICR_ACCTDMA) == 0)
+		return;
+
+	/* Get all finished requests */
+	CESA_LOCK(sc, requests);
+	STAILQ_INIT(&requests);
+	STAILQ_CONCAT(&requests, &sc->sc_queued_requests);
+	STAILQ_INIT(&sc->sc_queued_requests);
+	CESA_UNLOCK(sc, requests);
+
+	/* Execute all ready requests */
+	cesa_execute(sc);
+
+	/* Process completed requests */
+	cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD |
+	    BUS_DMASYNC_POSTWRITE);
+
+	STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) {
+		bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap,
+		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+		cr->cr_crp->crp_etype = sc->sc_error;
+		if (cr->cr_mac)
+			crypto_copyback(cr->cr_crp->crp_flags,
+			    cr->cr_crp->crp_buf, cr->cr_mac->crd_inject,
+			    cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
+
+		crypto_done(cr->cr_crp);
+		cesa_free_request(sc, cr);
+	}
+
+	cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD |
+	    BUS_DMASYNC_PREWRITE);
+
+	sc->sc_error = 0;
+
+	/* Unblock driver if it ran out of resources */
+	CESA_LOCK(sc, sc);
+	blocked = sc->sc_blocked;
+	sc->sc_blocked = 0;
+	CESA_UNLOCK(sc, sc);
+
+	if (blocked)
+		crypto_unblock(sc->sc_cid, blocked);
+}
+
+static int
+cesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
+{
+	struct cesa_session *cs;
+	struct cesa_softc *sc;
+	struct cryptoini *enc;
+	struct cryptoini *mac;
+	int error;
+ 
+	sc = device_get_softc(dev);
+	enc = NULL;
+	mac = NULL;
+	error = 0;
+
+	/* Check and parse input */
+	if (cesa_is_hash(cri->cri_alg))
+		mac = cri;
+	else
+		enc = cri;
+
+	cri = cri->cri_next;
+
+	if (cri) {
+		if (!enc && !cesa_is_hash(cri->cri_alg))
+			enc = cri;
+
+		if (!mac && cesa_is_hash(cri->cri_alg))
+			mac = cri;
+
+		if (cri->cri_next || !(enc && mac))
+			return (EINVAL);
+	}
+
+	if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) ||
+	    (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN))
+		return (E2BIG);
+
+	/* Allocate session */
+	cs = cesa_alloc_session(sc);
+	if (!cs)
+		return (ENOMEM);
+
+	/* Prepare CESA configuration */
+	cs->cs_config = 0;
+	cs->cs_ivlen = 1;
+	cs->cs_mblen = 1;
+
+	if (enc) {
+		switch (enc->cri_alg) {
+		case CRYPTO_AES_CBC:
+			cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
+			cs->cs_ivlen = AES_BLOCK_LEN;
+			break;
+		case CRYPTO_DES_CBC:
+			cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
+			cs->cs_ivlen = DES_BLOCK_LEN;
+			break;
+		case CRYPTO_3DES_CBC:
+			cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
+			    CESA_CSHD_CBC;
+			cs->cs_ivlen = DES3_BLOCK_LEN;
+			break;
+		default:
+			error = EINVAL;
+			break;
+		}
+	}
+
+	if (!error && mac) {
+		switch (mac->cri_alg) {
+		case CRYPTO_MD5:
+			cs->cs_config |= CESA_CSHD_MD5;
+			cs->cs_mblen = 1;
+			cs->cs_hlen = MD5_HASH_LEN;
+			break;
+		case CRYPTO_MD5_HMAC:
+			cs->cs_config |= CESA_CSHD_MD5_HMAC;
+			cs->cs_mblen = MD5_HMAC_BLOCK_LEN;
+			cs->cs_hlen = CESA_HMAC_HASH_LENGTH;
+			break;
+		case CRYPTO_SHA1:
+			cs->cs_config |= CESA_CSHD_SHA1;
+			cs->cs_mblen = 1;
+			cs->cs_hlen = SHA1_HASH_LEN;
+			break;
+		case CRYPTO_SHA1_HMAC:
+			cs->cs_config |= CESA_CSHD_SHA1_HMAC;
+			cs->cs_mblen = SHA1_HMAC_BLOCK_LEN;
+			cs->cs_hlen = CESA_HMAC_HASH_LENGTH;
+			break;
+		default:
+			error = EINVAL;
+			break;
+		}
+	}
+
+	/* Save cipher key */
+	if (!error && enc && enc->cri_key) {
+		cs->cs_klen = enc->cri_klen / 8;
+		memcpy(cs->cs_key, enc->cri_key, cs->cs_klen);
+		if (enc->cri_alg == CRYPTO_AES_CBC)
+			error = cesa_prep_aes_key(cs);
+	}
+
+	/* Save digest key */
+	if (!error && mac && mac->cri_key)
+		error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key,
+		    mac->cri_klen / 8);
+
+	if (error) {
+		cesa_free_session(sc, cs);
+		return (EINVAL);
+	}
+
+	*sidp = cs->cs_sid;
+
+	return (0);
+}
+
+static int
+cesa_freesession(device_t dev, uint64_t tid)
+{
+	struct cesa_session *cs;
+	struct cesa_softc *sc;
+ 
+	sc = device_get_softc(dev);
+	cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid));
+	if (!cs)
+		return (EINVAL);
+
+	/* Free session */
+	cesa_free_session(sc, cs);
+
+	return (0);
+}
+
+static int
+cesa_process(device_t dev, struct cryptop *crp, int hint)
+{
+	struct cesa_request *cr;
+	struct cesa_session *cs;
+	struct cryptodesc *crd;
+	struct cryptodesc *enc;
+	struct cryptodesc *mac;
+	struct cesa_softc *sc;
+	int error;
+
+	sc = device_get_softc(dev);
+	crd = crp->crp_desc;
+	enc = NULL;
+	mac = NULL;
+	error = 0;
+
+	/* Check session ID */
+	cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid));
+	if (!cs) {
+		crp->crp_etype = EINVAL;
+		crypto_done(crp);
+		return (0);
+	}
+
+	/* Check and parse input */
+	if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) {
+		crp->crp_etype = E2BIG;
+		crypto_done(crp);
+		return (0);
+	}
+
+	if (cesa_is_hash(crd->crd_alg))
+		mac = crd;
+	else
+		enc = crd;
+
+	crd = crd->crd_next;
+
+	if (crd) {
+		if (!enc && !cesa_is_hash(crd->crd_alg))
+			enc = crd;
+
+		if (!mac && cesa_is_hash(crd->crd_alg))
+			mac = crd;
+
+		if (crd->crd_next || !(enc && mac)) {
+			crp->crp_etype = EINVAL;
+			crypto_done(crp);
+			return (0);
+		}
+	}
+
+	/*
+	 * Get request descriptor. Block driver if there is no free
+	 * descriptors in pool.
+	 */
+	cr = cesa_alloc_request(sc);
+	if (!cr) {
+		CESA_LOCK(sc, sc);
+		sc->sc_blocked = CRYPTO_SYMQ;
+		CESA_UNLOCK(sc, sc);
+		return (ERESTART);
+	}
+
+	/* Prepare request */
+	cr->cr_crp = crp;
+	cr->cr_enc = enc;
+	cr->cr_mac = mac;
+	cr->cr_cs = cs;
+
+	CESA_LOCK(sc, sessions);
+	cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
+		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
+		else
+			arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0);
+
+		if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
+			crypto_copyback(crp->crp_flags, crp->crp_buf,
+			    enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
+	} else if (enc) {
+		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
+		else
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
+	}
+
+	if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
+		if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) {
+			cs->cs_klen = enc->crd_klen / 8;
+			memcpy(cs->cs_key, enc->crd_key, cs->cs_klen);
+			if (enc->crd_alg == CRYPTO_AES_CBC)
+				error = cesa_prep_aes_key(cs);
+		} else
+			error = E2BIG;
+	}
+
+	if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
+		if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN)
+			error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key,
+			    mac->crd_klen / 8);
+		else
+			error = E2BIG;
+	}
+
+	/* Convert request to chain of TDMA and SA descriptors */
+	if (!error)
+		error = cesa_create_chain(sc, cr);
+
+	cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	CESA_UNLOCK(sc, sessions);
+
+	if (error) {
+		cesa_free_request(sc, cr);
+		crp->crp_etype = error;
+		crypto_done(crp);
+		return (0);
+	}
+
+	bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD |
+	    BUS_DMASYNC_PREWRITE);
+
+	/* Enqueue request to execution */
+	cesa_enqueue_request(sc, cr);
+
+	/* Start execution, if we have no more requests in queue */
+	if ((hint & CRYPTO_HINT_MORE) == 0)
+		cesa_execute(sc);
+
+	return (0);
+}
diff --git a/sys/dev/cesa/cesa.h b/sys/dev/cesa/cesa.h
new file mode 100644
index 0000000..2c953f0
--- /dev/null
+++ b/sys/dev/cesa/cesa.h
@@ -0,0 +1,350 @@
+/*-
+ * Copyright (C) 2009-2011 Semihalf.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _DEV_CESA_H_
+#define _DEV_CESA_H_
+
+/* Maximum number of allocated sessions */
+#define CESA_SESSIONS			64
+
+/* Maximum number of queued requests */
+#define CESA_REQUESTS			256
+
+/*
+ * CESA is able to process data only in CESA SRAM, which is quite small (2 kB).
+ * We have to fit a packet there, which contains SA descriptor, keys, IV
+ * and data to be processed. Every request must be converted into chain of
+ * packets and each packet can hold about 1.75 kB of data.
+ *
+ * To process each packet we need at least 1 SA descriptor and at least 4 TDMA
+ * descriptors. However there are cases when we use 2 SA and 8 TDMA descriptors
+ * per packet. Number of used TDMA descriptors can increase beyond given values
+ * if data in the request is fragmented in physical memory.
+ *
+ * The driver uses preallocated SA and TDMA descriptors pools to get best
+ * performace. Size of these pools should match expected request size. Example:
+ *
+ * Expected average request size:			1.5 kB (Ethernet MTU)
+ * Packets per average request:				(1.5 kB / 1.75 kB) = 1
+ * SA decriptors per average request (worst case):	1 * 2 = 2
+ * TDMA desctiptors per average request (worst case):	1 * 8 = 8
+ *
+ * More TDMA descriptors should be allocated, if data fragmentation is expected
+ * (for example while processing mbufs larger than MCLBYTES). The driver may use
+ * 2 additional TDMA descriptors per each discontinuity in the physical data
+ * layout.
+ */
+
+/* Values below are optimized for requests containing about 1.5 kB of data */
+#define CESA_SA_DESC_PER_REQ		2
+#define CESA_TDMA_DESC_PER_REQ		8
+
+#define CESA_SA_DESCRIPTORS		(CESA_SA_DESC_PER_REQ * CESA_REQUESTS)
+#define CESA_TDMA_DESCRIPTORS		(CESA_TDMA_DESC_PER_REQ * CESA_REQUESTS)
+
+/* Useful constants */
+#define CESA_HMAC_HASH_LENGTH		12
+#define CESA_MAX_FRAGMENTS		64
+#define CESA_SRAM_SIZE			2048
+
+/*
+ * CESA_MAX_HASH_LEN is maximum length of hash generated by CESA.
+ * As CESA suports only MD5 and SHA1 this equals to 20 bytes.
+ * However we increase the value to 24 bytes to meet alignment
+ * requirements in cesa_sa_data structure.
+ */
+#define CESA_MAX_HASH_LEN		24
+#define CESA_MAX_KEY_LEN		32
+#define CESA_MAX_IV_LEN			16
+#define CESA_MAX_HMAC_BLOCK_LEN		64
+#define CESA_MAX_MKEY_LEN		CESA_MAX_HMAC_BLOCK_LEN
+#define CESA_MAX_PACKET_SIZE		(CESA_SRAM_SIZE - CESA_DATA(0))
+#define CESA_MAX_REQUEST_SIZE		65535
+
+/* Locking macros */
+#define CESA_LOCK(sc, what)		mtx_lock(&(sc)->sc_ ## what ## _lock)
+#define CESA_UNLOCK(sc, what)		mtx_unlock(&(sc)->sc_ ## what ## _lock)
+#define CESA_LOCK_ASSERT(sc, what)	\
+	mtx_assert(&(sc)->sc_ ## what ## _lock, MA_OWNED)
+
+/* Registers read/write macros */
+#define CESA_READ(sc, reg)		\
+	bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg))
+#define CESA_WRITE(sc, reg, val)	\
+	bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
+
+/* Generic allocator for objects */
+#define CESA_GENERIC_ALLOC_LOCKED(sc, obj, pool) do {		\
+	CESA_LOCK(sc, pool);					\
+								\
+	if (STAILQ_EMPTY(&(sc)->sc_free_ ## pool))		\
+		obj = NULL;					\
+	else {							\
+		obj = STAILQ_FIRST(&(sc)->sc_free_ ## pool);	\
+		STAILQ_REMOVE_HEAD(&(sc)->sc_free_ ## pool,	\
+		    obj ## _stq);				\
+	}							\
+								\
+	CESA_UNLOCK(sc, pool);					\
+} while (0)
+
+#define CESA_GENERIC_FREE_LOCKED(sc, obj, pool) do {		\
+	CESA_LOCK(sc, pool);					\
+	STAILQ_INSERT_TAIL(&(sc)->sc_free_ ## pool, obj,	\
+	    obj ## _stq);					\
+	CESA_UNLOCK(sc, pool);					\
+} while (0)
+
+/* CESA SRAM offset calculation macros */
+#define CESA_SA_DATA(member)					\
+	(sizeof(struct cesa_sa_hdesc) + offsetof(struct cesa_sa_data, member))
+#define CESA_DATA(offset)					\
+	(sizeof(struct cesa_sa_hdesc) + sizeof(struct cesa_sa_data) + offset)
+
+struct cesa_tdma_hdesc {
+	uint16_t	cthd_byte_count;
+	uint16_t	cthd_flags;
+	uint32_t	cthd_src;
+	uint32_t	cthd_dst;
+	uint32_t	cthd_next;
+};
+
+struct cesa_sa_hdesc {
+	uint32_t	cshd_config;
+	uint16_t	cshd_enc_src;
+	uint16_t	cshd_enc_dst;
+	uint32_t	cshd_enc_dlen;
+	uint32_t	cshd_enc_key;
+	uint16_t	cshd_enc_iv;
+	uint16_t	cshd_enc_iv_buf;
+	uint16_t	cshd_mac_src;
+	uint16_t	cshd_mac_total_dlen;
+	uint16_t	cshd_mac_dst;
+	uint16_t	cshd_mac_dlen;
+	uint16_t	cshd_mac_iv_in;
+	uint16_t	cshd_mac_iv_out;
+};
+
+struct cesa_sa_data {
+	uint8_t		csd_key[CESA_MAX_KEY_LEN];
+	uint8_t		csd_iv[CESA_MAX_IV_LEN];
+	uint8_t		csd_hiv_in[CESA_MAX_HASH_LEN];
+	uint8_t		csd_hiv_out[CESA_MAX_HASH_LEN];
+	uint8_t		csd_hash[CESA_MAX_HASH_LEN];
+};
+
+struct cesa_dma_mem {
+	void		*cdm_vaddr;
+	bus_addr_t	cdm_paddr;
+	bus_dma_tag_t	cdm_tag;
+	bus_dmamap_t	cdm_map;
+};
+
+struct cesa_tdma_desc {
+	struct cesa_tdma_hdesc		*ctd_cthd;
+	bus_addr_t			ctd_cthd_paddr;
+
+	STAILQ_ENTRY(cesa_tdma_desc)	ctd_stq;
+};
+
+struct cesa_sa_desc {
+	struct cesa_sa_hdesc		*csd_cshd;
+	bus_addr_t			csd_cshd_paddr;
+
+	STAILQ_ENTRY(cesa_sa_desc)	csd_stq;
+};
+
+struct cesa_session {
+	uint32_t			cs_sid;
+	uint32_t			cs_config;
+	unsigned int			cs_klen;
+	unsigned int			cs_ivlen;
+	unsigned int			cs_hlen;
+	unsigned int			cs_mblen;
+	uint8_t				cs_key[CESA_MAX_KEY_LEN];
+	uint8_t				cs_aes_dkey[CESA_MAX_KEY_LEN];
+	uint8_t				cs_hiv_in[CESA_MAX_HASH_LEN];
+	uint8_t				cs_hiv_out[CESA_MAX_HASH_LEN];
+
+	STAILQ_ENTRY(cesa_session)	cs_stq;
+};
+
+struct cesa_request {
+	struct cesa_sa_data		*cr_csd;
+	bus_addr_t			cr_csd_paddr;
+	struct cryptop			*cr_crp;
+	struct cryptodesc		*cr_enc;
+	struct cryptodesc		*cr_mac;
+	struct cesa_session		*cr_cs;
+	bus_dmamap_t			cr_dmap;
+	int				cr_dmap_loaded;
+
+	STAILQ_HEAD(, cesa_tdma_desc)	cr_tdesc;
+	STAILQ_HEAD(, cesa_sa_desc)	cr_sdesc;
+
+	STAILQ_ENTRY(cesa_request)	cr_stq;
+};
+
+struct cesa_packet {
+	STAILQ_HEAD(, cesa_tdma_desc)	cp_copyin;
+	STAILQ_HEAD(, cesa_tdma_desc)	cp_copyout;
+	unsigned int			cp_size;
+	unsigned int			cp_offset;
+};
+
+struct cesa_softc {
+	device_t			sc_dev;
+	int32_t				sc_cid;
+	struct resource			*sc_res[2];
+	void				*sc_icookie;
+	bus_dma_tag_t			sc_data_dtag;
+	bus_space_tag_t			sc_bst;
+	bus_space_handle_t		sc_bsh;
+	int				sc_error;
+	int				sc_tperr;
+
+	struct mtx			sc_sc_lock;
+	int				sc_blocked;
+
+	/* TDMA descriptors pool */
+	struct mtx			sc_tdesc_lock;
+	struct cesa_tdma_desc		sc_tdesc[CESA_TDMA_DESCRIPTORS];
+	struct cesa_dma_mem		sc_tdesc_cdm;
+	STAILQ_HEAD(, cesa_tdma_desc)	sc_free_tdesc;
+
+	/* SA descriptors pool */
+	struct mtx			sc_sdesc_lock;
+	struct cesa_sa_desc		sc_sdesc[CESA_SA_DESCRIPTORS];
+	struct cesa_dma_mem		sc_sdesc_cdm;
+	STAILQ_HEAD(, cesa_sa_desc)	sc_free_sdesc;
+
+	/* Requests pool */
+	struct mtx			sc_requests_lock;
+	struct cesa_request		sc_requests[CESA_REQUESTS];
+	struct cesa_dma_mem		sc_requests_cdm;
+	STAILQ_HEAD(, cesa_request)	sc_free_requests;
+	STAILQ_HEAD(, cesa_request)	sc_ready_requests;
+	STAILQ_HEAD(, cesa_request)	sc_queued_requests;
+
+	/* Sessions pool */
+	struct mtx			sc_sessions_lock;
+	struct cesa_session		sc_sessions[CESA_SESSIONS];
+	STAILQ_HEAD(, cesa_session)	sc_free_sessions;
+
+	/* CESA SRAM Address */
+	bus_addr_t			sc_sram_base;
+};
+
+struct cesa_chain_info {
+	struct cesa_softc		*cci_sc;
+	struct cesa_request		*cci_cr;
+	struct cryptodesc		*cci_enc;
+	struct cryptodesc		*cci_mac;
+	uint32_t			cci_config;
+	int				cci_error;
+};
+
+/* CESA descriptors flags definitions */
+#define CESA_CTHD_OWNED			(1 << 15)
+
+#define CESA_CSHD_MAC			(0 << 0)
+#define CESA_CSHD_ENC			(1 << 0)
+#define CESA_CSHD_MAC_AND_ENC		(2 << 0)
+#define CESA_CSHD_ENC_AND_MAC		(3 << 0)
+#define CESA_CSHD_OP_MASK		(3 << 0)
+
+#define CESA_CSHD_MD5			(4 << 4)
+#define CESA_CSHD_SHA1			(5 << 4)
+#define CESA_CSHD_MD5_HMAC		((6 << 4) | (1 << 7))
+#define CESA_CSHD_SHA1_HMAC		((7 << 4) | (1 << 7))
+
+#define CESA_CSHD_DES			(1 << 8)
+#define CESA_CSHD_3DES			(2 << 8)
+#define CESA_CSHD_AES			(3 << 8)
+
+#define CESA_CSHD_DECRYPT		(1 << 12)
+#define CESA_CSHD_CBC			(1 << 16)
+#define CESA_CSHD_3DES_EDE		(1 << 20)
+
+#define CESA_CSH_AES_KLEN_128		(0 << 24)
+#define CESA_CSH_AES_KLEN_192		(1 << 24)
+#define CESA_CSH_AES_KLEN_256		(2 << 24)
+#define CESA_CSH_AES_KLEN_MASK		(3 << 24)
+
+#define CESA_CSHD_FRAG_FIRST		(1 << 30)
+#define CESA_CSHD_FRAG_LAST		(2 << 30)
+#define CESA_CSHD_FRAG_MIDDLE		(3 << 30)
+
+/* CESA registers definitions */
+#define CESA_ICR			0xDE20
+#define CESA_ICR_ACCTDMA		(1 << 7)
+#define CESA_ICR_TPERR			(1 << 12)
+
+#define CESA_ICM			0xDE24
+#define CESA_ICM_ACCTDMA		CESA_ICR_ACCTDMA
+#define CESA_ICM_TPERR			CESA_ICR_TPERR
+
+/* CESA TDMA registers definitions */
+#define CESA_TDMA_ND			0x0830
+
+#define CESA_TDMA_CR			0x0840
+#define CESA_TDMA_CR_DBL128		(4 << 0)
+#define CESA_TDMA_CR_ORDEN		(1 << 4)
+#define CESA_TDMA_CR_SBL128		(4 << 6)
+#define CESA_TDMA_CR_NBS		(1 << 11)
+#define CESA_TDMA_CR_ENABLE		(1 << 12)
+#define CESA_TDMA_CR_FETCHND		(1 << 13)
+#define CESA_TDMA_CR_ACTIVE		(1 << 14)
+
+#define CESA_TDMA_ECR			0x08C8
+#define CESA_TDMA_ECR_MISS		(1 << 0)
+#define CESA_TDMA_ECR_DOUBLE_HIT	(1 << 1)
+#define CESA_TDMA_ECR_BOTH_HIT		(1 << 2)
+#define CESA_TDMA_ECR_DATA_ERROR	(1 << 3)
+
+#define CESA_TDMA_EMR			0x08CC
+#define CESA_TDMA_EMR_MISS		CESA_TDMA_ECR_MISS
+#define CESA_TDMA_EMR_DOUBLE_HIT	CESA_TDMA_ECR_DOUBLE_HIT
+#define CESA_TDMA_EMR_BOTH_HIT		CESA_TDMA_ECR_BOTH_HIT
+#define CESA_TDMA_EMR_DATA_ERROR	CESA_TDMA_ECR_DATA_ERROR
+
+/* CESA SA registers definitions */
+#define CESA_SA_CMD			0xDE00
+#define CESA_SA_CMD_ACTVATE		(1 << 0)
+
+#define CESA_SA_DPR			0xDE04
+
+#define CESA_SA_CR			0xDE08
+#define CESA_SA_CR_WAIT_FOR_TDMA	(1 << 7)
+#define CESA_SA_CR_ACTIVATE_TDMA	(1 << 9)
+#define CESA_SA_CR_MULTI_MODE		(1 << 11)
+
+#define CESA_SA_SR			0xDE0C
+#define CESA_SA_SR_ACTIVE		(1 << 0)
+
+#endif
diff --git a/sys/dev/ciss/ciss.c b/sys/dev/ciss/ciss.c
index 016d5f3..b530de1 100644
--- a/sys/dev/ciss/ciss.c
+++ b/sys/dev/ciss/ciss.c
@@ -105,7 +105,8 @@
 #include <dev/ciss/cissio.h>
 #include <dev/ciss/cissvar.h>
 
-MALLOC_DEFINE(CISS_MALLOC_CLASS, "ciss_data", "ciss internal data buffers");
+static MALLOC_DEFINE(CISS_MALLOC_CLASS, "ciss_data",
+    "ciss internal data buffers");
 
 /* pci interface */
 static int	ciss_lookup(device_t dev);
diff --git a/sys/dev/cs/if_cs.c b/sys/dev/cs/if_cs.c
index 3b75efc..3789ea4 100644
--- a/sys/dev/cs/if_cs.c
+++ b/sys/dev/cs/if_cs.c
@@ -103,7 +103,7 @@ devclass_t cs_devclass;
 driver_intr_t	csintr;
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, cs, CTLFLAG_RD, 0, "cs device parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, cs, CTLFLAG_RD, 0, "cs device parameters");
 
 int	cs_ignore_cksum_failure = 0;
 TUNABLE_INT("hw.cs.ignore_checksum_failure", &cs_ignore_cksum_failure);
diff --git a/sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c b/sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c
index be22676..926b445 100644
--- a/sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c
+++ b/sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c
@@ -78,7 +78,7 @@ __FBSDID("$FreeBSD$");
 
 /* Avoid clutter in the hw.* space, keep all toe tunables within hw.cxgb */
 SYSCTL_DECL(_hw_cxgb);
-SYSCTL_NODE(_hw_cxgb, OID_AUTO, toe, CTLFLAG_RD, 0, "TOE parameters");
+static SYSCTL_NODE(_hw_cxgb, OID_AUTO, toe, CTLFLAG_RD, 0, "TOE parameters");
 
 static struct tom_tunables default_tunable_vals = {
 	.max_host_sndbuf = 32 * 1024,
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 18b813d..b7cb09a 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -129,7 +129,8 @@ MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
 /*
  * Tunables.
  */
-SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0, "cxgbe driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0,
+    "cxgbe driver parameters");
 
 static int force_firmware_install = 0;
 TUNABLE_INT("hw.cxgbe.force_firmware_install", &force_firmware_install);
diff --git a/sys/dev/dc/dcphy.c b/sys/dev/dc/dcphy.c
index 5c60ad5..ab44993 100644
--- a/sys/dev/dc/dcphy.c
+++ b/sys/dev/dc/dcphy.c
@@ -294,7 +294,7 @@ static void
 dcphy_status(struct mii_softc *sc)
 {
 	struct mii_data *mii = sc->mii_pdata;
-	int reg, anlpar, tstat = 0;
+	int anlpar, tstat;
 	struct dc_softc		*dc_sc;
 
 	dc_sc = mii->mii_ifp->if_softc;
@@ -305,13 +305,12 @@ dcphy_status(struct mii_softc *sc)
 	if ((mii->mii_ifp->if_flags & IFF_UP) == 0)
 		return;
 
-	reg = CSR_READ_4(dc_sc, DC_10BTSTAT);
-	if (!(reg & DC_TSTAT_LS10) || !(reg & DC_TSTAT_LS100))
+	tstat = CSR_READ_4(dc_sc, DC_10BTSTAT);
+	if (!(tstat & DC_TSTAT_LS10) || !(tstat & DC_TSTAT_LS100))
 		mii->mii_media_status |= IFM_ACTIVE;
 
 	if (CSR_READ_4(dc_sc, DC_10BTCTRL) & DC_TCTL_AUTONEGENBL) {
 		/* Erg, still trying, I guess... */
-		tstat = CSR_READ_4(dc_sc, DC_10BTSTAT);
 		if ((tstat & DC_TSTAT_ANEGSTAT) != DC_ASTAT_AUTONEGCMP) {
 			if ((DC_IS_MACRONIX(dc_sc) || DC_IS_PNICII(dc_sc)) &&
 			    (tstat & DC_TSTAT_ANEGSTAT) == DC_ASTAT_DISABLE)
@@ -351,9 +350,9 @@ dcphy_status(struct mii_softc *sc)
 		 * and hope that the user is clever enough to manually
 		 * change the media settings if we're wrong.
 		 */
-		if (!(reg & DC_TSTAT_LS100))
+		if (!(tstat & DC_TSTAT_LS100))
 			mii->mii_media_active |= IFM_100_TX | IFM_HDX;
-		else if (!(reg & DC_TSTAT_LS10))
+		else if (!(tstat & DC_TSTAT_LS10))
 			mii->mii_media_active |= IFM_10_T | IFM_HDX;
 		else
 			mii->mii_media_active |= IFM_NONE;
diff --git a/sys/dev/dc/if_dc.c b/sys/dev/dc/if_dc.c
index 184fa8f..b9ea19a 100644
--- a/sys/dev/dc/if_dc.c
+++ b/sys/dev/dc/if_dc.c
@@ -122,6 +122,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -149,7 +150,7 @@ MODULE_DEPEND(dc, miibus, 1, 1, 1);
 /*
  * Various supported device vendors/types and their names.
  */
-static const struct dc_type dc_devs[] = {
+static const struct dc_type const dc_devs[] = {
 	{ DC_DEVID(DC_VENDORID_DEC, DC_DEVICEID_21143), 0,
 		"Intel 21143 10/100BaseTX" },
 	{ DC_DEVID(DC_VENDORID_DAVICOM, DC_DEVICEID_DM9009), 0,
@@ -272,12 +273,6 @@ static void dc_eeprom_getword_xircom(struct dc_softc *, int, uint16_t *);
 static void dc_eeprom_width(struct dc_softc *);
 static void dc_read_eeprom(struct dc_softc *, caddr_t, int, int, int);
 
-static void dc_mii_writebit(struct dc_softc *, int);
-static int dc_mii_readbit(struct dc_softc *);
-static void dc_mii_sync(struct dc_softc *);
-static void dc_mii_send(struct dc_softc *, uint32_t, int);
-static int dc_mii_readreg(struct dc_softc *, struct dc_mii_frame *);
-static int dc_mii_writereg(struct dc_softc *, struct dc_mii_frame *);
 static int dc_miibus_readreg(device_t, int, int);
 static int dc_miibus_writereg(device_t, int, int, int);
 static void dc_miibus_statchg(device_t);
@@ -307,6 +302,24 @@ static int dc_decode_leaf_sym(struct dc_softc *, struct dc_eblock_sym *);
 static void dc_apply_fixup(struct dc_softc *, int);
 static int dc_check_multiport(struct dc_softc *);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t dc_mii_bitbang_read(device_t);
+static void dc_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops dc_mii_bitbang_ops = {
+	dc_mii_bitbang_read,
+	dc_mii_bitbang_write,
+	{
+		DC_SIO_MII_DATAOUT,	/* MII_BIT_MDO */
+		DC_SIO_MII_DATAIN,	/* MII_BIT_MDI */
+		DC_SIO_MII_CLK,		/* MII_BIT_MDC */
+		0,			/* MII_BIT_DIR_HOST_PHY */
+		DC_SIO_MII_DIR,		/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 #ifdef DC_USEIOSPACE
 #define	DC_RES			SYS_RES_IOPORT
 #define	DC_RID			DC_PCI_CFBIO
@@ -611,185 +624,45 @@ dc_read_eeprom(struct dc_softc *sc, caddr_t dest, int off, int cnt, int be)
 }
 
 /*
- * The following two routines are taken from the Macronix 98713
- * Application Notes pp.19-21.
- */
-/*
- * Write a bit to the MII bus.
+ * Write the MII serial port for the MII bit-bang module.
  */
 static void
-dc_mii_writebit(struct dc_softc *sc, int bit)
+dc_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	uint32_t reg;
+	struct dc_softc *sc;
 
-	reg = DC_SIO_ROMCTL_WRITE | (bit != 0 ? DC_SIO_MII_DATAOUT : 0);
-	CSR_WRITE_4(sc, DC_SIO, reg);
-	CSR_BARRIER_4(sc, DC_SIO,
-	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
+	sc = device_get_softc(dev);
 
-	CSR_WRITE_4(sc, DC_SIO, reg | DC_SIO_MII_CLK);
-	CSR_BARRIER_4(sc, DC_SIO,
-	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
-	CSR_WRITE_4(sc, DC_SIO, reg);
+	CSR_WRITE_4(sc, DC_SIO, val);
 	CSR_BARRIER_4(sc, DC_SIO,
 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
 }
 
 /*
- * Read a bit from the MII bus.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static int
-dc_mii_readbit(struct dc_softc *sc)
+static uint32_t
+dc_mii_bitbang_read(device_t dev)
 {
-	uint32_t reg;
-
-	reg = DC_SIO_ROMCTL_READ | DC_SIO_MII_DIR;
-	CSR_WRITE_4(sc, DC_SIO, reg);
-	CSR_BARRIER_4(sc, DC_SIO,
-	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
-	(void)CSR_READ_4(sc, DC_SIO);
-	CSR_WRITE_4(sc, DC_SIO, reg | DC_SIO_MII_CLK);
-	CSR_BARRIER_4(sc, DC_SIO,
-	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
-	CSR_WRITE_4(sc, DC_SIO, reg);
-	CSR_BARRIER_4(sc, DC_SIO,
-	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
-	if (CSR_READ_4(sc, DC_SIO) & DC_SIO_MII_DATAIN)
-		return (1);
-
-	return (0);
-}
+	struct dc_softc *sc;
+	uint32_t val;
 
-/*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
- */
-static void
-dc_mii_sync(struct dc_softc *sc)
-{
-	int i;
+	sc = device_get_softc(dev);
 
-	CSR_WRITE_4(sc, DC_SIO, DC_SIO_ROMCTL_WRITE);
+	val = CSR_READ_4(sc, DC_SIO);
 	CSR_BARRIER_4(sc, DC_SIO,
 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
-	DELAY(1);
-
-	for (i = 0; i < 32; i++)
-		dc_mii_writebit(sc, 1);
-}
-
-/*
- * Clock a series of bits through the MII.
- */
-static void
-dc_mii_send(struct dc_softc *sc, uint32_t bits, int cnt)
-{
-	int i;
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1)
-		dc_mii_writebit(sc, bits & i);
-}
-
-/*
- * Read an PHY register through the MII.
- */
-static int
-dc_mii_readreg(struct dc_softc *sc, struct dc_mii_frame *frame)
-{
-	int i;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = DC_MII_STARTDELIM;
-	frame->mii_opcode = DC_MII_READOP;
-
-	/*
-	 * Sync the PHYs.
-	 */
-	dc_mii_sync(sc);
-
-	/*
-	 * Send command/address info.
-	 */
-	dc_mii_send(sc, frame->mii_stdelim, 2);
-	dc_mii_send(sc, frame->mii_opcode, 2);
-	dc_mii_send(sc, frame->mii_phyaddr, 5);
-	dc_mii_send(sc, frame->mii_regaddr, 5);
-
-	/*
-	 * Now try reading data bits.  If the turnaround failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	frame->mii_turnaround = dc_mii_readbit(sc);
-	if (frame->mii_turnaround != 0) {
-		for (i = 0; i < 16; i++)
-			dc_mii_readbit(sc);
-		goto fail;
-	}
-	for (i = 0x8000; i; i >>= 1) {
-		if (dc_mii_readbit(sc))
-			frame->mii_data |= i;
-	}
-
-fail:
 
-	/* Clock the idle bits. */
-	dc_mii_writebit(sc, 0);
-	dc_mii_writebit(sc, 0);
-
-	if (frame->mii_turnaround != 0)
-		return (1);
-	return (0);
-}
-
-/*
- * Write to a PHY register through the MII.
- */
-static int
-dc_mii_writereg(struct dc_softc *sc, struct dc_mii_frame *frame)
-{
-
-	/*
-	 * Set up frame for TX.
-	 */
-	frame->mii_stdelim = DC_MII_STARTDELIM;
-	frame->mii_opcode = DC_MII_WRITEOP;
-	frame->mii_turnaround = DC_MII_TURNAROUND;
-
-	/*
-	 * Sync the PHYs.
-	 */
-	dc_mii_sync(sc);
-
-	dc_mii_send(sc, frame->mii_stdelim, 2);
-	dc_mii_send(sc, frame->mii_opcode, 2);
-	dc_mii_send(sc, frame->mii_phyaddr, 5);
-	dc_mii_send(sc, frame->mii_regaddr, 5);
-	dc_mii_send(sc, frame->mii_turnaround, 2);
-	dc_mii_send(sc, frame->mii_data, 16);
-
-	/* Clock the idle bits. */
-	dc_mii_writebit(sc, 0);
-	dc_mii_writebit(sc, 0);
-
-	return (0);
+	return (val);
 }
 
 static int
 dc_miibus_readreg(device_t dev, int phy, int reg)
 {
-	struct dc_mii_frame frame;
-	struct dc_softc	 *sc;
+	struct dc_softc *sc;
 	int i, rval, phy_reg = 0;
 
 	sc = device_get_softc(dev);
-	bzero(&frame, sizeof(frame));
 
 	if (sc->dc_pmode != DC_PMODE_MII) {
 		if (phy == (MII_NPHY - 1)) {
@@ -881,34 +754,29 @@ dc_miibus_readreg(device_t dev, int phy, int reg)
 		}
 
 		rval = CSR_READ_4(sc, phy_reg) & 0x0000FFFF;
-
 		if (rval == 0xFFFF)
 			return (0);
 		return (rval);
 	}
 
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
 	if (sc->dc_type == DC_TYPE_98713) {
 		phy_reg = CSR_READ_4(sc, DC_NETCFG);
 		CSR_WRITE_4(sc, DC_NETCFG, phy_reg & ~DC_NETCFG_PORTSEL);
 	}
-	dc_mii_readreg(sc, &frame);
+	rval = mii_bitbang_readreg(dev, &dc_mii_bitbang_ops, phy, reg);
 	if (sc->dc_type == DC_TYPE_98713)
 		CSR_WRITE_4(sc, DC_NETCFG, phy_reg);
 
-	return (frame.mii_data);
+	return (rval);
 }
 
 static int
 dc_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	struct dc_softc *sc;
-	struct dc_mii_frame frame;
 	int i, phy_reg = 0;
 
 	sc = device_get_softc(dev);
-	bzero(&frame, sizeof(frame));
 
 	if (DC_IS_PNIC(sc)) {
 		CSR_WRITE_4(sc, DC_PN_MII, DC_PN_MIIOPCODE_WRITE |
@@ -964,15 +832,11 @@ dc_miibus_writereg(device_t dev, int phy, int reg, int data)
 		return (0);
 	}
 
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
-
 	if (sc->dc_type == DC_TYPE_98713) {
 		phy_reg = CSR_READ_4(sc, DC_NETCFG);
 		CSR_WRITE_4(sc, DC_NETCFG, phy_reg & ~DC_NETCFG_PORTSEL);
 	}
-	dc_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &dc_mii_bitbang_ops, phy, reg, data);
 	if (sc->dc_type == DC_TYPE_98713)
 		CSR_WRITE_4(sc, DC_NETCFG, phy_reg);
 
@@ -996,12 +860,11 @@ dc_miibus_statchg(device_t dev)
 		return;
 
 	ifm = &mii->mii_media;
-	if (DC_IS_DAVICOM(sc) &&
-	    IFM_SUBTYPE(ifm->ifm_media) == IFM_HPNA_1) {
+	if (DC_IS_DAVICOM(sc) && IFM_SUBTYPE(ifm->ifm_media) == IFM_HPNA_1) {
 		dc_setcfg(sc, ifm->ifm_media);
-		sc->dc_if_media = ifm->ifm_media;
 		return;
-	}
+	} else if (!DC_IS_ADMTEK(sc))
+		dc_setcfg(sc, mii->mii_media_active);
 
 	sc->dc_link = 0;
 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
@@ -1011,17 +874,8 @@ dc_miibus_statchg(device_t dev)
 		case IFM_100_TX:
 			sc->dc_link = 1;
 			break;
-		default:
-			break;
 		}
 	}
-	if (sc->dc_link == 0)
-		return;
-
-	sc->dc_if_media = mii->mii_media_active;
-	if (DC_IS_ADMTEK(sc))
-		return;
-	dc_setcfg(sc, mii->mii_media_active);
 }
 
 /*
@@ -2602,9 +2456,6 @@ dc_attach(device_t dev)
 		if (sc->dc_pmode != DC_PMODE_SIA)
 			sc->dc_pmode = DC_PMODE_SYM;
 		sc->dc_flags |= DC_21143_NWAY;
-		mii_attach(dev, &sc->dc_miibus, ifp, dc_ifmedia_upd,
-		    dc_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY,
-		    MII_OFFSET_ANY, 0);
 		/*
 		 * For non-MII cards, we need to have the 21143
 		 * drive the LEDs. Except there are some systems
@@ -2615,7 +2466,9 @@ dc_attach(device_t dev)
 		if (!(pci_get_subvendor(dev) == 0x1033 &&
 		    pci_get_subdevice(dev) == 0x8028))
 			sc->dc_flags |= DC_TULIP_LEDS;
-		error = 0;
+		error = mii_attach(dev, &sc->dc_miibus, ifp, dc_ifmedia_upd,
+		    dc_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY,
+		    MII_OFFSET_ANY, 0);
 	}
 
 	if (error) {
@@ -2684,7 +2537,7 @@ dc_detach(device_t dev)
 	ifp = sc->dc_ifp;
 
 #ifdef DEVICE_POLLING
-	if (ifp->if_capenable & IFCAP_POLLING)
+	if (ifp != NULL && ifp->if_capenable & IFCAP_POLLING)
 		ether_poll_deregister(ifp);
 #endif
 
@@ -2708,7 +2561,7 @@ dc_detach(device_t dev)
 	if (sc->dc_res)
 		bus_release_resource(dev, DC_RES, DC_RID, sc->dc_res);
 
-	if (ifp)
+	if (ifp != NULL)
 		if_free(ifp);
 
 	dc_dma_free(sc);
@@ -2758,7 +2611,6 @@ dc_list_tx_init(struct dc_softc *sc)
 	return (0);
 }
 
-
 /*
  * Initialize the RX descriptors and allocate mbufs for them. Note that
  * we arrange the descriptors in a closed ring, so that the last descriptor
diff --git a/sys/dev/dc/if_dcreg.h b/sys/dev/dc/if_dcreg.h
index acd80c0..e290881 100644
--- a/sys/dev/dc/if_dcreg.h
+++ b/sys/dev/dc/if_dcreg.h
@@ -531,27 +531,9 @@ struct dc_mediainfo {
 struct dc_type {
 	uint32_t		dc_devid;
 	uint8_t			dc_minrev;
-	char			*dc_name;
+	const char		*dc_name;
 };
 
-struct dc_mii_frame {
-	uint8_t			mii_stdelim;
-	uint8_t			mii_opcode;
-	uint8_t			mii_phyaddr;
-	uint8_t			mii_regaddr;
-	uint8_t			mii_turnaround;
-	uint16_t		mii_data;
-};
-
-/*
- * MII constants
- */
-#define	DC_MII_STARTDELIM	0x01
-#define	DC_MII_READOP		0x02
-#define	DC_MII_WRITEOP		0x01
-#define	DC_MII_TURNAROUND	0x02
-
-
 /*
  * Registers specific to clone devices.
  * This mainly relates to RX filter programming: not all 21x4x clones
@@ -778,7 +760,6 @@ struct dc_softc {
 	int			dc_pnic_rx_bug_save;
 	unsigned char		*dc_pnic_rx_buf;
 	int			dc_if_flags;
-	int			dc_if_media;
 	uint32_t		dc_flags;
 	uint32_t		dc_txthresh;
 	uint32_t		dc_eaddr[2];
@@ -827,7 +808,7 @@ struct dc_softc {
 #define	CSR_READ_4(sc, reg)		\
 	bus_space_read_4(sc->dc_btag, sc->dc_bhandle, reg)
 
-#define	CSR_BARRIER_4(sc, reg, flags)		\
+#define	CSR_BARRIER_4(sc, reg, flags)					\
 	bus_space_barrier(sc->dc_btag, sc->dc_bhandle, reg, 4, flags)
 
 #define	DC_TIMEOUT		1000
diff --git a/sys/dev/dcons/dcons_os.c b/sys/dev/dcons/dcons_os.c
index b7de414..d638d41 100644
--- a/sys/dev/dcons/dcons_os.c
+++ b/sys/dev/dcons/dcons_os.c
@@ -93,7 +93,7 @@ static int poll_hz = DCONS_POLL_HZ;
 
 static struct dcons_softc sc[DCONS_NPORT];
 
-SYSCTL_NODE(_kern, OID_AUTO, dcons, CTLFLAG_RD, 0, "Dumb Console");
+static SYSCTL_NODE(_kern, OID_AUTO, dcons, CTLFLAG_RD, 0, "Dumb Console");
 SYSCTL_INT(_kern_dcons, OID_AUTO, poll_hz, CTLFLAG_RW, &poll_hz, 0,
 				"dcons polling rate");
 
diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c
index 4d8ba65..10d556e 100644
--- a/sys/dev/e1000/if_em.c
+++ b/sys/dev/e1000/if_em.c
@@ -330,7 +330,7 @@ MODULE_DEPEND(em, ether, 1, 1, 1);
 #define CSUM_TSO	0
 #endif
 
-SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
 
 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c
index ff2e424..4ae4204 100644
--- a/sys/dev/e1000/if_igb.c
+++ b/sys/dev/e1000/if_igb.c
@@ -300,7 +300,7 @@ MODULE_DEPEND(igb, ether, 1, 1, 1);
  *  Tunable default values.
  *********************************************************************/
 
-SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
 
 /* Descriptor defaults */
 static int igb_rxd = IGB_DEFAULT_RXD;
diff --git a/sys/dev/esp/am53c974reg.h b/sys/dev/esp/am53c974reg.h
new file mode 100644
index 0000000..86260fd
--- /dev/null
+++ b/sys/dev/esp/am53c974reg.h
@@ -0,0 +1,72 @@
+/*	$NetBSD: pcscpreg.h,v 1.2 2008/04/28 20:23:55 martin Exp $	*/
+
+/*-
+ * Copyright (c) 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Izumi Tsutsui.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _AM53C974_H_
+#define	_AM53C974_H_
+
+/*
+ * Am53c974 DMA engine registers
+ */
+
+#define	DMA_CMD		0x40 		/* Command */
+#define	 DMACMD_RSVD	0xFFFFFF28	/* reserved */
+#define	 DMACMD_DIR	0x00000080	/* Transfer Direction (read:1) */
+#define	 DMACMD_INTE	0x00000040	/* DMA Interrupt Enable	*/
+#define	 DMACMD_MDL	0x00000010	/* Map to Memory Description List */
+#define	 DMACMD_DIAG	0x00000004	/* Diagnostic */
+#define	 DMACMD_CMD	0x00000003	/* Command Code Bit */
+#define	  DMACMD_IDLE	0x00000000	/*  Idle */
+#define	  DMACMD_BLAST	0x00000001	/*  Blast */
+#define	  DMACMD_ABORT	0x00000002	/*  Abort */
+#define	  DMACMD_START	0x00000003	/*  Start */
+
+#define	DMA_STC		0x44		/* Start Transfer Count */
+#define	DMA_SPA		0x48		/* Start Physical Address */
+#define	DMA_WBC		0x4C		/* Working Byte Counter */
+#define	DMA_WAC		0x50		/* Working Address Counter */
+
+#define	DMA_STAT	0x54		/* Status Register */
+#define	 DMASTAT_RSVD	0xFFFFFF80	/* reserved */
+#define	 DMASTAT_PABT	0x00000040	/* PCI master/target Abort */
+#define	 DMASTAT_BCMP	0x00000020	/* BLAST Complete */
+#define	 DMASTAT_SINT	0x00000010	/* SCSI Interrupt */
+#define	 DMASTAT_DONE	0x00000008	/* DMA Transfer Terminated */
+#define	 DMASTAT_ABT	0x00000004	/* DMA Transfer Aborted */
+#define	 DMASTAT_ERR	0x00000002	/* DMA Transfer Error */
+#define	 DMASTAT_PWDN	0x00000001	/* Power Down Indicator */
+
+#define	DMA_SMDLA	0x58	/* Starting Memory Descpritor List Address */
+#define	DMA_WMAC	0x5C	/* Working MDL Counter */
+#define	DMA_SBAC	0x70	/* SCSI Bus and Control */
+
+#endif /* _AM53C974_H_ */
diff --git a/sys/dev/esp/esp_pci.c b/sys/dev/esp/esp_pci.c
new file mode 100644
index 0000000..2fa2030
--- /dev/null
+++ b/sys/dev/esp/esp_pci.c
@@ -0,0 +1,654 @@
+/*-
+ * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*	$NetBSD: pcscp.c,v 1.45 2010/11/13 13:52:08 uebayasi Exp $	*/
+
+/*-
+ * Copyright (c) 1997, 1998, 1999 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center; Izumi Tsutsui.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * esp_pci.c: device dependent code for AMD Am53c974 (PCscsi-PCI)
+ * written by Izumi Tsutsui <tsutsui@NetBSD.org>
+ *
+ * Technical manual available at
+ * http://www.amd.com/files/connectivitysolutions/networking/archivednetworking/19113.pdf
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/resource.h>
+#include <sys/rman.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/scsi/scsi_all.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <dev/esp/ncr53c9xreg.h>
+#include <dev/esp/ncr53c9xvar.h>
+
+#include <dev/esp/am53c974reg.h>
+
+#define	PCI_DEVICE_ID_AMD53C974	0x20201022
+
+struct esp_pci_softc {
+	struct ncr53c9x_softc	sc_ncr53c9x;	/* glue to MI code */
+	struct device		*sc_dev;
+
+	struct resource *sc_res[2];
+#define	ESP_PCI_RES_INTR	0
+#define	ESP_PCI_RES_IO		1
+
+	bus_dma_tag_t		sc_pdmat;
+
+	bus_dma_tag_t		sc_xferdmat;	/* DMA tag for transfers */
+	bus_dmamap_t		sc_xferdmam;	/* DMA map for transfers */
+
+	void			*sc_ih;		/* interrupt handler */
+
+	size_t			sc_dmasize;	/* DMA size */
+	void			**sc_dmaaddr;	/* DMA address */
+	size_t			*sc_dmalen;	/* DMA length */
+	int			sc_active;	/* DMA state */
+	int			sc_datain;	/* DMA Data Direction */
+};
+
+static struct resource_spec esp_pci_res_spec[] = {
+	{ SYS_RES_IRQ, 0, RF_SHAREABLE | RF_ACTIVE },	/* ESP_PCI_RES_INTR */
+	{ SYS_RES_IOPORT, PCIR_BAR(0), RF_ACTIVE },	/* ESP_PCI_RES_IO */
+	{ -1, 0 }
+};
+
+#define	READ_DMAREG(sc, reg)						\
+	bus_read_4((sc)->sc_res[ESP_PCI_RES_IO], (reg))
+#define	WRITE_DMAREG(sc, reg, var)					\
+	bus_write_4((sc)->sc_res[ESP_PCI_RES_IO], (reg), (var))
+
+#define	READ_ESPREG(sc, reg)						\
+	bus_read_1((sc)->sc_res[ESP_PCI_RES_IO], (reg) << 2)
+#define	WRITE_ESPREG(sc, reg, val)					\
+	bus_write_1((sc)->sc_res[ESP_PCI_RES_IO], (reg) << 2, (val))
+
+static int	esp_pci_probe(device_t);
+static int	esp_pci_attach(device_t);
+static int	esp_pci_detach(device_t);
+static int	esp_pci_suspend(device_t);
+static int	esp_pci_resume(device_t);
+
+static device_method_t esp_pci_methods[] = {
+	DEVMETHOD(device_probe,		esp_pci_probe),
+	DEVMETHOD(device_attach,	esp_pci_attach),
+	DEVMETHOD(device_detach,	esp_pci_detach),
+	DEVMETHOD(device_suspend,	esp_pci_suspend),
+	DEVMETHOD(device_resume,	esp_pci_resume),
+
+	KOBJMETHOD_END
+};
+
+static driver_t esp_pci_driver = {
+	"esp",
+	esp_pci_methods,
+	sizeof(struct esp_pci_softc)
+};
+
+DRIVER_MODULE(esp, pci, esp_pci_driver, esp_devclass, 0, 0);
+MODULE_DEPEND(esp, pci, 1, 1, 1);
+
+/*
+ * Functions and the switch for the MI code
+ */
+static void	esp_pci_dma_go(struct ncr53c9x_softc *);
+static int	esp_pci_dma_intr(struct ncr53c9x_softc *);
+static int	esp_pci_dma_isactive(struct ncr53c9x_softc *);
+
+static int	esp_pci_dma_isintr(struct ncr53c9x_softc *);
+static void	esp_pci_dma_reset(struct ncr53c9x_softc *);
+static int	esp_pci_dma_setup(struct ncr53c9x_softc *, void **, size_t *,
+		    int, size_t *);
+static void	esp_pci_dma_stop(struct ncr53c9x_softc *);
+static void	esp_pci_write_reg(struct ncr53c9x_softc *, int, uint8_t);
+static uint8_t	esp_pci_read_reg(struct ncr53c9x_softc *, int);
+static void	esp_pci_xfermap(void *arg, bus_dma_segment_t *segs, int nseg,
+		    int error);
+
+static struct ncr53c9x_glue esp_pci_glue = {
+	esp_pci_read_reg,
+	esp_pci_write_reg,
+	esp_pci_dma_isintr,
+	esp_pci_dma_reset,
+	esp_pci_dma_intr,
+	esp_pci_dma_setup,
+	esp_pci_dma_go,
+	esp_pci_dma_stop,
+	esp_pci_dma_isactive,
+};
+
+static int
+esp_pci_probe(device_t dev)
+{
+
+	if (pci_get_devid(dev) == PCI_DEVICE_ID_AMD53C974) {
+		device_set_desc(dev, "AMD Am53C974 Fast-SCSI");
+		return (BUS_PROBE_DEFAULT);
+	}
+
+	return (ENXIO);
+}
+
+/*
+ * Attach this instance, and then all the sub-devices
+ */
+static int
+esp_pci_attach(device_t dev)
+{
+	struct esp_pci_softc *esc;
+	struct ncr53c9x_softc *sc;
+	int error;
+
+	esc = device_get_softc(dev);
+	sc = &esc->sc_ncr53c9x;
+
+	NCR_LOCK_INIT(sc);
+
+	esc->sc_dev = dev;
+	sc->sc_glue = &esp_pci_glue;
+
+	pci_enable_busmaster(dev);
+
+	error = bus_alloc_resources(dev, esp_pci_res_spec, esc->sc_res);
+	if (error != 0) {
+		device_printf(dev, "failed to allocate resources\n");
+		bus_release_resources(dev, esp_pci_res_spec, esc->sc_res);
+		return (error);
+	}
+
+	error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
+	    BUS_SPACE_MAXSIZE_32BIT, BUS_SPACE_UNRESTRICTED,
+	    BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL, &esc->sc_pdmat);
+	if (error != 0) {
+		device_printf(dev, "cannot create parent DMA tag\n");
+		goto fail_res;
+	}
+
+	/*
+	 * XXX More of this should be in ncr53c9x_attach(), but
+	 * XXX should we really poke around the chip that much in
+	 * XXX the MI code?  Think about this more...
+	 */
+
+	/*
+	 * Set up static configuration info.
+	 *
+	 * XXX we should read the configuration from the EEPROM.
+	 */
+	sc->sc_id = 7;
+	sc->sc_cfg1 = sc->sc_id | NCRCFG1_PARENB;
+	sc->sc_cfg2 = NCRCFG2_SCSI2 | NCRCFG2_FE;
+	sc->sc_cfg3 = NCRAMDCFG3_IDM | NCRAMDCFG3_FCLK;
+	sc->sc_cfg4 = NCRAMDCFG4_GE12NS | NCRAMDCFG4_RADE;
+	sc->sc_rev = NCR_VARIANT_AM53C974;
+	sc->sc_features = NCR_F_FASTSCSI | NCR_F_DMASELECT;
+	sc->sc_cfg3_fscsi = NCRAMDCFG3_FSCSI;
+	sc->sc_freq = 40; /* MHz */
+
+	/*
+	 * This is the value used to start sync negotiations
+	 * Note that the NCR register "SYNCTP" is programmed
+	 * in "clocks per byte", and has a minimum value of 4.
+	 * The SCSI period used in negotiation is one-fourth
+	 * of the time (in nanoseconds) needed to transfer one byte.
+	 * Since the chip's clock is given in MHz, we have the following
+	 * formula: 4 * period = (1000 / freq) * 4
+	 */
+	sc->sc_minsync = 1000 / sc->sc_freq;
+
+	sc->sc_maxxfer = DFLTPHYS;	/* see below */
+	sc->sc_maxoffset = 15;
+	sc->sc_extended_geom = 1;
+
+#define	MDL_SEG_SIZE	0x1000	/* 4kbyte per segment */
+
+	/*
+	 * Create the DMA tag and map for the data transfers.
+	 *
+	 * Note: given that bus_dma(9) only adheres to the requested alignment
+	 * for the first segment (and that also only for bus_dmamem_alloc()ed
+	 * DMA maps) we can't use the Memory Descriptor List.  However, also
+	 * when not using the MDL, the maximum transfer size apparently is
+	 * limited to 4k so we have to split transfers up, which plain sucks.
+	 */
+	error = bus_dma_tag_create(esc->sc_pdmat, PAGE_SIZE, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
+	    MDL_SEG_SIZE, 1, MDL_SEG_SIZE, BUS_DMA_ALLOCNOW,
+	    busdma_lock_mutex, &sc->sc_lock, &esc->sc_xferdmat);
+	if (error != 0) {
+		device_printf(dev, "cannot create transfer DMA tag\n");
+		goto fail_pdmat;
+	}
+	error = bus_dmamap_create(esc->sc_xferdmat, 0, &esc->sc_xferdmam);
+	if (error != 0) {
+		device_printf(dev, "cannnot create transfer DMA map\n");
+		goto fail_xferdmat;
+	}
+
+	error = bus_setup_intr(dev, esc->sc_res[ESP_PCI_RES_INTR],
+	    INTR_MPSAFE | INTR_TYPE_CAM, NULL, ncr53c9x_intr, sc,
+	    &esc->sc_ih);
+	if (error != 0) {
+		device_printf(dev, "cannot set up interrupt\n");
+		goto fail_xferdmam;
+	}
+
+	/* Do the common parts of attachment. */
+	sc->sc_dev = esc->sc_dev;
+	error = ncr53c9x_attach(sc);
+	if (error != 0) {
+		device_printf(esc->sc_dev, "ncr53c9x_attach failed\n");
+		goto fail_intr;
+	}
+
+	return (0);
+
+ fail_intr:
+	 bus_teardown_intr(esc->sc_dev, esc->sc_res[ESP_PCI_RES_INTR],
+	    esc->sc_ih);
+ fail_xferdmam:
+	bus_dmamap_destroy(esc->sc_xferdmat, esc->sc_xferdmam);
+ fail_xferdmat:
+	bus_dma_tag_destroy(esc->sc_xferdmat);
+ fail_pdmat:
+	bus_dma_tag_destroy(esc->sc_pdmat);
+ fail_res:
+	bus_release_resources(dev, esp_pci_res_spec, esc->sc_res);
+	NCR_LOCK_DESTROY(sc);
+
+	return (error);
+}
+
+static int
+esp_pci_detach(device_t dev)
+{
+	struct ncr53c9x_softc *sc;
+	struct esp_pci_softc *esc;
+	int error;
+
+	esc = device_get_softc(dev);
+	sc = &esc->sc_ncr53c9x;
+
+	bus_teardown_intr(esc->sc_dev, esc->sc_res[ESP_PCI_RES_INTR],
+	    esc->sc_ih);
+	error = ncr53c9x_detach(sc);
+	if (error != 0)
+		return (error);
+	bus_dmamap_destroy(esc->sc_xferdmat, esc->sc_xferdmam);
+	bus_dma_tag_destroy(esc->sc_xferdmat);
+	bus_dma_tag_destroy(esc->sc_pdmat);
+	bus_release_resources(dev, esp_pci_res_spec, esc->sc_res);
+	NCR_LOCK_DESTROY(sc);
+
+	return (0);
+}
+
+static int
+esp_pci_suspend(device_t dev)
+{
+
+	return (ENXIO);
+}
+
+static int
+esp_pci_resume(device_t dev)
+{
+
+	return (ENXIO);
+}
+
+static void
+esp_pci_xfermap(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)arg;
+
+	if (error != 0)
+		return;
+
+	KASSERT(nsegs == 1, ("%s: bad transfer segment count %d", __func__,
+	    nsegs));
+	KASSERT(segs[0].ds_len <= MDL_SEG_SIZE,
+	    ("%s: bad transfer segment length %ld", __func__,
+	    (long)segs[0].ds_len));
+
+	/* Program the DMA Starting Physical Address. */
+	WRITE_DMAREG(esc, DMA_SPA, segs[0].ds_addr);
+}
+
+/*
+ * Glue functions
+ */
+
+static uint8_t
+esp_pci_read_reg(struct ncr53c9x_softc *sc, int reg)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+
+	return (READ_ESPREG(esc, reg));
+}
+
+static void
+esp_pci_write_reg(struct ncr53c9x_softc *sc, int reg, uint8_t v)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+
+	WRITE_ESPREG(esc, reg, v);
+}
+
+static int
+esp_pci_dma_isintr(struct ncr53c9x_softc *sc)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+
+	return (READ_ESPREG(esc, NCR_STAT) & NCRSTAT_INT) != 0;
+}
+
+static void
+esp_pci_dma_reset(struct ncr53c9x_softc *sc)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+
+	WRITE_DMAREG(esc, DMA_CMD, DMACMD_IDLE);
+
+	esc->sc_active = 0;
+}
+
+static int
+esp_pci_dma_intr(struct ncr53c9x_softc *sc)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+	bus_dma_tag_t xferdmat;
+	bus_dmamap_t xferdmam;
+	size_t dmasize;
+	int datain, i, resid, trans;
+	uint32_t dmastat;
+	char *p = NULL;
+
+	xferdmat = esc->sc_xferdmat;
+	xferdmam = esc->sc_xferdmam;
+	datain = esc->sc_datain;
+
+	dmastat = READ_DMAREG(esc, DMA_STAT);
+
+	if ((dmastat & DMASTAT_ERR) != 0) {
+		/* XXX not tested... */
+		WRITE_DMAREG(esc, DMA_CMD, DMACMD_ABORT | (datain != 0 ?
+		    DMACMD_DIR : 0));
+
+		device_printf(esc->sc_dev, "DMA error detected; Aborting.\n");
+		bus_dmamap_sync(xferdmat, xferdmam, datain != 0 ?
+		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(xferdmat, xferdmam);
+		return (-1);
+	}
+
+	if ((dmastat & DMASTAT_ABT) != 0) {
+		/* XXX what should be done? */
+		device_printf(esc->sc_dev, "DMA aborted.\n");
+		WRITE_DMAREG(esc, DMA_CMD, DMACMD_IDLE | (datain != 0 ?
+		    DMACMD_DIR : 0));
+		esc->sc_active = 0;
+		return (0);
+	}
+
+	KASSERT(esc->sc_active != 0, ("%s: DMA wasn't active", __func__));
+
+	/* DMA has stopped. */
+
+	esc->sc_active = 0;
+
+	dmasize = esc->sc_dmasize;
+	if (dmasize == 0) {
+		/* A "Transfer Pad" operation completed. */
+		NCR_DMA(("%s: discarded %d bytes (tcl=%d, tcm=%d)\n",
+		    __func__, READ_ESPREG(esc, NCR_TCL) |
+		    (READ_ESPREG(esc, NCR_TCM) << 8),
+		    READ_ESPREG(esc, NCR_TCL), READ_ESPREG(esc, NCR_TCM)));
+		return (0);
+	}
+
+	resid = 0;
+	/*
+	 * If a transfer onto the SCSI bus gets interrupted by the device
+	 * (e.g. for a SAVEPOINTER message), the data in the FIFO counts
+	 * as residual since the ESP counter registers get decremented as
+	 * bytes are clocked into the FIFO.
+	 */
+	if (datain == 0 &&
+	    (resid = (READ_ESPREG(esc, NCR_FFLAG) & NCRFIFO_FF)) != 0)
+		NCR_DMA(("%s: empty esp FIFO of %d ", __func__, resid));
+
+	if ((sc->sc_espstat & NCRSTAT_TC) == 0) {
+		/*
+		 * "Terminal count" is off, so read the residue
+		 * out of the ESP counter registers.
+		 */
+		if (datain != 0) {
+			resid = READ_ESPREG(esc, NCR_FFLAG) & NCRFIFO_FF;
+			while (resid > 1)
+				resid =
+				    READ_ESPREG(esc, NCR_FFLAG) & NCRFIFO_FF;
+			WRITE_DMAREG(esc, DMA_CMD, DMACMD_BLAST | DMACMD_DIR);
+
+			for (i = 0; i < 0x8000; i++) /* XXX 0x8000 ? */
+				if ((READ_DMAREG(esc, DMA_STAT) &
+				    DMASTAT_BCMP) != 0)
+					break;
+
+			/* See the below comments... */
+			if (resid != 0)
+				p = *esc->sc_dmaaddr;
+		}
+
+		resid += READ_ESPREG(esc, NCR_TCL) |
+		    (READ_ESPREG(esc, NCR_TCM) << 8) |
+		    (READ_ESPREG(esc, NCR_TCH) << 16);
+	} else
+		while ((dmastat & DMASTAT_DONE) == 0)
+			dmastat = READ_DMAREG(esc, DMA_STAT);
+
+	WRITE_DMAREG(esc, DMA_CMD, DMACMD_IDLE | (datain != 0 ?
+	    DMACMD_DIR : 0));
+
+	/* Sync the transfer buffer. */
+	bus_dmamap_sync(xferdmat, xferdmam, datain != 0 ?
+	    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
+	bus_dmamap_unload(xferdmat, xferdmam);
+
+	trans = dmasize - resid;
+
+	/*
+	 * From the technical manual notes:
+	 *
+	 * "In some odd byte conditions, one residual byte will be left
+	 *  in the SCSI FIFO, and the FIFO flags will never count to 0.
+	 *  When this happens, the residual byte should be retrieved
+	 *  via PIO following completion of the BLAST operation."
+	 */
+	if (p != NULL) {
+		p += trans;
+		*p = READ_ESPREG(esc, NCR_FIFO);
+		trans++;
+	}
+
+	if (trans < 0) {			/* transferred < 0 ? */
+#if 0
+		/*
+		 * This situation can happen in perfectly normal operation
+		 * if the ESP is reselected while using DMA to select
+		 * another target.  As such, don't print the warning.
+		 */
+		device_printf(dev, "xfer (%d) > req (%d)\n", trans, dmasize);
+#endif
+		trans = dmasize;
+	}
+
+	NCR_DMA(("%s: tcl=%d, tcm=%d, tch=%d; trans=%d, resid=%d\n", __func__,
+	    READ_ESPREG(esc, NCR_TCL), READ_ESPREG(esc, NCR_TCM),
+	    READ_ESPREG(esc, NCR_TCH), trans, resid));
+
+	*esc->sc_dmalen -= trans;
+	*esc->sc_dmaaddr = (char *)*esc->sc_dmaaddr + trans;
+
+	return (0);
+}
+
+static int
+esp_pci_dma_setup(struct ncr53c9x_softc *sc, void **addr, size_t *len,
+    int datain, size_t *dmasize)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+	int error;
+
+	WRITE_DMAREG(esc, DMA_CMD, DMACMD_IDLE | (datain != 0 ? DMACMD_DIR :
+	    0));
+
+	*dmasize = esc->sc_dmasize = ulmin(*dmasize, MDL_SEG_SIZE);
+	esc->sc_dmaaddr = addr;
+	esc->sc_dmalen = len;
+	esc->sc_datain = datain;
+
+	/*
+	 * There's no need to set up DMA for a "Transfer Pad" operation.
+	 */
+	if (*dmasize == 0)
+		return (0);
+
+	/* Set the transfer length. */
+	WRITE_DMAREG(esc, DMA_STC, *dmasize);
+
+	/*
+	 * Load the transfer buffer and program the DMA address.
+	 * Note that the NCR53C9x core can't handle EINPROGRESS so we set
+	 * BUS_DMA_NOWAIT.
+	 */
+	error = bus_dmamap_load(esc->sc_xferdmat, esc->sc_xferdmam,
+	    *esc->sc_dmaaddr, *dmasize, esp_pci_xfermap, sc, BUS_DMA_NOWAIT);
+
+	return (error);
+}
+
+static void
+esp_pci_dma_go(struct ncr53c9x_softc *sc)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+	int datain;
+
+	datain = esc->sc_datain;
+
+	/* No DMA transfer for a "Transfer Pad" operation */
+	if (esc->sc_dmasize == 0)
+		return;
+
+	/* Sync the transfer buffer. */
+	bus_dmamap_sync(esc->sc_xferdmat, esc->sc_xferdmam, datain != 0 ?
+	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
+
+	/* Set the DMA engine to the IDLE state. */
+	/* XXX DMA Transfer Interrupt Enable bit is broken? */
+	WRITE_DMAREG(esc, DMA_CMD, DMACMD_IDLE | /* DMACMD_INTE | */
+	    (datain != 0 ? DMACMD_DIR : 0));
+
+	/* Issue a DMA start command. */
+	WRITE_DMAREG(esc, DMA_CMD, DMACMD_START | /* DMACMD_INTE | */
+	    (datain != 0 ? DMACMD_DIR : 0));
+
+	esc->sc_active = 1;
+}
+
+static void
+esp_pci_dma_stop(struct ncr53c9x_softc *sc)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+
+	/* DMA stop */
+	/* XXX what should we do here ? */
+	WRITE_DMAREG(esc, DMA_CMD,
+	    DMACMD_ABORT | (esc->sc_datain != 0 ? DMACMD_DIR : 0));
+	bus_dmamap_unload(esc->sc_xferdmat, esc->sc_xferdmam);
+
+	esc->sc_active = 0;
+}
+
+static int
+esp_pci_dma_isactive(struct ncr53c9x_softc *sc)
+{
+	struct esp_pci_softc *esc = (struct esp_pci_softc *)sc;
+
+	/* XXX should we check esc->sc_active? */
+	if ((READ_DMAREG(esc, DMA_CMD) & DMACMD_CMD) != DMACMD_IDLE)
+		return (1);
+
+	return (0);
+}
diff --git a/sys/dev/esp/esp_sbus.c b/sys/dev/esp/esp_sbus.c
index 62a4592..c1ff021 100644
--- a/sys/dev/esp/esp_sbus.c
+++ b/sys/dev/esp/esp_sbus.c
@@ -68,13 +68,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
+#include <sys/rman.h>
 
 #include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/openfirm.h>
 #include <machine/bus.h>
 #include <machine/ofw_machdep.h>
 #include <machine/resource.h>
-#include <sys/rman.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
@@ -92,7 +92,7 @@ __FBSDID("$FreeBSD$");
 
 struct esp_softc {
 	struct ncr53c9x_softc	sc_ncr53c9x;	/* glue to MI code */
-	struct device		*sc_dev;
+	device_t		sc_dev;
 
 	struct resource		*sc_res;
 
@@ -102,8 +102,6 @@ struct esp_softc {
 	struct lsi64854_softc	*sc_dma;	/* pointer to my DMA */
 };
 
-static devclass_t	esp_devclass;
-
 static int	esp_probe(device_t);
 static int	esp_dma_attach(device_t);
 static int	esp_dma_detach(device_t);
@@ -118,7 +116,8 @@ static device_method_t esp_dma_methods[] = {
 	DEVMETHOD(device_detach,	esp_dma_detach),
 	DEVMETHOD(device_suspend,	esp_suspend),
 	DEVMETHOD(device_resume,	esp_resume),
-	{0, 0}
+
+	KOBJMETHOD_END
 };
 
 static driver_t esp_dma_driver = {
@@ -136,7 +135,8 @@ static device_method_t esp_sbus_methods[] = {
 	DEVMETHOD(device_detach,	esp_sbus_detach),
 	DEVMETHOD(device_suspend,	esp_suspend),
 	DEVMETHOD(device_resume,	esp_resume),
-	{0, 0}
+
+	KOBJMETHOD_END	
 };
 
 static driver_t esp_sbus_driver = {
@@ -175,7 +175,6 @@ static const struct ncr53c9x_glue const esp_sbus_glue = {
 	esp_dma_go,
 	esp_dma_stop,
 	esp_dma_isactive,
-	NULL,			/* gl_clear_latched_intr */
 };
 
 static int
@@ -245,9 +244,9 @@ esp_sbus_attach(device_t dev)
 		    BUS_SPACE_MAXADDR,		/* lowaddr */
 		    BUS_SPACE_MAXADDR,		/* highaddr */
 		    NULL, NULL,			/* filter, filterarg */
-		    BUS_SPACE_MAXSIZE_32BIT,	/* maxsize */
-		    0,				/* nsegments */
-		    BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
+		    BUS_SPACE_MAXSIZE,		/* maxsize */
+		    BUS_SPACE_UNRESTRICTED,	/* nsegments */
+		    BUS_SPACE_MAXSIZE,		/* maxsegsize */
 		    0,				/* flags */
 		    NULL, NULL,			/* no locking */
 		    &lsc->sc_parent_dmat);
@@ -292,8 +291,10 @@ esp_sbus_attach(device_t dev)
 		}
 		for (i = 0; i < nchildren; i++) {
 			if (device_is_attached(children[i]) &&
-			    sbus_get_slot(children[i]) == sbus_get_slot(dev) &&
-			    strcmp(ofw_bus_get_name(children[i]), "dma") == 0) {
+			    sbus_get_slot(children[i]) ==
+			    sbus_get_slot(dev) &&
+			    strcmp(ofw_bus_get_name(children[i]),
+			    "dma") == 0) {
 				/* XXX hackery */
 				esc->sc_dma = (struct lsi64854_softc *)
 				    device_get_softc(children[i]);
@@ -453,13 +454,6 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 
 	NCR_LOCK_INIT(sc);
 
-	/* Attach the DMA engine. */
-	error = lsi64854_attach(esc->sc_dma);
-	if (error != 0) {
-		device_printf(esc->sc_dev, "lsi64854_attach failed\n");
-		goto fail_lock;
-	}
-
 	sc->sc_id = OF_getscsinitid(esc->sc_dev);
 
 #ifdef ESP_SBUS_DEBUG
@@ -516,9 +510,9 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 	NCR_WRITE_REG(sc, NCR_CFG2, sc->sc_cfg2);
 
 	if ((NCR_READ_REG(sc, NCR_CFG2) & ~NCRCFG2_RSVD) !=
-	    (NCRCFG2_SCSI2 | NCRCFG2_RPE)) {
+	    (NCRCFG2_SCSI2 | NCRCFG2_RPE))
 		sc->sc_rev = NCR_VARIANT_ESP100;
-	} else {
+	else {
 		sc->sc_cfg2 = NCRCFG2_SCSI2;
 		NCR_WRITE_REG(sc, NCR_CFG2, sc->sc_cfg2);
 		sc->sc_cfg3 = 0;
@@ -526,9 +520,9 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 		sc->sc_cfg3 = (NCRCFG3_CDB | NCRCFG3_FCLK);
 		NCR_WRITE_REG(sc, NCR_CFG3, sc->sc_cfg3);
 		if (NCR_READ_REG(sc, NCR_CFG3) !=
-		    (NCRCFG3_CDB | NCRCFG3_FCLK)) {
+		    (NCRCFG3_CDB | NCRCFG3_FCLK))
 			sc->sc_rev = NCR_VARIANT_ESP100A;
-		} else {
+		else {
 			/* NCRCFG2_FE enables > 64K transfers. */
 			sc->sc_cfg2 |= NCRCFG2_FE;
 			sc->sc_cfg3 = 0;
@@ -543,9 +537,11 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 
 				case 0x02:
 					if ((uid & 0x07) == 0x02)
-						sc->sc_rev = NCR_VARIANT_FAS216;
+						sc->sc_rev =
+						    NCR_VARIANT_FAS216;
 					else
-						sc->sc_rev = NCR_VARIANT_FAS236;
+						sc->sc_rev =
+						    NCR_VARIANT_FAS236;
 					break;
 
 				case 0x0a:
@@ -560,7 +556,8 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 					 */
 					device_printf(esc->sc_dev,
 					    "Unknown chip\n");
-					goto fail_lsi;
+					error = ENXIO;
+					goto fail_lock;
 				}
 			}
 		}
@@ -571,12 +568,6 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 #endif
 
 	/*
-	 * XXX minsync and maxxfer _should_ be set up in MI code,
-	 * XXX but it appears to have some dependency on what sort
-	 * XXX of DMA we're hooked up to, etc.
-	 */
-
-	/*
 	 * This is the value used to start sync negotiations
 	 * Note that the NCR register "SYNCTP" is programmed
 	 * in "clocks per byte", and has a minimum value of 4.
@@ -587,31 +578,27 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 	 */
 	sc->sc_minsync = 1000 / sc->sc_freq;
 
+	/*
+	 * Except for some variants the maximum transfer size is 64k.
+	 */
+	sc->sc_maxxfer = 64 * 1024;
 	sc->sc_maxoffset = 15;
 	sc->sc_extended_geom = 1;
 
 	/*
 	 * Alas, we must now modify the value a bit, because it's
-	 * only valid when can switch on FASTCLK and FASTSCSI bits
-	 * in config register 3...
+	 * only valid when we can switch on FASTCLK and FASTSCSI bits
+	 * in the config register 3...
 	 */
 	switch (sc->sc_rev) {
 	case NCR_VARIANT_ESP100:
 		sc->sc_maxwidth = MSG_EXT_WDTR_BUS_8_BIT;
-		sc->sc_maxxfer = 64 * 1024;
 		sc->sc_minsync = 0;	/* No synch on old chip? */
 		break;
 
 	case NCR_VARIANT_ESP100A:
-		sc->sc_maxwidth = MSG_EXT_WDTR_BUS_8_BIT;
-		sc->sc_maxxfer = 64 * 1024;
-		/* Min clocks/byte is 5 */
-		sc->sc_minsync = ncr53c9x_cpb2stp(sc, 5);
-		break;
-
 	case NCR_VARIANT_ESP200:
 		sc->sc_maxwidth = MSG_EXT_WDTR_BUS_8_BIT;
-		sc->sc_maxxfer = 16 * 1024 * 1024;
 		/* Min clocks/byte is 5 */
 		sc->sc_minsync = ncr53c9x_cpb2stp(sc, 5);
 		break;
@@ -642,6 +629,19 @@ espattach(struct esp_softc *esc, const struct ncr53c9x_glue *gluep)
 		break;
 	}
 
+	/*
+	 * Given that we allocate resources based on sc->sc_maxxfer it doesn't
+	 * make sense to supply a value higher than the maximum actually used.
+	 */
+	sc->sc_maxxfer = min(sc->sc_maxxfer, MAXPHYS);
+
+	/* Attach the DMA engine. */
+	error = lsi64854_attach(esc->sc_dma);
+	if (error != 0) {
+		device_printf(esc->sc_dev, "lsi64854_attach failed\n");
+		goto fail_lock;
+	}
+
 	/* Establish interrupt channel. */
 	i = 0;
 	if ((esc->sc_irqres = bus_alloc_resource_any(esc->sc_dev, SYS_RES_IRQ,
diff --git a/sys/dev/esp/ncr53c9x.c b/sys/dev/esp/ncr53c9x.c
index 1d46318..b9d4990 100644
--- a/sys/dev/esp/ncr53c9x.c
+++ b/sys/dev/esp/ncr53c9x.c
@@ -123,6 +123,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/esp/ncr53c9xreg.h>
 #include <dev/esp/ncr53c9xvar.h>
 
+devclass_t esp_devclass;
+
 MODULE_DEPEND(esp, cam, 1, 1, 1);
 
 #ifdef NCR53C9X_DEBUG
@@ -179,8 +181,7 @@ static inline int	ncr53c9x_stp2cpb(struct ncr53c9x_softc *sc,
 #define	NCR_SET_COUNT(sc, size) do {					\
 		NCR_WRITE_REG((sc), NCR_TCL, (size));			\
 		NCR_WRITE_REG((sc), NCR_TCM, (size) >> 8);		\
-		if ((sc->sc_cfg2 & NCRCFG2_FE) ||			\
-		    (sc->sc_rev == NCR_VARIANT_FAS366))			\
+		if ((sc->sc_features & NCR_F_LARGEXFER) != 0)		\
 			NCR_WRITE_REG((sc), NCR_TCH, (size) >> 16);	\
 		if (sc->sc_rev == NCR_VARIANT_FAS366)			\
 			NCR_WRITE_REG(sc, NCR_RCH, 0);			\
@@ -391,6 +392,7 @@ ncr53c9x_attach(struct ncr53c9x_softc *sc)
 		ecb = &sc->ecb_array[i];
 		ecb->sc = sc;
 		ecb->tag_id = i;
+		callout_init_mtx(&ecb->ch, &sc->sc_lock, 0);
 		TAILQ_INSERT_HEAD(&sc->free_list, ecb, free_links);
 	}
 
@@ -449,10 +451,10 @@ ncr53c9x_detach(struct ncr53c9x_softc *sc)
 	xpt_register_async(0, ncr53c9x_async, sc->sc_sim, sc->sc_path);
 	xpt_free_path(sc->sc_path);
 	xpt_bus_deregister(cam_sim_path(sc->sc_sim));
+	cam_sim_free(sc->sc_sim, TRUE);
 
 	NCR_UNLOCK(sc);
 
-	cam_sim_free(sc->sc_sim, TRUE);
 	free(sc->ecb_array, M_DEVBUF);
 	free(sc->sc_tinfo, M_DEVBUF);
 	if (sc->sc_imess_self)
@@ -504,6 +506,8 @@ ncr53c9x_reset(struct ncr53c9x_softc *sc)
 		/* FALLTHROUGH */
 	case NCR_VARIANT_ESP100A:
 		sc->sc_features |= NCR_F_SELATN3;
+		if ((sc->sc_cfg2 & NCRCFG2_FE) != 0)
+			sc->sc_features |= NCR_F_LARGEXFER;
 		NCR_WRITE_REG(sc, NCR_CFG2, sc->sc_cfg2);
 		/* FALLTHROUGH */
 	case NCR_VARIANT_ESP100:
@@ -514,8 +518,8 @@ ncr53c9x_reset(struct ncr53c9x_softc *sc)
 		break;
 
 	case NCR_VARIANT_FAS366:
-		sc->sc_features |=
-		    NCR_F_HASCFG3 | NCR_F_FASTSCSI | NCR_F_SELATN3;
+		sc->sc_features |= NCR_F_HASCFG3 | NCR_F_FASTSCSI |
+		    NCR_F_SELATN3 | NCR_F_LARGEXFER;
 		sc->sc_cfg3 = NCRFASCFG3_FASTCLK | NCRFASCFG3_OBAUTO;
 		if (sc->sc_id > 7)
 			sc->sc_cfg3 |= NCRFASCFG3_IDBIT3;
@@ -711,9 +715,6 @@ ncr53c9x_readregs(struct ncr53c9x_softc *sc)
 
 	sc->sc_espintr = NCR_READ_REG(sc, NCR_INTR);
 
-	if (sc->sc_glue->gl_clear_latched_intr != NULL)
-		(*sc->sc_glue->gl_clear_latched_intr)(sc);
-
 	/*
 	 * Determine the SCSI bus phase, return either a real SCSI bus phase
 	 * or some pseudo phase we use to detect certain exceptions.
@@ -806,7 +807,7 @@ ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	struct ncr53c9x_tinfo *ti;
 	uint8_t *cmd;
 	size_t dmasize;
-	int clen, selatn3, selatns;
+	int clen, error, selatn3, selatns;
 	int lun = ecb->ccb->ccb_h.target_lun;
 	int target = ecb->ccb->ccb_h.target_id;
 
@@ -887,13 +888,19 @@ ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 		dmasize = clen;
 		sc->sc_cmdlen = clen;
 		sc->sc_cmdp = cmd;
-		NCRDMA_SETUP(sc, &sc->sc_cmdp, &sc->sc_cmdlen, 0, &dmasize);
+		error = NCRDMA_SETUP(sc, &sc->sc_cmdp, &sc->sc_cmdlen, 0,
+		    &dmasize);
+		if (error != 0) {
+			sc->sc_cmdlen = 0;
+			sc->sc_cmdp = NULL;
+			goto cmd;
+		}
+
 		/* Program the SCSI counter. */
 		NCR_SET_COUNT(sc, dmasize);
 
 		/* Load the count in. */
-		/* if (sc->sc_rev != NCR_VARIANT_FAS366) */
-			NCRCMD(sc, NCRCMD_NOP | NCRCMD_DMA);
+		NCRCMD(sc, NCRCMD_NOP | NCRCMD_DMA);
 
 		/* And get the target's attention. */
 		if (selatn3) {
@@ -906,6 +913,7 @@ ncr53c9x_select(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 		return;
 	}
 
+cmd:
 	/*
 	 * Who am I?  This is where we tell the target that we are
 	 * happy for it to disconnect etc.
@@ -989,13 +997,11 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 	case XPT_RESET_BUS:
 		ncr53c9x_init(sc, 1);
 		ccb->ccb_h.status = CAM_REQ_CMP;
-		xpt_done(ccb);
-		return;
+		break;
 
 	case XPT_CALC_GEOMETRY:
 		cam_calc_geometry(&ccb->ccg, sc->sc_extended_geom);
-		xpt_done(ccb);
-		return;
+		break;
 
 	case XPT_PATH_INQ:
 		cpi = &ccb->cpi;
@@ -1009,19 +1015,19 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 		cpi->max_target = sc->sc_ntarg - 1;
 		cpi->max_lun = 7;
 		cpi->initiator_id = sc->sc_id;
-		cpi->bus_id = 0;
-		cpi->base_transfer_speed = 3300;
 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
-		strncpy(cpi->hba_vid, "Sun", HBA_IDLEN);
+		strncpy(cpi->hba_vid, "NCR", HBA_IDLEN);
 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 		cpi->unit_number = cam_sim_unit(sim);
-		cpi->transport = XPORT_SPI;
-		cpi->transport_version = 2;
+		cpi->bus_id = 0;
+		cpi->base_transfer_speed = 3300;
 		cpi->protocol = PROTO_SCSI;
 		cpi->protocol_version = SCSI_REV_2;
+		cpi->transport = XPORT_SPI;
+		cpi->transport_version = 2;
+		cpi->maxio = sc->sc_maxxfer;
 		ccb->ccb_h.status = CAM_REQ_CMP;
-		xpt_done(ccb);
-		return;
+		break;
 
 	case XPT_GET_TRAN_SETTINGS:
 		cts = &ccb->cts;
@@ -1064,28 +1070,24 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 		    CTS_SPI_VALID_DISC;
 		scsi->valid = CTS_SCSI_VALID_TQ;
 		ccb->ccb_h.status = CAM_REQ_CMP;
-		xpt_done(ccb);
-		return;
+		break;
 
 	case XPT_ABORT:
 		device_printf(sc->sc_dev, "XPT_ABORT called\n");
 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
-		xpt_done(ccb);
-		return;
+		break;
 
 	case XPT_TERM_IO:
 		device_printf(sc->sc_dev, "XPT_TERM_IO called\n");
 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
-		xpt_done(ccb);
-		return;
+		break;
 
 	case XPT_RESET_DEV:
 	case XPT_SCSI_IO:
 		if (ccb->ccb_h.target_id < 0 ||
 		    ccb->ccb_h.target_id >= sc->sc_ntarg) {
 			ccb->ccb_h.status = CAM_PATH_INVALID;
-			xpt_done(ccb);
-			return;
+			goto done;
 		}
 		/* Get an ECB to use. */
 		ecb = ncr53c9x_get_ecb(sc);
@@ -1097,8 +1099,7 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 			xpt_freeze_simq(sim, 1);
 			ccb->ccb_h.status = CAM_REQUEUE_REQ;
 			device_printf(sc->sc_dev, "unable to allocate ecb\n");
-			xpt_done(ccb);
-			return;
+			goto done;
 		}
 
 		/* Initialize ecb. */
@@ -1127,7 +1128,7 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 		ecb->flags |= ECB_READY;
 		if (sc->sc_state == NCR_IDLE)
 			ncr53c9x_sched(sc);
-		break;
+		return;
 
 	case XPT_SET_TRAN_SETTINGS:
 		cts = &ccb->cts;
@@ -1165,16 +1166,16 @@ ncr53c9x_action(struct cam_sim *sim, union ccb *ccb)
 		}
 
 		ccb->ccb_h.status = CAM_REQ_CMP;
-		xpt_done(ccb);
-		return;
+		break;
 
 	default:
 		device_printf(sc->sc_dev, "Unhandled function code %d\n",
 		    ccb->ccb_h.func_code);
 		ccb->ccb_h.status = CAM_PROVIDE_FAIL;
-		xpt_done(ccb);
-		return;
 	}
+
+done:
+	xpt_done(ccb);
 }
 
 /*
@@ -1329,11 +1330,10 @@ ncr53c9x_sched(struct ncr53c9x_softc *sc)
 			sc->sc_nexus = ecb;
 			ncr53c9x_select(sc, ecb);
 			break;
-		} else {
+		} else
 			NCR_TRACE(("[%s %d:%d busy] \n", __func__,
 			    ecb->ccb->ccb_h.target_id,
 			    ecb->ccb->ccb_h.target_lun));
-		}
 	}
 }
 
@@ -1412,10 +1412,10 @@ ncr53c9x_done(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 	 */
 	if (ccb->ccb_h.status == CAM_REQ_CMP) {
 		ccb->csio.scsi_status = ecb->stat;
-		if ((ecb->flags & ECB_ABORT) != 0) {
+		if ((ecb->flags & ECB_ABORT) != 0)
 			ccb->ccb_h.status = CAM_CMD_TIMEOUT;
-		} else if ((ecb->flags & ECB_SENSE) != 0 &&
-			   (ecb->stat != SCSI_STATUS_CHECK_COND)) {
+		else if ((ecb->flags & ECB_SENSE) != 0 &&
+		   (ecb->stat != SCSI_STATUS_CHECK_COND)) {
 			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
 			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR |
 			    CAM_AUTOSNS_VALID;
@@ -1439,13 +1439,15 @@ ncr53c9x_done(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 				}
 				ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 			}
-		} else {
+		} else
 			ccb->csio.resid = ecb->dleft;
-		}
 		if (ecb->stat == SCSI_STATUS_QUEUE_FULL)
 			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 		else if (ecb->stat == SCSI_STATUS_BUSY)
 			ccb->ccb_h.status = CAM_SCSI_BUSY;
+	} else if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
+		ccb->ccb_h.status |= CAM_DEV_QFRZN;
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
 	}
 
 #ifdef NCR53C9X_DEBUG
@@ -1473,7 +1475,7 @@ ncr53c9x_done(struct ncr53c9x_softc *sc, struct ncr53c9x_ecb *ecb)
 		}
 	}
 
-	if (ccb->ccb_h.status == CAM_SEL_TIMEOUT) {
+	if ((ccb->ccb_h.status & CAM_SEL_TIMEOUT) != 0) {
 		/* Selection timeout -- discard this LUN if empty. */
 		if (li->untagged == NULL && li->used == 0) {
 			if (lun < NCR_NLUN)
@@ -2030,8 +2032,8 @@ gotit:
 
 			default:
 				xpt_print_path(ecb->ccb->ccb_h.path);
-				printf("unrecognized MESSAGE EXTENDED;"
-				    " sending REJECT\n");
+				printf("unrecognized MESSAGE EXTENDED 0x%x;"
+				    " sending REJECT\n", sc->sc_imess[2]);
 				goto reject;
 			}
 			break;
@@ -2039,7 +2041,8 @@ gotit:
 		default:
 			NCR_MSGS(("ident "));
 			xpt_print_path(ecb->ccb->ccb_h.path);
-			printf("unrecognized MESSAGE; sending REJECT\n");
+			printf("unrecognized MESSAGE 0x%x; sending REJECT\n",
+			    sc->sc_imess[0]);
 			/* FALLTHROUGH */
 		reject:
 			ncr53c9x_sched_msgout(SEND_REJECT);
@@ -2109,6 +2112,7 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 	struct ncr53c9x_tinfo *ti;
 	struct ncr53c9x_ecb *ecb;
 	size_t size;
+	int error;
 #ifdef NCR53C9X_DEBUG
 	int i;
 #endif
@@ -2246,17 +2250,14 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 		NCR_MSGS(("> "));
 	}
 #endif
-	if (sc->sc_rev == NCR_VARIANT_FAS366) {
-		/*
-		 * XXX FIFO size
-		 */
-		ncr53c9x_flushfifo(sc);
-		ncr53c9x_wrfifo(sc, sc->sc_omp, sc->sc_omlen);
-		NCRCMD(sc, NCRCMD_TRANS);
-	} else {
+
+	if (sc->sc_rev != NCR_VARIANT_FAS366) {
 		/* (Re)send the message. */
 		size = ulmin(sc->sc_omlen, sc->sc_maxxfer);
-		NCRDMA_SETUP(sc, &sc->sc_omp, &sc->sc_omlen, 0, &size);
+		error = NCRDMA_SETUP(sc, &sc->sc_omp, &sc->sc_omlen, 0, &size);
+		if (error != 0)
+			goto cmd;
+
 		/* Program the SCSI counter. */
 		NCR_SET_COUNT(sc, size);
 
@@ -2264,7 +2265,16 @@ ncr53c9x_msgout(struct ncr53c9x_softc *sc)
 		NCRCMD(sc, NCRCMD_NOP | NCRCMD_DMA);
 		NCRCMD(sc, NCRCMD_TRANS | NCRCMD_DMA);
 		NCRDMA_GO(sc);
+		return;
 	}
+
+cmd:
+	/*
+	 * XXX FIFO size
+	 */
+	ncr53c9x_flushfifo(sc);
+	ncr53c9x_wrfifo(sc, sc->sc_omp, sc->sc_omlen);
+	NCRCMD(sc, NCRCMD_TRANS);
 }
 
 void
@@ -2299,7 +2309,7 @@ ncr53c9x_intr1(struct ncr53c9x_softc *sc)
 	struct ncr53c9x_tinfo *ti;
 	struct timeval cur, wait;
 	size_t size;
-	int i, nfifo;
+	int error, i, nfifo;
 	uint8_t msg;
 
 	NCR_LOCK_ASSERT(sc, MA_OWNED);
@@ -2974,8 +2984,14 @@ msgin:
 			size = ecb->clen;
 			sc->sc_cmdlen = size;
 			sc->sc_cmdp = (void *)&ecb->cmd.cmd;
-			NCRDMA_SETUP(sc, &sc->sc_cmdp, &sc->sc_cmdlen,
+			error = NCRDMA_SETUP(sc, &sc->sc_cmdp, &sc->sc_cmdlen,
 			    0, &size);
+			if (error != 0) {
+				sc->sc_cmdlen = 0;
+				sc->sc_cmdp = NULL;
+				goto cmd;
+			}
+
 			/* Program the SCSI counter. */
 			NCR_SET_COUNT(sc, size);
 
@@ -2985,30 +3001,51 @@ msgin:
 			/* Start the command transfer. */
 			NCRCMD(sc, NCRCMD_TRANS | NCRCMD_DMA);
 			NCRDMA_GO(sc);
-		} else {
-			ncr53c9x_wrfifo(sc, (uint8_t *)&ecb->cmd.cmd,
-			    ecb->clen);
-			NCRCMD(sc, NCRCMD_TRANS);
+			sc->sc_prevphase = COMMAND_PHASE;
+			break;
 		}
+cmd:
+		ncr53c9x_wrfifo(sc, (uint8_t *)&ecb->cmd.cmd, ecb->clen);
+		NCRCMD(sc, NCRCMD_TRANS);
 		sc->sc_prevphase = COMMAND_PHASE;
 		break;
 
 	case DATA_OUT_PHASE:
 		NCR_PHASE(("DATA_OUT_PHASE [%ld] ", (long)sc->sc_dleft));
+		sc->sc_prevphase = DATA_OUT_PHASE;
 		NCRCMD(sc, NCRCMD_FLUSH);
 		size = ulmin(sc->sc_dleft, sc->sc_maxxfer);
-		NCRDMA_SETUP(sc, &sc->sc_dp, &sc->sc_dleft, 0, &size);
-		sc->sc_prevphase = DATA_OUT_PHASE;
+		error = NCRDMA_SETUP(sc, &sc->sc_dp, &sc->sc_dleft, 0, &size);
 		goto setup_xfer;
 
 	case DATA_IN_PHASE:
 		NCR_PHASE(("DATA_IN_PHASE "));
+		sc->sc_prevphase = DATA_IN_PHASE;
 		if (sc->sc_rev == NCR_VARIANT_ESP100)
 			NCRCMD(sc, NCRCMD_FLUSH);
 		size = ulmin(sc->sc_dleft, sc->sc_maxxfer);
-		NCRDMA_SETUP(sc, &sc->sc_dp, &sc->sc_dleft, 1, &size);
-		sc->sc_prevphase = DATA_IN_PHASE;
-	setup_xfer:
+		error = NCRDMA_SETUP(sc, &sc->sc_dp, &sc->sc_dleft, 1, &size);
+setup_xfer:
+		if (error != 0) {
+			switch (error) {
+			case EFBIG:
+				ecb->ccb->ccb_h.status |= CAM_REQ_TOO_BIG;
+				break;
+			case EINPROGRESS:
+				panic("%s: cannot deal with deferred DMA",
+				    __func__);
+			case EINVAL:
+				ecb->ccb->ccb_h.status |= CAM_REQ_INVALID;
+				break;
+			case ENOMEM:
+				ecb->ccb->ccb_h.status |= CAM_REQUEUE_REQ;
+				break;
+			default:
+				ecb->ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
+			}
+			goto finish;
+		}
+
 		/* Target returned to data phase: wipe "done" memory */
 		ecb->flags &= ~ECB_TENTATIVE_DONE;
 
diff --git a/sys/dev/esp/ncr53c9xreg.h b/sys/dev/esp/ncr53c9xreg.h
index ab03421..dffbf7c 100644
--- a/sys/dev/esp/ncr53c9xreg.h
+++ b/sys/dev/esp/ncr53c9xreg.h
@@ -31,6 +31,9 @@
 
 /* $FreeBSD$ */
 
+#ifndef _NCR53C9XREG_H_
+#define	_NCR53C9XREG_H_
+
 /*
  * Register addresses, relative to some base address
  */
@@ -288,3 +291,4 @@
 #define	NCRFAS_STAT2_OSHUTTLE 0x40	   /* next byte	from FIFO is MSB */
 #define	NCRFAS_STAT2_EMPTY    0x80	   /* FIFO is empty */
 
+#endif /* _NCR53C9XREG_H_ */
diff --git a/sys/dev/esp/ncr53c9xvar.h b/sys/dev/esp/ncr53c9xvar.h
index 38285fc..88ae049 100644
--- a/sys/dev/esp/ncr53c9xvar.h
+++ b/sys/dev/esp/ncr53c9xvar.h
@@ -68,8 +68,8 @@
 
 /* $FreeBSD$ */
 
-#ifndef _DEV_IC_NCR53C9XVAR_H_
-#define	_DEV_IC_NCR53C9XVAR_H_
+#ifndef _NCR53C9XVAR_H_
+#define	_NCR53C9XVAR_H_
 
 #include <sys/lock.h>
 
@@ -115,7 +115,8 @@
  * scsi_status,sense_data}.
  */
 struct ncr53c9x_ecb {
-	/* These fields are preserved between alloc and free */
+	/* These fields are preserved between alloc and free. */
+	struct callout ch;
 	struct ncr53c9x_softc *sc;
 	int tag_id;
 	int flags;
@@ -130,7 +131,6 @@ struct ncr53c9x_ecb {
 #define	ECB_RESET		0x80
 #define	ECB_TENTATIVE_DONE	0x100
 	int timeout;
-	struct callout ch;
 
 	struct {
 		uint8_t	msg[3];			/* Selection Id msg and tags */
@@ -290,7 +290,7 @@ extern int ncr53c9x_debug;
 struct ncr53c9x_softc;
 
 /*
- * Function switch used as glue to MD code.
+ * Function switch used as glue to MD code
  */
 struct ncr53c9x_glue {
 	/* Mandatory entry points. */
@@ -304,9 +304,6 @@ struct ncr53c9x_glue {
 	void	(*gl_dma_go)(struct ncr53c9x_softc *);
 	void	(*gl_dma_stop)(struct ncr53c9x_softc *);
 	int	(*gl_dma_isactive)(struct ncr53c9x_softc *);
-
-	/* Optional entry points. */
-	void	(*gl_clear_latched_intr)(struct ncr53c9x_softc *);
 };
 
 struct ncr53c9x_softc {
@@ -330,7 +327,7 @@ struct ncr53c9x_softc {
 	uint8_t	sc_ccf;				/* Clock Conversion */
 	uint8_t	sc_timeout;
 
-	/* register copies, see espreadregs() */
+	/* register copies, see ncr53c9x_readregs() */
 	uint8_t	sc_espintr;
 	uint8_t	sc_espstat;
 	uint8_t	sc_espstep;
@@ -415,6 +412,7 @@ struct ncr53c9x_softc {
 #define	NCR_F_FASTSCSI	0x02	/* chip supports Fast mode */
 #define	NCR_F_DMASELECT 0x04	/* can do dmaselect */
 #define	NCR_F_SELATN3	0x08	/* chip supports SELATN3 command */
+#define	NCR_F_LARGEXFER	0x10	/* chip supports transfers > 64k */
 
 /* values for sc_msgout */
 #define	SEND_DEV_RESET		0x0001
@@ -499,8 +497,10 @@ struct ncr53c9x_softc {
 #define	ncr53c9x_cpb2stp(sc, cpb)					\
 	((250 * (cpb)) / (sc)->sc_freq)
 
+extern devclass_t esp_devclass;
+
 int	ncr53c9x_attach(struct ncr53c9x_softc *sc);
 int	ncr53c9x_detach(struct ncr53c9x_softc *sc);
 void	ncr53c9x_intr(void *arg);
 
-#endif /* _DEV_IC_NCR53C9XVAR_H_ */
+#endif /* _NCR53C9XVAR_H_ */
diff --git a/sys/dev/fb/vesa.c b/sys/dev/fb/vesa.c
index 0c05699..ee97ff1 100644
--- a/sys/dev/fb/vesa.c
+++ b/sys/dev/fb/vesa.c
@@ -95,7 +95,7 @@ static size_t vesa_bios_size = 0;
 /* VESA video adapter */
 static video_adapter_t *vesa_adp = NULL;
 
-SYSCTL_NODE(_debug, OID_AUTO, vesa, CTLFLAG_RD, NULL, "VESA debugging");
+static SYSCTL_NODE(_debug, OID_AUTO, vesa, CTLFLAG_RD, NULL, "VESA debugging");
 static int vesa_shadow_rom = 0;
 TUNABLE_INT("debug.vesa.shadow_rom", &vesa_shadow_rom);
 SYSCTL_INT(_debug_vesa, OID_AUTO, shadow_rom, CTLFLAG_RDTUN, &vesa_shadow_rom,
diff --git a/sys/dev/fdc/fdc.c b/sys/dev/fdc/fdc.c
index 01998ec..00cdcd6 100644
--- a/sys/dev/fdc/fdc.c
+++ b/sys/dev/fdc/fdc.c
@@ -267,7 +267,7 @@ static driver_filter_t fdc_intr_fast;
 static void fdc_reset(struct fdc_data *);
 static int fd_probe_disk(struct fd_data *, int *);
 
-SYSCTL_NODE(_debug, OID_AUTO, fdc, CTLFLAG_RW, 0, "fdc driver");
+static SYSCTL_NODE(_debug, OID_AUTO, fdc, CTLFLAG_RW, 0, "fdc driver");
 
 static int fifo_threshold = 8;
 SYSCTL_INT(_debug_fdc, OID_AUTO, fifo, CTLFLAG_RW, &fifo_threshold, 0,
diff --git a/sys/dev/firewire/fwmem.c b/sys/dev/firewire/fwmem.c
index 1799682..a342d14 100644
--- a/sys/dev/firewire/fwmem.c
+++ b/sys/dev/firewire/fwmem.c
@@ -73,7 +73,7 @@ __FBSDID("$FreeBSD$");
 static int fwmem_speed=2, fwmem_debug=0;
 static struct fw_eui64 fwmem_eui64;
 SYSCTL_DECL(_hw_firewire);
-SYSCTL_NODE(_hw_firewire, OID_AUTO, fwmem, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_hw_firewire, OID_AUTO, fwmem, CTLFLAG_RD, 0,
 	"FireWire Memory Access");
 SYSCTL_UINT(_hw_firewire_fwmem, OID_AUTO, eui64_hi, CTLFLAG_RW,
 	&fwmem_eui64.hi, 0, "Fwmem target EUI64 high");
@@ -84,7 +84,7 @@ SYSCTL_INT(_hw_firewire_fwmem, OID_AUTO, speed, CTLFLAG_RW, &fwmem_speed, 0,
 SYSCTL_INT(_debug, OID_AUTO, fwmem_debug, CTLFLAG_RW, &fwmem_debug, 0,
 	"Fwmem driver debug flag");
 
-MALLOC_DEFINE(M_FWMEM, "fwmem", "fwmem/FireWire");
+static MALLOC_DEFINE(M_FWMEM, "fwmem", "fwmem/FireWire");
 
 #define MAXLEN (512 << fwmem_speed)
 
diff --git a/sys/dev/firewire/if_fwe.c b/sys/dev/firewire/if_fwe.c
index 62616cf..52a9c8d 100644
--- a/sys/dev/firewire/if_fwe.c
+++ b/sys/dev/firewire/if_fwe.c
@@ -86,10 +86,10 @@ static int stream_ch = 1;
 static int tx_speed = 2;
 static int rx_queue_len = FWMAXQUEUE;
 
-MALLOC_DEFINE(M_FWE, "if_fwe", "Ethernet over FireWire interface");
+static MALLOC_DEFINE(M_FWE, "if_fwe", "Ethernet over FireWire interface");
 SYSCTL_INT(_debug, OID_AUTO, if_fwe_debug, CTLFLAG_RW, &fwedebug, 0, "");
 SYSCTL_DECL(_hw_firewire);
-SYSCTL_NODE(_hw_firewire, OID_AUTO, fwe, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_hw_firewire, OID_AUTO, fwe, CTLFLAG_RD, 0,
 	"Ethernet emulation subsystem");
 SYSCTL_INT(_hw_firewire_fwe, OID_AUTO, stream_ch, CTLFLAG_RW, &stream_ch, 0,
 	"Stream channel to use");
diff --git a/sys/dev/firewire/if_fwip.c b/sys/dev/firewire/if_fwip.c
index 66cdb4a..e7c4a66 100644
--- a/sys/dev/firewire/if_fwip.c
+++ b/sys/dev/firewire/if_fwip.c
@@ -99,10 +99,10 @@ static int broadcast_channel = 0xc0 | 0x1f; /*  tag | channel(XXX) */
 static int tx_speed = 2;
 static int rx_queue_len = FWMAXQUEUE;
 
-MALLOC_DEFINE(M_FWIP, "if_fwip", "IP over FireWire interface");
+static MALLOC_DEFINE(M_FWIP, "if_fwip", "IP over FireWire interface");
 SYSCTL_INT(_debug, OID_AUTO, if_fwip_debug, CTLFLAG_RW, &fwipdebug, 0, "");
 SYSCTL_DECL(_hw_firewire);
-SYSCTL_NODE(_hw_firewire, OID_AUTO, fwip, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_hw_firewire, OID_AUTO, fwip, CTLFLAG_RD, 0,
 	"Firewire ip subsystem");
 SYSCTL_INT(_hw_firewire_fwip, OID_AUTO, rx_queue_len, CTLFLAG_RW, &rx_queue_len,
 	0, "Length of the receive queue");
diff --git a/sys/dev/firewire/sbp.c b/sys/dev/firewire/sbp.c
index 2387bf9..9204449 100644
--- a/sys/dev/firewire/sbp.c
+++ b/sys/dev/firewire/sbp.c
@@ -132,7 +132,8 @@ static int use_doorbell = 0;
 static int sbp_tags = 0;
 
 SYSCTL_DECL(_hw_firewire);
-SYSCTL_NODE(_hw_firewire, OID_AUTO, sbp, CTLFLAG_RD, 0, "SBP-II Subsystem");
+static SYSCTL_NODE(_hw_firewire, OID_AUTO, sbp, CTLFLAG_RD, 0,
+	"SBP-II Subsystem");
 SYSCTL_INT(_debug, OID_AUTO, sbp_debug, CTLFLAG_RW, &debug, 0,
 	"SBP debug flag");
 SYSCTL_INT(_hw_firewire_sbp, OID_AUTO, auto_login, CTLFLAG_RW, &auto_login, 0,
@@ -276,7 +277,7 @@ static void sbp_mgm_timeout (void *arg);
 static void sbp_timeout (void *arg);
 static void sbp_mgm_orb (struct sbp_dev *, int, struct sbp_ocb *);
 
-MALLOC_DEFINE(M_SBP, "sbp", "SBP-II/FireWire");
+static MALLOC_DEFINE(M_SBP, "sbp", "SBP-II/FireWire");
 
 /* cam related functions */
 static void	sbp_action(struct cam_sim *sim, union ccb *ccb);
diff --git a/sys/dev/firewire/sbp_targ.c b/sys/dev/firewire/sbp_targ.c
index f04b968..9be31bc4 100644
--- a/sys/dev/firewire/sbp_targ.c
+++ b/sys/dev/firewire/sbp_targ.c
@@ -95,7 +95,7 @@
 #define F_HOLD		(1 << 3)
 #define F_FREEZED	(1 << 4)
 
-MALLOC_DEFINE(M_SBP_TARG, "sbp_targ", "SBP-II/FireWire target mode");
+static MALLOC_DEFINE(M_SBP_TARG, "sbp_targ", "SBP-II/FireWire target mode");
 
 static int debug = 0;
 
diff --git a/sys/dev/gpio/gpiobus.c b/sys/dev/gpio/gpiobus.c
index d973694..54644a2 100644
--- a/sys/dev/gpio/gpiobus.c
+++ b/sys/dev/gpio/gpiobus.c
@@ -219,8 +219,7 @@ static int
 gpiobus_detach(device_t dev)
 {
 	struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev);
-	int err, ndevs, i;
-	device_t *devlist;
+	int err;
 
 	KASSERT(mtx_initialized(&sc->sc_mtx),
 	    ("gpiobus mutex not initialized"));
@@ -228,16 +227,14 @@ gpiobus_detach(device_t dev)
 
 	if ((err = bus_generic_detach(dev)) != 0)
 		return (err);
-	if ((err = device_get_children(dev, &devlist, &ndevs)) != 0)
-		return (err);
-	for (i = 0; i < ndevs; i++)
-		device_delete_child(dev, devlist[i]);
+
+	/* detach and delete all children */
+	device_delete_all_children(dev);
 
 	if (sc->sc_pins_mapped) {
 		free(sc->sc_pins_mapped, M_DEVBUF);
 		sc->sc_pins_mapped = NULL;
 	}
-	free(devlist, M_TEMP);
 
 	return (0);
 }
diff --git a/sys/dev/hifn/hifn7751.c b/sys/dev/hifn/hifn7751.c
index 2416f2b..2da5a9e 100644
--- a/sys/dev/hifn/hifn7751.c
+++ b/sys/dev/hifn/hifn7751.c
@@ -184,7 +184,8 @@ READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
 }
 #define	WRITE_REG_1(sc, reg, val)	hifn_write_reg_1(sc, reg, val)
 
-SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0, "Hifn driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0,
+	    "Hifn driver parameters");
 
 #ifdef HIFN_DEBUG
 static	int hifn_debug = 0;
diff --git a/sys/dev/hwpmc/hwpmc_mips24k.c b/sys/dev/hwpmc/hwpmc_mips24k.c
index 0b2a117..4970171 100644
--- a/sys/dev/hwpmc/hwpmc_mips24k.c
+++ b/sys/dev/hwpmc/hwpmc_mips24k.c
@@ -254,6 +254,8 @@ mips24k_allocate_pmc(int cpu, int ri, struct pmc *pm,
 		config |= MIPS24K_PMC_USER_ENABLE;
 	if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
 		config |= MIPS24K_PMC_ENABLE;
+	if (caps & PMC_CAP_INTERRUPT)
+		config |= MIPS24K_PMC_INTERRUPT_ENABLE;
 
 	pm->pm_md.pm_mips24k.pm_mips24k_evsel = config;
 
@@ -404,7 +406,65 @@ mips24k_release_pmc(int cpu, int ri, struct pmc *pmc)
 static int
 mips24k_intr(int cpu, struct trapframe *tf)
 {
-	return 0;
+	int error;
+	int retval, ri;
+	struct pmc *pm;
+	struct mips24k_cpu *pc;
+	uint32_t r, r0, r2;
+
+	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+	    ("[mips24k,%d] CPU %d out of range", __LINE__, cpu));
+
+	retval = 0;
+	pc = mips24k_pcpu[cpu];
+
+	/* Stop PMCs without clearing the counter */
+	r0 = mips_rd_perfcnt0();
+	mips_wr_perfcnt0(r0 & ~(0x1f));
+	r2 = mips_rd_perfcnt2();
+	mips_wr_perfcnt2(r2 & ~(0x1f));
+
+	for (ri = 0; ri < mips24k_npmcs; ri++) {
+		pm = mips24k_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
+		if (pm == NULL)
+			continue;
+		if (! PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
+			continue;
+
+		r = mips24k_pmcn_read(ri);
+
+		/* If bit 31 is set, the counter has overflowed */
+		if ((r & 0x80000000) == 0)
+			continue;
+
+		retval = 1;
+		if (pm->pm_state != PMC_STATE_RUNNING)
+			continue;
+		error = pmc_process_interrupt(cpu, pm, tf,
+		    TRAPF_USERMODE(tf));
+		if (error) {
+			/* Clear/disable the relevant counter */
+			if (ri == 0)
+				r0 = 0;
+			else if (ri == 1)
+				r2 = 0;
+			mips24k_stop_pmc(cpu, ri);
+		}
+
+		/* Reload sampling count */
+		mips24k_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount);
+	}
+
+	/*
+	 * Re-enable the PMC counters where they left off.
+	 *
+	 * Any counter which overflowed will have its sample count
+	 * reloaded in the loop above.
+	 */
+	mips_wr_perfcnt0(r0);
+	mips_wr_perfcnt2(r2);
+
+	return retval;
 }
 
 static int
diff --git a/sys/dev/iicbus/ad7417.c b/sys/dev/iicbus/ad7417.c
index 6ae16be..acd2d89 100644
--- a/sys/dev/iicbus/ad7417.c
+++ b/sys/dev/iicbus/ad7417.c
@@ -121,7 +121,7 @@ static driver_t ad7417_driver = {
 static devclass_t ad7417_devclass;
 
 DRIVER_MODULE(ad7417, iicbus, ad7417_driver, ad7417_devclass, 0, 0);
-MALLOC_DEFINE(M_AD7417, "ad7417", "Supply-Monitor AD7417");
+static MALLOC_DEFINE(M_AD7417, "ad7417", "Supply-Monitor AD7417");
 
 
 static int
diff --git a/sys/dev/iicbus/max6690.c b/sys/dev/iicbus/max6690.c
index 83f3b50..ab312e7 100644
--- a/sys/dev/iicbus/max6690.c
+++ b/sys/dev/iicbus/max6690.c
@@ -101,7 +101,7 @@ static driver_t max6690_driver = {
 static devclass_t max6690_devclass;
 
 DRIVER_MODULE(max6690, iicbus, max6690_driver, max6690_devclass, 0, 0);
-MALLOC_DEFINE(M_MAX6690, "max6690", "Temp-Monitor MAX6690");
+static MALLOC_DEFINE(M_MAX6690, "max6690", "Temp-Monitor MAX6690");
 
 static int
 max6690_read(device_t dev, uint32_t addr, uint8_t reg, uint8_t *data)
diff --git a/sys/dev/iir/iir.c b/sys/dev/iir/iir.c
index 2be7c39..057194a 100644
--- a/sys/dev/iir/iir.c
+++ b/sys/dev/iir/iir.c
@@ -69,7 +69,7 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/iir/iir.h>
 
-MALLOC_DEFINE(M_GDTBUF, "iirbuf", "iir driver buffer");
+static MALLOC_DEFINE(M_GDTBUF, "iirbuf", "iir driver buffer");
 
 struct gdt_softc *gdt_wait_gdt;
 int     gdt_wait_index;
diff --git a/sys/dev/ipmi/ipmi.c b/sys/dev/ipmi/ipmi.c
index 74a7d03..6b7d464 100644
--- a/sys/dev/ipmi/ipmi.c
+++ b/sys/dev/ipmi/ipmi.c
@@ -63,7 +63,8 @@ static void ipmi_dtor(void *arg);
 int ipmi_attached = 0;
 
 static int on = 1;
-SYSCTL_NODE(_hw, OID_AUTO, ipmi, CTLFLAG_RD, 0, "IPMI driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, ipmi, CTLFLAG_RD, 0,
+    "IPMI driver parameters");
 SYSCTL_INT(_hw_ipmi, OID_AUTO, on, CTLFLAG_RW,
 	&on, 0, "");
 
@@ -75,7 +76,7 @@ static struct cdevsw ipmi_cdevsw = {
 	.d_name =	"ipmi",
 };
 
-MALLOC_DEFINE(M_IPMI, "ipmi", "ipmi");
+static MALLOC_DEFINE(M_IPMI, "ipmi", "ipmi");
 
 static int
 ipmi_open(struct cdev *cdev, int flags, int fmt, struct thread *td)
diff --git a/sys/dev/iscsi/initiator/isc_subr.c b/sys/dev/iscsi/initiator/isc_subr.c
index 7a8138b..4d75a87 100644
--- a/sys/dev/iscsi/initiator/isc_subr.c
+++ b/sys/dev/iscsi/initiator/isc_subr.c
@@ -58,7 +58,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/iscsi/initiator/iscsi.h>
 #include <dev/iscsi/initiator/iscsivar.h>
 
-MALLOC_DEFINE(M_ISC, "iSC", "iSCSI driver options");
+static MALLOC_DEFINE(M_ISC, "iSC", "iSCSI driver options");
 
 static char *
 i_strdupin(char *s, size_t maxlen)
diff --git a/sys/dev/iscsi/initiator/iscsi.c b/sys/dev/iscsi/initiator/iscsi.c
index 292ce8f..4c06b98 100644
--- a/sys/dev/iscsi/initiator/iscsi.c
+++ b/sys/dev/iscsi/initiator/iscsi.c
@@ -66,7 +66,7 @@ static struct isc_softc *isc;
 
 MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI driver");
 MALLOC_DEFINE(M_ISCSIBUF, "iSCbuf", "iSCSI buffers");
-MALLOC_DEFINE(M_TMP, "iSCtmp", "iSCSI tmp");
+static MALLOC_DEFINE(M_TMP, "iSCtmp", "iSCSI tmp");
 
 #ifdef ISCSI_INITIATOR_DEBUG
 int iscsi_debug = ISCSI_INITIATOR_DEBUG;
diff --git a/sys/dev/iscsi/initiator/iscsivar.h b/sys/dev/iscsi/initiator/iscsivar.h
index 0a56f80..875d7ad 100644
--- a/sys/dev/iscsi/initiator/iscsivar.h
+++ b/sys/dev/iscsi/initiator/iscsivar.h
@@ -64,7 +64,6 @@ typedef uint32_t digest_t(const void *, int len, uint32_t ocrc);
 
 MALLOC_DECLARE(M_ISCSI);
 MALLOC_DECLARE(M_ISCSIBUF);
-MALLOC_DECLARE(M_PDU);
 
 #define ISOK2DIG(dig, pp)	((dig != NULL) && ((pp->ipdu.bhs.opcode & 0x1f) != ISCSI_LOGIN_CMD))
 
diff --git a/sys/dev/isp/DriverManual.txt b/sys/dev/isp/DriverManual.txt
index 0ed5a1b..3d2f0fc 100644
--- a/sys/dev/isp/DriverManual.txt
+++ b/sys/dev/isp/DriverManual.txt
@@ -327,7 +327,7 @@ here in clarifying some of this.
 
 A succesful execution of isp_init will lead to the driver 'registering'
 itself with this platform's SCSI subsystem. One assumed action for this
-is the registry of a function the SCSI subsystem for this platform
+is the registry of a function that the SCSI subsystem for this platform
 will call when it has a SCSI command to run.
 
 The platform specific module function that receives this will do whatever
diff --git a/sys/dev/isp/isp.c b/sys/dev/isp/isp.c
index dfba62e..00a04b6 100644
--- a/sys/dev/isp/isp.c
+++ b/sys/dev/isp/isp.c
@@ -748,11 +748,13 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 
 	if (dodnld && IS_24XX(isp)) {
 		const uint32_t *ptr = isp->isp_mdvec->dv_ispfw;
+		int wordload;
 
 		/*
 		 * Keep loading until we run out of f/w.
 		 */
 		code_org = ptr[2];	/* 1st load address is our start addr */
+		wordload = 0;
 
 		for (;;) {
 			uint32_t la, wi, wl;
@@ -777,6 +779,7 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 					wl--;
 				}
 				MEMORYBARRIER(isp, SYNC_REQUEST, 0, ISP_QUEUE_SIZE(RQUEST_QUEUE_LEN(isp)), -1);
+	again:
 				ISP_MEMZERO(&mbs, sizeof (mbs));
 				if (la < 0x10000 && nw < 0x10000) {
 					mbs.param[0] = MBOX_LOAD_RISC_RAM_2100;
@@ -786,6 +789,23 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 					mbs.param[4] = nw;
 					mbs.param[6] = DMA_WD3(isp->isp_rquest_dma);
 					mbs.param[7] = DMA_WD2(isp->isp_rquest_dma);
+					isp_prt(isp, ISP_LOGDEBUG0, "LOAD RISC RAM 2100 %u words at load address 0x%x", nw, la);
+				} else if (wordload) {
+					union {
+						const uint32_t *cp;
+						uint32_t *np;
+					} ucd;
+					ucd.cp = (const uint32_t *)cp;
+					mbs.param[0] = MBOX_WRITE_RAM_WORD_EXTENDED;
+					mbs.param[1] = la;
+					mbs.param[2] = (*ucd.np);
+					mbs.param[3] = (*ucd.np) >> 16;
+					mbs.param[8] = la >> 16;
+					isp->isp_mbxwrk0 = nw - 1;
+					isp->isp_mbxworkp = ucd.np+1;
+					isp->isp_mbxwrk1 = (la + 1);
+					isp->isp_mbxwrk8 = (la + 1) >> 16;
+					isp_prt(isp, ISP_LOGDEBUG0, "WRITE RAM WORD EXTENDED %u words at load address 0x%x", nw, la);
 				} else {
 					mbs.param[0] = MBOX_LOAD_RISC_RAM;
 					mbs.param[1] = la;
@@ -796,10 +816,16 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 					mbs.param[6] = DMA_WD3(isp->isp_rquest_dma);
 					mbs.param[7] = DMA_WD2(isp->isp_rquest_dma);
 					mbs.param[8] = la >> 16;
+					isp_prt(isp, ISP_LOGDEBUG0, "LOAD RISC RAM %u words at load address 0x%x", nw, la);
 				}
 				mbs.logval = MBLOGALL;
 				isp_mboxcmd(isp, &mbs);
 				if (mbs.param[0] != MBOX_COMMAND_COMPLETE) {
+					if (mbs.param[0] == MBOX_HOST_INTERFACE_ERROR) {
+						isp_prt(isp, ISP_LOGERR, "switching to word load");
+						wordload = 1;
+						goto again;
+					}
 					isp_prt(isp, ISP_LOGERR, "F/W Risc Ram Load Failed");
 					ISP_RESET0(isp);
 					return;
@@ -855,6 +881,7 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 					mbs.param[4] = nw;
 					mbs.param[6] = DMA_WD3(isp->isp_rquest_dma);
 					mbs.param[7] = DMA_WD2(isp->isp_rquest_dma);
+					isp_prt(isp, ISP_LOGDEBUG1, "LOAD RISC RAM 2100 %u words at load address 0x%x\n", nw, la);
 				} else {
 					mbs.param[0] = MBOX_LOAD_RISC_RAM;
 					mbs.param[1] = la;
@@ -864,6 +891,7 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 					mbs.param[6] = DMA_WD3(isp->isp_rquest_dma);
 					mbs.param[7] = DMA_WD2(isp->isp_rquest_dma);
 					mbs.param[8] = la >> 16;
+					isp_prt(isp, ISP_LOGDEBUG1, "LOAD RISC RAM %u words at load address 0x%x\n", nw, la);
 				}
 				mbs.logval = MBLOGALL;
 				isp_mboxcmd(isp, &mbs);
@@ -910,6 +938,7 @@ isp_reset(ispsoftc_t *isp, int do_load_defaults)
 		mbs.param[1] = code_org;
 		mbs.param[2] = ucd.np[0];
 		mbs.logval = MBLOGNONE;
+		isp_prt(isp, ISP_LOGDEBUG1, "WRITE RAM %u words at load address 0x%x\n", ucd.np[3], code_org);
 		isp_mboxcmd(isp, &mbs);
 		if (mbs.param[0] != MBOX_COMMAND_COMPLETE) {
 			isp_prt(isp, ISP_LOGERR, "F/W download failed at word %d", isp->isp_mbxwrk1 - code_org);
@@ -6589,23 +6618,39 @@ isp_mbox_continue(ispsoftc_t *isp)
 		mbs.param[1] = isp->isp_mbxwrk1++;
 		break;
 	case MBOX_WRITE_RAM_WORD_EXTENDED:
+		if (IS_24XX(isp)) {
+			uint32_t *lptr = (uint32_t *)ptr;
+			mbs.param[2] = lptr[0];
+			mbs.param[3] = lptr[0] >> 16;
+			lptr++;
+			ptr = (uint16_t *)lptr;
+		} else {
+			mbs.param[2] = *ptr++;
+		}
 		offset = isp->isp_mbxwrk1;
 		offset |= isp->isp_mbxwrk8 << 16;
-
-		mbs.param[2] = *ptr++;
 		mbs.param[1] = offset;
 		mbs.param[8] = offset >> 16;
-		isp->isp_mbxwrk1 = ++offset;
+		offset++;
+		isp->isp_mbxwrk1 = offset;
 		isp->isp_mbxwrk8 = offset >> 16;
 		break;
 	case MBOX_READ_RAM_WORD_EXTENDED:
+		if (IS_24XX(isp)) {
+			uint32_t *lptr = (uint32_t *)ptr;
+			uint32_t val = isp->isp_mboxtmp[2];
+			val |= (isp->isp_mboxtmp[3]) << 16;
+			*lptr++ = val;
+			ptr = (uint16_t *)lptr;
+		} else {
+			*ptr++ = isp->isp_mboxtmp[2];
+		}
 		offset = isp->isp_mbxwrk1;
 		offset |= isp->isp_mbxwrk8 << 16;
-
-		*ptr++ = isp->isp_mboxtmp[2];
 		mbs.param[1] = offset;
 		mbs.param[8] = offset >> 16;
-		isp->isp_mbxwrk1 = ++offset;
+		offset++;
+		isp->isp_mbxwrk1 = offset;
 		isp->isp_mbxwrk8 = offset >> 16;
 		break;
 	}
@@ -6830,7 +6875,7 @@ static const uint32_t mbpfc[] = {
 	ISPOPMAP(0x00, 0x00),	/* 0x0c: */
 	ISPOPMAP(0x10f, 0x01),	/* 0x0d: MBOX_WRITE_RAM_WORD_EXTENDED */
 	ISPOPMAP(0x01, 0x05),	/* 0x0e: MBOX_CHECK_FIRMWARE */
-	ISPOPMAP(0x10f, 0x05),	/* 0x0f: MBOX_READ_RAM_WORD_EXTENDED */
+	ISPOPMAP(0x103, 0x0d),	/* 0x0f: MBOX_READ_RAM_WORD_EXTENDED */
 	ISPOPMAP(0x1f, 0x11),	/* 0x10: MBOX_INIT_REQ_QUEUE */
 	ISPOPMAP(0x2f, 0x21),	/* 0x11: MBOX_INIT_RES_QUEUE */
 	ISPOPMAP(0x0f, 0x01),	/* 0x12: MBOX_EXECUTE_IOCB */
@@ -6962,13 +7007,13 @@ static const char *fc_mbcmd_names[] = {
 	"MAILBOX REG TEST",
 	"VERIFY CHECKSUM",
 	"ABOUT FIRMWARE",
-	"LOAD RAM",
+	"LOAD RAM (2100)",
 	"DUMP RAM",
-	"WRITE RAM WORD EXTENDED",
+	"LOAD RISC RAM",
 	NULL,
-	"READ RAM WORD EXTENDED",
+	"WRITE RAM WORD EXTENDED",
 	"CHECK FIRMWARE",
-	NULL,
+	"READ RAM WORD EXTENDED",
 	"INIT REQUEST QUEUE",
 	"INIT RESULT QUEUE",
 	"EXECUTE IOCB",
diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index 8d62f5e..3d676ab 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -175,6 +175,14 @@ isp_attach_chan(ispsoftc_t *isp, struct cam_devq *devq, int chan)
 			isp_prt(isp, ISP_LOGERR, "cannot create test target thread");
 		}
 #endif
+		if (chan == 0) {
+			struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(isp->isp_osinfo.dev);
+			struct sysctl_oid *tree = device_get_sysctl_tree(isp->isp_osinfo.dev);
+			SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwnn", CTLFLAG_RD, &FCPARAM(isp, 0)->isp_wwnn, "World Wide Node Name");
+			SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwpn", CTLFLAG_RD, &FCPARAM(isp, 0)->isp_wwpn, "World Wide Port Name");
+			SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "loop_down_limit", CTLFLAG_RW, &ISP_FC_PC(isp, 0)->loop_down_limit, 0, "Loop Down Limit");
+			SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "gone_device_time", CTLFLAG_RW, &ISP_FC_PC(isp, 0)->gone_device_time, 0, "Gone Device Time");
+		}
 	}
 	return (0);
 }
diff --git a/sys/dev/isp/isp_freebsd.h b/sys/dev/isp/isp_freebsd.h
index 5d1b500..0875e64 100644
--- a/sys/dev/isp/isp_freebsd.h
+++ b/sys/dev/isp/isp_freebsd.h
@@ -38,6 +38,7 @@
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/condvar.h>
+#include <sys/sysctl.h>
 
 #include <sys/proc.h>
 #include <sys/bus.h>
diff --git a/sys/dev/isp/isp_pci.c b/sys/dev/isp/isp_pci.c
index b6b23fa..9a0bc4a 100644
--- a/sys/dev/isp/isp_pci.c
+++ b/sys/dev/isp/isp_pci.c
@@ -1458,6 +1458,7 @@ imc(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 		imushp->error = EINVAL;
 		return;
 	}
+	isp_prt(imushp->isp, ISP_LOGDEBUG0, "request/result area @ 0x%jx/0x%jx", (uintmax_t) segs->ds_addr, (uintmax_t) segs->ds_len);
 	imushp->isp->isp_rquest = imushp->vbase;
 	imushp->isp->isp_rquest_dma = segs->ds_addr;
 	segs->ds_addr += ISP_QUEUE_SIZE(RQUEST_QUEUE_LEN(imushp->isp));
@@ -1487,6 +1488,7 @@ imc1(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 		imushp->error = EINVAL;
 		return;
 	}
+	isp_prt(imushp->isp, ISP_LOGDEBUG0, "scdma @ 0x%jx/0x%jx", (uintmax_t) segs->ds_addr, (uintmax_t) segs->ds_len);
 	FCPARAM(imushp->isp, imushp->chan)->isp_scdma = segs->ds_addr;
 	FCPARAM(imushp->isp, imushp->chan)->isp_scratch = imushp->vbase;
 }
diff --git a/sys/dev/kbd/kbd.c b/sys/dev/kbd/kbd.c
index 01e262e..8036762 100644
--- a/sys/dev/kbd/kbd.c
+++ b/sys/dev/kbd/kbd.c
@@ -81,7 +81,7 @@ static keyboard_switch_t *kbdsw_ini;
        keyboard_switch_t **kbdsw = &kbdsw_ini;
 
 static int keymap_restrict_change;
-SYSCTL_NODE(_hw, OID_AUTO, kbd, CTLFLAG_RD, 0, "kbd");
+static SYSCTL_NODE(_hw, OID_AUTO, kbd, CTLFLAG_RD, 0, "kbd");
 SYSCTL_INT(_hw_kbd, OID_AUTO, keymap_restrict_change, CTLFLAG_RW,
     &keymap_restrict_change, 0, "restrict ability to change keymap");
 
diff --git a/sys/dev/lmc/if_lmc.c b/sys/dev/lmc/if_lmc.c
index d5fb9ba..40dc1e5 100644
--- a/sys/dev/lmc/if_lmc.c
+++ b/sys/dev/lmc/if_lmc.c
@@ -4945,7 +4945,9 @@ lmc_ifnet_detach(softc_t *sc)
   /* Detach from the ifnet kernel interface. */
   if_detach(sc->ifp);
 
-# if (__FreeBSD_version >= 600000)
+# if (defined(__FreeBSD__) && __FreeBSD_version >= 800082)
+  if_free(sc->ifp);
+# elif (defined(__FreeBSD__) && __FreeBSD_version >= 600000)
   if_free_type(sc->ifp, NSPPP ? IFT_PPP : IFT_OTHER);
 # endif
   }
diff --git a/sys/dev/malo/if_malo.c b/sys/dev/malo/if_malo.c
index ae6ef88..6a7a786 100644
--- a/sys/dev/malo/if_malo.c
+++ b/sys/dev/malo/if_malo.c
@@ -123,7 +123,7 @@ enum {
 } while (0)
 #endif
 
-MALLOC_DEFINE(M_MALODEV, "malodev", "malo driver dma buffers");
+static MALLOC_DEFINE(M_MALODEV, "malodev", "malo driver dma buffers");
 
 static struct ieee80211vap *malo_vap_create(struct ieee80211com *ic,
 	const char name[IFNAMSIZ], int unit, int opmode, int flags,
diff --git a/sys/dev/malo/if_malo_pci.c b/sys/dev/malo/if_malo_pci.c
index 9c2027b..de9c39d 100644
--- a/sys/dev/malo/if_malo_pci.c
+++ b/sys/dev/malo/if_malo_pci.c
@@ -79,7 +79,7 @@ struct malo_pci_softc {
  * Tunable variables.
  */
 SYSCTL_DECL(_hw_malo);
-SYSCTL_NODE(_hw_malo, OID_AUTO, pci, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_hw_malo, OID_AUTO, pci, CTLFLAG_RD, 0,
     "Marvell 88W8335 driver PCI parameters");
 
 static int msi_disable = 0;				/* MSI disabled  */
diff --git a/sys/dev/md/md.c b/sys/dev/md/md.c
index 078eca8..35d5832 100644
--- a/sys/dev/md/md.c
+++ b/sys/dev/md/md.c
@@ -1370,6 +1370,11 @@ g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 			    indent, (uintmax_t) mp->fwsectors);
 			sbuf_printf(sb, "%s<length>%ju</length>\n",
 			    indent, (uintmax_t) mp->mediasize);
+			sbuf_printf(sb, "%s<compression>%s</compression>\n", indent,
+			    (mp->flags & MD_COMPRESS) == 0 ? "off": "on");
+			sbuf_printf(sb, "%s<access>%s</access>\n", indent,
+			    (mp->flags & MD_READONLY) == 0 ? "read-write":
+			    "read-only");
 			sbuf_printf(sb, "%s<type>%s</type>\n", indent,
 			    type);
 			if (mp->type == MD_VNODE && mp->vnode != NULL)
diff --git a/sys/dev/mfi/mfi.c b/sys/dev/mfi/mfi.c
index 8a3d277..e55d7da 100644
--- a/sys/dev/mfi/mfi.c
+++ b/sys/dev/mfi/mfi.c
@@ -484,15 +484,8 @@ mfi_attach(struct mfi_softc *sc)
 	mtx_unlock(&sc->mfi_io_lock);
 
 	/*
-	 * Set up the interrupt handler.  XXX This should happen in
-	 * mfi_pci.c
+	 * Set up the interrupt handler.
 	 */
-	sc->mfi_irq_rid = 0;
-	if ((sc->mfi_irq = bus_alloc_resource_any(sc->mfi_dev, SYS_RES_IRQ,
-	    &sc->mfi_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-		device_printf(sc->mfi_dev, "Cannot allocate interrupt\n");
-		return (EINVAL);
-	}
 	if (bus_setup_intr(sc->mfi_dev, sc->mfi_irq, INTR_MPSAFE|INTR_TYPE_BIO,
 	    NULL, mfi_intr, sc, &sc->mfi_intr)) {
 		device_printf(sc->mfi_dev, "Cannot set up interrupt\n");
@@ -932,6 +925,12 @@ mfi_intr(void *arg)
 	if (sc->mfi_check_clear_intr(sc))
 		return;
 
+	/*
+	 * Do a dummy read to flush the interrupt ACK that we just performed,
+	 * ensuring that everything is really, truly consistent.
+	 */
+	(void)sc->mfi_read_fw_status(sc);
+
 	pi = sc->mfi_comms->hw_pi;
 	ci = sc->mfi_comms->hw_ci;
 	mtx_lock(&sc->mfi_io_lock);
diff --git a/sys/dev/mfi/mfi_cam.c b/sys/dev/mfi/mfi_cam.c
index c49daff..aa76a10 100644
--- a/sys/dev/mfi/mfi_cam.c
+++ b/sys/dev/mfi/mfi_cam.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/uio.h>
 #include <sys/proc.h>
 #include <sys/signalvar.h>
+#include <sys/sysctl.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
diff --git a/sys/dev/mfi/mfi_debug.c b/sys/dev/mfi/mfi_debug.c
index 515e6ed..8fe7778 100644
--- a/sys/dev/mfi/mfi_debug.c
+++ b/sys/dev/mfi/mfi_debug.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/malloc.h>
 #include <sys/selinfo.h>
+#include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/uio.h>
 #include <machine/resource.h>
diff --git a/sys/dev/mfi/mfi_disk.c b/sys/dev/mfi/mfi_disk.c
index 4594ca2..ad656d3c 100644
--- a/sys/dev/mfi/mfi_disk.c
+++ b/sys/dev/mfi/mfi_disk.c
@@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/selinfo.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
+#include <sys/sysctl.h>
 #include <sys/uio.h>
 
 #include <sys/bio.h>
diff --git a/sys/dev/mfi/mfi_pci.c b/sys/dev/mfi/mfi_pci.c
index 685aa0b..8de2e12 100644
--- a/sys/dev/mfi/mfi_pci.c
+++ b/sys/dev/mfi/mfi_pci.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/bio.h>
 #include <sys/malloc.h>
+#include <sys/sysctl.h>
 #include <sys/uio.h>
 
 #include <machine/bus.h>
@@ -107,6 +108,11 @@ static devclass_t	mfi_devclass;
 DRIVER_MODULE(mfi, pci, mfi_pci_driver, mfi_devclass, 0, 0);
 MODULE_VERSION(mfi, 1);
 
+static int	mfi_msi = 0;
+TUNABLE_INT("hw.mfi.msi", &mfi_msi);
+SYSCTL_INT(_hw_mfi, OID_AUTO, msi, CTLFLAG_RDTUN, &mfi_msi, 0,
+    "Enable use of MSI interrupts");
+
 struct mfi_ident {
 	uint16_t	vendor;
 	uint16_t	device;
@@ -169,7 +175,7 @@ mfi_pci_attach(device_t dev)
 	struct mfi_softc *sc;
 	struct mfi_ident *m;
 	uint32_t command;
-	int error;
+	int count, error;
 
 	sc = device_get_softc(dev);
 	bzero(sc, sizeof(*sc));
@@ -226,6 +232,20 @@ mfi_pci_attach(device_t dev)
 		goto out;
 	}
 
+	/* Allocate IRQ resource. */
+	sc->mfi_irq_rid = 0;
+	count = 1;
+	if (mfi_msi && pci_alloc_msi(sc->mfi_dev, &count) == 0) {
+		device_printf(sc->mfi_dev, "Using MSI\n");
+		sc->mfi_irq_rid = 1;
+	}
+	if ((sc->mfi_irq = bus_alloc_resource_any(sc->mfi_dev, SYS_RES_IRQ,
+	    &sc->mfi_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
+		device_printf(sc->mfi_dev, "Cannot allocate interrupt\n");
+		error = EINVAL;
+		goto out;
+	}
+
 	error = mfi_attach(sc);
 out:
 	if (error) {
@@ -280,6 +300,8 @@ mfi_pci_free(struct mfi_softc *sc)
 		bus_release_resource(sc->mfi_dev, SYS_RES_MEMORY,
 		    sc->mfi_regs_rid, sc->mfi_regs_resource);
 	}
+	if (sc->mfi_irq_rid != 0)
+		pci_release_msi(sc->mfi_dev);
 
 	return;
 }
diff --git a/sys/dev/mfi/mfivar.h b/sys/dev/mfi/mfivar.h
index 14ea884..9fcd0d9 100644
--- a/sys/dev/mfi/mfivar.h
+++ b/sys/dev/mfi/mfivar.h
@@ -352,12 +352,29 @@ mfi_dequeue_bio(struct mfi_softc *sc)
 	return (bp);
 }
 
+/*
+ * This is from the original scsi_extract_sense() in CAM.  It's copied
+ * here because CAM now uses a non-inline version that follows more complex
+ * additions to the SPC spec, and we don't want to force a dependency on
+ * the CAM module for such a trivial action.
+ */
+static __inline void
+mfi_extract_sense(struct scsi_sense_data_fixed *sense,
+    int *error_code, int *sense_key, int *asc, int *ascq)
+{
+
+	*error_code = sense->error_code & SSD_ERRCODE;
+	*sense_key = sense->flags & SSD_KEY;
+	*asc = (sense->extra_len >= 5) ? sense->add_sense_code : 0;
+	*ascq = (sense->extra_len >= 6) ? sense->add_sense_code_qual : 0;
+}
+
 static __inline void
 mfi_print_sense(struct mfi_softc *sc, void *sense)
 {
 	int error, key, asc, ascq;
 
-	scsi_extract_sense((struct scsi_sense_data *)sense,
+	mfi_extract_sense((struct scsi_sense_data_fixed *)sense,
 	    &error, &key, &asc, &ascq);
 	device_printf(sc->mfi_dev, "sense error %d, sense_key %d, "
 	    "asc %d, ascq %d\n", error, key, asc, ascq);
@@ -378,6 +395,7 @@ mfi_print_sense(struct mfi_softc *sc, void *sense)
 	(sc)->mfi_bhandle, (reg))
 
 MALLOC_DECLARE(M_MFIBUF);
+SYSCTL_DECL(_hw_mfi);
 
 #define MFI_CMD_TIMEOUT 30
 #define MFI_MAXPHYS (128 * 1024)
diff --git a/sys/dev/mii/brgphy.c b/sys/dev/mii/brgphy.c
index 88090b7..2c14ea5 100644
--- a/sys/dev/mii/brgphy.c
+++ b/sys/dev/mii/brgphy.c
@@ -141,6 +141,7 @@ static const struct mii_phydesc brgphys[] = {
 	MII_PHY_DESC(BROADCOM2, BCM5784),
 	MII_PHY_DESC(BROADCOM3, BCM5717C),
 	MII_PHY_DESC(BROADCOM3, BCM5719C),
+	MII_PHY_DESC(BROADCOM3, BCM5720C),
 	MII_PHY_DESC(BROADCOM3, BCM57765),
 	MII_PHY_DESC(xxBROADCOM_ALT1, BCM5906),
 	MII_PHY_END
diff --git a/sys/dev/mii/mii.c b/sys/dev/mii/mii.c
index 224f85d..2c99bc0 100644
--- a/sys/dev/mii/mii.c
+++ b/sys/dev/mii/mii.c
@@ -48,7 +48,6 @@ __FBSDID("$FreeBSD$");
 
 #include <net/if.h>
 #include <net/if_media.h>
-#include <net/route.h>
 
 #include <dev/mii/mii.h>
 #include <dev/mii/miivar.h>
@@ -57,18 +56,20 @@ MODULE_VERSION(miibus, 1);
 
 #include "miibus_if.h"
 
-static int miibus_print_child(device_t dev, device_t child);
-static int miibus_read_ivar(device_t dev, device_t child, int which,
-    uintptr_t *result);
-static int miibus_child_location_str(device_t bus, device_t child, char *buf,
-    size_t buflen);
-static int miibus_child_pnpinfo_str(device_t bus, device_t child, char *buf,
-    size_t buflen);
-static int miibus_readreg(device_t, int, int);
-static int miibus_writereg(device_t, int, int, int);
-static void miibus_statchg(device_t);
-static void miibus_linkchg(device_t);
-static void miibus_mediainit(device_t);
+static device_attach_t miibus_attach;
+static bus_child_location_str_t miibus_child_location_str;
+static bus_child_pnpinfo_str_t miibus_child_pnpinfo_str;
+static device_detach_t miibus_detach;
+static bus_hinted_child_t miibus_hinted_child;
+static bus_print_child_t miibus_print_child;
+static device_probe_t miibus_probe;
+static bus_read_ivar_t miibus_read_ivar;
+static miibus_readreg_t miibus_readreg;
+static miibus_statchg_t miibus_statchg;
+static miibus_writereg_t miibus_writereg;
+static miibus_linkchg_t miibus_linkchg;
+static miibus_mediainit_t miibus_mediainit;
+
 static unsigned char mii_bitreverse(unsigned char x);
 
 static device_method_t miibus_methods[] = {
@@ -84,6 +85,7 @@ static device_method_t miibus_methods[] = {
 	DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
 	DEVMETHOD(bus_child_pnpinfo_str, miibus_child_pnpinfo_str),
 	DEVMETHOD(bus_child_location_str, miibus_child_location_str),
+	DEVMETHOD(bus_hinted_child,	miibus_hinted_child),
 
 	/* MII interface */
 	DEVMETHOD(miibus_readreg,	miibus_readreg),
@@ -107,10 +109,11 @@ struct miibus_ivars {
 	struct ifnet	*ifp;
 	ifm_change_cb_t	ifmedia_upd;
 	ifm_stat_cb_t	ifmedia_sts;
-	int		mii_flags;
+	u_int		mii_flags;
+	u_int		mii_offset;
 };
 
-int
+static int
 miibus_probe(device_t dev)
 {
 
@@ -119,7 +122,7 @@ miibus_probe(device_t dev)
 	return (BUS_PROBE_SPECIFIC);
 }
 
-int
+static int
 miibus_attach(device_t dev)
 {
 	struct miibus_ivars	*ivars;
@@ -129,7 +132,6 @@ miibus_attach(device_t dev)
 	int			i, nchildren;
 
 	mii = device_get_softc(dev);
-	nchildren = 0;
 	if (device_get_children(dev, &children, &nchildren) == 0) {
 		for (i = 0; i < nchildren; i++) {
 			ma = device_get_ivars(children[i]);
@@ -152,7 +154,7 @@ miibus_attach(device_t dev)
 	return (bus_generic_attach(dev));
 }
 
-int
+static int
 miibus_detach(device_t dev)
 {
 	struct mii_data		*mii;
@@ -201,7 +203,7 @@ miibus_read_ivar(device_t dev, device_t child __unused, int which,
 }
 
 static int
-miibus_child_pnpinfo_str(device_t bus __unused, device_t child, char *buf,
+miibus_child_pnpinfo_str(device_t dev __unused, device_t child, char *buf,
     size_t buflen)
 {
 	struct mii_attach_args *ma;
@@ -214,7 +216,7 @@ miibus_child_pnpinfo_str(device_t bus __unused, device_t child, char *buf,
 }
 
 static int
-miibus_child_location_str(device_t bus __unused, device_t child, char *buf,
+miibus_child_location_str(device_t dev __unused, device_t child, char *buf,
     size_t buflen)
 {
 	struct mii_attach_args *ma;
@@ -224,6 +226,60 @@ miibus_child_location_str(device_t bus __unused, device_t child, char *buf,
 	return (0);
 }
 
+static void
+miibus_hinted_child(device_t dev, const char *name, int unit)
+{
+	struct miibus_ivars *ivars;
+	struct mii_attach_args *args, *ma;
+	device_t *children, phy;
+	int i, nchildren;
+	u_int val;
+
+	if (resource_int_value(name, unit, "phyno", &val) != 0)
+		return;
+	if (device_get_children(dev, &children, &nchildren) != 0)
+		return;
+	ma = NULL;
+	for (i = 0; i < nchildren; i++) {
+		args = device_get_ivars(children[i]);
+		if (args->mii_phyno == val) {
+			ma = args;
+			break;
+		}
+	}
+	free(children, M_TEMP);
+
+	/*
+	 * Don't add a PHY that was automatically identified by having media
+	 * in its BMSR twice, only allow to alter its attach arguments.
+	 */
+	if (ma == NULL) {
+		ma = malloc(sizeof(struct mii_attach_args), M_DEVBUF,
+		    M_NOWAIT);
+		if (ma == NULL)
+			return;
+		phy = device_add_child(dev, name, unit);
+		if (phy == NULL) {
+			free(ma, M_DEVBUF);
+			return;
+		}
+		ivars = device_get_ivars(dev);
+		ma->mii_phyno = val;
+		ma->mii_offset = ivars->mii_offset++;
+		ma->mii_id1 = 0;
+		ma->mii_id2 = 0;
+		ma->mii_capmask = BMSR_DEFCAPMASK;
+		device_set_ivars(phy, ma);
+	}
+
+	if (resource_int_value(name, unit, "id1", &val) == 0)
+		ma->mii_id1 = val;
+	if (resource_int_value(name, unit, "id2", &val) == 0)
+		ma->mii_id2 = val;
+	if (resource_int_value(name, unit, "capmask", &val) == 0)
+		ma->mii_capmask = val;
+}
+
 static int
 miibus_readreg(device_t dev, int phy, int reg)
 {
@@ -307,9 +363,10 @@ mii_attach(device_t dev, device_t *miibus, struct ifnet *ifp,
     int phyloc, int offloc, int flags)
 {
 	struct miibus_ivars *ivars;
-	struct mii_attach_args ma, *args;
+	struct mii_attach_args *args, ma;
 	device_t *children, phy;
-	int bmsr, first, i, nchildren, offset, phymax, phymin, rv;
+	int bmsr, first, i, nchildren, phymax, phymin, rv;
+	uint32_t phymask;
 
 	if (phyloc != MII_PHY_ANY && offloc != MII_OFFSET_ANY) {
 		printf("%s: phyloc and offloc specified\n", __func__);
@@ -366,27 +423,30 @@ mii_attach(device_t dev, device_t *miibus, struct ifnet *ifp,
 
 	ma.mii_capmask = capmask;
 
-	phy = NULL;
-	offset = 0;
+	if (resource_int_value(device_get_name(*miibus),
+	    device_get_unit(*miibus), "phymask", &phymask) != 0)
+		phymask = 0xffffffff;
+
+	if (device_get_children(*miibus, &children, &nchildren) != 0) {
+		children = NULL;
+		nchildren = 0;
+	}
+	ivars->mii_offset = 0;
 	for (ma.mii_phyno = phymin; ma.mii_phyno <= phymax; ma.mii_phyno++) {
 		/*
 		 * Make sure we haven't already configured a PHY at this
 		 * address.  This allows mii_attach() to be called
 		 * multiple times.
 		 */
-		if (device_get_children(*miibus, &children, &nchildren) == 0) {
-			for (i = 0; i < nchildren; i++) {
-				args = device_get_ivars(children[i]);
-				if (args->mii_phyno == ma.mii_phyno) {
-					/*
-					 * Yes, there is already something
-					 * configured at this address.
-					 */
-					free(children, M_TEMP);
-					goto skip;
-				}
+		for (i = 0; i < nchildren; i++) {
+			args = device_get_ivars(children[i]);
+			if (args->mii_phyno == ma.mii_phyno) {
+				/*
+				 * Yes, there is already something
+				 * configured at this address.
+				 */
+				goto skip;
 			}
-			free(children, M_TEMP);
 		}
 
 		/*
@@ -405,18 +465,24 @@ mii_attach(device_t dev, device_t *miibus, struct ifnet *ifp,
 		 * There is a PHY at this address.  If we were given an
 		 * `offset' locator, skip this PHY if it doesn't match.
 		 */
-		if (offloc != MII_OFFSET_ANY && offloc != offset)
+		if (offloc != MII_OFFSET_ANY && offloc != ivars->mii_offset)
+			goto skip;
+
+		/*
+		 * Skip this PHY if it's not included in the phymask hint.
+		 */
+		if ((phymask & (1 << ma.mii_phyno)) == 0)
 			goto skip;
 
 		/*
-		 * Extract the IDs. Braindead PHYs will be handled by
+		 * Extract the IDs.  Braindead PHYs will be handled by
 		 * the `ukphy' driver, as we have no ID information to
 		 * match on.
 		 */
 		ma.mii_id1 = MIIBUS_READREG(dev, ma.mii_phyno, MII_PHYIDR1);
 		ma.mii_id2 = MIIBUS_READREG(dev, ma.mii_phyno, MII_PHYIDR2);
 
-		ma.mii_offset = offset;
+		ma.mii_offset = ivars->mii_offset;
 		args = malloc(sizeof(struct mii_attach_args), M_DEVBUF,
 		    M_NOWAIT);
 		if (args == NULL)
@@ -429,15 +495,24 @@ mii_attach(device_t dev, device_t *miibus, struct ifnet *ifp,
 		}
 		device_set_ivars(phy, args);
  skip:
-		offset++;
+		ivars->mii_offset++;
 	}
+	free(children, M_TEMP);
 
 	if (first != 0) {
-		if (phy == NULL) {
+		rv = device_probe(*miibus);
+		if (rv != 0)
+			goto fail;
+		bus_enumerate_hinted_children(*miibus);
+		rv = device_get_children(*miibus, &children, &nchildren);
+		if (rv != 0)
+			goto fail;
+		free(children, M_TEMP);
+		if (nchildren == 0) {
 			rv = ENXIO;
 			goto fail;
 		}
-		rv = bus_generic_attach(dev);
+		rv = device_attach(*miibus);
 		if (rv != 0)
 			goto fail;
 
diff --git a/sys/dev/mii/mii_bitbang.c b/sys/dev/mii/mii_bitbang.c
new file mode 100644
index 0000000..6a5f5dc
--- /dev/null
+++ b/sys/dev/mii/mii_bitbang.c
@@ -0,0 +1,180 @@
+/*	$NetBSD: mii_bitbang.c,v 1.12 2008/05/04 17:06:09 xtraeme Exp $	*/
+
+/*-
+ * Copyright (c) 1999 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following didevlaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following didevlaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Common module for bit-bang'ing the MII.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/module.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
+
+MODULE_VERSION(mii_bitbang, 1);
+
+static void mii_bitbang_sendbits(device_t dev, mii_bitbang_ops_t ops,
+    uint32_t data, int nbits);
+
+#define	MWRITE(x)							\
+do {									\
+	ops->mbo_write(dev, (x));					\
+	DELAY(1);							\
+} while (/* CONSTCOND */ 0)
+
+#define	MREAD		ops->mbo_read(dev)
+
+#define	MDO		ops->mbo_bits[MII_BIT_MDO]
+#define	MDI		ops->mbo_bits[MII_BIT_MDI]
+#define	MDC		ops->mbo_bits[MII_BIT_MDC]
+#define	MDIRPHY		ops->mbo_bits[MII_BIT_DIR_HOST_PHY]
+#define	MDIRHOST	ops->mbo_bits[MII_BIT_DIR_PHY_HOST]
+
+/*
+ * mii_bitbang_sync:
+ *
+ *	Synchronize the MII.
+ */
+void
+mii_bitbang_sync(device_t dev, mii_bitbang_ops_t ops)
+{
+	int i;
+	uint32_t v;
+
+	v = MDIRPHY | MDO;
+
+	MWRITE(v);
+	for (i = 0; i < 32; i++) {
+		MWRITE(v | MDC);
+		MWRITE(v);
+	}
+}
+
+/*
+ * mii_bitbang_sendbits:
+ *
+ *	Send a series of bits to the MII.
+ */
+static void
+mii_bitbang_sendbits(device_t dev, mii_bitbang_ops_t ops, uint32_t data,
+    int nbits)
+{
+	int i;
+	uint32_t v;
+
+	v = MDIRPHY;
+	MWRITE(v);
+
+	for (i = 1 << (nbits - 1); i != 0; i >>= 1) {
+		if (data & i)
+			v |= MDO;
+		else
+			v &= ~MDO;
+		MWRITE(v);
+		MWRITE(v | MDC);
+		MWRITE(v);
+	}
+}
+
+/*
+ * mii_bitbang_readreg:
+ *
+ *	Read a PHY register by bit-bang'ing the MII.
+ */
+int
+mii_bitbang_readreg(device_t dev, mii_bitbang_ops_t ops, int phy, int reg)
+{
+	int i, error, val;
+
+	mii_bitbang_sync(dev, ops);
+
+	mii_bitbang_sendbits(dev, ops, MII_COMMAND_START, 2);
+	mii_bitbang_sendbits(dev, ops, MII_COMMAND_READ, 2);
+	mii_bitbang_sendbits(dev, ops, phy, 5);
+	mii_bitbang_sendbits(dev, ops, reg, 5);
+
+	/* Switch direction to PHY->host, without a clock transition. */
+	MWRITE(MDIRHOST);
+
+	/* Turnaround clock. */
+	MWRITE(MDIRHOST | MDC);
+	MWRITE(MDIRHOST);
+
+	/* Check for error. */
+	error = MREAD & MDI;
+
+	/* Idle clock. */
+	MWRITE(MDIRHOST | MDC);
+	MWRITE(MDIRHOST);
+
+	val = 0;
+	for (i = 0; i < 16; i++) {
+		val <<= 1;
+		/* Read data prior to clock low-high transition. */
+		if (error == 0 && (MREAD & MDI) != 0)
+			val |= 1;
+
+		MWRITE(MDIRHOST | MDC);
+		MWRITE(MDIRHOST);
+	}
+
+	/* Set direction to host->PHY, without a clock transition. */
+	MWRITE(MDIRPHY);
+
+	return (error != 0 ? 0 : val);
+}
+
+/*
+ * mii_bitbang_writereg:
+ *
+ *	Write a PHY register by bit-bang'ing the MII.
+ */
+void
+mii_bitbang_writereg(device_t dev, mii_bitbang_ops_t ops, int phy, int reg,
+    int val)
+{
+
+	mii_bitbang_sync(dev, ops);
+
+	mii_bitbang_sendbits(dev, ops, MII_COMMAND_START, 2);
+	mii_bitbang_sendbits(dev, ops, MII_COMMAND_WRITE, 2);
+	mii_bitbang_sendbits(dev, ops, phy, 5);
+	mii_bitbang_sendbits(dev, ops, reg, 5);
+	mii_bitbang_sendbits(dev, ops, MII_COMMAND_ACK, 2);
+	mii_bitbang_sendbits(dev, ops, val, 16);
+
+	MWRITE(MDIRPHY);
+}
diff --git a/sys/dev/mii/mii_bitbang.h b/sys/dev/mii/mii_bitbang.h
new file mode 100644
index 0000000..2bc7427
--- /dev/null
+++ b/sys/dev/mii/mii_bitbang.h
@@ -0,0 +1,54 @@
+/*	$NetBSD: mii_bitbang.h,v 1.6 2009/05/12 14:31:27 cegger Exp $	*/
+
+/*-
+ * Copyright (c) 1999 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define	MII_BIT_MDO		0	/* data out (host->PHY) */
+#define	MII_BIT_MDI		1	/* data in (PHY->host) */
+#define	MII_BIT_MDC		2	/* clock */
+#define	MII_BIT_DIR_HOST_PHY	3	/* set direction: host->PHY */
+#define	MII_BIT_DIR_PHY_HOST	4	/* set direction: PHY->host */
+#define	MII_NBITS		5
+
+struct mii_bitbang_ops {
+	uint32_t	(*mbo_read)(device_t);
+	void		(*mbo_write)(device_t, uint32_t);
+	uint32_t	mbo_bits[MII_NBITS];
+};
+
+typedef	const struct mii_bitbang_ops *mii_bitbang_ops_t;
+
+int	mii_bitbang_readreg(device_t dev, mii_bitbang_ops_t ops,
+	    int phy, int reg);
+void	mii_bitbang_sync(device_t dev, mii_bitbang_ops_t ops);
+void	mii_bitbang_writereg(device_t dev, mii_bitbang_ops_t ops,
+	    int phy, int reg, int val);
diff --git a/sys/dev/mii/miidevs b/sys/dev/mii/miidevs
index 1b98542..d069c68 100644
--- a/sys/dev/mii/miidevs
+++ b/sys/dev/mii/miidevs
@@ -179,6 +179,7 @@ model BROADCOM2 BCM5709S	0x003f BCM5709S 1000/2500baseSX PHY
 model BROADCOM3 BCM5717C	0x0020 BCM5717C 1000BASE-T media interface
 model BROADCOM3 BCM5719C	0x0022 BCM5719C 1000BASE-T media interface
 model BROADCOM3 BCM57765	0x0024 BCM57765 1000BASE-T media interface
+model BROADCOM3 BCM5720C	0x0036 BCM5720C 1000BASE-T media interface
 model xxBROADCOM_ALT1 BCM5906	0x0004 BCM5906 10/100baseTX media interface
 
 /* Cicada Semiconductor PHYs (now owned by Vitesse?) */
diff --git a/sys/dev/mii/miivar.h b/sys/dev/mii/miivar.h
index 9af5b4e..34b0e9ed 100644
--- a/sys/dev/mii/miivar.h
+++ b/sys/dev/mii/miivar.h
@@ -246,10 +246,6 @@ MIIBUS_ACCESSOR(flags,		FLAGS,		u_int)
 extern devclass_t	miibus_devclass;
 extern driver_t		miibus_driver;
 
-int	miibus_probe(device_t);
-int	miibus_attach(device_t);
-int	miibus_detach(device_t);
-
 int	mii_attach(device_t, device_t *, struct ifnet *, ifm_change_cb_t,
 	    ifm_stat_cb_t, int, int, int, int);
 void	mii_down(struct mii_data *);
diff --git a/sys/dev/mmc/mmc.c b/sys/dev/mmc/mmc.c
index 6ff47c3..a5dfe92 100644
--- a/sys/dev/mmc/mmc.c
+++ b/sys/dev/mmc/mmc.c
@@ -105,7 +105,7 @@ struct mmc_ivars {
 
 #define CMD_RETRIES	3
 
-SYSCTL_NODE(_hw, OID_AUTO, mmc, CTLFLAG_RD, NULL, "mmc driver");
+static SYSCTL_NODE(_hw, OID_AUTO, mmc, CTLFLAG_RD, NULL, "mmc driver");
 
 static int mmc_debug;
 SYSCTL_INT(_hw_mmc, OID_AUTO, debug, CTLFLAG_RW, &mmc_debug, 0, "Debug level");
diff --git a/sys/dev/mps/mps_sas.c b/sys/dev/mps/mps_sas.c
index 28ddca1..cf4757a 100644
--- a/sys/dev/mps/mps_sas.c
+++ b/sys/dev/mps/mps_sas.c
@@ -119,7 +119,7 @@ struct mpssas_devprobe {
 #define MPSSAS_DISCOVERY_TIMEOUT	20
 #define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
 
-MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
+static MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
 
 static __inline int mpssas_set_lun(uint8_t *lun, u_int ccblun);
 static struct mpssas_target * mpssas_alloc_target(struct mpssas_softc *,
diff --git a/sys/dev/mpt/mpt.c b/sys/dev/mpt/mpt.c
index 9dfd73b..5f0e0cf 100644
--- a/sys/dev/mpt/mpt.c
+++ b/sys/dev/mpt/mpt.c
@@ -2084,7 +2084,7 @@ mpt_send_port_enable(struct mpt_softc *mpt, int port)
 
 	mpt_send_cmd(mpt, req);
 	error = mpt_wait_req(mpt, req, REQ_STATE_DONE, REQ_STATE_DONE,
-	    FALSE, (mpt->is_sas || mpt->is_fc)? 30000 : 3000);
+	    FALSE, (mpt->is_sas || mpt->is_fc)? 300000 : 30000);
 	if (error != 0) {
 		mpt_prt(mpt, "port %d enable timed out\n", port);
 		return (-1);
diff --git a/sys/dev/msk/if_msk.c b/sys/dev/msk/if_msk.c
index b060949..36407a2 100644
--- a/sys/dev/msk/if_msk.c
+++ b/sys/dev/msk/if_msk.c
@@ -700,7 +700,7 @@ msk_init_rx_ring(struct msk_if_softc *sc_if)
 {
 	struct msk_ring_data *rd;
 	struct msk_rxdesc *rxd;
-	int i, prod;
+	int i, nbuf, prod;
 
 	MSK_IF_LOCK_ASSERT(sc_if);
 
@@ -710,11 +710,18 @@ msk_init_rx_ring(struct msk_if_softc *sc_if)
 
 	rd = &sc_if->msk_rdata;
 	bzero(rd->msk_rx_ring, sizeof(struct msk_rx_desc) * MSK_RX_RING_CNT);
-	prod = sc_if->msk_cdata.msk_rx_prod;
-	i = 0;
+	for (i = prod = 0; i < MSK_RX_RING_CNT; i++) {
+		rxd = &sc_if->msk_cdata.msk_rxdesc[prod];
+		rxd->rx_m = NULL;
+		rxd->rx_le = &rd->msk_rx_ring[prod];
+		MSK_INC(prod, MSK_RX_RING_CNT);
+	}
+	nbuf = MSK_RX_BUF_CNT;
+	prod = 0;
 	/* Have controller know how to compute Rx checksum. */
 	if ((sc_if->msk_flags & MSK_FLAG_DESCV2) == 0 &&
 	    (sc_if->msk_ifp->if_capenable & IFCAP_RXCSUM) != 0) {
+#ifdef MSK_64BIT_DMA
 		rxd = &sc_if->msk_cdata.msk_rxdesc[prod];
 		rxd->rx_m = NULL;
 		rxd->rx_le = &rd->msk_rx_ring[prod];
@@ -723,15 +730,21 @@ msk_init_rx_ring(struct msk_if_softc *sc_if)
 		rxd->rx_le->msk_control = htole32(OP_TCPSTART | HW_OWNER);
 		MSK_INC(prod, MSK_RX_RING_CNT);
 		MSK_INC(sc_if->msk_cdata.msk_rx_cons, MSK_RX_RING_CNT);
-		i++;
-	}
-	for (; i < MSK_RX_RING_CNT; i++) {
+#endif
 		rxd = &sc_if->msk_cdata.msk_rxdesc[prod];
 		rxd->rx_m = NULL;
 		rxd->rx_le = &rd->msk_rx_ring[prod];
+		rxd->rx_le->msk_addr = htole32(ETHER_HDR_LEN << 16 |
+		    ETHER_HDR_LEN);
+		rxd->rx_le->msk_control = htole32(OP_TCPSTART | HW_OWNER);
+		MSK_INC(prod, MSK_RX_RING_CNT);
+		MSK_INC(sc_if->msk_cdata.msk_rx_cons, MSK_RX_RING_CNT);
+		nbuf--;
+	}
+	for (i = 0; i < nbuf; i++) {
 		if (msk_newbuf(sc_if, prod) != 0)
 			return (ENOBUFS);
-		MSK_INC(prod, MSK_RX_RING_CNT);
+		MSK_RX_INC(prod, MSK_RX_RING_CNT);
 	}
 
 	bus_dmamap_sync(sc_if->msk_cdata.msk_rx_ring_tag,
@@ -739,10 +752,11 @@ msk_init_rx_ring(struct msk_if_softc *sc_if)
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	/* Update prefetch unit. */
-	sc_if->msk_cdata.msk_rx_prod = MSK_RX_RING_CNT - 1;
+	sc_if->msk_cdata.msk_rx_prod = prod;
 	CSR_WRITE_2(sc_if->msk_softc,
 	    Y2_PREF_Q_ADDR(sc_if->msk_rxq, PREF_UNIT_PUT_IDX_REG),
-	    sc_if->msk_cdata.msk_rx_prod);
+	    (sc_if->msk_cdata.msk_rx_prod + MSK_RX_RING_CNT - 1) %
+	    MSK_RX_RING_CNT);
 	if (msk_rx_fill(sc_if, 0) != 0)
 		return (ENOBUFS);
 	return (0);
@@ -753,7 +767,7 @@ msk_init_jumbo_rx_ring(struct msk_if_softc *sc_if)
 {
 	struct msk_ring_data *rd;
 	struct msk_rxdesc *rxd;
-	int i, prod;
+	int i, nbuf, prod;
 
 	MSK_IF_LOCK_ASSERT(sc_if);
 
@@ -764,11 +778,18 @@ msk_init_jumbo_rx_ring(struct msk_if_softc *sc_if)
 	rd = &sc_if->msk_rdata;
 	bzero(rd->msk_jumbo_rx_ring,
 	    sizeof(struct msk_rx_desc) * MSK_JUMBO_RX_RING_CNT);
-	prod = sc_if->msk_cdata.msk_rx_prod;
-	i = 0;
+	for (i = prod = 0; i < MSK_JUMBO_RX_RING_CNT; i++) {
+		rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[prod];
+		rxd->rx_m = NULL;
+		rxd->rx_le = &rd->msk_jumbo_rx_ring[prod];
+		MSK_INC(prod, MSK_JUMBO_RX_RING_CNT);
+	}
+	nbuf = MSK_RX_BUF_CNT;
+	prod = 0;
 	/* Have controller know how to compute Rx checksum. */
 	if ((sc_if->msk_flags & MSK_FLAG_DESCV2) == 0 &&
 	    (sc_if->msk_ifp->if_capenable & IFCAP_RXCSUM) != 0) {
+#ifdef MSK_64BIT_DMA
 		rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[prod];
 		rxd->rx_m = NULL;
 		rxd->rx_le = &rd->msk_jumbo_rx_ring[prod];
@@ -777,25 +798,33 @@ msk_init_jumbo_rx_ring(struct msk_if_softc *sc_if)
 		rxd->rx_le->msk_control = htole32(OP_TCPSTART | HW_OWNER);
 		MSK_INC(prod, MSK_JUMBO_RX_RING_CNT);
 		MSK_INC(sc_if->msk_cdata.msk_rx_cons, MSK_JUMBO_RX_RING_CNT);
-		i++;
-	}
-	for (; i < MSK_JUMBO_RX_RING_CNT; i++) {
+#endif
 		rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[prod];
 		rxd->rx_m = NULL;
 		rxd->rx_le = &rd->msk_jumbo_rx_ring[prod];
+		rxd->rx_le->msk_addr = htole32(ETHER_HDR_LEN << 16 |
+		    ETHER_HDR_LEN);
+		rxd->rx_le->msk_control = htole32(OP_TCPSTART | HW_OWNER);
+		MSK_INC(prod, MSK_JUMBO_RX_RING_CNT);
+		MSK_INC(sc_if->msk_cdata.msk_rx_cons, MSK_JUMBO_RX_RING_CNT);
+		nbuf--;
+	}
+	for (i = 0; i < nbuf; i++) {
 		if (msk_jumbo_newbuf(sc_if, prod) != 0)
 			return (ENOBUFS);
-		MSK_INC(prod, MSK_JUMBO_RX_RING_CNT);
+		MSK_RX_INC(prod, MSK_JUMBO_RX_RING_CNT);
 	}
 
 	bus_dmamap_sync(sc_if->msk_cdata.msk_jumbo_rx_ring_tag,
 	    sc_if->msk_cdata.msk_jumbo_rx_ring_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
-	sc_if->msk_cdata.msk_rx_prod = MSK_JUMBO_RX_RING_CNT - 1;
+	/* Update prefetch unit. */
+	sc_if->msk_cdata.msk_rx_prod = prod;
 	CSR_WRITE_2(sc_if->msk_softc,
 	    Y2_PREF_Q_ADDR(sc_if->msk_rxq, PREF_UNIT_PUT_IDX_REG),
-	    sc_if->msk_cdata.msk_rx_prod);
+	    (sc_if->msk_cdata.msk_rx_prod + MSK_JUMBO_RX_RING_CNT - 1) %
+	    MSK_JUMBO_RX_RING_CNT);
 	if (msk_rx_fill(sc_if, 1) != 0)
 		return (ENOBUFS);
 	return (0);
@@ -813,6 +842,7 @@ msk_init_tx_ring(struct msk_if_softc *sc_if)
 	sc_if->msk_cdata.msk_tx_prod = 0;
 	sc_if->msk_cdata.msk_tx_cons = 0;
 	sc_if->msk_cdata.msk_tx_cnt = 0;
+	sc_if->msk_cdata.msk_tx_high_addr = 0;
 
 	rd = &sc_if->msk_rdata;
 	bzero(rd->msk_tx_ring, sizeof(struct msk_tx_desc) * MSK_TX_RING_CNT);
@@ -834,6 +864,12 @@ msk_discard_rxbuf(struct msk_if_softc *sc_if, int idx)
 	struct msk_rxdesc *rxd;
 	struct mbuf *m;
 
+#ifdef MSK_64BIT_DMA
+	rxd = &sc_if->msk_cdata.msk_rxdesc[idx];
+	rx_le = rxd->rx_le;
+	rx_le->msk_control = htole32(OP_ADDR64 | HW_OWNER);
+	MSK_INC(idx, MSK_RX_RING_CNT);
+#endif
 	rxd = &sc_if->msk_cdata.msk_rxdesc[idx];
 	m = rxd->rx_m;
 	rx_le = rxd->rx_le;
@@ -847,6 +883,12 @@ msk_discard_jumbo_rxbuf(struct msk_if_softc *sc_if, int	idx)
 	struct msk_rxdesc *rxd;
 	struct mbuf *m;
 
+#ifdef MSK_64BIT_DMA
+	rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[idx];
+	rx_le = rxd->rx_le;
+	rx_le->msk_control = htole32(OP_ADDR64 | HW_OWNER);
+	MSK_INC(idx, MSK_JUMBO_RX_RING_CNT);
+#endif
 	rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[idx];
 	m = rxd->rx_m;
 	rx_le = rxd->rx_le;
@@ -884,10 +926,18 @@ msk_newbuf(struct msk_if_softc *sc_if, int idx)
 	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
 
 	rxd = &sc_if->msk_cdata.msk_rxdesc[idx];
+#ifdef MSK_64BIT_DMA
+	rx_le = rxd->rx_le;
+	rx_le->msk_addr = htole32(MSK_ADDR_HI(segs[0].ds_addr));
+	rx_le->msk_control = htole32(OP_ADDR64 | HW_OWNER);
+	MSK_INC(idx, MSK_RX_RING_CNT);
+	rxd = &sc_if->msk_cdata.msk_rxdesc[idx];
+#endif
 	if (rxd->rx_m != NULL) {
 		bus_dmamap_sync(sc_if->msk_cdata.msk_rx_tag, rxd->rx_dmamap,
 		    BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc_if->msk_cdata.msk_rx_tag, rxd->rx_dmamap);
+		rxd->rx_m = NULL;
 	}
 	map = rxd->rx_dmamap;
 	rxd->rx_dmamap = sc_if->msk_cdata.msk_rx_sparemap;
@@ -937,11 +987,19 @@ msk_jumbo_newbuf(struct msk_if_softc *sc_if, int idx)
 	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
 
 	rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[idx];
+#ifdef MSK_64BIT_DMA
+	rx_le = rxd->rx_le;
+	rx_le->msk_addr = htole32(MSK_ADDR_HI(segs[0].ds_addr));
+	rx_le->msk_control = htole32(OP_ADDR64 | HW_OWNER);
+	MSK_INC(idx, MSK_JUMBO_RX_RING_CNT);
+	rxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[idx];
+#endif
 	if (rxd->rx_m != NULL) {
 		bus_dmamap_sync(sc_if->msk_cdata.msk_jumbo_rx_tag,
 		    rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc_if->msk_cdata.msk_jumbo_rx_tag,
 		    rxd->rx_dmamap);
+		rxd->rx_m = NULL;
 	}
 	map = rxd->rx_dmamap;
 	rxd->rx_dmamap = sc_if->msk_cdata.msk_jumbo_rx_sparemap;
@@ -1472,7 +1530,7 @@ mskc_reset(struct msk_softc *sc)
 
 	/* Clear status list. */
 	bzero(sc->msk_stat_ring,
-	    sizeof(struct msk_stat_desc) * MSK_STAT_RING_CNT);
+	    sizeof(struct msk_stat_desc) * sc->msk_stat_count);
 	sc->msk_stat_cons = 0;
 	bus_dmamap_sync(sc->msk_stat_tag, sc->msk_stat_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
@@ -1483,7 +1541,7 @@ mskc_reset(struct msk_softc *sc)
 	CSR_WRITE_4(sc, STAT_LIST_ADDR_LO, MSK_ADDR_LO(addr));
 	CSR_WRITE_4(sc, STAT_LIST_ADDR_HI, MSK_ADDR_HI(addr));
 	/* Set the status list last index. */
-	CSR_WRITE_2(sc, STAT_LAST_IDX, MSK_STAT_RING_CNT - 1);
+	CSR_WRITE_2(sc, STAT_LAST_IDX, sc->msk_stat_count - 1);
 	if (sc->msk_hw_id == CHIP_ID_YUKON_EC &&
 	    sc->msk_hw_rev == CHIP_REV_YU_EC_A1) {
 		/* WA for dev. #4.3 */
@@ -2083,17 +2141,29 @@ static int
 msk_status_dma_alloc(struct msk_softc *sc)
 {
 	struct msk_dmamap_arg ctx;
-	int error;
+	bus_size_t stat_sz;
+	int count, error;
 
+	/*
+	 * It seems controller requires number of status LE entries
+	 * is power of 2 and the maximum number of status LE entries
+	 * is 4096.  For dual-port controllers, the number of status
+	 * LE entries should be large enough to hold both port's
+	 * status updates.
+	 */
+	count = 3 * MSK_RX_RING_CNT + MSK_TX_RING_CNT;
+	count = imin(4096, roundup2(count, 1024));
+	sc->msk_stat_count = count;
+	stat_sz = count * sizeof(struct msk_stat_desc);
 	error = bus_dma_tag_create(
 		    bus_get_dma_tag(sc->msk_dev),	/* parent */
 		    MSK_STAT_ALIGN, 0,		/* alignment, boundary */
 		    BUS_SPACE_MAXADDR,		/* lowaddr */
 		    BUS_SPACE_MAXADDR,		/* highaddr */
 		    NULL, NULL,			/* filter, filterarg */
-		    MSK_STAT_RING_SZ,		/* maxsize */
+		    stat_sz,			/* maxsize */
 		    1,				/* nsegments */
-		    MSK_STAT_RING_SZ,		/* maxsegsize */
+		    stat_sz,			/* maxsegsize */
 		    0,				/* flags */
 		    NULL, NULL,			/* lockfunc, lockarg */
 		    &sc->msk_stat_tag);
@@ -2114,9 +2184,8 @@ msk_status_dma_alloc(struct msk_softc *sc)
 	}
 
 	ctx.msk_busaddr = 0;
-	error = bus_dmamap_load(sc->msk_stat_tag,
-	    sc->msk_stat_map, sc->msk_stat_ring, MSK_STAT_RING_SZ,
-	    msk_dmamap_cb, &ctx, 0);
+	error = bus_dmamap_load(sc->msk_stat_tag, sc->msk_stat_map,
+	    sc->msk_stat_ring, stat_sz, msk_dmamap_cb, &ctx, BUS_DMA_NOWAIT);
 	if (error != 0) {
 		device_printf(sc->msk_dev,
 		    "failed to load DMA'able memory for status ring\n");
@@ -2157,27 +2226,10 @@ msk_txrx_dma_alloc(struct msk_if_softc *sc_if)
 	int error, i;
 
 	/* Create parent DMA tag. */
-	/*
-	 * XXX
-	 * It seems that Yukon II supports full 64bits DMA operations. But
-	 * it needs two descriptors(list elements) for 64bits DMA operations.
-	 * Since we don't know what DMA address mappings(32bits or 64bits)
-	 * would be used in advance for each mbufs, we limits its DMA space
-	 * to be in range of 32bits address space. Otherwise, we should check
-	 * what DMA address is used and chain another descriptor for the
-	 * 64bits DMA operation. This also means descriptor ring size is
-	 * variable. Limiting DMA address to be in 32bit address space greatly
-	 * simplifies descriptor handling and possibly would increase
-	 * performance a bit due to efficient handling of descriptors.
-	 * Apart from harassing checksum offloading mechanisms, it seems
-	 * it's really bad idea to use a separate descriptor for 64bit
-	 * DMA operation to save small descriptor memory. Anyway, I've
-	 * never seen these exotic scheme on ethernet interface hardware.
-	 */
 	error = bus_dma_tag_create(
 		    bus_get_dma_tag(sc_if->msk_if_dev),	/* parent */
 		    1, 0,			/* alignment, boundary */
-		    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+		    BUS_SPACE_MAXADDR,		/* lowaddr */
 		    BUS_SPACE_MAXADDR,		/* highaddr */
 		    NULL, NULL,			/* filter, filterarg */
 		    BUS_SPACE_MAXSIZE_32BIT,	/* maxsize */
@@ -2283,7 +2335,7 @@ msk_txrx_dma_alloc(struct msk_if_softc *sc_if)
 	ctx.msk_busaddr = 0;
 	error = bus_dmamap_load(sc_if->msk_cdata.msk_tx_ring_tag,
 	    sc_if->msk_cdata.msk_tx_ring_map, sc_if->msk_rdata.msk_tx_ring,
-	    MSK_TX_RING_SZ, msk_dmamap_cb, &ctx, 0);
+	    MSK_TX_RING_SZ, msk_dmamap_cb, &ctx, BUS_DMA_NOWAIT);
 	if (error != 0) {
 		device_printf(sc_if->msk_if_dev,
 		    "failed to load DMA'able memory for Tx ring\n");
@@ -2304,7 +2356,7 @@ msk_txrx_dma_alloc(struct msk_if_softc *sc_if)
 	ctx.msk_busaddr = 0;
 	error = bus_dmamap_load(sc_if->msk_cdata.msk_rx_ring_tag,
 	    sc_if->msk_cdata.msk_rx_ring_map, sc_if->msk_rdata.msk_rx_ring,
-	    MSK_RX_RING_SZ, msk_dmamap_cb, &ctx, 0);
+	    MSK_RX_RING_SZ, msk_dmamap_cb, &ctx, BUS_DMA_NOWAIT);
 	if (error != 0) {
 		device_printf(sc_if->msk_if_dev,
 		    "failed to load DMA'able memory for Rx ring\n");
@@ -2421,7 +2473,7 @@ msk_rx_dma_jalloc(struct msk_if_softc *sc_if)
 	error = bus_dmamap_load(sc_if->msk_cdata.msk_jumbo_rx_ring_tag,
 	    sc_if->msk_cdata.msk_jumbo_rx_ring_map,
 	    sc_if->msk_rdata.msk_jumbo_rx_ring, MSK_JUMBO_RX_RING_SZ,
-	    msk_dmamap_cb, &ctx, 0);
+	    msk_dmamap_cb, &ctx, BUS_DMA_NOWAIT);
 	if (error != 0) {
 		device_printf(sc_if->msk_if_dev,
 		    "failed to load DMA'able memory for jumbo Rx ring\n");
@@ -2781,6 +2833,18 @@ msk_encap(struct msk_if_softc *sc_if, struct mbuf **m_head)
 		}
 	}
 
+#ifdef MSK_64BIT_DMA
+	if (MSK_ADDR_HI(txsegs[0].ds_addr) !=
+	    sc_if->msk_cdata.msk_tx_high_addr) {
+		sc_if->msk_cdata.msk_tx_high_addr =
+		    MSK_ADDR_HI(txsegs[0].ds_addr);
+		tx_le = &sc_if->msk_rdata.msk_tx_ring[prod];
+		tx_le->msk_addr = htole32(MSK_ADDR_HI(txsegs[0].ds_addr));
+		tx_le->msk_control = htole32(OP_ADDR64 | HW_OWNER);
+		sc_if->msk_cdata.msk_tx_cnt++;
+		MSK_INC(prod, MSK_TX_RING_CNT);
+	}
+#endif
 	si = prod;
 	tx_le = &sc_if->msk_rdata.msk_tx_ring[prod];
 	tx_le->msk_addr = htole32(MSK_ADDR_LO(txsegs[0].ds_addr));
@@ -2795,6 +2859,20 @@ msk_encap(struct msk_if_softc *sc_if, struct mbuf **m_head)
 
 	for (i = 1; i < nseg; i++) {
 		tx_le = &sc_if->msk_rdata.msk_tx_ring[prod];
+#ifdef MSK_64BIT_DMA
+		if (MSK_ADDR_HI(txsegs[i].ds_addr) !=
+		    sc_if->msk_cdata.msk_tx_high_addr) {
+			sc_if->msk_cdata.msk_tx_high_addr =
+			    MSK_ADDR_HI(txsegs[i].ds_addr);
+			tx_le = &sc_if->msk_rdata.msk_tx_ring[prod];
+			tx_le->msk_addr =
+			    htole32(MSK_ADDR_HI(txsegs[i].ds_addr));
+			tx_le->msk_control = htole32(OP_ADDR64 | HW_OWNER);
+			sc_if->msk_cdata.msk_tx_cnt++;
+			MSK_INC(prod, MSK_TX_RING_CNT);
+			tx_le = &sc_if->msk_rdata.msk_tx_ring[prod];
+		}
+#endif
 		tx_le->msk_addr = htole32(MSK_ADDR_LO(txsegs[i].ds_addr));
 		tx_le->msk_control = htole32(txsegs[i].ds_len | control |
 		    OP_BUFFER | HW_OWNER);
@@ -3147,7 +3225,12 @@ msk_rxeof(struct msk_if_softc *sc_if, uint32_t status, uint32_t control,
 			msk_discard_rxbuf(sc_if, cons);
 			break;
 		}
+#ifdef MSK_64BIT_DMA
+		rxd = &sc_if->msk_cdata.msk_rxdesc[(cons + 1) %
+		    MSK_RX_RING_CNT];
+#else
 		rxd = &sc_if->msk_cdata.msk_rxdesc[cons];
+#endif
 		m = rxd->rx_m;
 		if (msk_newbuf(sc_if, cons) != 0) {
 			ifp->if_iqdrops++;
@@ -3175,8 +3258,8 @@ msk_rxeof(struct msk_if_softc *sc_if, uint32_t status, uint32_t control,
 		MSK_IF_LOCK(sc_if);
 	} while (0);
 
-	MSK_INC(sc_if->msk_cdata.msk_rx_cons, MSK_RX_RING_CNT);
-	MSK_INC(sc_if->msk_cdata.msk_rx_prod, MSK_RX_RING_CNT);
+	MSK_RX_INC(sc_if->msk_cdata.msk_rx_cons, MSK_RX_RING_CNT);
+	MSK_RX_INC(sc_if->msk_cdata.msk_rx_prod, MSK_RX_RING_CNT);
 }
 
 static void
@@ -3207,7 +3290,12 @@ msk_jumbo_rxeof(struct msk_if_softc *sc_if, uint32_t status, uint32_t control,
 			msk_discard_jumbo_rxbuf(sc_if, cons);
 			break;
 		}
+#ifdef MSK_64BIT_DMA
+		jrxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[(cons + 1) %
+		    MSK_JUMBO_RX_RING_CNT];
+#else
 		jrxd = &sc_if->msk_cdata.msk_jumbo_rxdesc[cons];
+#endif
 		m = jrxd->rx_m;
 		if (msk_jumbo_newbuf(sc_if, cons) != 0) {
 			ifp->if_iqdrops++;
@@ -3235,8 +3323,8 @@ msk_jumbo_rxeof(struct msk_if_softc *sc_if, uint32_t status, uint32_t control,
 		MSK_IF_LOCK(sc_if);
 	} while (0);
 
-	MSK_INC(sc_if->msk_cdata.msk_rx_cons, MSK_JUMBO_RX_RING_CNT);
-	MSK_INC(sc_if->msk_cdata.msk_rx_prod, MSK_JUMBO_RX_RING_CNT);
+	MSK_RX_INC(sc_if->msk_cdata.msk_rx_cons, MSK_JUMBO_RX_RING_CNT);
+	MSK_RX_INC(sc_if->msk_cdata.msk_rx_prod, MSK_JUMBO_RX_RING_CNT);
 }
 
 static void
@@ -3581,7 +3669,7 @@ msk_handle_events(struct msk_softc *sc)
 			    control & STLE_OP_MASK);
 			break;
 		}
-		MSK_INC(cons, MSK_STAT_RING_CNT);
+		MSK_INC(cons, sc->msk_stat_count);
 		if (rxprog > sc->msk_process_limit)
 			break;
 	}
diff --git a/sys/dev/msk/if_mskreg.h b/sys/dev/msk/if_mskreg.h
index 583f5e8..5465deb 100644
--- a/sys/dev/msk/if_mskreg.h
+++ b/sys/dev/msk/if_mskreg.h
@@ -2315,35 +2315,48 @@ struct msk_stat_desc {
 #define BMU_UDP_CHECK	(0x57<<16)	/* Descr with UDP ext (YUKON only) */
 #define BMU_BBC		0xffff	/* Bit 15.. 0:	Buffer Byte Counter */
 
+/*
+ * Controller requires an additional LE op code for 64bit DMA operation.
+ * Driver uses fixed number of RX buffers such that this limitation
+ * reduces number of available RX buffers with 64bit DMA so double
+ * number of RX buffers on platforms that support 64bit DMA. For TX
+ * side, controller requires an additional OP_ADDR64 op code if a TX
+ * buffer uses different high address value than previously used one.
+ * Driver monitors high DMA address change in TX and inserts an
+ * OP_ADDR64 op code if the high DMA address is changed.  Driver
+ * allocates 50% more total TX buffers on platforms that support 64bit
+ * DMA.
+ */
+#if (BUS_SPACE_MAXADDR > 0xFFFFFFFF)
+#define	MSK_64BIT_DMA
+#define MSK_TX_RING_CNT		384
+#define MSK_RX_RING_CNT		512
+#else
+#undef	MSK_64BIT_DMA
 #define MSK_TX_RING_CNT		256
 #define MSK_RX_RING_CNT		256
+#endif
 #define	MSK_RX_BUF_ALIGN	8
 #define MSK_JUMBO_RX_RING_CNT	MSK_RX_RING_CNT
-#define	MSK_STAT_RING_CNT	((1 + 3) * (MSK_TX_RING_CNT + MSK_RX_RING_CNT))
 #define MSK_MAXTXSEGS		32
 #define	MSK_TSO_MAXSGSIZE	4096
 #define	MSK_TSO_MAXSIZE		(65535 + sizeof(struct ether_vlan_header))
 
 /*
- * It seems that the hardware requires extra decriptors(LEs) to offload
- * TCP/UDP checksum, VLAN hardware tag inserstion and TSO.
+ * It seems that the hardware requires extra descriptors(LEs) to offload
+ * TCP/UDP checksum, VLAN hardware tag insertion and TSO.
  *
  * 1 descriptor for TCP/UDP checksum offload.
  * 1 descriptor VLAN hardware tag insertion.
  * 1 descriptor for TSO(TCP Segmentation Offload)
- * 1 descriptor for 64bits DMA : Not applicatable due to the use of
- *  BUS_SPACE_MAXADDR_32BIT in parent DMA tag creation.
+ * 1 descriptor for each 64bits DMA transfers 
  */
+#ifdef MSK_64BIT_DMA
+#define	MSK_RESERVED_TX_DESC_CNT	(MSK_MAXTXSEGS + 3)
+#else
 #define	MSK_RESERVED_TX_DESC_CNT	3
+#endif
 
-/*
- * Jumbo buffer stuff. Note that we must allocate more jumbo
- * buffers than there are descriptors in the receive ring. This
- * is because we don't know how long it will take for a packet
- * to be released after we hand it off to the upper protocol
- * layers. To be safe, we allocate 1.5 times the number of
- * receive descriptors.
- */
 #define MSK_JUMBO_FRAMELEN	9022
 #define MSK_JUMBO_MTU		(MSK_JUMBO_FRAMELEN-ETHER_HDR_LEN-ETHER_CRC_LEN)
 #define MSK_MAX_FRAMELEN		\
@@ -2380,6 +2393,7 @@ struct msk_chain_data {
 	bus_dmamap_t		msk_jumbo_rx_sparemap;
 	uint16_t		msk_tso_mtu;
 	uint32_t		msk_last_csum;
+	uint32_t		msk_tx_high_addr;
 	int			msk_tx_prod;
 	int			msk_tx_cons;
 	int			msk_tx_cnt;
@@ -2411,10 +2425,17 @@ struct msk_ring_data {
     (sizeof(struct msk_rx_desc) * MSK_RX_RING_CNT)
 #define MSK_JUMBO_RX_RING_SZ		\
     (sizeof(struct msk_rx_desc) * MSK_JUMBO_RX_RING_CNT)
-#define MSK_STAT_RING_SZ		\
-    (sizeof(struct msk_stat_desc) * MSK_STAT_RING_CNT)
 
 #define MSK_INC(x, y)	(x) = (x + 1) % y
+#ifdef MSK_64BIT_DMA
+#define MSK_RX_INC(x, y)	(x) = (x + 2) % y
+#define MSK_RX_BUF_CNT		(MSK_RX_RING_CNT / 2)
+#define MSK_JUMBO_RX_BUF_CNT	(MSK_JUMBO_RX_RING_CNT / 2)
+#else
+#define MSK_RX_INC(x, y)	(x) = (x + 1) % y
+#define MSK_RX_BUF_CNT		MSK_RX_RING_CNT
+#define MSK_JUMBO_RX_BUF_CNT	MSK_JUMBO_RX_RING_CNT
+#endif
 
 #define	MSK_PCI_BUS	0
 #define	MSK_PCIX_BUS	1
@@ -2519,6 +2540,7 @@ struct msk_softc {
 	int			msk_int_holdoff;
 	int			msk_process_limit;
 	int			msk_stat_cons;
+	int			msk_stat_count;
 	struct mtx		msk_mtx;
 };
 
diff --git a/sys/dev/mvs/mvs.c b/sys/dev/mvs/mvs.c
index 54808c5..e128616 100644
--- a/sys/dev/mvs/mvs.c
+++ b/sys/dev/mvs/mvs.c
@@ -94,7 +94,7 @@ static void mvs_process_request_sense(device_t dev, union ccb *ccb);
 static void mvsaction(struct cam_sim *sim, union ccb *ccb);
 static void mvspoll(struct cam_sim *sim);
 
-MALLOC_DEFINE(M_MVS, "MVS driver", "MVS driver data buffers");
+static MALLOC_DEFINE(M_MVS, "MVS driver", "MVS driver data buffers");
 
 #define recovery_type		spriv_field0
 #define RECOVERY_NONE		0
diff --git a/sys/dev/mvs/mvs_pci.c b/sys/dev/mvs/mvs_pci.c
index e2e37da..36dd93d 100644
--- a/sys/dev/mvs/mvs_pci.c
+++ b/sys/dev/mvs/mvs_pci.c
@@ -177,15 +177,10 @@ static int
 mvs_detach(device_t dev)
 {
 	struct mvs_controller *ctlr = device_get_softc(dev);
-	device_t *children;
-	int nchildren, i;
 
 	/* Detach & delete all children */
-	if (!device_get_children(dev, &children, &nchildren)) {
-		for (i = 0; i < nchildren; i++)
-			device_delete_child(dev, children[i]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
+
 	/* Free interrupt. */
 	if (ctlr->irq.r_irq) {
 		bus_teardown_intr(dev, ctlr->irq.r_irq,
diff --git a/sys/dev/mvs/mvs_soc.c b/sys/dev/mvs/mvs_soc.c
index 5c1116c..670bfec 100644
--- a/sys/dev/mvs/mvs_soc.c
+++ b/sys/dev/mvs/mvs_soc.c
@@ -173,15 +173,10 @@ static int
 mvs_detach(device_t dev)
 {
 	struct mvs_controller *ctlr = device_get_softc(dev);
-	device_t *children;
-	int nchildren, i;
 
 	/* Detach & delete all children */
-	if (!device_get_children(dev, &children, &nchildren)) {
-		for (i = 0; i < nchildren; i++)
-			device_delete_child(dev, children[i]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
+
 	/* Free interrupt. */
 	if (ctlr->irq.r_irq) {
 		bus_teardown_intr(dev, ctlr->irq.r_irq,
diff --git a/sys/dev/mwl/if_mwl.c b/sys/dev/mwl/if_mwl.c
index 662f201..5af96ba 100644
--- a/sys/dev/mwl/if_mwl.c
+++ b/sys/dev/mwl/if_mwl.c
@@ -262,7 +262,7 @@ static	void mwl_printtxbuf(const struct mwl_txbuf *bf, u_int qnum, u_int ix);
 } while (0)
 #endif
 
-MALLOC_DEFINE(M_MWLDEV, "mwldev", "mwl driver dma buffers");
+static MALLOC_DEFINE(M_MWLDEV, "mwldev", "mwl driver dma buffers");
 
 /*
  * Each packet has fixed front matter: a 2-byte length
diff --git a/sys/dev/mwl/mwlhal.c b/sys/dev/mwl/mwlhal.c
index ffc956f..4547fe5 100644
--- a/sys/dev/mwl/mwlhal.c
+++ b/sys/dev/mwl/mwlhal.c
@@ -190,7 +190,8 @@ static void dumpresult(struct mwl_hal_priv *, int showresult);
 #endif /* MWLHAL_DEBUG */
 
 SYSCTL_DECL(_hw_mwl);
-SYSCTL_NODE(_hw_mwl, OID_AUTO, hal, CTLFLAG_RD, 0, "Marvell HAL parameters");
+static SYSCTL_NODE(_hw_mwl, OID_AUTO, hal, CTLFLAG_RD, 0,
+    "Marvell HAL parameters");
 
 static __inline void
 MWL_HAL_LOCK(struct mwl_hal_priv *mh)
diff --git a/sys/dev/netmap/head.diff b/sys/dev/netmap/head.diff
new file mode 100644
index 0000000..51a8e34
--- /dev/null
+++ b/sys/dev/netmap/head.diff
@@ -0,0 +1,654 @@
+Index: conf/NOTES
+===================================================================
+--- conf/NOTES	(revision 227552)
++++ conf/NOTES	(working copy)
+@@ -799,6 +799,12 @@
+ #  option.  DHCP requires bpf.
+ device		bpf
+ 
++#  The `netmap' device implements memory-mapped access to network
++#  devices from userspace, enabling wire-speed packet capture and
++#  generation even at 10Gbit/s. Requires support in the device
++#  driver. Supported drivers are ixgbe, e1000, re.
++device		netmap
++
+ #  The `disc' device implements a minimal network interface,
+ #  which throws away all packets sent and never receives any.  It is
+ #  included for testing and benchmarking purposes.
+Index: conf/files
+===================================================================
+--- conf/files	(revision 227552)
++++ conf/files	(working copy)
+@@ -1507,6 +1507,7 @@
+ dev/my/if_my.c			optional my
+ dev/ncv/ncr53c500.c		optional ncv
+ dev/ncv/ncr53c500_pccard.c	optional ncv pccard
++dev/netmap/netmap.c		optional netmap
+ dev/nge/if_nge.c		optional nge
+ dev/nxge/if_nxge.c		optional nxge
+ dev/nxge/xgehal/xgehal-device.c	optional nxge
+Index: conf/options
+===================================================================
+--- conf/options	(revision 227552)
++++ conf/options	(working copy)
+@@ -689,6 +689,7 @@
+ 
+ # various 'device presence' options.
+ DEV_BPF			opt_bpf.h
++DEV_NETMAP		opt_global.h
+ DEV_MCA			opt_mca.h
+ DEV_CARP		opt_carp.h
+ DEV_SPLASH		opt_splash.h
+Index: dev/e1000/if_igb.c
+===================================================================
+--- dev/e1000/if_igb.c	(revision 227552)
++++ dev/e1000/if_igb.c	(working copy)
+@@ -369,6 +369,9 @@
+     &igb_rx_process_limit, 0,
+     "Maximum number of received packets to process at a time, -1 means unlimited");
+ 
++#ifdef DEV_NETMAP
++#include <dev/netmap/if_igb_netmap.h>
++#endif /* DEV_NETMAP */
+ /*********************************************************************
+  *  Device identification routine
+  *
+@@ -664,6 +667,9 @@
+ 	adapter->led_dev = led_create(igb_led_func, adapter,
+ 	    device_get_nameunit(dev));
+ 
++#ifdef DEV_NETMAP
++	igb_netmap_attach(adapter);
++#endif /* DEV_NETMAP */
+ 	INIT_DEBUGOUT("igb_attach: end");
+ 
+ 	return (0);
+@@ -742,6 +748,9 @@
+ 
+ 	callout_drain(&adapter->timer);
+ 
++#ifdef DEV_NETMAP
++	netmap_detach(adapter->ifp);
++#endif /* DEV_NETMAP */
+ 	igb_free_pci_resources(adapter);
+ 	bus_generic_detach(dev);
+ 	if_free(ifp);
+@@ -3212,6 +3221,10 @@
+ 	struct adapter *adapter = txr->adapter;
+ 	struct igb_tx_buffer *txbuf;
+ 	int i;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(adapter->ifp),
++		NR_TX, txr->me, 0);
++#endif
+ 
+ 	/* Clear the old descriptor contents */
+ 	IGB_TX_LOCK(txr);
+@@ -3231,6 +3244,13 @@
+ 			m_freem(txbuf->m_head);
+ 			txbuf->m_head = NULL;
+ 		}
++#ifdef DEV_NETMAP
++		if (slot) {
++			netmap_load_map(txr->txtag, txbuf->map,
++				NMB(slot), adapter->rx_mbuf_sz);
++			slot++;
++		}
++#endif /* DEV_NETMAP */
+ 		/* clear the watch index */
+ 		txbuf->next_eop = -1;
+         }
+@@ -3626,6 +3646,19 @@
+ 
+ 	IGB_TX_LOCK_ASSERT(txr);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		struct netmap_adapter *na = NA(ifp);
++
++		selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
++		IGB_TX_UNLOCK(txr);
++		IGB_CORE_LOCK(adapter);
++		selwakeuppri(&na->tx_rings[na->num_queues + 1].si, PI_NET);
++		IGB_CORE_UNLOCK(adapter);
++		IGB_TX_LOCK(txr); // the caller is supposed to own the lock
++		return FALSE;
++	}
++#endif /* DEV_NETMAP */
+         if (txr->tx_avail == adapter->num_tx_desc) {
+ 		txr->queue_status = IGB_QUEUE_IDLE;
+                 return FALSE;
+@@ -3949,6 +3982,10 @@
+ 	bus_dma_segment_t	pseg[1], hseg[1];
+ 	struct lro_ctrl		*lro = &rxr->lro;
+ 	int			rsize, nsegs, error = 0;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(rxr->adapter->ifp),
++				NR_RX, rxr->me, 0);
++#endif
+ 
+ 	adapter = rxr->adapter;
+ 	dev = adapter->dev;
+@@ -3974,6 +4011,18 @@
+ 		struct mbuf	*mh, *mp;
+ 
+ 		rxbuf = &rxr->rx_buffers[j];
++#ifdef DEV_NETMAP
++		if (slot) {
++			netmap_load_map(rxr->ptag,
++					rxbuf->pmap, NMB(slot),
++					adapter->rx_mbuf_sz);
++			/* Update descriptor */
++			rxr->rx_base[j].read.pkt_addr =
++				htole64(vtophys(NMB(slot)));
++			slot++;
++			continue;
++		}
++#endif /* DEV_NETMAP */
+ 		if (rxr->hdr_split == FALSE)
+ 			goto skip_head;
+ 
+@@ -4436,6 +4485,19 @@
+ 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+ 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		struct netmap_adapter *na = NA(ifp);
++
++		selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
++		IGB_RX_UNLOCK(rxr);
++		IGB_CORE_LOCK(adapter);
++		selwakeuppri(&na->rx_rings[na->num_queues + 1].si, PI_NET);
++		IGB_CORE_UNLOCK(adapter);
++		return (0);
++	}
++#endif /* DEV_NETMAP */
++
+ 	/* Main clean loop */
+ 	for (i = rxr->next_to_check; count != 0;) {
+ 		struct mbuf		*sendmp, *mh, *mp;
+Index: dev/e1000/if_lem.c
+===================================================================
+--- dev/e1000/if_lem.c	(revision 227552)
++++ dev/e1000/if_lem.c	(working copy)
+@@ -316,6 +316,10 @@
+ /* Global used in WOL setup with multiport cards */
+ static int global_quad_port_a = 0;
+ 
++#ifdef DEV_NETMAP
++#include <dev/netmap/if_lem_netmap.h>
++#endif /* DEV_NETMAP */
++
+ /*********************************************************************
+  *  Device identification routine
+  *
+@@ -646,6 +650,9 @@
+ 	adapter->led_dev = led_create(lem_led_func, adapter,
+ 	    device_get_nameunit(dev));
+ 
++#ifdef DEV_NETMAP
++	lem_netmap_attach(adapter);
++#endif /* DEV_NETMAP */
+ 	INIT_DEBUGOUT("lem_attach: end");
+ 
+ 	return (0);
+@@ -724,6 +731,9 @@
+ 	callout_drain(&adapter->timer);
+ 	callout_drain(&adapter->tx_fifo_timer);
+ 
++#ifdef DEV_NETMAP
++	netmap_detach(ifp);
++#endif /* DEV_NETMAP */
+ 	lem_free_pci_resources(adapter);
+ 	bus_generic_detach(dev);
+ 	if_free(ifp);
+@@ -2637,6 +2647,9 @@
+ lem_setup_transmit_structures(struct adapter *adapter)
+ {
+ 	struct em_buffer *tx_buffer;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), NR_TX, 0, 0);
++#endif
+ 
+ 	/* Clear the old ring contents */
+ 	bzero(adapter->tx_desc_base,
+@@ -2650,6 +2663,15 @@
+ 		bus_dmamap_unload(adapter->txtag, tx_buffer->map);
+ 		m_freem(tx_buffer->m_head);
+ 		tx_buffer->m_head = NULL;
++#ifdef DEV_NETMAP
++		if (slot) {
++			/* reload the map for netmap mode */
++			netmap_load_map(adapter->txtag,
++				tx_buffer->map, NMB(slot),
++				NA(adapter->ifp)->buff_size);
++			slot++;
++		}
++#endif /* DEV_NETMAP */
+ 		tx_buffer->next_eop = -1;
+ 	}
+ 
+@@ -2951,6 +2973,12 @@
+ 
+ 	EM_TX_LOCK_ASSERT(adapter);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		selwakeuppri(&NA(ifp)->tx_rings[0].si, PI_NET);
++		return;
++	}
++#endif /* DEV_NETMAP */
+         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
+                 return;
+ 
+@@ -3181,6 +3209,9 @@
+ {
+ 	struct em_buffer *rx_buffer;
+ 	int i, error;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(adapter->ifp), NR_RX, 0, 0);
++#endif
+ 
+ 	/* Reset descriptor ring */
+ 	bzero(adapter->rx_desc_base,
+@@ -3200,6 +3231,18 @@
+ 
+ 	/* Allocate new ones. */
+ 	for (i = 0; i < adapter->num_rx_desc; i++) {
++#ifdef DEV_NETMAP
++		if (slot) {
++			netmap_load_map(adapter->rxtag,
++				rx_buffer->map, NMB(slot),
++				NA(adapter->ifp)->buff_size);
++			/* Update descriptor */
++			adapter->rx_desc_base[i].buffer_addr =
++				htole64(vtophys(NMB(slot)));
++			slot++;
++			continue;
++		}
++#endif /* DEV_NETMAP */
+ 		error = lem_get_buf(adapter, i);
+ 		if (error)
+                         return (error);
+@@ -3407,6 +3450,14 @@
+ 	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
+ 	    BUS_DMASYNC_POSTREAD);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		selwakeuppri(&NA(ifp)->rx_rings[0].si, PI_NET);
++		EM_RX_UNLOCK(adapter);
++		return (0);
++	}
++#endif /* DEV_NETMAP */
++
+ 	if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
+ 		if (done != NULL)
+ 			*done = rx_sent;
+Index: dev/e1000/if_em.c
+===================================================================
+--- dev/e1000/if_em.c	(revision 227552)
++++ dev/e1000/if_em.c	(working copy)
+@@ -399,6 +399,10 @@
+ /* Global used in WOL setup with multiport cards */
+ static int global_quad_port_a = 0;
+ 
++#ifdef DEV_NETMAP
++#include <dev/netmap/if_em_netmap.h>
++#endif /* DEV_NETMAP */
++
+ /*********************************************************************
+  *  Device identification routine
+  *
+@@ -714,6 +718,9 @@
+ 
+ 	adapter->led_dev = led_create(em_led_func, adapter,
+ 	    device_get_nameunit(dev));
++#ifdef DEV_NETMAP
++	em_netmap_attach(adapter);
++#endif /* DEV_NETMAP */
+ 
+ 	INIT_DEBUGOUT("em_attach: end");
+ 
+@@ -785,6 +792,10 @@
+ 	ether_ifdetach(adapter->ifp);
+ 	callout_drain(&adapter->timer);
+ 
++#ifdef DEV_NETMAP
++	netmap_detach(ifp);
++#endif /* DEV_NETMAP */
++
+ 	em_free_pci_resources(adapter);
+ 	bus_generic_detach(dev);
+ 	if_free(ifp);
+@@ -3213,6 +3224,10 @@
+ 	struct adapter *adapter = txr->adapter;
+ 	struct em_buffer *txbuf;
+ 	int i;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(adapter->ifp),
++		NR_TX, txr->me, 0);
++#endif
+ 
+ 	/* Clear the old descriptor contents */
+ 	EM_TX_LOCK(txr);
+@@ -3232,6 +3247,16 @@
+ 			m_freem(txbuf->m_head);
+ 			txbuf->m_head = NULL;
+ 		}
++#ifdef DEV_NETMAP
++		if (slot) {
++			/* reload the map for netmap mode */
++			netmap_load_map(txr->txtag,
++					txbuf->map, NMB(slot),
++					adapter->rx_mbuf_sz);
++			slot++;
++		}
++#endif /* DEV_NETMAP */
++
+ 		/* clear the watch index */
+ 		txbuf->next_eop = -1;
+         }
+@@ -3682,6 +3707,12 @@
+ 	struct ifnet   *ifp = adapter->ifp;
+ 
+ 	EM_TX_LOCK_ASSERT(txr);
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		selwakeuppri(&NA(ifp)->tx_rings[txr->me].si, PI_NET);
++		return (FALSE);
++	}
++#endif /* DEV_NETMAP */
+ 
+ 	/* No work, make sure watchdog is off */
+         if (txr->tx_avail == adapter->num_tx_desc) {
+@@ -3978,6 +4009,33 @@
+ 		if (++j == adapter->num_rx_desc)
+ 			j = 0;
+ 	}
++#ifdef DEV_NETMAP
++    {
++	/* slot is NULL if we are not in netmap mode */
++	struct netmap_slot *slot = netmap_reset(NA(adapter->ifp),
++		NR_RX, rxr->me, rxr->next_to_check);
++	/*
++	 * we need to restore all buffer addresses in the ring as they might
++	 * be in the wrong state if we are exiting from netmap mode.
++	 */
++	for (j = 0; j != adapter->num_rx_desc; ++j) {
++		void *addr;
++		rxbuf = &rxr->rx_buffers[j];
++		if (rxbuf->m_head == NULL && !slot)
++			continue;
++		addr = slot ? NMB(slot) : rxbuf->m_head->m_data;
++		// XXX load or reload ?
++		netmap_load_map(rxr->rxtag, rxbuf->map, addr, adapter->rx_mbuf_sz);
++		/* Update descriptor */
++		rxr->rx_base[j].buffer_addr = htole64(vtophys(addr));
++		bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD);
++		if (slot)
++			slot++;
++	}
++	/* Setup our descriptor indices */
++	NA(adapter->ifp)->rx_rings[rxr->me].nr_hwcur = rxr->next_to_check;
++    }
++#endif /* DEV_NETMAP */
+ 
+ fail:
+ 	rxr->next_to_refresh = i;
+@@ -4247,6 +4305,14 @@
+ 
+ 	EM_RX_LOCK(rxr);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		selwakeuppri(&NA(ifp)->rx_rings[rxr->me].si, PI_NET);
++		EM_RX_UNLOCK(rxr);
++		return (0);
++	}
++#endif /* DEV_NETMAP */
++
+ 	for (i = rxr->next_to_check, processed = 0; count != 0;) {
+ 
+ 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+Index: dev/re/if_re.c
+===================================================================
+--- dev/re/if_re.c	(revision 227552)
++++ dev/re/if_re.c	(working copy)
+@@ -291,6 +291,10 @@
+ static void re_setwol		(struct rl_softc *);
+ static void re_clrwol		(struct rl_softc *);
+ 
++#ifdef DEV_NETMAP
++#include <dev/netmap/if_re_netmap.h>
++#endif /* !DEV_NETMAP */
++
+ #ifdef RE_DIAG
+ static int re_diag		(struct rl_softc *);
+ #endif
+@@ -1583,6 +1587,9 @@
+ 	 */
+ 	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
+ 
++#ifdef DEV_NETMAP
++	re_netmap_attach(sc);
++#endif /* DEV_NETMAP */
+ #ifdef RE_DIAG
+ 	/*
+ 	 * Perform hardware diagnostic on the original RTL8169.
+@@ -1778,6 +1785,9 @@
+ 		bus_dma_tag_destroy(sc->rl_ldata.rl_stag);
+ 	}
+ 
++#ifdef DEV_NETMAP
++	netmap_detach(ifp);
++#endif /* DEV_NETMAP */
+ 	if (sc->rl_parent_tag)
+ 		bus_dma_tag_destroy(sc->rl_parent_tag);
+ 
+@@ -1952,6 +1962,9 @@
+ 	    sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc));
+ 	for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++)
+ 		sc->rl_ldata.rl_tx_desc[i].tx_m = NULL;
++#ifdef DEV_NETMAP
++	re_netmap_tx_init(sc);
++#endif /* DEV_NETMAP */
+ 	/* Set EOR. */
+ 	desc = &sc->rl_ldata.rl_tx_list[sc->rl_ldata.rl_tx_desc_cnt - 1];
+ 	desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOR);
+@@ -1979,6 +1992,9 @@
+ 		if ((error = re_newbuf(sc, i)) != 0)
+ 			return (error);
+ 	}
++#ifdef DEV_NETMAP
++	re_netmap_rx_init(sc);
++#endif /* DEV_NETMAP */
+ 
+ 	/* Flush the RX descriptors */
+ 
+@@ -2035,6 +2051,12 @@
+ 	RL_LOCK_ASSERT(sc);
+ 
+ 	ifp = sc->rl_ifp;
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		selwakeuppri(&NA(ifp)->rx_rings->si, PI_NET);
++		return 0;
++	}
++#endif /* DEV_NETMAP */
+ 	if (ifp->if_mtu > RL_MTU && (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
+ 		jumbo = 1;
+ 	else
+@@ -2276,6 +2298,12 @@
+ 		return;
+ 
+ 	ifp = sc->rl_ifp;
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		selwakeuppri(&NA(ifp)->tx_rings[0].si, PI_NET);
++		return;
++	}
++#endif /* DEV_NETMAP */
+ 	/* Invalidate the TX descriptor list */
+ 	bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
+ 	    sc->rl_ldata.rl_tx_list_map,
+@@ -2794,6 +2822,20 @@
+ 
+ 	sc = ifp->if_softc;
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		struct netmap_kring *kring = &NA(ifp)->tx_rings[0];
++		if (sc->rl_ldata.rl_tx_prodidx != kring->nr_hwcur) {
++			/* kick the tx unit */
++			CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
++#ifdef RE_TX_MODERATION
++			CSR_WRITE_4(sc, RL_TIMERCNT, 1);
++#endif
++			sc->rl_watchdog_timer = 5;
++		}
++		return;
++	}
++#endif /* DEV_NETMAP */
+ 	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
+ 	    IFF_DRV_RUNNING || (sc->rl_flags & RL_FLAG_LINK) == 0)
+ 		return;
+Index: dev/ixgbe/ixgbe.c
+===================================================================
+--- dev/ixgbe/ixgbe.c	(revision 227552)
++++ dev/ixgbe/ixgbe.c	(working copy)
+@@ -313,6 +313,10 @@
+ static int fdir_pballoc = 1;
+ #endif
+ 
++#ifdef DEV_NETMAP
++#include <dev/netmap/ixgbe_netmap.h>
++#endif /* DEV_NETMAP */
++
+ /*********************************************************************
+  *  Device identification routine
+  *
+@@ -578,6 +582,9 @@
+ 
+ 	ixgbe_add_hw_stats(adapter);
+ 
++#ifdef DEV_NETMAP
++	ixgbe_netmap_attach(adapter);
++#endif /* DEV_NETMAP */
+ 	INIT_DEBUGOUT("ixgbe_attach: end");
+ 	return (0);
+ err_late:
+@@ -652,6 +659,9 @@
+ 
+ 	ether_ifdetach(adapter->ifp);
+ 	callout_drain(&adapter->timer);
++#ifdef DEV_NETMAP
++	netmap_detach(adapter->ifp);
++#endif /* DEV_NETMAP */
+ 	ixgbe_free_pci_resources(adapter);
+ 	bus_generic_detach(dev);
+ 	if_free(adapter->ifp);
+@@ -1719,6 +1729,7 @@
+ 		if (++i == adapter->num_tx_desc)
+ 			i = 0;
+ 
++		// XXX should we sync each buffer ?
+ 		txbuf->m_head = NULL;
+ 		txbuf->eop_index = -1;
+ 	}
+@@ -2813,6 +2824,10 @@
+ 	struct adapter *adapter = txr->adapter;
+ 	struct ixgbe_tx_buf *txbuf;
+ 	int i;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(adapter->ifp),
++		NR_TX, txr->me, 0);
++#endif
+ 
+ 	/* Clear the old ring contents */
+ 	IXGBE_TX_LOCK(txr);
+@@ -2832,6 +2847,13 @@
+ 			m_freem(txbuf->m_head);
+ 			txbuf->m_head = NULL;
+ 		}
++#ifdef DEV_NETMAP
++		if (slot) {
++			netmap_load_map(txr->txtag, txbuf->map,
++				NMB(slot), adapter->rx_mbuf_sz);
++			slot++;
++		}
++#endif /* DEV_NETMAP */
+ 		/* Clear the EOP index */
+ 		txbuf->eop_index = -1;
+         }
+@@ -3310,6 +3332,20 @@
+ 
+ 	mtx_assert(&txr->tx_mtx, MA_OWNED);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		struct netmap_adapter *na = NA(ifp);
++
++		selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
++		IXGBE_TX_UNLOCK(txr);
++		IXGBE_CORE_LOCK(adapter);
++		selwakeuppri(&na->tx_rings[na->num_queues + 1].si, PI_NET);
++		IXGBE_CORE_UNLOCK(adapter);
++		IXGBE_TX_LOCK(txr); // the caller is supposed to own the lock
++		return (FALSE);
++	}
++#endif /* DEV_NETMAP */
++
+ 	if (txr->tx_avail == adapter->num_tx_desc) {
+ 		txr->queue_status = IXGBE_QUEUE_IDLE;
+ 		return FALSE;
+@@ -3698,6 +3734,10 @@
+ 	bus_dma_segment_t	pseg[1], hseg[1];
+ 	struct lro_ctrl		*lro = &rxr->lro;
+ 	int			rsize, nsegs, error = 0;
++#ifdef DEV_NETMAP
++	struct netmap_slot *slot = netmap_reset(NA(rxr->adapter->ifp),
++				NR_RX, rxr->me, 0);
++#endif /* DEV_NETMAP */
+ 
+ 	adapter = rxr->adapter;
+ 	ifp = adapter->ifp;
+@@ -3721,6 +3761,18 @@
+ 		struct mbuf	*mh, *mp;
+ 
+ 		rxbuf = &rxr->rx_buffers[j];
++#ifdef DEV_NETMAP
++		if (slot) {
++			netmap_load_map(rxr->ptag,
++					rxbuf->pmap, NMB(slot),
++					adapter->rx_mbuf_sz);
++			/* Update descriptor */
++			rxr->rx_base[j].read.pkt_addr =
++				htole64(vtophys(NMB(slot)));
++			slot++;
++			continue;
++		}
++#endif /* DEV_NETMAP */
+ 		/*
+ 		** Don't allocate mbufs if not
+ 		** doing header split, its wasteful
+@@ -4148,6 +4200,18 @@
+ 
+ 	IXGBE_RX_LOCK(rxr);
+ 
++#ifdef DEV_NETMAP
++	if (ifp->if_capenable & IFCAP_NETMAP) {
++		struct netmap_adapter *na = NA(ifp);
++
++		selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
++		IXGBE_RX_UNLOCK(rxr);
++		IXGBE_CORE_LOCK(adapter);
++		selwakeuppri(&na->rx_rings[na->num_queues + 1].si, PI_NET);
++		IXGBE_CORE_UNLOCK(adapter);
++		return (0);
++	}
++#endif /* DEV_NETMAP */
+ 	for (i = rxr->next_to_check; count != 0;) {
+ 		struct mbuf	*sendmp, *mh, *mp;
+ 		u32		rsc, ptype;
diff --git a/sys/dev/netmap/if_em_netmap.h b/sys/dev/netmap/if_em_netmap.h
new file mode 100644
index 0000000..0e220e7
--- /dev/null
+++ b/sys/dev/netmap/if_em_netmap.h
@@ -0,0 +1,383 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: if_em_netmap.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * netmap changes for if_em.
+ */
+
+#include <net/netmap.h>
+#include <sys/selinfo.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>    /* vtophys ? */
+#include <dev/netmap/netmap_kern.h>
+
+static void	em_netmap_block_tasks(struct adapter *);
+static void	em_netmap_unblock_tasks(struct adapter *);
+static int	em_netmap_reg(struct ifnet *, int onoff);
+static int	em_netmap_txsync(void *, u_int, int);
+static int	em_netmap_rxsync(void *, u_int, int);
+static void	em_netmap_lock_wrapper(void *, int, u_int);
+
+static void
+em_netmap_attach(struct adapter *adapter)
+{
+	struct netmap_adapter na;
+
+	bzero(&na, sizeof(na));
+
+	na.ifp = adapter->ifp;
+	na.separate_locks = 1;
+	na.num_tx_desc = adapter->num_tx_desc;
+	na.num_rx_desc = adapter->num_rx_desc;
+	na.nm_txsync = em_netmap_txsync;
+	na.nm_rxsync = em_netmap_rxsync;
+	na.nm_lock = em_netmap_lock_wrapper;
+	na.nm_register = em_netmap_reg;
+	/*
+	 * adapter->rx_mbuf_sz is set by SIOCSETMTU, but in netmap mode
+	 * we allocate the buffers on the first register. So we must
+	 * disallow a SIOCSETMTU when if_capenable & IFCAP_NETMAP is set.
+	 */
+	na.buff_size = MCLBYTES;
+	netmap_attach(&na, adapter->num_queues);
+}
+
+
+/*
+ * wrapper to export locks to the generic code
+ */
+static void
+em_netmap_lock_wrapper(void *_a, int what, u_int queueid)
+{
+	struct adapter *adapter = _a;
+
+	ASSERT(queueid < adapter->num_queues);
+	switch (what) {
+		case NETMAP_CORE_LOCK:
+		EM_CORE_LOCK(adapter);
+		break;
+	case NETMAP_CORE_UNLOCK:
+		EM_CORE_UNLOCK(adapter);
+		break;
+	case NETMAP_TX_LOCK:
+		EM_TX_LOCK(&adapter->tx_rings[queueid]);
+		break;
+	case NETMAP_TX_UNLOCK:
+		EM_TX_UNLOCK(&adapter->tx_rings[queueid]);
+		break;
+	case NETMAP_RX_LOCK:
+		EM_RX_LOCK(&adapter->rx_rings[queueid]);
+		break;
+	case NETMAP_RX_UNLOCK:
+		EM_RX_UNLOCK(&adapter->rx_rings[queueid]);
+		break;
+	}
+}
+
+
+static void
+em_netmap_block_tasks(struct adapter *adapter)
+{
+	if (adapter->msix > 1) { /* MSIX */
+		int i;
+		struct tx_ring *txr = adapter->tx_rings;
+		struct rx_ring *rxr = adapter->rx_rings;
+
+		for (i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
+			taskqueue_block(txr->tq);
+			taskqueue_drain(txr->tq, &txr->tx_task);
+			taskqueue_block(rxr->tq);
+			taskqueue_drain(rxr->tq, &rxr->rx_task);
+		}
+	} else {	/* legacy */
+		taskqueue_block(adapter->tq);
+		taskqueue_drain(adapter->tq, &adapter->link_task);
+		taskqueue_drain(adapter->tq, &adapter->que_task);
+	}
+}
+
+
+static void
+em_netmap_unblock_tasks(struct adapter *adapter)
+{
+	if (adapter->msix > 1) {
+		struct tx_ring *txr = adapter->tx_rings;
+		struct rx_ring *rxr = adapter->rx_rings;
+		int i;
+
+		for (i = 0; i < adapter->num_queues; i++) {
+			taskqueue_unblock(txr->tq);
+			taskqueue_unblock(rxr->tq);
+		}
+	} else { /* legacy */
+		taskqueue_unblock(adapter->tq);
+	}
+}
+
+/*
+ * register-unregister routine
+ */
+static int
+em_netmap_reg(struct ifnet *ifp, int onoff)
+{
+	struct adapter *adapter = ifp->if_softc;
+	struct netmap_adapter *na = NA(ifp);
+	int error = 0;
+
+	if (na == NULL)
+		return EINVAL;	/* no netmap support here */
+
+	em_disable_intr(adapter);
+
+	/* Tell the stack that the interface is no longer active */
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+
+	em_netmap_block_tasks(adapter);
+
+	if (onoff) {
+		ifp->if_capenable |= IFCAP_NETMAP;
+
+		/* save if_transmit for later restore.
+		 * XXX also if_start and if_qflush ?
+		 */
+		na->if_transmit = ifp->if_transmit;
+		ifp->if_transmit = netmap_start;
+
+		em_init_locked(adapter);
+		if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) {
+			error = ENOMEM;
+			goto fail;
+		}
+	} else {
+fail:
+		/* restore if_transmit */
+		ifp->if_transmit = na->if_transmit;
+		ifp->if_capenable &= ~IFCAP_NETMAP;
+		em_init_locked(adapter);	/* also enable intr */
+
+	}
+	em_netmap_unblock_tasks(adapter);
+	return (error);
+}
+
+/*
+ * Reconcile hardware and user view of the transmit ring, see
+ * ixgbe.c for details.
+ */
+static int
+em_netmap_txsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct tx_ring *txr = &adapter->tx_rings[ring_nr];
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->tx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	/* generate an interrupt approximately every half ring */
+	int report_frequency = kring->nkr_num_slots >> 1;
+
+	k = ring->cur;
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		EM_TX_LOCK(txr);
+	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
+			BUS_DMASYNC_POSTREAD);
+
+	/* record completed transmissions TODO
+	 *
+	 * instead of using TDH, we could read the transmitted status bit.
+	 */
+	j = E1000_READ_REG(&adapter->hw, E1000_TDH(ring_nr));
+	if (j >= kring->nkr_num_slots) { /* XXX can happen */
+		D("TDH wrap %d", j);
+		j -= kring->nkr_num_slots;
+	}
+	int delta = j - txr->next_to_clean;
+	if (delta) {
+		/* new transmissions were completed, increment
+		   ring->nr_hwavail. */
+		if (delta < 0)
+			delta += kring->nkr_num_slots;
+		txr->next_to_clean = j;
+		kring->nr_hwavail += delta;
+	}
+
+	/* update avail to what the hardware knows */
+	ring->avail = kring->nr_hwavail;
+
+	j = kring->nr_hwcur;
+	if (j != k) {	/* we have packets to send */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct e1000_tx_desc *curr = &txr->tx_base[j];
+			struct em_buffer *txbuf = &txr->tx_buffers[j];
+			int flags = ((slot->flags & NS_REPORT) ||
+				j == 0 || j == report_frequency) ?
+					E1000_TXD_CMD_RS : 0;
+			void *addr = NMB(slot);
+			int len = slot->len;
+			if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) {
+				if (do_lock)
+					EM_TX_UNLOCK(txr);
+				return netmap_ring_reinit(kring);
+			}
+
+			slot->flags &= ~NS_REPORT;
+			curr->upper.data = 0;
+			curr->lower.data = 
+			    htole32(
+				adapter->txd_cmd |
+				(E1000_TXD_CMD_EOP | flags) |
+				slot->len);
+			if (slot->flags & NS_BUF_CHANGED) {
+				curr->buffer_addr = htole64(vtophys(addr));
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(txr->txtag, txbuf->map,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(txr->txtag, txbuf->map,
+				BUS_DMASYNC_PREWRITE);
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwcur = ring->cur;
+
+		/* decrease avail by number of sent packets */
+		ring->avail -= n;
+		kring->nr_hwavail = ring->avail;
+
+		bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+		E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me),
+			ring->cur);
+	}
+	if (do_lock)
+		EM_TX_UNLOCK(txr);
+	return 0;
+}
+
+/*
+ * Reconcile kernel and user view of the receive ring, see ixgbe.c
+ */
+static int
+em_netmap_rxsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct rx_ring *rxr = &adapter->rx_rings[ring_nr];
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->rx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	k = ring->cur;
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+ 
+	if (do_lock)
+		EM_RX_LOCK(rxr);
+	/* XXX check sync modes */
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	/* acknowledge all the received packets. */
+	j = rxr->next_to_check;
+	for (n = 0; ; n++) {
+		struct e1000_rx_desc *curr = &rxr->rx_base[j];
+
+		if ((curr->status & E1000_RXD_STAT_DD) == 0)
+			break;
+		ring->slot[j].len = le16toh(curr->length);
+		bus_dmamap_sync(rxr->tag, rxr->rx_buffers[j].map,
+			BUS_DMASYNC_POSTREAD);
+		j = (j == lim) ? 0 : j + 1;
+	}
+	if (n) {
+		rxr->next_to_check = j;
+		kring->nr_hwavail += n;
+	}
+
+	/* skip past packets that userspace has already processed:
+	 * making them available for reception.
+	 * advance nr_hwcur and issue a bus_dmamap_sync on the
+	 * buffers so it is safe to write to them.
+	 * Also increase nr_hwavail
+         */
+	j = kring->nr_hwcur;
+	if (j != k) { /* userspace has read some packets. */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct e1000_rx_desc *curr = &rxr->rx_base[j];
+			struct em_buffer *rxbuf = &rxr->rx_buffers[j];
+			void *addr = NMB(slot);
+
+			if (addr == netmap_buffer_base) { /* bad buf */
+				if (do_lock)
+					EM_RX_UNLOCK(rxr);
+				return netmap_ring_reinit(kring);
+			}
+
+			curr->status = 0;
+			if (slot->flags & NS_BUF_CHANGED) {
+				curr->buffer_addr = htole64(vtophys(addr));
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(rxr->rxtag, rxbuf->map,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(rxr->rxtag, rxbuf->map,
+				BUS_DMASYNC_PREREAD);
+
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwavail -= n;
+		kring->nr_hwcur = ring->cur;
+		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+		/*
+		 * IMPORTANT: we must leave one free slot in the ring,
+		 * so move j back by one unit
+		 */
+		j = (j == 0) ? lim : j - 1;
+		E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), j);
+	}
+	/* tell userspace that there are new packets */
+	ring->avail = kring->nr_hwavail ;
+	if (do_lock)
+		EM_RX_UNLOCK(rxr);
+	return 0;
+}
diff --git a/sys/dev/netmap/if_igb_netmap.h b/sys/dev/netmap/if_igb_netmap.h
new file mode 100644
index 0000000..0c14706
--- /dev/null
+++ b/sys/dev/netmap/if_igb_netmap.h
@@ -0,0 +1,378 @@
+/*
+ * Copyright (C) 2011 Universita` di Pisa. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: if_igb_netmap.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * netmap modifications for igb
+ * contribured by Ahmed Kooli
+ */
+
+#include <net/netmap.h>
+#include <sys/selinfo.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>    /* vtophys ? */
+#include <dev/netmap/netmap_kern.h>
+
+static int	igb_netmap_reg(struct ifnet *, int onoff);
+static int	igb_netmap_txsync(void *, u_int, int);
+static int	igb_netmap_rxsync(void *, u_int, int);
+static void	igb_netmap_lock_wrapper(void *, int, u_int);
+
+
+static void
+igb_netmap_attach(struct adapter *adapter)
+{
+	struct netmap_adapter na;
+
+	bzero(&na, sizeof(na));
+
+	na.ifp = adapter->ifp;
+	na.separate_locks = 1;
+	na.num_tx_desc = adapter->num_tx_desc;
+	na.num_rx_desc = adapter->num_rx_desc;
+	na.nm_txsync = igb_netmap_txsync;
+	na.nm_rxsync = igb_netmap_rxsync;
+	na.nm_lock = igb_netmap_lock_wrapper;
+	na.nm_register = igb_netmap_reg;
+	/*
+	 * adapter->rx_mbuf_sz is set by SIOCSETMTU, but in netmap mode
+	 * we allocate the buffers on the first register. So we must
+	 * disallow a SIOCSETMTU when if_capenable & IFCAP_NETMAP is set.
+	 */
+	na.buff_size = MCLBYTES;
+	netmap_attach(&na, adapter->num_queues);
+}	
+
+
+/*
+ * wrapper to export locks to the generic code
+ */
+static void
+igb_netmap_lock_wrapper(void *_a, int what, u_int queueid)
+{
+	struct adapter *adapter = _a;
+
+	ASSERT(queueid < adapter->num_queues);
+	switch (what) {
+	case NETMAP_CORE_LOCK:
+		IGB_CORE_LOCK(adapter);
+		break;
+	case NETMAP_CORE_UNLOCK:
+		IGB_CORE_UNLOCK(adapter);
+		break;
+	case NETMAP_TX_LOCK:
+		IGB_TX_LOCK(&adapter->tx_rings[queueid]);
+		break;
+	case NETMAP_TX_UNLOCK:
+		IGB_TX_UNLOCK(&adapter->tx_rings[queueid]);
+		break;
+	case NETMAP_RX_LOCK:
+		IGB_RX_LOCK(&adapter->rx_rings[queueid]);
+		break;
+	case NETMAP_RX_UNLOCK:
+		IGB_RX_UNLOCK(&adapter->rx_rings[queueid]);
+		break;
+	}
+}
+
+
+/*
+ * support for netmap register/unregisted. We are already under core lock.
+ * only called on the first init or the last unregister.
+ */
+static int
+igb_netmap_reg(struct ifnet *ifp, int onoff)
+{
+	struct adapter *adapter = ifp->if_softc;
+	struct netmap_adapter *na = NA(ifp);
+	int error = 0;
+
+	if (!na)
+		return EINVAL;
+
+	igb_disable_intr(adapter);
+
+	/* Tell the stack that the interface is no longer active */
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+
+	if (onoff) {
+		ifp->if_capenable |= IFCAP_NETMAP;
+
+		/* save if_transmit to restore it later */
+		na->if_transmit = ifp->if_transmit;
+		ifp->if_transmit = netmap_start;
+
+		igb_init_locked(adapter);
+		if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) {
+			error = ENOMEM;
+			goto fail;
+		}
+	} else {
+fail:
+		/* restore if_transmit */
+		ifp->if_transmit = na->if_transmit;
+		ifp->if_capenable &= ~IFCAP_NETMAP;
+		igb_init_locked(adapter);	/* also enables intr */
+	}
+	return (error);
+}
+
+
+/*
+ * Reconcile kernel and user view of the transmit ring.
+ *
+ * Userspace has filled tx slots up to cur (excluded).
+ * The last unused slot previously known to the kernel was nr_hwcur,
+ * and the last interrupt reported nr_hwavail slots available
+ * (using the special value -1 to indicate idle transmit ring).
+ * The function must first update avail to what the kernel
+ * knows, subtract the newly used slots (cur - nr_hwcur)
+ * from both avail and nr_hwavail, and set nr_hwcur = cur
+ * issuing a dmamap_sync on all slots.
+ *
+ * Check parameters in the struct netmap_ring.
+ * We don't use avail, only check for bogus values.
+ * Make sure cur is valid, and same goes for buffer indexes and lengths.
+ * To avoid races, read the values once, and never use those from
+ * the ring afterwards.
+ */
+static int
+igb_netmap_txsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct tx_ring *txr = &adapter->tx_rings[ring_nr];
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->tx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	/* generate an interrupt approximately every half ring */
+	int report_frequency = kring->nkr_num_slots >> 1;
+
+	k = ring->cur;	/* ring is not protected by any lock */
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		IGB_TX_LOCK(txr);
+	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
+			BUS_DMASYNC_POSTREAD);
+
+	/* record completed transmissions. TODO
+	 *
+	 * Instead of reading from the TDH register, we could and try to check
+	 * the status bit of descriptor packets.
+	 */
+	j = E1000_READ_REG(&adapter->hw, E1000_TDH(ring_nr));
+	if (j >= kring->nkr_num_slots) /* XXX can it happen ? */
+		j -= kring->nkr_num_slots;
+	int delta = j - txr->next_to_clean;
+	if (delta) {
+		/* new tx were completed */
+		if (delta < 0)
+			delta += kring->nkr_num_slots;
+		txr->next_to_clean = j;
+		kring->nr_hwavail += delta;
+	}
+
+	/* update avail to what the hardware knows */
+	ring->avail = kring->nr_hwavail;
+
+	j = kring->nr_hwcur;
+	if (j != k) {	/* we have new packets to send */
+		u32 olinfo_status = 0;
+		n = 0;
+
+		/* 82575 needs the queue index added */
+		if (adapter->hw.mac.type == e1000_82575)
+			olinfo_status |= txr->me << 4;
+
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct igb_tx_buffer *txbuf = &txr->tx_buffers[j];
+			union e1000_adv_tx_desc *curr =
+				(union e1000_adv_tx_desc *)&txr->tx_base[j];
+			void *addr = NMB(slot);
+			int flags = ((slot->flags & NS_REPORT) ||
+				j == 0 || j == report_frequency) ?
+					E1000_ADVTXD_DCMD_RS : 0;
+			int len = slot->len;
+
+			if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) {
+				if (do_lock)
+					IGB_TX_UNLOCK(txr);
+				return netmap_ring_reinit(kring);
+			}
+
+			slot->flags &= ~NS_REPORT;
+			curr->read.buffer_addr = htole64(vtophys(addr));
+			curr->read.olinfo_status =
+			    htole32(olinfo_status |
+				(len<< E1000_ADVTXD_PAYLEN_SHIFT));
+			curr->read.cmd_type_len =
+			    htole32(len | E1000_ADVTXD_DTYP_DATA |
+				    E1000_ADVTXD_DCMD_IFCS |
+				    E1000_ADVTXD_DCMD_DEXT |
+				    E1000_ADVTXD_DCMD_EOP | flags);
+			if (slot->flags & NS_BUF_CHANGED) {
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(txr->txtag, txbuf->map,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(txr->txtag, txbuf->map,
+				BUS_DMASYNC_PREWRITE);
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwcur = k;
+
+		/* decrease avail by number of sent packets */
+		ring->avail -= n;
+		kring->nr_hwavail = ring->avail;
+
+		/* Set the watchdog */
+		txr->queue_status = IGB_QUEUE_WORKING;
+		txr->watchdog_time = ticks;
+
+		bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+		E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), k);
+	}
+	if (do_lock)
+		IGB_TX_UNLOCK(txr);
+	return 0;
+}
+
+
+/*
+ * Reconcile kernel and user view of the receive ring.
+ *
+ * Userspace has read rx slots up to cur (excluded).
+ * The last unread slot previously known to the kernel was nr_hwcur,
+ * and the last interrupt reported nr_hwavail slots available.
+ * We must subtract the newly consumed slots (cur - nr_hwcur)
+ * from nr_hwavail, clearing the descriptors for the next
+ * read, tell the hardware that they are available,
+ * and set nr_hwcur = cur and avail = nr_hwavail.
+ * issuing a dmamap_sync on all slots.
+ */
+static int
+igb_netmap_rxsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct rx_ring *rxr = &adapter->rx_rings[ring_nr];
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->rx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	k = ring->cur;	/* ring is not protected by any lock */
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		IGB_RX_LOCK(rxr);
+
+	/* Sync the ring. */
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	j = rxr->next_to_check;
+	for (n = 0; ; n++) {
+		union e1000_adv_rx_desc *curr = &rxr->rx_base[j];
+		uint32_t staterr = le32toh(curr->wb.upper.status_error);
+
+		if ((staterr & E1000_RXD_STAT_DD) == 0)
+			break;
+		ring->slot[j].len = le16toh(curr->wb.upper.length);
+		
+		bus_dmamap_sync(rxr->ptag,
+			rxr->rx_buffers[j].pmap, BUS_DMASYNC_POSTREAD);
+		j = (j == lim) ? 0 : j + 1;
+	}
+	if (n) {
+		rxr->next_to_check = j;
+		kring->nr_hwavail += n;
+		if (kring->nr_hwavail >= lim - 10) {
+			ND("rx ring %d almost full %d", ring_nr, kring->nr_hwavail);
+		}
+	}
+
+	/* skip past packets that userspace has already processed,
+	 * making them available for reception.
+	 * advance nr_hwcur and issue a bus_dmamap_sync on the
+	 * buffers so it is safe to write to them.
+	 * Also increase nr_hwavail
+	 */
+	j = kring->nr_hwcur;
+	if (j != k) {	/* userspace has read some packets. */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = ring->slot + j;
+			union e1000_adv_rx_desc *curr = &rxr->rx_base[j];
+			struct igb_rx_buf *rxbuf = rxr->rx_buffers + j;
+			void *addr = NMB(slot);
+
+			if (addr == netmap_buffer_base) { /* bad buf */
+				if (do_lock)
+					IGB_RX_UNLOCK(rxr);
+				return netmap_ring_reinit(kring);
+			}
+
+			curr->wb.upper.status_error = 0;
+			curr->read.pkt_addr = htole64(vtophys(addr));
+			if (slot->flags & NS_BUF_CHANGED) {
+				netmap_reload_map(rxr->ptag, rxbuf->pmap,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
+				BUS_DMASYNC_PREREAD);
+
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwavail -= n;
+		kring->nr_hwcur = ring->cur;
+		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+		/* IMPORTANT: we must leave one free slot in the ring,
+		 * so move j back by one unit
+		 */
+		j = (j == 0) ? lim : j - 1;
+		E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), j);
+	}
+	/* tell userspace that there are new packets */
+	ring->avail = kring->nr_hwavail ;
+	if (do_lock)
+		IGB_RX_UNLOCK(rxr);
+	return 0;
+}
diff --git a/sys/dev/netmap/if_lem_netmap.h b/sys/dev/netmap/if_lem_netmap.h
new file mode 100644
index 0000000..a8f3498
--- /dev/null
+++ b/sys/dev/netmap/if_lem_netmap.h
@@ -0,0 +1,344 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: if_lem_netmap.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * netmap support for if_lem.c
+ */
+
+#include <net/netmap.h>
+#include <sys/selinfo.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>    /* vtophys ? */
+#include <dev/netmap/netmap_kern.h>
+
+static int	lem_netmap_reg(struct ifnet *, int onoff);
+static int	lem_netmap_txsync(void *, u_int, int);
+static int	lem_netmap_rxsync(void *, u_int, int);
+static void	lem_netmap_lock_wrapper(void *, int, u_int);
+
+
+SYSCTL_NODE(_dev, OID_AUTO, lem, CTLFLAG_RW, 0, "lem card");
+
+static void
+lem_netmap_attach(struct adapter *adapter)
+{
+	struct netmap_adapter na;
+
+	bzero(&na, sizeof(na));
+
+	na.ifp = adapter->ifp;
+	na.separate_locks = 1;
+	na.num_tx_desc = adapter->num_tx_desc;
+	na.num_rx_desc = adapter->num_rx_desc;
+	na.nm_txsync = lem_netmap_txsync;
+	na.nm_rxsync = lem_netmap_rxsync;
+	na.nm_lock = lem_netmap_lock_wrapper;
+	na.nm_register = lem_netmap_reg;
+	na.buff_size = MCLBYTES;
+	netmap_attach(&na, 1);
+}
+
+
+static void
+lem_netmap_lock_wrapper(void *_a, int what, u_int ringid)
+{
+	struct adapter *adapter = _a;
+
+	/* only one ring here so ignore the ringid */
+	switch (what) {
+	case NETMAP_CORE_LOCK:
+		EM_CORE_LOCK(adapter);
+		break;
+	case NETMAP_CORE_UNLOCK:
+		EM_CORE_UNLOCK(adapter);
+		break;
+	case NETMAP_TX_LOCK:
+		EM_TX_LOCK(adapter);
+		break;
+	case NETMAP_TX_UNLOCK:
+		EM_TX_UNLOCK(adapter);
+		break;
+	case NETMAP_RX_LOCK:
+		EM_RX_LOCK(adapter);
+		break;
+	case NETMAP_RX_UNLOCK:
+		EM_RX_UNLOCK(adapter);
+		break;
+	}
+}
+
+
+/*
+ * Reconcile kernel and user view of the transmit ring. see ixgbe.c
+ */
+static int
+lem_netmap_txsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->tx_rings[0];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	/* generate an interrupt approximately every half ring */
+	int report_frequency = kring->nkr_num_slots >> 1;
+
+	k = ring->cur;
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		EM_TX_LOCK(adapter);
+	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
+			BUS_DMASYNC_POSTREAD);
+
+	/* record completed transmissions TODO
+	 *
+	 * instead of using TDH, we could read the transmitted status bit.
+	 */
+	j = E1000_READ_REG(&adapter->hw, E1000_TDH(0));
+	if (j >= kring->nkr_num_slots) { /* can it happen ? */
+		D("bad TDH %d", j);
+		j -= kring->nkr_num_slots;
+	}
+	int delta = j - adapter->next_tx_to_clean;
+	if (delta) {
+		if (delta < 0)
+			delta += kring->nkr_num_slots;
+		adapter->next_tx_to_clean = j;
+		kring->nr_hwavail += delta;
+	}
+
+	/* update avail to what the hardware knows */
+	ring->avail = kring->nr_hwavail;
+
+	j = kring->nr_hwcur;
+	if (j != k) {	/* we have new packets to send */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct e1000_tx_desc *curr = &adapter->tx_desc_base[j];
+			struct em_buffer *txbuf = &adapter->tx_buffer_area[j];
+			void *addr = NMB(slot);
+			int flags = ((slot->flags & NS_REPORT) ||
+				j == 0 || j == report_frequency) ?
+					E1000_TXD_CMD_RS : 0;
+			int len = slot->len;
+
+			if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) {
+				if (do_lock)
+					EM_TX_UNLOCK(adapter);
+				return netmap_ring_reinit(kring);
+			}
+
+			curr->upper.data = 0;
+			/* always interrupt. XXX make it conditional */
+			curr->lower.data =
+			    htole32( adapter->txd_cmd | len |
+				(E1000_TXD_CMD_EOP | flags) );
+			if (slot->flags & NS_BUF_CHANGED) {
+				curr->buffer_addr = htole64(vtophys(addr));
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(adapter->txtag, txbuf->map,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(adapter->txtag, txbuf->map,
+				BUS_DMASYNC_PREWRITE);
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwcur = ring->cur;
+
+		/* decrease avail by number of sent packets */
+		ring->avail -= n;
+		kring->nr_hwavail = ring->avail;
+
+		bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
+				BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+		E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), ring->cur);
+	}
+	if (do_lock)
+		EM_TX_UNLOCK(adapter);
+	return 0;
+}
+
+
+/*
+ * Reconcile kernel and user view of the receive ring. see ixgbe.c
+ */
+static int
+lem_netmap_rxsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->rx_rings[0];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	k = ring->cur;
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		EM_RX_LOCK(adapter);
+	/* XXX check sync modes */
+	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
+			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	/* acknowldge all the received packets. */
+	j = adapter->next_rx_desc_to_check;
+	for (n = 0; ; n++) {
+		struct e1000_rx_desc *curr = &adapter->rx_desc_base[j];
+		int len = le16toh(adapter->rx_desc_base[j].length) - 4; // CRC
+
+		if ((curr->status & E1000_RXD_STAT_DD) == 0)
+			break;
+
+		if (len < 0) {
+			D("bogus pkt size at %d", j);
+			len = 0;
+		}
+		ring->slot[j].len = len;
+		bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[j].map,
+			BUS_DMASYNC_POSTREAD);
+		j = (j == lim) ? 0 : j + 1;
+	}
+	if (n) {
+		adapter->next_rx_desc_to_check = j;
+		kring->nr_hwavail += n;
+	}
+
+	/* skip past packets that userspace has already processed,
+	 * making them available for reception. We don't need to set
+	 * the length as it is the same for all slots.
+	 */
+	j = kring->nr_hwcur;
+	if (j != k) {	/* userspace has read some packets. */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct e1000_rx_desc *curr = &adapter->rx_desc_base[j];
+			struct em_buffer *rxbuf = &adapter->rx_buffer_area[j];
+			void *addr = NMB(slot);
+
+			if (addr == netmap_buffer_base) { /* bad buf */
+				if (do_lock)
+					EM_RX_UNLOCK(adapter);
+				return netmap_ring_reinit(kring);
+			}
+			curr = &adapter->rx_desc_base[j];
+			curr->status = 0;
+			if (slot->flags & NS_BUF_CHANGED) {
+				curr->buffer_addr = htole64(vtophys(addr));
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(adapter->rxtag, rxbuf->map,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(adapter->rxtag, rxbuf->map,
+				BUS_DMASYNC_PREREAD);
+
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwavail -= n;
+		kring->nr_hwcur = ring->cur;
+		bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+		/*
+		 * IMPORTANT: we must leave one free slot in the ring,
+		 * so move j back by one unit
+		 */
+		j = (j == 0) ? lim : j - 1;
+		E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), j);
+	}
+
+	/* tell userspace that there are new packets */
+	ring->avail = kring->nr_hwavail ;
+	if (do_lock)
+		EM_RX_UNLOCK(adapter);
+	return 0;
+}
+
+
+/*
+ * Register/unregister routine
+ */
+static int
+lem_netmap_reg(struct ifnet *ifp, int onoff)
+{
+	struct adapter *adapter = ifp->if_softc;
+	struct netmap_adapter *na = NA(ifp);
+	int error = 0;
+
+	if (!na)
+		return EINVAL;
+
+	lem_disable_intr(adapter);
+
+	/* Tell the stack that the interface is no longer active */
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+
+	/* lem_netmap_block_tasks(adapter); */
+#ifndef EM_LEGACY_IRQ
+	taskqueue_block(adapter->tq);
+	taskqueue_drain(adapter->tq, &adapter->rxtx_task);
+	taskqueue_drain(adapter->tq, &adapter->link_task);
+#endif /* !EM_LEGCY_IRQ */
+	if (onoff) {
+		ifp->if_capenable |= IFCAP_NETMAP;
+
+		/* save if_transmit to restore it when exiting.
+		 * XXX what about if_start and if_qflush ?
+		 */
+		na->if_transmit = ifp->if_transmit;
+		ifp->if_transmit = netmap_start;
+
+		lem_init_locked(adapter);
+		if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) {
+			error = ENOMEM;
+			goto fail;
+		}
+	} else {
+fail:
+		/* restore non-netmap mode */
+		ifp->if_transmit = na->if_transmit;
+		ifp->if_capenable &= ~IFCAP_NETMAP;
+		lem_init_locked(adapter);	/* also enables intr */
+	}
+
+#ifndef EM_LEGACY_IRQ
+	taskqueue_unblock(adapter->tq);
+#endif /* !EM_LEGCY_IRQ */
+
+	return (error);
+}
diff --git a/sys/dev/netmap/if_re_netmap.h b/sys/dev/netmap/if_re_netmap.h
new file mode 100644
index 0000000..efccf3a
--- /dev/null
+++ b/sys/dev/netmap/if_re_netmap.h
@@ -0,0 +1,415 @@
+/*
+ * Copyright (C) 2011 Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: if_re_netmap.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * netmap support for if_re
+ */
+
+#include <net/netmap.h>
+#include <sys/selinfo.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>    /* vtophys ? */
+#include <dev/netmap/netmap_kern.h>
+
+static int re_netmap_reg(struct ifnet *, int onoff);
+static int re_netmap_txsync(void *, u_int, int);
+static int re_netmap_rxsync(void *, u_int, int);
+static void re_netmap_lock_wrapper(void *, int, u_int);
+
+static void
+re_netmap_attach(struct rl_softc *sc)
+{
+	struct netmap_adapter na;
+
+	bzero(&na, sizeof(na));
+
+	na.ifp = sc->rl_ifp;
+	na.separate_locks = 0;
+	na.num_tx_desc = sc->rl_ldata.rl_tx_desc_cnt;
+	na.num_rx_desc = sc->rl_ldata.rl_rx_desc_cnt;
+	na.nm_txsync = re_netmap_txsync;
+	na.nm_rxsync = re_netmap_rxsync;
+	na.nm_lock = re_netmap_lock_wrapper;
+	na.nm_register = re_netmap_reg;
+	na.buff_size = MCLBYTES;
+	netmap_attach(&na, 1);
+}
+
+
+/*
+ * wrapper to export locks to the generic code
+ * We should not use the tx/rx locks
+ */
+static void
+re_netmap_lock_wrapper(void *_a, int what, u_int queueid)
+{
+	struct rl_softc *adapter = _a;
+
+	switch (what) {
+	case NETMAP_CORE_LOCK:
+		RL_LOCK(adapter);
+		break;
+	case NETMAP_CORE_UNLOCK:
+		RL_UNLOCK(adapter);
+		break;
+
+	case NETMAP_TX_LOCK:
+	case NETMAP_RX_LOCK:
+	case NETMAP_TX_UNLOCK:
+	case NETMAP_RX_UNLOCK:
+		D("invalid lock call %d, no tx/rx locks here", what);
+		break;
+	}
+}
+
+
+/*
+ * support for netmap register/unregisted. We are already under core lock.
+ * only called on the first register or the last unregister.
+ */
+static int
+re_netmap_reg(struct ifnet *ifp, int onoff)
+{
+	struct rl_softc *adapter = ifp->if_softc;
+	struct netmap_adapter *na = NA(ifp);
+	int error = 0;
+
+	if (!na)
+		return EINVAL;
+	/* Tell the stack that the interface is no longer active */
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+
+	re_stop(adapter);
+
+	if (onoff) {
+		ifp->if_capenable |= IFCAP_NETMAP;
+
+		/* save if_transmit and restore it */
+		na->if_transmit = ifp->if_transmit;
+		/* XXX if_start and if_qflush ??? */
+		ifp->if_transmit = netmap_start;
+
+		re_init_locked(adapter);
+
+		if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) {
+			error = ENOMEM;
+			goto fail;
+		}
+	} else {
+fail:
+		/* restore if_transmit */
+		ifp->if_transmit = na->if_transmit;
+		ifp->if_capenable &= ~IFCAP_NETMAP;
+		re_init_locked(adapter);	/* also enables intr */
+	}
+    return (error);
+
+}
+
+
+/*
+ * Reconcile kernel and user view of the transmit ring.
+ *
+ * Userspace has filled tx slots up to cur (excluded).
+ * The last unused slot previously known to the kernel was nr_hwcur,
+ * and the last interrupt reported nr_hwavail slots available
+ * (using the special value -1 to indicate idle transmit ring).
+ * The function must first update avail to what the kernel
+ * knows (translating the -1 to nkr_num_slots - 1),
+ * subtract the newly used slots (cur - nr_hwcur)
+ * from both avail and nr_hwavail, and set nr_hwcur = cur
+ * issuing a dmamap_sync on all slots.
+ */
+static int
+re_netmap_txsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct rl_softc *sc = a;
+	struct rl_txdesc *txd = sc->rl_ldata.rl_tx_desc;
+	struct netmap_adapter *na = NA(sc->rl_ifp);
+	struct netmap_kring *kring = &na->tx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	k = ring->cur;
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		RL_LOCK(sc);
+
+	/* Sync the TX descriptor list */
+	bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
+            sc->rl_ldata.rl_tx_list_map,
+            BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	/* record completed transmissions */
+        for (n = 0, j = sc->rl_ldata.rl_tx_considx;
+	    j != sc->rl_ldata.rl_tx_prodidx;
+	    n++, j = RL_TX_DESC_NXT(sc, j)) {
+		uint32_t cmdstat =
+			le32toh(sc->rl_ldata.rl_tx_list[j].rl_cmdstat);
+		if (cmdstat & RL_TDESC_STAT_OWN)
+			break;
+	}
+	if (n > 0) {
+		sc->rl_ldata.rl_tx_considx = j;
+		sc->rl_ldata.rl_tx_free += n;
+		kring->nr_hwavail += n;
+	}
+
+	/* update avail to what the hardware knows */
+	ring->avail = kring->nr_hwavail;
+	
+	/* we trust prodidx, not hwcur */
+	j = kring->nr_hwcur = sc->rl_ldata.rl_tx_prodidx;
+	if (j != k) {	/* we have new packets to send */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct rl_desc *desc = &sc->rl_ldata.rl_tx_list[j];
+			int cmd = slot->len | RL_TDESC_CMD_EOF |
+				RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF ;
+			void *addr = NMB(slot);
+			int len = slot->len;
+
+			if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) {
+				if (do_lock)
+					RL_UNLOCK(sc);
+				return netmap_ring_reinit(kring);
+			}
+			
+			if (j == lim)	/* mark end of ring */
+				cmd |= RL_TDESC_CMD_EOR;
+
+			if (slot->flags & NS_BUF_CHANGED) {
+				uint64_t paddr = vtophys(addr);
+				desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr));
+				desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr));
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(sc->rl_ldata.rl_tx_mtag,
+					txd[j].tx_dmamap, addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+			slot->flags &= ~NS_REPORT;
+			desc->rl_cmdstat = htole32(cmd);
+			bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
+				txd[j].tx_dmamap, BUS_DMASYNC_PREWRITE);
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		sc->rl_ldata.rl_tx_prodidx = kring->nr_hwcur = ring->cur;
+
+		/* decrease avail by number of sent packets */
+		ring->avail -= n;
+		kring->nr_hwavail = ring->avail;
+
+		bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
+		    sc->rl_ldata.rl_tx_list_map,
+		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
+
+		/* start ? */
+		CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
+	}
+	if (do_lock)
+		RL_UNLOCK(sc);
+	return 0;
+}
+
+
+/*
+ * Reconcile kernel and user view of the receive ring.
+ *
+ * Userspace has read rx slots up to cur (excluded).
+ * The last unread slot previously known to the kernel was nr_hwcur,
+ * and the last interrupt reported nr_hwavail slots available.
+ * We must subtract the newly consumed slots (cur - nr_hwcur)
+ * from nr_hwavail, clearing the descriptors for the next
+ * read, tell the hardware that they are available,
+ * and set nr_hwcur = cur and avail = nr_hwavail.
+ * issuing a dmamap_sync on all slots.
+ */
+static int
+re_netmap_rxsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct rl_softc *sc = a;
+	struct rl_rxdesc *rxd = sc->rl_ldata.rl_rx_desc;
+	struct netmap_adapter *na = NA(sc->rl_ifp);
+	struct netmap_kring *kring = &na->rx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	k = ring->cur;
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		RL_LOCK(sc);
+	/* XXX check sync modes */
+	bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
+	    sc->rl_ldata.rl_rx_list_map,
+	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	/*
+	 * The device uses all the buffers in the ring, so we need
+	 * another termination condition in addition to RL_RDESC_STAT_OWN
+	 * cleared (all buffers could have it cleared. The easiest one
+	 * is to limit the amount of data reported up to 'lim'
+	 */
+	j = sc->rl_ldata.rl_rx_prodidx;
+	for (n = kring->nr_hwavail; n < lim ; n++) {
+		struct rl_desc *cur_rx = &sc->rl_ldata.rl_rx_list[j];
+		uint32_t rxstat = le32toh(cur_rx->rl_cmdstat);
+		uint32_t total_len;
+
+		if ((rxstat & RL_RDESC_STAT_OWN) != 0)
+			break;
+		total_len = rxstat & sc->rl_rxlenmask;
+		/* XXX subtract crc */
+		total_len = (total_len < 4) ? 0 : total_len - 4;
+		kring->ring->slot[j].len = total_len;
+		/*  sync was in re_newbuf() */
+		bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
+		    rxd[j].rx_dmamap, BUS_DMASYNC_POSTREAD);
+		j = RL_RX_DESC_NXT(sc, j);
+	}
+	if (n != kring->nr_hwavail) {
+		sc->rl_ldata.rl_rx_prodidx = j;
+		sc->rl_ifp->if_ipackets += n - kring->nr_hwavail;
+		kring->nr_hwavail = n;
+	}
+
+	/* skip past packets that userspace has already processed,
+	 * making them available for reception.
+	 * advance nr_hwcur and issue a bus_dmamap_sync on the
+	 * buffers so it is safe to write to them.
+	 * Also increase nr_hwavail
+	 */
+	j = kring->nr_hwcur;
+	if (j != k) {	/* userspace has read some packets. */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = ring->slot + j;
+			struct rl_desc *desc = &sc->rl_ldata.rl_rx_list[j];
+			int cmd = na->buff_size | RL_RDESC_CMD_OWN;
+			void *addr = NMB(slot);
+
+			if (addr == netmap_buffer_base) { /* bad buf */
+				if (do_lock)
+					RL_UNLOCK(sc);
+				return netmap_ring_reinit(kring);
+			}
+
+			if (j == lim)	/* mark end of ring */
+				cmd |= RL_RDESC_CMD_EOR;
+
+			desc->rl_cmdstat = htole32(cmd);
+			slot->flags &= ~NS_REPORT;
+			if (slot->flags & NS_BUF_CHANGED) {
+				uint64_t paddr = vtophys(addr);
+				desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr));
+				desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr));
+				netmap_reload_map(sc->rl_ldata.rl_rx_mtag,
+					rxd[j].rx_dmamap, addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+			bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
+				rxd[j].rx_dmamap, BUS_DMASYNC_PREREAD);
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwavail -= n;
+		kring->nr_hwcur = k;
+		/* Flush the RX DMA ring */
+
+		bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
+		    sc->rl_ldata.rl_rx_list_map,
+		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
+	}
+	/* tell userspace that there are new packets */
+	ring->avail = kring->nr_hwavail ;
+	if (do_lock)
+		RL_UNLOCK(sc);
+	return 0;
+}
+
+static void
+re_netmap_tx_init(struct rl_softc *sc)
+{   
+	struct rl_txdesc *txd;
+	struct rl_desc *desc;
+	int i;
+	struct netmap_adapter *na = NA(sc->rl_ifp);
+	struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0);
+
+	/* slot is NULL if we are not in netmap mode */
+	if (!slot)
+		return;
+	/* in netmap mode, overwrite addresses and maps */
+	txd = sc->rl_ldata.rl_tx_desc;
+	desc = sc->rl_ldata.rl_tx_list;
+
+	for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
+		void *addr = NMB(slot+i);
+		uint64_t paddr = vtophys(addr);
+
+		desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr));
+		desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr));
+		netmap_load_map(sc->rl_ldata.rl_tx_mtag,
+			txd[i].tx_dmamap, addr, na->buff_size);
+	}
+}
+
+static void
+re_netmap_rx_init(struct rl_softc *sc)
+{
+	/* slot is NULL if we are not in netmap mode */  
+	struct netmap_adapter *na = NA(sc->rl_ifp);
+	struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0);
+	struct rl_desc *desc = sc->rl_ldata.rl_rx_list;
+	uint32_t cmdstat;
+	int i;
+
+	if (!slot)
+		return;
+
+	for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
+		void *addr = NMB(slot+i);
+		uint64_t paddr = vtophys(addr);
+
+		desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr));
+		desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr));
+		cmdstat = slot[i].len = na->buff_size; // XXX
+		if (i == sc->rl_ldata.rl_rx_desc_cnt - 1)
+			cmdstat |= RL_RDESC_CMD_EOR;
+		desc[i].rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
+
+		netmap_reload_map(sc->rl_ldata.rl_rx_mtag,
+			sc->rl_ldata.rl_rx_desc[i].rx_dmamap,
+			addr, na->buff_size);
+	}
+}
diff --git a/sys/dev/netmap/ixgbe_netmap.h b/sys/dev/netmap/ixgbe_netmap.h
new file mode 100644
index 0000000..a4d5491
--- /dev/null
+++ b/sys/dev/netmap/ixgbe_netmap.h
@@ -0,0 +1,376 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: ixgbe_netmap.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * netmap modifications for ixgbe
+ */
+
+#include <net/netmap.h>
+#include <sys/selinfo.h>
+// #include <vm/vm.h>
+// #include <vm/pmap.h>    /* vtophys ? */
+#include <dev/netmap/netmap_kern.h>
+
+static int	ixgbe_netmap_reg(struct ifnet *, int onoff);
+static int	ixgbe_netmap_txsync(void *, u_int, int);
+static int	ixgbe_netmap_rxsync(void *, u_int, int);
+static void	ixgbe_netmap_lock_wrapper(void *, int, u_int);
+
+
+SYSCTL_NODE(_dev, OID_AUTO, ixgbe, CTLFLAG_RW, 0, "ixgbe card");
+
+static void
+ixgbe_netmap_attach(struct adapter *adapter)
+{
+	struct netmap_adapter na;
+
+	bzero(&na, sizeof(na));
+
+	na.ifp = adapter->ifp;
+	na.separate_locks = 1;
+	na.num_tx_desc = adapter->num_tx_desc;
+	na.num_rx_desc = adapter->num_rx_desc;
+	na.nm_txsync = ixgbe_netmap_txsync;
+	na.nm_rxsync = ixgbe_netmap_rxsync;
+	na.nm_lock = ixgbe_netmap_lock_wrapper;
+	na.nm_register = ixgbe_netmap_reg;
+	/*
+	 * adapter->rx_mbuf_sz is set by SIOCSETMTU, but in netmap mode
+	 * we allocate the buffers on the first register. So we must
+	 * disallow a SIOCSETMTU when if_capenable & IFCAP_NETMAP is set.
+	 */
+	na.buff_size = MCLBYTES;
+	netmap_attach(&na, adapter->num_queues);
+}	
+
+
+/*
+ * wrapper to export locks to the generic code
+ */
+static void
+ixgbe_netmap_lock_wrapper(void *_a, int what, u_int queueid)
+{
+	struct adapter *adapter = _a;
+
+	ASSERT(queueid < adapter->num_queues);
+	switch (what) {
+	case NETMAP_CORE_LOCK:
+		IXGBE_CORE_LOCK(adapter);
+		break;
+	case NETMAP_CORE_UNLOCK:
+		IXGBE_CORE_UNLOCK(adapter);
+		break;
+	case NETMAP_TX_LOCK:
+		IXGBE_TX_LOCK(&adapter->tx_rings[queueid]);
+		break;
+	case NETMAP_TX_UNLOCK:
+		IXGBE_TX_UNLOCK(&adapter->tx_rings[queueid]);
+		break;
+	case NETMAP_RX_LOCK:
+		IXGBE_RX_LOCK(&adapter->rx_rings[queueid]);
+		break;
+	case NETMAP_RX_UNLOCK:
+		IXGBE_RX_UNLOCK(&adapter->rx_rings[queueid]);
+		break;
+	}
+}
+
+
+/*
+ * support for netmap register/unregisted. We are already under core lock.
+ * only called on the first init or the last unregister.
+ */
+static int
+ixgbe_netmap_reg(struct ifnet *ifp, int onoff)
+{
+	struct adapter *adapter = ifp->if_softc;
+	struct netmap_adapter *na = NA(ifp);
+	int error = 0;
+
+	if (!na)
+		return EINVAL;
+
+	ixgbe_disable_intr(adapter);
+
+	/* Tell the stack that the interface is no longer active */
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+
+	if (onoff) {
+		ifp->if_capenable |= IFCAP_NETMAP;
+
+		/* save if_transmit to restore it later */
+		na->if_transmit = ifp->if_transmit;
+		ifp->if_transmit = netmap_start;
+
+		ixgbe_init_locked(adapter);
+		if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) {
+			error = ENOMEM;
+			goto fail;
+		}
+	} else {
+fail:
+		/* restore if_transmit */
+		ifp->if_transmit = na->if_transmit;
+		ifp->if_capenable &= ~IFCAP_NETMAP;
+		ixgbe_init_locked(adapter);	/* also enables intr */
+	}
+	return (error);
+}
+
+
+/*
+ * Reconcile kernel and user view of the transmit ring.
+ *
+ * Userspace has filled tx slots up to cur (excluded).
+ * The last unused slot previously known to the kernel was nr_hwcur,
+ * and the last interrupt reported nr_hwavail slots available
+ * (using the special value -1 to indicate idle transmit ring).
+ * The function must first update avail to what the kernel
+ * knows, subtract the newly used slots (cur - nr_hwcur)
+ * from both avail and nr_hwavail, and set nr_hwcur = cur
+ * issuing a dmamap_sync on all slots.
+ *
+ * Check parameters in the struct netmap_ring.
+ * We don't use avail, only check for bogus values.
+ * Make sure cur is valid, and same goes for buffer indexes and lengths.
+ * To avoid races, read the values once, and never use those from
+ * the ring afterwards.
+ */
+static int
+ixgbe_netmap_txsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct tx_ring *txr = &adapter->tx_rings[ring_nr];
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->tx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n = 0, lim = kring->nkr_num_slots - 1;
+
+	/* generate an interrupt approximately every half ring */
+	int report_frequency = kring->nkr_num_slots >> 1;
+
+	k = ring->cur;	/* ring is not protected by any lock */
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		IXGBE_TX_LOCK(txr);
+	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
+			BUS_DMASYNC_POSTREAD);
+
+	/* update avail to what the hardware knows */
+	ring->avail = kring->nr_hwavail;
+
+	j = kring->nr_hwcur;
+	if (j != k) {	/* we have new packets to send */
+		while (j != k) {
+			struct netmap_slot *slot = &ring->slot[j];
+			struct ixgbe_tx_buf *txbuf = &txr->tx_buffers[j];
+			union ixgbe_adv_tx_desc *curr = &txr->tx_base[j];
+			void *addr = NMB(slot);
+			int flags = ((slot->flags & NS_REPORT) ||
+				j == 0 || j == report_frequency) ?
+					IXGBE_TXD_CMD_RS : 0;
+			int len = slot->len;
+
+			if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) {
+				if (do_lock)
+					IXGBE_TX_UNLOCK(txr);
+				return netmap_ring_reinit(kring);
+			}
+
+			slot->flags &= ~NS_REPORT;
+			curr->read.buffer_addr = htole64(vtophys(addr));
+			curr->read.olinfo_status = 0;
+			curr->read.cmd_type_len =
+			    htole32(txr->txd_cmd | len |
+				(IXGBE_ADVTXD_DTYP_DATA |
+				    IXGBE_ADVTXD_DCMD_IFCS |
+				    IXGBE_TXD_CMD_EOP | flags) );
+			if (slot->flags & NS_BUF_CHANGED) {
+				/* buffer has changed, unload and reload map */
+				netmap_reload_map(txr->txtag, txbuf->map,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(txr->txtag, txbuf->map,
+				BUS_DMASYNC_PREWRITE);
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwcur = k;
+
+		/* decrease avail by number of sent packets */
+		ring->avail -= n;
+		kring->nr_hwavail = ring->avail;
+
+		bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+		IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), k);
+	}
+
+	if (n == 0 || kring->nr_hwavail < 1) {
+		/* record completed transmissions. TODO
+		 *
+		 * The datasheet discourages the use of TDH to find out the
+		 * number of sent packets; the right way to do so, is to check
+		 * the DD bit inside the status of a packet descriptor.  On the
+		 * other hand, we avoid to set the `report status' bit for
+		 * *all* outgoing packets (kind of interrupt mitigation),
+		 * consequently the DD bit is not guaranteed to be set for all
+		 * the packets: thats way, for the moment we continue to use
+		 * TDH.
+		 */
+		j = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(ring_nr));
+		if (j >= kring->nkr_num_slots) { /* XXX can happen */
+			D("TDH wrap %d", j);
+			j -= kring->nkr_num_slots;
+		}
+		int delta = j - txr->next_to_clean;
+		if (delta) {
+			/* new transmissions were completed, increment
+			   ring->nr_hwavail. */
+			if (delta < 0)
+				delta += kring->nkr_num_slots;
+			txr->next_to_clean = j;
+			kring->nr_hwavail += delta;
+			ring->avail = kring->nr_hwavail;
+		}
+	}
+
+	if (do_lock)
+		IXGBE_TX_UNLOCK(txr);
+	return 0;
+}
+
+
+/*
+ * Reconcile kernel and user view of the receive ring.
+ *
+ * Userspace has read rx slots up to cur (excluded).
+ * The last unread slot previously known to the kernel was nr_hwcur,
+ * and the last interrupt reported nr_hwavail slots available.
+ * We must subtract the newly consumed slots (cur - nr_hwcur)
+ * from nr_hwavail, clearing the descriptors for the next
+ * read, tell the hardware that they are available,
+ * and set nr_hwcur = cur and avail = nr_hwavail.
+ * issuing a dmamap_sync on all slots.
+ */
+static int
+ixgbe_netmap_rxsync(void *a, u_int ring_nr, int do_lock)
+{
+	struct adapter *adapter = a;
+	struct rx_ring *rxr = &adapter->rx_rings[ring_nr];
+	struct netmap_adapter *na = NA(adapter->ifp);
+	struct netmap_kring *kring = &na->rx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	int j, k, n, lim = kring->nkr_num_slots - 1;
+
+	k = ring->cur;	/* ring is not protected by any lock */
+	if ( (kring->nr_kflags & NR_REINIT) || k > lim)
+		return netmap_ring_reinit(kring);
+
+	if (do_lock)
+		IXGBE_RX_LOCK(rxr);
+	/* XXX check sync modes */
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	j = rxr->next_to_check;
+	for (n = 0; ; n++) {
+		union ixgbe_adv_rx_desc *curr = &rxr->rx_base[j];
+		uint32_t staterr = le32toh(curr->wb.upper.status_error);
+
+		if ((staterr & IXGBE_RXD_STAT_DD) == 0)
+			break;
+		ring->slot[j].len = le16toh(curr->wb.upper.length);
+		bus_dmamap_sync(rxr->ptag,
+			rxr->rx_buffers[j].pmap, BUS_DMASYNC_POSTREAD);
+		j = (j == lim) ? 0 : j + 1;
+	}
+	if (n) {
+		rxr->next_to_check = j;
+		kring->nr_hwavail += n;
+		if (kring->nr_hwavail >= lim - 10) {
+			ND("rx ring %d almost full %d", ring_nr, kring->nr_hwavail);
+		}
+	}
+
+	/* skip past packets that userspace has already processed,
+	 * making them available for reception.
+	 * advance nr_hwcur and issue a bus_dmamap_sync on the
+	 * buffers so it is safe to write to them.
+	 * Also increase nr_hwavail
+	 */
+	j = kring->nr_hwcur;
+	if (j != k) {	/* userspace has read some packets. */
+		n = 0;
+		while (j != k) {
+			struct netmap_slot *slot = ring->slot + j;
+			union ixgbe_adv_rx_desc *curr = &rxr->rx_base[j];
+			struct ixgbe_rx_buf *rxbuf = rxr->rx_buffers + j;
+			void *addr = NMB(slot);
+
+			if (addr == netmap_buffer_base) { /* bad buf */
+				if (do_lock)
+					IXGBE_RX_UNLOCK(rxr);
+				return netmap_ring_reinit(kring);
+			}
+
+			curr->wb.upper.status_error = 0;
+			curr->read.pkt_addr = htole64(vtophys(addr));
+			if (slot->flags & NS_BUF_CHANGED) {
+				netmap_reload_map(rxr->ptag, rxbuf->pmap,
+					addr, na->buff_size);
+				slot->flags &= ~NS_BUF_CHANGED;
+			}
+
+			bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
+				BUS_DMASYNC_PREREAD);
+
+			j = (j == lim) ? 0 : j + 1;
+			n++;
+		}
+		kring->nr_hwavail -= n;
+		kring->nr_hwcur = ring->cur;
+		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+		/* IMPORTANT: we must leave one free slot in the ring,
+		 * so move j back by one unit
+		 */
+		j = (j == 0) ? lim : j - 1;
+		IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), j);
+	}
+	/* tell userspace that there are new packets */
+	ring->avail = kring->nr_hwavail ;
+	if (do_lock)
+		IXGBE_RX_UNLOCK(rxr);
+	return 0;
+}
diff --git a/sys/dev/netmap/netmap.c b/sys/dev/netmap/netmap.c
new file mode 100644
index 0000000..7645a4e
--- /dev/null
+++ b/sys/dev/netmap/netmap.c
@@ -0,0 +1,1762 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: netmap.c 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * This module supports memory mapped access to network devices,
+ * see netmap(4).
+ *
+ * The module uses a large, memory pool allocated by the kernel
+ * and accessible as mmapped memory by multiple userspace threads/processes.
+ * The memory pool contains packet buffers and "netmap rings",
+ * i.e. user-accessible copies of the interface's queues.
+ *
+ * Access to the network card works like this:
+ * 1. a process/thread issues one or more open() on /dev/netmap, to create
+ *    select()able file descriptor on which events are reported.
+ * 2. on each descriptor, the process issues an ioctl() to identify
+ *    the interface that should report events to the file descriptor.
+ * 3. on each descriptor, the process issues an mmap() request to
+ *    map the shared memory region within the process' address space.
+ *    The list of interesting queues is indicated by a location in
+ *    the shared memory region.
+ * 4. using the functions in the netmap(4) userspace API, a process
+ *    can look up the occupation state of a queue, access memory buffers,
+ *    and retrieve received packets or enqueue packets to transmit.
+ * 5. using some ioctl()s the process can synchronize the userspace view
+ *    of the queue with the actual status in the kernel. This includes both
+ *    receiving the notification of new packets, and transmitting new
+ *    packets on the output interface.
+ * 6. select() or poll() can be used to wait for events on individual
+ *    transmit or receive queues (or all queues for a given interface).
+ */
+
+#include <sys/cdefs.h> /* prerequisite */
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/module.h>
+#include <sys/errno.h>
+#include <sys/param.h>	/* defines used in kernel.h */
+#include <sys/kernel.h>	/* types used in module initialization */
+#include <sys/conf.h>	/* cdevsw struct */
+#include <sys/uio.h>	/* uio struct */
+#include <sys/sockio.h>
+#include <sys/socketvar.h>	/* struct socket */
+#include <sys/malloc.h>
+#include <sys/mman.h>	/* PROT_EXEC */
+#include <sys/poll.h>
+#include <vm/vm.h>	/* vtophys */
+#include <vm/pmap.h>	/* vtophys */
+#include <sys/socket.h> /* sockaddrs */
+#include <machine/bus.h>
+#include <sys/selinfo.h>
+#include <sys/sysctl.h>
+#include <net/if.h>
+#include <net/bpf.h>		/* BIOCIMMEDIATE */
+#include <net/netmap.h>
+#include <dev/netmap/netmap_kern.h>
+#include <machine/bus.h>	/* bus_dmamap_* */
+
+MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map");
+
+/*
+ * lock and unlock for the netmap memory allocator
+ */
+#define NMA_LOCK()	mtx_lock(&netmap_mem_d->nm_mtx);
+#define NMA_UNLOCK()	mtx_unlock(&netmap_mem_d->nm_mtx);
+
+/*
+ * Default amount of memory pre-allocated by the module.
+ * We start with a large size and then shrink our demand
+ * according to what is avalable when the module is loaded.
+ * At the moment the block is contiguous, but we can easily
+ * restrict our demand to smaller units (16..64k)
+ */
+#define NETMAP_MEMORY_SIZE (64 * 1024 * PAGE_SIZE)
+static void * netmap_malloc(size_t size, const char *msg);
+static void netmap_free(void *addr, const char *msg);
+
+/*
+ * Allocator for a pool of packet buffers. For each buffer we have
+ * one entry in the bitmap to signal the state. Allocation scans
+ * the bitmap, but since this is done only on attach, we are not
+ * too worried about performance
+ * XXX if we need to allocate small blocks, a translation
+ * table is used both for kernel virtual address and physical
+ * addresses.
+ */
+struct netmap_buf_pool {
+	u_int total_buffers;	/* total buffers. */
+	u_int free;
+	u_int bufsize;
+	char *base;		/* buffer base address */
+	uint32_t *bitmap;	/* one bit per buffer, 1 means free */
+};
+struct netmap_buf_pool nm_buf_pool;
+/* XXX move these two vars back into netmap_buf_pool */
+u_int netmap_total_buffers;
+char *netmap_buffer_base;
+
+/* user-controlled variables */
+int netmap_verbose;
+
+static int no_timestamp; /* don't timestamp on rxsync */
+
+SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args");
+SYSCTL_INT(_dev_netmap, OID_AUTO, verbose,
+    CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode");
+SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp,
+    CTLFLAG_RW, &no_timestamp, 0, "no_timestamp");
+SYSCTL_INT(_dev_netmap, OID_AUTO, total_buffers,
+    CTLFLAG_RD, &nm_buf_pool.total_buffers, 0, "total_buffers");
+SYSCTL_INT(_dev_netmap, OID_AUTO, free_buffers,
+    CTLFLAG_RD, &nm_buf_pool.free, 0, "free_buffers");
+
+/*
+ * Allocate n buffers from the ring, and fill the slot.
+ * Buffer 0 is the 'junk' buffer.
+ */
+static void
+netmap_new_bufs(struct netmap_buf_pool *p, struct netmap_slot *slot, u_int n)
+{
+	uint32_t bi = 0;		/* index in the bitmap */
+	uint32_t mask, j, i = 0;	/* slot counter */
+
+	if (n > p->free) {
+		D("only %d out of %d buffers available", i, n);
+		return;
+	}
+	/* termination is guaranteed by p->free */
+	while (i < n && p->free > 0) {
+		uint32_t cur = p->bitmap[bi];
+		if (cur == 0) { /* bitmask is fully used */
+			bi++;
+			continue;
+		}
+		/* locate a slot */
+		for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) ;
+		p->bitmap[bi] &= ~mask;		/* slot in use */
+		p->free--;
+		slot[i].buf_idx = bi*32+j;
+		slot[i].len = p->bufsize;
+		slot[i].flags = NS_BUF_CHANGED;
+		i++;
+	}
+	ND("allocated %d buffers, %d available", n, p->free);
+}
+
+
+static void
+netmap_free_buf(struct netmap_buf_pool *p, uint32_t i)
+{
+	uint32_t pos, mask;
+	if (i >= p->total_buffers) {
+		D("invalid free index %d", i);
+		return;
+	}
+	pos = i / 32;
+	mask = 1 << (i % 32);
+	if (p->bitmap[pos] & mask) {
+		D("slot %d already free", i);
+		return;
+	}
+	p->bitmap[pos] |= mask;
+	p->free++;
+}
+
+
+/* Descriptor of the memory objects handled by our memory allocator. */
+struct netmap_mem_obj {
+	TAILQ_ENTRY(netmap_mem_obj) nmo_next; /* next object in the
+						 chain. */
+	int nmo_used; /* flag set on used memory objects. */
+	size_t nmo_size; /* size of the memory area reserved for the
+			    object. */
+	void *nmo_data; /* pointer to the memory area. */
+};
+
+/* Wrap our memory objects to make them ``chainable``. */
+TAILQ_HEAD(netmap_mem_obj_h, netmap_mem_obj);
+
+
+/* Descriptor of our custom memory allocator. */
+struct netmap_mem_d {
+	struct mtx nm_mtx; /* lock used to handle the chain of memory
+			      objects. */
+	struct netmap_mem_obj_h nm_molist; /* list of memory objects */
+	size_t nm_size; /* total amount of memory used for rings etc. */
+	size_t nm_totalsize; /* total amount of allocated memory
+		(the difference is used for buffers) */
+	size_t nm_buf_start; /* offset of packet buffers.
+			This is page-aligned. */
+	size_t nm_buf_len; /* total memory for buffers */
+	void *nm_buffer; /* pointer to the whole pre-allocated memory
+			    area. */
+};
+
+
+/* Structure associated to each thread which registered an interface. */
+struct netmap_priv_d {
+	struct netmap_if *np_nifp;	/* netmap interface descriptor. */
+
+	struct ifnet	*np_ifp;	/* device for which we hold a reference */
+	int		np_ringid;	/* from the ioctl */
+	u_int		np_qfirst, np_qlast;	/* range of rings to scan */
+	uint16_t	np_txpoll;
+};
+
+
+static struct cdev *netmap_dev; /* /dev/netmap character device. */
+static struct netmap_mem_d *netmap_mem_d; /* Our memory allocator. */
+
+
+static d_mmap_t netmap_mmap;
+static d_ioctl_t netmap_ioctl;
+static d_poll_t netmap_poll;
+
+#ifdef NETMAP_KEVENT
+static d_kqfilter_t netmap_kqfilter;
+#endif
+
+static struct cdevsw netmap_cdevsw = {
+	.d_version = D_VERSION,
+	.d_name = "netmap",
+	.d_mmap = netmap_mmap,
+	.d_ioctl = netmap_ioctl,
+	.d_poll = netmap_poll,
+#ifdef NETMAP_KEVENT
+	.d_kqfilter = netmap_kqfilter,
+#endif
+};
+
+#ifdef NETMAP_KEVENT
+static int              netmap_kqread(struct knote *, long);
+static int              netmap_kqwrite(struct knote *, long);
+static void             netmap_kqdetach(struct knote *);
+
+static struct filterops netmap_read_filterops = {
+	.f_isfd =       1,
+	.f_attach =     NULL,
+	.f_detach =     netmap_kqdetach,
+	.f_event =      netmap_kqread,
+};
+  
+static struct filterops netmap_write_filterops = {
+	.f_isfd =       1,
+	.f_attach =     NULL,
+	.f_detach =     netmap_kqdetach,
+	.f_event =      netmap_kqwrite,
+};
+
+/*
+ * support for the kevent() system call.
+ *
+ * This is the kevent filter, and is executed each time a new event
+ * is triggered on the device. This function execute some operation
+ * depending on the received filter.
+ *
+ * The implementation should test the filters and should implement
+ * filter operations we are interested on (a full list in /sys/event.h).
+ *
+ * On a match we should:
+ * - set kn->kn_fop
+ * - set kn->kn_hook
+ * - call knlist_add() to deliver the event to the application.
+ *
+ * Return 0 if the event should be delivered to the application.
+ */
+static int
+netmap_kqfilter(struct cdev *dev, struct knote *kn)
+{
+	/* declare variables needed to read/write */
+
+	switch(kn->kn_filter) {
+	case EVFILT_READ:
+		if (netmap_verbose)
+			D("%s kqfilter: EVFILT_READ" ifp->if_xname);
+
+		/* read operations */
+		kn->kn_fop = &netmap_read_filterops;
+		break;
+
+	case EVFILT_WRITE:
+		if (netmap_verbose)
+			D("%s kqfilter: EVFILT_WRITE" ifp->if_xname);
+
+		/* write operations */
+		kn->kn_fop = &netmap_write_filterops;
+		break;
+  
+	default:
+		if (netmap_verbose)
+			D("%s kqfilter: invalid filter" ifp->if_xname);
+		return(EINVAL);
+	}
+  
+	kn->kn_hook = 0;//
+	knlist_add(&netmap_sc->tun_rsel.si_note, kn, 0);
+
+	return (0);
+}
+#endif /* NETMAP_KEVENT */
+
+/*
+ * File descriptor's private data destructor.
+ *
+ * Call nm_register(ifp,0) to stop netmap mode on the interface and
+ * revert to normal operation. We expect that np_ifp has not gone.
+ */
+static void
+netmap_dtor(void *data)
+{
+	struct netmap_priv_d *priv = data;
+	struct ifnet *ifp = priv->np_ifp;
+	struct netmap_adapter *na = NA(ifp);
+	struct netmap_if *nifp = priv->np_nifp;
+
+	if (0)
+	    printf("%s starting for %p ifp %p\n", __FUNCTION__, priv,
+		priv ? priv->np_ifp : NULL);
+
+	na->nm_lock(ifp->if_softc, NETMAP_CORE_LOCK, 0); 
+
+	na->refcount--;
+	if (na->refcount <= 0) {	/* last instance */
+		u_int i;
+
+		D("deleting last netmap instance for %s", ifp->if_xname);
+		/*
+		 * there is a race here with *_netmap_task() and
+		 * netmap_poll(), which don't run under NETMAP_CORE_LOCK.
+		 * na->refcount == 0 && na->ifp->if_capenable & IFCAP_NETMAP
+		 * (aka NETMAP_DELETING(na)) are a unique marker that the
+		 * device is dying.
+		 * Before destroying stuff we sleep a bit, and then complete
+		 * the job. NIOCREG should realize the condition and
+		 * loop until they can continue; the other routines
+		 * should check the condition at entry and quit if
+		 * they cannot run.
+		 */
+		na->nm_lock(ifp->if_softc, NETMAP_CORE_UNLOCK, 0);
+		tsleep(na, 0, "NIOCUNREG", 4);
+		na->nm_lock(ifp->if_softc, NETMAP_CORE_LOCK, 0);
+		na->nm_register(ifp, 0); /* off, clear IFCAP_NETMAP */
+		/* Wake up any sleeping threads. netmap_poll will
+		 * then return POLLERR
+		 */
+		for (i = 0; i < na->num_queues + 2; i++) {
+			selwakeuppri(&na->tx_rings[i].si, PI_NET);
+			selwakeuppri(&na->rx_rings[i].si, PI_NET);
+		}
+		/* release all buffers */
+		NMA_LOCK();
+		for (i = 0; i < na->num_queues + 1; i++) {
+			int j, lim;
+			struct netmap_ring *ring;
+
+			ND("tx queue %d", i);
+			ring = na->tx_rings[i].ring;
+			lim = na->tx_rings[i].nkr_num_slots;
+			for (j = 0; j < lim; j++)
+				netmap_free_buf(&nm_buf_pool,
+					ring->slot[j].buf_idx);
+
+			ND("rx queue %d", i);
+			ring = na->rx_rings[i].ring;
+			lim = na->rx_rings[i].nkr_num_slots;
+			for (j = 0; j < lim; j++)
+				netmap_free_buf(&nm_buf_pool,
+					ring->slot[j].buf_idx);
+		}
+		NMA_UNLOCK();
+		netmap_free(na->tx_rings[0].ring, "shadow rings");
+		wakeup(na);
+	}
+	netmap_free(nifp, "nifp");
+
+	na->nm_lock(ifp->if_softc, NETMAP_CORE_UNLOCK, 0); 
+
+	if_rele(ifp);
+
+	bzero(priv, sizeof(*priv));	/* XXX for safety */
+	free(priv, M_DEVBUF);
+}
+
+
+
+/*
+ * Create and return a new ``netmap_if`` object, and possibly also
+ * rings and packet buffors.
+ *
+ * Return NULL on failure.
+ */
+static void *
+netmap_if_new(const char *ifname, struct netmap_adapter *na)
+{
+	struct netmap_if *nifp;
+	struct netmap_ring *ring;
+	char *buff;
+	u_int i, len, ofs;
+	u_int n = na->num_queues + 1; /* shorthand, include stack queue */
+
+	/*
+	 * the descriptor is followed inline by an array of offsets
+	 * to the tx and rx rings in the shared memory region.
+	 */
+	len = sizeof(struct netmap_if) + 2 * n * sizeof(ssize_t);
+	nifp = netmap_malloc(len, "nifp");
+	if (nifp == NULL)
+		return (NULL);
+
+	/* initialize base fields */
+	*(int *)(uintptr_t)&nifp->ni_num_queues = na->num_queues;
+	strncpy(nifp->ni_name, ifname, IFNAMSIZ);
+
+	(na->refcount)++;	/* XXX atomic ? we are under lock */
+	if (na->refcount > 1)
+		goto final;
+
+	/*
+	 * If this is the first instance, allocate the shadow rings and
+	 * buffers for this card (one for each hw queue, one for the host).
+	 * The rings are contiguous, but have variable size.
+	 * The entire block is reachable at
+	 *	na->tx_rings[0].ring
+	 */
+
+	len = n * (2 * sizeof(struct netmap_ring) +
+		  (na->num_tx_desc + na->num_rx_desc) *
+		   sizeof(struct netmap_slot) );
+	buff = netmap_malloc(len, "shadow rings");
+	if (buff == NULL) {
+		D("failed to allocate %d bytes for %s shadow ring",
+			len, ifname);
+error:
+		(na->refcount)--;
+		netmap_free(nifp, "nifp, rings failed");
+		return (NULL);
+	}
+	/* do we have the bufers ? we are in need of num_tx_desc buffers for
+	 * each tx ring and num_tx_desc buffers for each rx ring. */
+	len = n * (na->num_tx_desc + na->num_rx_desc);
+	NMA_LOCK();
+	if (nm_buf_pool.free < len) {
+		NMA_UNLOCK();
+		netmap_free(buff, "not enough bufs");
+		goto error;
+	}
+	/*
+	 * in the kring, store the pointers to the shared rings
+	 * and initialize the rings. We are under NMA_LOCK().
+	 */
+	ofs = 0;
+	for (i = 0; i < n; i++) {
+		struct netmap_kring *kring;
+		int numdesc;
+
+		/* Transmit rings */
+		kring = &na->tx_rings[i];
+		numdesc = na->num_tx_desc;
+		bzero(kring, sizeof(*kring));
+		kring->na = na;
+
+		ring = kring->ring = (struct netmap_ring *)(buff + ofs);
+		*(ssize_t *)(uintptr_t)&ring->buf_ofs =
+			nm_buf_pool.base - (char *)ring;
+		ND("txring[%d] at %p ofs %d", i, ring, ring->buf_ofs);
+		*(int *)(int *)(uintptr_t)&ring->num_slots =
+			kring->nkr_num_slots = numdesc;
+
+		/*
+		 * IMPORTANT:
+		 * Always keep one slot empty, so we can detect new
+		 * transmissions comparing cur and nr_hwcur (they are
+		 * the same only if there are no new transmissions).
+		 */
+		ring->avail = kring->nr_hwavail = numdesc - 1;
+		ring->cur = kring->nr_hwcur = 0;
+		netmap_new_bufs(&nm_buf_pool, ring->slot, numdesc);
+
+		ofs += sizeof(struct netmap_ring) +
+			numdesc * sizeof(struct netmap_slot);
+
+		/* Receive rings */
+		kring = &na->rx_rings[i];
+		numdesc = na->num_rx_desc;
+		bzero(kring, sizeof(*kring));
+		kring->na = na;
+
+		ring = kring->ring = (struct netmap_ring *)(buff + ofs);
+		*(ssize_t *)(uintptr_t)&ring->buf_ofs =
+			nm_buf_pool.base - (char *)ring;
+		ND("rxring[%d] at %p offset %d", i, ring, ring->buf_ofs);
+		*(int *)(int *)(uintptr_t)&ring->num_slots =
+			kring->nkr_num_slots = numdesc;
+		ring->cur = kring->nr_hwcur = 0;
+		ring->avail = kring->nr_hwavail = 0; /* empty */
+		netmap_new_bufs(&nm_buf_pool, ring->slot, numdesc);
+		ofs += sizeof(struct netmap_ring) +
+			numdesc * sizeof(struct netmap_slot);
+	}
+	NMA_UNLOCK();
+	for (i = 0; i < n+1; i++) {
+		// XXX initialize the selrecord structs.
+	}
+final:
+	/*
+	 * fill the slots for the rx and tx queues. They contain the offset
+	 * between the ring and nifp, so the information is usable in
+	 * userspace to reach the ring from the nifp.
+	 */
+	for (i = 0; i < n; i++) {
+		char *base = (char *)nifp;
+		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] =
+			(char *)na->tx_rings[i].ring - base;
+		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+n] =
+			(char *)na->rx_rings[i].ring - base;
+	}
+	return (nifp);
+}
+
+
+/*
+ * mmap(2) support for the "netmap" device.
+ *
+ * Expose all the memory previously allocated by our custom memory
+ * allocator: this way the user has only to issue a single mmap(2), and
+ * can work on all the data structures flawlessly.
+ *
+ * Return 0 on success, -1 otherwise.
+ */
+static int
+#if __FreeBSD_version < 900000
+netmap_mmap(__unused struct cdev *dev, vm_offset_t offset, vm_paddr_t *paddr,
+	    int nprot)
+#else
+netmap_mmap(__unused struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr,
+	    int nprot, __unused vm_memattr_t *memattr)
+#endif
+{
+	if (nprot & PROT_EXEC)
+		return (-1);	// XXX -1 or EINVAL ?
+	ND("request for offset 0x%x", (uint32_t)offset);
+	*paddr = vtophys(netmap_mem_d->nm_buffer) + offset;
+
+	return (0);
+}
+
+
+/*
+ * handler for synchronization of the queues from/to the host
+ */
+static void
+netmap_sync_to_host(struct netmap_adapter *na)
+{
+	struct netmap_kring *kring = &na->tx_rings[na->num_queues];
+	struct netmap_ring *ring = kring->ring;
+	struct mbuf *head = NULL, *tail = NULL, *m;
+	u_int n, lim = kring->nkr_num_slots - 1;
+
+	na->nm_lock(na->ifp->if_softc, NETMAP_CORE_LOCK, 0);
+
+	/* Take packets from hwcur to cur and pass them up.
+	 * In case of no buffers we give up. At the end of the loop,
+	 * the queue is drained in all cases.
+	 */
+	for (n = kring->nr_hwcur; n != ring->cur;) {
+		struct netmap_slot *slot = &ring->slot[n];
+
+		n = (n == lim) ? 0 : n + 1;
+		if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE) {
+			D("bad pkt at %d len %d", n, slot->len);
+			continue;
+		}
+		m = m_devget(NMB(slot), slot->len, 0, na->ifp, NULL);
+
+		if (m == NULL)
+			break;
+		if (tail)
+			tail->m_nextpkt = m;
+		else
+			head = m;
+		tail = m;
+		m->m_nextpkt = NULL;
+	}
+	kring->nr_hwcur = ring->cur;
+	kring->nr_hwavail = ring->avail = lim;
+	na->nm_lock(na->ifp->if_softc, NETMAP_CORE_UNLOCK, 0);
+
+	/* send packets up, outside the lock */
+	while ((m = head) != NULL) {
+		head = head->m_nextpkt;
+		m->m_nextpkt = NULL;
+		m->m_pkthdr.rcvif = na->ifp;
+		if (netmap_verbose & NM_VERB_HOST)
+			D("sending up pkt %p size %d", m, m->m_pkthdr.len);
+		(na->ifp->if_input)(na->ifp, m);
+	}
+}
+
+/*
+ * This routine also does the selrecord if called from the poll handler
+ * (we know because td != NULL).
+ */
+static void
+netmap_sync_from_host(struct netmap_adapter *na, struct thread *td)
+{
+	struct netmap_kring *kring = &na->rx_rings[na->num_queues];
+	struct netmap_ring *ring = kring->ring;
+	int delta;
+
+	na->nm_lock(na->ifp->if_softc, NETMAP_CORE_LOCK, 0);
+
+	/* skip past packets processed by userspace,
+	 * and then sync cur/avail with hwcur/hwavail
+	 */
+	delta = ring->cur - kring->nr_hwcur;
+	if (delta < 0)
+		delta += kring->nkr_num_slots;
+	kring->nr_hwavail -= delta;
+	kring->nr_hwcur = ring->cur;
+	ring->avail = kring->nr_hwavail;
+	if (ring->avail == 0 && td)
+		selrecord(td, &kring->si);
+	if (ring->avail && (netmap_verbose & NM_VERB_HOST))
+		D("%d pkts from stack", ring->avail);
+	na->nm_lock(na->ifp->if_softc, NETMAP_CORE_UNLOCK, 0);
+}
+
+
+/*
+ * get a refcounted reference to an interface.
+ * Return ENXIO if the interface does not exist, EINVAL if netmap
+ * is not supported by the interface.
+ * If successful, hold a reference.
+ */
+static int
+get_ifp(const char *name, struct ifnet **ifp)
+{
+	*ifp = ifunit_ref(name);
+	if (*ifp == NULL)
+		return (ENXIO);
+	/* can do this if the capability exists and if_pspare[0]
+	 * points to the netmap descriptor.
+	 */
+	if ((*ifp)->if_capabilities & IFCAP_NETMAP && NA(*ifp))
+		return 0;	/* valid pointer, we hold the refcount */
+	if_rele(*ifp);
+	return EINVAL;	// not NETMAP capable
+}
+
+
+/*
+ * Error routine called when txsync/rxsync detects an error.
+ * Can't do much more than resetting cur = hwcur, avail = hwavail.
+ * Return 1 on reinit.
+ */
+int
+netmap_ring_reinit(struct netmap_kring *kring)
+{
+	struct netmap_ring *ring = kring->ring;
+	u_int i, lim = kring->nkr_num_slots - 1;
+	int errors = 0;
+
+	D("called for %s", kring->na->ifp->if_xname);
+	if (ring->cur > lim)
+		errors++;
+	for (i = 0; i <= lim; i++) {
+		u_int idx = ring->slot[i].buf_idx;
+		u_int len = ring->slot[i].len;
+		if (idx < 2 || idx >= netmap_total_buffers) {
+			if (!errors++)
+				D("bad buffer at slot %d idx %d len %d ", i, idx, len);
+			ring->slot[i].buf_idx = 0;
+			ring->slot[i].len = 0;
+		} else if (len > NETMAP_BUF_SIZE) {
+			ring->slot[i].len = 0;
+			if (!errors++)
+				D("bad len %d at slot %d idx %d",
+					len, i, idx);
+		}
+	}
+	if (errors) {
+		int pos = kring - kring->na->tx_rings;
+		int n = kring->na->num_queues + 2;
+
+		D("total %d errors", errors);
+		errors++;
+		D("%s %s[%d] reinit, cur %d -> %d avail %d -> %d",
+			kring->na->ifp->if_xname,
+			pos < n ?  "TX" : "RX", pos < n ? pos : pos - n, 
+			ring->cur, kring->nr_hwcur,
+			ring->avail, kring->nr_hwavail);
+		ring->cur = kring->nr_hwcur;
+		ring->avail = kring->nr_hwavail;
+		ring->flags |= NR_REINIT;
+		kring->na->flags |= NR_REINIT;
+	}
+	return (errors ? 1 : 0);
+}
+
+/*
+ * Clean the reinit flag for our rings.
+ * XXX at the moment, clear for all rings
+ */
+static void
+netmap_clean_reinit(struct netmap_adapter *na)
+{
+	//struct netmap_kring *kring;
+	u_int i;
+
+	na->flags &= ~NR_REINIT;
+	D("--- NR_REINIT reset on %s", na->ifp->if_xname);
+	for (i = 0; i < na->num_queues + 1; i++) {
+		na->tx_rings[i].ring->flags &= ~NR_REINIT;
+		na->rx_rings[i].ring->flags &= ~NR_REINIT;
+	}
+}
+
+/*
+ * Set the ring ID. For devices with a single queue, a request
+ * for all rings is the same as a single ring.
+ */
+static int
+netmap_set_ringid(struct netmap_priv_d *priv, u_int ringid)
+{
+	struct ifnet *ifp = priv->np_ifp;
+	struct netmap_adapter *na = NA(ifp);
+	void *adapter = na->ifp->if_softc;	/* shorthand */
+	u_int i = ringid & NETMAP_RING_MASK;
+	/* first time we don't lock */
+	int need_lock = (priv->np_qfirst != priv->np_qlast);
+
+	if ( (ringid & NETMAP_HW_RING) && i >= na->num_queues) {
+		D("invalid ring id %d", i);
+		return (EINVAL);
+	}
+	if (need_lock)
+		na->nm_lock(adapter, NETMAP_CORE_LOCK, 0);
+	priv->np_ringid = ringid;
+	if (ringid & NETMAP_SW_RING) {
+		priv->np_qfirst = na->num_queues;
+		priv->np_qlast = na->num_queues + 1;
+	} else if (ringid & NETMAP_HW_RING) {
+		priv->np_qfirst = i;
+		priv->np_qlast = i + 1;
+	} else {
+		priv->np_qfirst = 0;
+		priv->np_qlast = na->num_queues;
+	}
+	priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1;
+	if (need_lock)
+		na->nm_lock(adapter, NETMAP_CORE_UNLOCK, 0);
+	if (ringid & NETMAP_SW_RING)
+		D("ringid %s set to SW RING", ifp->if_xname);
+	else if (ringid & NETMAP_HW_RING)
+		D("ringid %s set to HW RING %d", ifp->if_xname,
+			priv->np_qfirst);
+	else
+		D("ringid %s set to all %d HW RINGS", ifp->if_xname,
+			priv->np_qlast);
+	return 0;
+}
+
+/*
+ * ioctl(2) support for the "netmap" device.
+ *
+ * Following a list of accepted commands:
+ * - NIOCGINFO
+ * - SIOCGIFADDR	just for convenience
+ * - NIOCREGIF
+ * - NIOCUNREGIF
+ * - NIOCTXSYNC
+ * - NIOCRXSYNC
+ *
+ * Return 0 on success, errno otherwise.
+ */
+static int
+netmap_ioctl(__unused struct cdev *dev, u_long cmd, caddr_t data,
+	__unused int fflag, __unused struct thread *td)
+{
+	struct netmap_priv_d *priv = NULL;
+	struct ifnet *ifp;
+	struct nmreq *nmr = (struct nmreq *) data;
+	struct netmap_adapter *na;
+	void *adapter;
+	int error;
+	u_int i;
+	struct netmap_if *nifp;
+
+	error = devfs_get_cdevpriv((void **)&priv);
+	if (error != ENOENT && error != 0)
+		return (error);
+
+	error = 0;	/* Could be ENOENT */
+	switch (cmd) {
+	case NIOCGINFO:		/* return capabilities etc */
+		/* memsize is always valid */
+		nmr->nr_memsize = netmap_mem_d->nm_totalsize;
+		nmr->nr_offset = 0;
+		nmr->nr_numrings = 0;
+		nmr->nr_numslots = 0;
+		if (nmr->nr_name[0] == '\0')	/* just get memory info */
+			break;
+		error = get_ifp(nmr->nr_name, &ifp); /* get a refcount */
+		if (error)
+			break;
+		na = NA(ifp); /* retrieve netmap_adapter */
+		nmr->nr_numrings = na->num_queues;
+		nmr->nr_numslots = na->num_tx_desc;
+		if_rele(ifp);	/* return the refcount */
+		break;
+
+	case NIOCREGIF:
+		if (priv != NULL)	/* thread already registered */
+			return netmap_set_ringid(priv, nmr->nr_ringid);
+		/* find the interface and a reference */
+		error = get_ifp(nmr->nr_name, &ifp); /* keep reference */
+		if (error)
+			break;
+		na = NA(ifp); /* retrieve netmap adapter */
+		adapter = na->ifp->if_softc;	/* shorthand */
+		/*
+		 * Allocate the private per-thread structure.
+		 * XXX perhaps we can use a blocking malloc ?
+		 */
+		priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF,
+			      M_NOWAIT | M_ZERO);
+		if (priv == NULL) {
+			error = ENOMEM;
+			if_rele(ifp);   /* return the refcount */
+			break;
+		}
+
+
+		for (i = 10; i > 0; i--) {
+			na->nm_lock(adapter, NETMAP_CORE_LOCK, 0);
+			if (!NETMAP_DELETING(na))
+				break;
+			na->nm_lock(adapter, NETMAP_CORE_UNLOCK, 0);
+			tsleep(na, 0, "NIOCREGIF", hz/10);
+		}
+		if (i == 0) {
+			D("too many NIOCREGIF attempts, give up");
+			error = EINVAL;
+			free(priv, M_DEVBUF);
+			if_rele(ifp);	/* return the refcount */
+			break;
+		}
+
+		priv->np_ifp = ifp;	/* store the reference */
+		error = netmap_set_ringid(priv, nmr->nr_ringid);
+		if (error)
+			goto error;
+		priv->np_nifp = nifp = netmap_if_new(nmr->nr_name, na);
+		if (nifp == NULL) { /* allocation failed */
+			error = ENOMEM;
+		} else if (ifp->if_capenable & IFCAP_NETMAP) {
+			/* was already set */
+		} else {
+			/* Otherwise set the card in netmap mode
+			 * and make it use the shared buffers.
+			 */
+			error = na->nm_register(ifp, 1); /* mode on */
+			if (error) {
+				/*
+				 * do something similar to netmap_dtor().
+				 */
+				netmap_free(na->tx_rings[0].ring, "rings, reg.failed");
+				free(na->tx_rings, M_DEVBUF);
+				na->tx_rings = na->rx_rings = NULL;
+				na->refcount--;
+				netmap_free(nifp, "nifp, rings failed");
+				nifp = NULL;
+			}
+		}
+
+		if (error) {	/* reg. failed, release priv and ref */
+error:
+			na->nm_lock(adapter, NETMAP_CORE_UNLOCK, 0);
+			free(priv, M_DEVBUF);
+			if_rele(ifp);	/* return the refcount */
+			break;
+		}
+
+		na->nm_lock(adapter, NETMAP_CORE_UNLOCK, 0);
+		error = devfs_set_cdevpriv(priv, netmap_dtor);
+
+		if (error != 0) {
+			/* could not assign the private storage for the
+			 * thread, call the destructor explicitly.
+			 */
+			netmap_dtor(priv);
+			break;
+		}
+
+		/* return the offset of the netmap_if object */
+		nmr->nr_numrings = na->num_queues;
+		nmr->nr_numslots = na->num_tx_desc;
+		nmr->nr_memsize = netmap_mem_d->nm_totalsize;
+		nmr->nr_offset =
+			((char *) nifp - (char *) netmap_mem_d->nm_buffer);
+		break;
+
+	case NIOCUNREGIF:
+		if (priv == NULL)
+			return (ENXIO);
+
+		/* the interface is unregistered inside the
+		   destructor of the private data. */
+		devfs_clear_cdevpriv();
+		break;
+
+	case NIOCTXSYNC:
+        case NIOCRXSYNC:
+		if (priv == NULL)
+			return (ENXIO);
+		ifp = priv->np_ifp;	/* we have a reference */
+		na = NA(ifp); /* retrieve netmap adapter */
+		adapter = ifp->if_softc;	/* shorthand */
+
+		if (na->flags & NR_REINIT)
+			netmap_clean_reinit(na);
+
+		if (priv->np_qfirst == na->num_queues) {
+			/* queues to/from host */
+			if (cmd == NIOCTXSYNC)
+				netmap_sync_to_host(na);
+			else
+				netmap_sync_from_host(na, NULL);
+			return error;
+		}
+
+		for (i = priv->np_qfirst; i < priv->np_qlast; i++) {
+		    if (cmd == NIOCTXSYNC) {
+			struct netmap_kring *kring = &na->tx_rings[i];
+			if (netmap_verbose & NM_VERB_TXSYNC)
+				D("sync tx ring %d cur %d hwcur %d",
+					i, kring->ring->cur,
+					kring->nr_hwcur);
+                        na->nm_txsync(adapter, i, 1 /* do lock */);
+			if (netmap_verbose & NM_VERB_TXSYNC)
+				D("after sync tx ring %d cur %d hwcur %d",
+					i, kring->ring->cur,
+					kring->nr_hwcur);
+		    } else {
+			na->nm_rxsync(adapter, i, 1 /* do lock */);
+			microtime(&na->rx_rings[i].ring->ts);
+		    }
+		}
+
+                break;
+
+	case BIOCIMMEDIATE:
+	case BIOCGHDRCMPLT:
+	case BIOCSHDRCMPLT:
+	case BIOCSSEESENT:
+		D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT");
+		break;
+
+	default:
+	    {
+		/*
+		 * allow device calls
+		 */
+		struct socket so;
+		bzero(&so, sizeof(so));
+		error = get_ifp(nmr->nr_name, &ifp); /* keep reference */
+		if (error)
+			break;
+		so.so_vnet = ifp->if_vnet;
+		// so->so_proto not null.
+		error = ifioctl(&so, cmd, data, td);
+		if_rele(ifp);
+	    }
+	}
+
+	return (error);
+}
+
+
+/*
+ * select(2) and poll(2) handlers for the "netmap" device.
+ *
+ * Can be called for one or more queues.
+ * Return true the event mask corresponding to ready events.
+ * If there are no ready events, do a selrecord on either individual
+ * selfd or on the global one.
+ * Device-dependent parts (locking and sync of tx/rx rings)
+ * are done through callbacks.
+ */
+static int
+netmap_poll(__unused struct cdev *dev, int events, struct thread *td)
+{
+	struct netmap_priv_d *priv = NULL;
+	struct netmap_adapter *na;
+	struct ifnet *ifp;
+	struct netmap_kring *kring;
+	u_int i, check_all, want_tx, want_rx, revents = 0;
+	void *adapter;
+
+	if (devfs_get_cdevpriv((void **)&priv) != 0 || priv == NULL)
+		return POLLERR;
+
+	ifp = priv->np_ifp;
+	// XXX check for deleting() ?
+	if ( (ifp->if_capenable & IFCAP_NETMAP) == 0)
+		return POLLERR;
+
+	if (netmap_verbose & 0x8000)
+		D("device %s events 0x%x", ifp->if_xname, events);
+	want_tx = events & (POLLOUT | POLLWRNORM);
+	want_rx = events & (POLLIN | POLLRDNORM);
+
+	adapter = ifp->if_softc;
+	na = NA(ifp); /* retrieve netmap adapter */
+
+	/* pending reinit, report up as a poll error. Pending
+	 * reads and writes are lost.
+	 */
+	if (na->flags & NR_REINIT) {
+		netmap_clean_reinit(na);
+		revents |= POLLERR;
+	}
+	/* how many queues we are scanning */
+	i = priv->np_qfirst;
+	if (i == na->num_queues) { /* from/to host */
+		if (priv->np_txpoll || want_tx) {
+			/* push any packets up, then we are always ready */
+			kring = &na->tx_rings[i];
+			netmap_sync_to_host(na);
+			revents |= want_tx;
+		}
+		if (want_rx) {
+			kring = &na->rx_rings[i];
+			if (kring->ring->avail == 0)
+				netmap_sync_from_host(na, td);
+			if (kring->ring->avail > 0) {
+				revents |= want_rx;
+			}
+		}
+		return (revents);
+	}
+
+	/*
+	 * check_all is set if the card has more than one queue and
+	 * the client is polling all of them. If true, we sleep on
+	 * the "global" selfd, otherwise we sleep on individual selfd
+	 * (we can only sleep on one of them per direction).
+	 * The interrupt routine in the driver should always wake on
+	 * the individual selfd, and also on the global one if the card
+	 * has more than one ring.
+	 *
+	 * If the card has only one lock, we just use that.
+	 * If the card has separate ring locks, we just use those
+	 * unless we are doing check_all, in which case the whole
+	 * loop is wrapped by the global lock.
+	 * We acquire locks only when necessary: if poll is called
+	 * when buffers are available, we can just return without locks.
+	 *
+	 * rxsync() is only called if we run out of buffers on a POLLIN.
+	 * txsync() is called if we run out of buffers on POLLOUT, or
+	 * there are pending packets to send. The latter can be disabled
+	 * passing NETMAP_NO_TX_POLL in the NIOCREG call.
+	 */
+	check_all = (i + 1 != priv->np_qlast);
+
+	/*
+	 * core_lock indicates what to do with the core lock.
+	 * The core lock is used when either the card has no individual
+	 * locks, or it has individual locks but we are cheking all
+	 * rings so we need the core lock to avoid missing wakeup events.
+	 *
+	 * It has three possible states:
+	 * NO_CL	we don't need to use the core lock, e.g.
+	 *		because we are protected by individual locks.
+	 * NEED_CL	we need the core lock. In this case, when we
+	 *		call the lock routine, move to LOCKED_CL
+	 *		to remember to release the lock once done.
+	 * LOCKED_CL	core lock is set, so we need to release it.
+	 */
+	enum {NO_CL, NEED_CL, LOCKED_CL };
+	int core_lock = (check_all || !na->separate_locks) ?
+			NEED_CL:NO_CL;
+	/*
+	 * We start with a lock free round which is good if we have
+	 * data available. If this fails, then lock and call the sync
+	 * routines.
+	 */
+	for (i = priv->np_qfirst; want_rx && i < priv->np_qlast; i++) {
+		kring = &na->rx_rings[i];
+		if (kring->ring->avail > 0) {
+			revents |= want_rx;
+			want_rx = 0;	/* also breaks the loop */
+		}
+	}
+	for (i = priv->np_qfirst; want_tx && i < priv->np_qlast; i++) {
+		kring = &na->tx_rings[i];
+		if (kring->ring->avail > 0) {
+			revents |= want_tx;
+			want_tx = 0;	/* also breaks the loop */
+		}
+	}
+
+	/*
+	 * If we to push packets out (priv->np_txpoll) or want_tx is
+	 * still set, we do need to run the txsync calls (on all rings,
+	 * to avoid that the tx rings stall).
+	 */
+	if (priv->np_txpoll || want_tx) {
+		for (i = priv->np_qfirst; i < priv->np_qlast; i++) {
+			kring = &na->tx_rings[i];
+			if (!want_tx && kring->ring->cur == kring->nr_hwcur)
+				continue;
+			if (core_lock == NEED_CL) {
+				na->nm_lock(adapter, NETMAP_CORE_LOCK, 0);
+				core_lock = LOCKED_CL;
+			}
+			if (na->separate_locks)
+				na->nm_lock(adapter, NETMAP_TX_LOCK, i);
+			if (netmap_verbose & NM_VERB_TXSYNC)
+				D("send %d on %s %d",
+					kring->ring->cur,
+					ifp->if_xname, i);
+			if (na->nm_txsync(adapter, i, 0 /* no lock */))
+				revents |= POLLERR;
+
+			if (want_tx) {
+				if (kring->ring->avail > 0) {
+					/* stop at the first ring. We don't risk
+					 * starvation.
+					 */
+					revents |= want_tx;
+					want_tx = 0;
+				} else if (!check_all)
+					selrecord(td, &kring->si);
+			}
+			if (na->separate_locks)
+				na->nm_lock(adapter, NETMAP_TX_UNLOCK, i);
+		}
+	}
+
+	/*
+	 * now if want_rx is still set we need to lock and rxsync.
+	 * Do it on all rings because otherwise we starve.
+	 */
+	if (want_rx) {
+		for (i = priv->np_qfirst; i < priv->np_qlast; i++) {
+			kring = &na->rx_rings[i];
+			if (core_lock == NEED_CL) {
+				na->nm_lock(adapter, NETMAP_CORE_LOCK, 0);
+				core_lock = LOCKED_CL;
+			}
+			if (na->separate_locks)
+				na->nm_lock(adapter, NETMAP_RX_LOCK, i);
+
+			if (na->nm_rxsync(adapter, i, 0 /* no lock */))
+				revents |= POLLERR;
+			if (no_timestamp == 0 ||
+					kring->ring->flags & NR_TIMESTAMP)
+				microtime(&kring->ring->ts);
+
+			if (kring->ring->avail > 0)
+				revents |= want_rx;
+			else if (!check_all)
+				selrecord(td, &kring->si);
+			if (na->separate_locks)
+				na->nm_lock(adapter, NETMAP_RX_UNLOCK, i);
+		}
+	}
+	if (check_all && revents == 0) {
+		i = na->num_queues + 1; /* the global queue */
+		if (want_tx)
+			selrecord(td, &na->tx_rings[i].si);
+		if (want_rx)
+			selrecord(td, &na->rx_rings[i].si);
+	}
+	if (core_lock == LOCKED_CL)
+		na->nm_lock(adapter, NETMAP_CORE_UNLOCK, 0);
+
+	return (revents);
+}
+
+/*------- driver support routines ------*/
+
+/*
+ * Initialize a ``netmap_adapter`` object created by driver on attach.
+ * We allocate a block of memory with room for a struct netmap_adapter
+ * plus two sets of N+2 struct netmap_kring (where N is the number
+ * of hardware rings):
+ * krings	0..N-1	are for the hardware queues.
+ * kring	N	is for the host stack queue
+ * kring	N+1	is only used for the selinfo for all queues.
+ * Return 0 on success, ENOMEM otherwise.
+ */
+int
+netmap_attach(struct netmap_adapter *na, int num_queues)
+{
+	int n = num_queues + 2;
+	int size = sizeof(*na) + 2 * n * sizeof(struct netmap_kring);
+	void *buf;
+	struct ifnet *ifp = na->ifp;
+
+	if (ifp == NULL) {
+		D("ifp not set, giving up");
+		return EINVAL;
+	}
+	na->refcount = 0;
+	na->num_queues = num_queues;
+
+	buf = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (buf) {
+		ifp->if_pspare[0] = buf;
+		na->tx_rings = (void *)((char *)buf + sizeof(*na));
+		na->rx_rings = na->tx_rings + n;
+		bcopy(na, buf, sizeof(*na));
+		ifp->if_capabilities |= IFCAP_NETMAP;
+	}
+	D("%s for %s", buf ? "ok" : "failed", ifp->if_xname);
+
+	return (buf ? 0 : ENOMEM);
+}
+
+
+/*
+ * Free the allocated memory linked to the given ``netmap_adapter``
+ * object.
+ */
+void
+netmap_detach(struct ifnet *ifp)
+{
+	u_int i;
+	struct netmap_adapter *na = NA(ifp);
+
+	if (!na)
+		return;
+
+	for (i = 0; i < na->num_queues + 2; i++) {
+		knlist_destroy(&na->tx_rings[i].si.si_note);
+		knlist_destroy(&na->rx_rings[i].si.si_note);
+	}
+	bzero(na, sizeof(*na));
+	ifp->if_pspare[0] = NULL;
+	free(na, M_DEVBUF);
+}
+
+
+/*
+ * intercept packets coming from the network stack and present
+ * them to netmap as incoming packets on a separate ring.
+ * We are not locked when called.
+ */
+int
+netmap_start(struct ifnet *ifp, struct mbuf *m)
+{
+	struct netmap_adapter *na = NA(ifp);
+	u_int i, len, n = na->num_queues;
+	int error = EBUSY;
+	struct netmap_kring *kring = &na->rx_rings[n];
+	struct netmap_slot *slot;
+
+	len = m->m_pkthdr.len;
+	if (netmap_verbose & NM_VERB_HOST)
+		D("%s packet %d len %d from the stack", ifp->if_xname,
+			kring->nr_hwcur + kring->nr_hwavail, len);
+	na->nm_lock(ifp->if_softc, NETMAP_CORE_LOCK, 0);
+	if (kring->nr_hwavail >= (int)kring->nkr_num_slots - 1) {
+		D("stack ring %s full\n", ifp->if_xname);
+		goto done;	/* no space */
+	}
+	if (len > na->buff_size) {
+		D("drop packet size %d > %d", len, na->buff_size);
+		goto done;	/* too long for us */
+	}
+
+	/* compute the insert position */
+	i = kring->nr_hwcur + kring->nr_hwavail;
+	if (i >= kring->nkr_num_slots)
+		i -= kring->nkr_num_slots;
+	slot = &kring->ring->slot[i];
+	m_copydata(m, 0, len, NMB(slot));
+	slot->len = len;
+	kring->nr_hwavail++;
+	if (netmap_verbose  & NM_VERB_HOST)
+		D("wake up host ring %s %d", na->ifp->if_xname, na->num_queues);
+	selwakeuppri(&kring->si, PI_NET);
+	error = 0;
+done:
+	na->nm_lock(ifp->if_softc, NETMAP_CORE_UNLOCK, 0);
+
+	/* release the mbuf in either cases of success or failure. As an
+	 * alternative, put the mbuf in a free list and free the list
+	 * only when really necessary.
+	 */
+	m_freem(m);
+
+	return (error);
+}
+
+
+/*
+ * netmap_reset() is called by the driver routines when reinitializing
+ * a ring. The driver is in charge of locking to protect the kring.
+ * If netmap mode is not set just return NULL.
+ * Otherwise set NR_REINIT (in the ring and in na) to signal
+ * that a ring has been reinitialized,
+ * set cur = hwcur = 0 and avail = hwavail = num_slots - 1 .
+ * IT IS IMPORTANT to leave one slot free even in the tx ring because
+ * we rely on cur=hwcur only for empty rings.
+ * These are good defaults but can be overridden later in the device
+ * specific code if, after a reinit, the ring does not start from 0
+ * (e.g. if_em.c does this).
+ *
+ * XXX we shouldn't be touching the ring, but there is a
+ * race anyways and this is our best option.
+ *
+ * XXX setting na->flags makes the syscall code faster, as there is
+ * only one place to check. On the other hand, we will need a better
+ * way to notify multiple threads that rings have been reset.
+ * One way is to increment na->rst_count at each ring reset.
+ * Each thread in its own priv structure will keep a matching counter,
+ * and on a reset will acknowledge and clean its own rings.
+ */
+struct netmap_slot *
+netmap_reset(struct netmap_adapter *na, enum txrx tx, int n,
+	u_int new_cur)
+{
+	struct netmap_kring *kring;
+	struct netmap_ring *ring;
+	struct netmap_slot *slot;
+	u_int i;
+
+	if (na == NULL)
+		return NULL;	/* no netmap support here */
+	if (!(na->ifp->if_capenable & IFCAP_NETMAP))
+		return NULL;	/* nothing to reinitialize */
+	kring = tx == NR_TX ?  na->tx_rings + n : na->rx_rings + n;
+	ring = kring->ring;
+    if (tx == NR_TX) {
+	/*
+	 * The last argument is the new value of next_to_clean.
+	 *
+	 * In the TX ring, we have P pending transmissions (from
+	 * next_to_clean to nr_hwcur) followed by nr_hwavail free slots.
+	 * Generally we can use all the slots in the ring so
+	 * P = ring_size - nr_hwavail hence (modulo ring_size):
+	 *	next_to_clean == nr_hwcur + nr_hwavail
+	 * 
+	 * If, upon a reset, nr_hwavail == ring_size and next_to_clean
+	 * does not change we have nothing to report. Otherwise some
+	 * pending packets may be lost, or newly injected packets will.
+	 */
+	/* if hwcur does not change, nothing to report.
+	 * otherwise remember the change so perhaps we can
+	 * shift the block at the next reinit
+	 */
+	if (new_cur == kring->nr_hwcur &&
+		    kring->nr_hwavail == kring->nkr_num_slots - 1) {
+		/* all ok */
+		D("+++ NR_REINIT ok on %s TX[%d]", na->ifp->if_xname, n);
+	} else {
+		D("+++ NR_REINIT set on %s TX[%d]", na->ifp->if_xname, n);
+	}
+		ring->flags |= NR_REINIT;
+		na->flags |= NR_REINIT;
+		ring->avail = kring->nr_hwavail = kring->nkr_num_slots - 1;
+		ring->cur = kring->nr_hwcur = new_cur;
+    } else {
+	/*
+	 * The last argument is the next free slot.
+	 * In the RX ring we have nr_hwavail full buffers starting
+	 * from nr_hwcur.
+	 * If nr_hwavail == 0 and nr_hwcur does not change we are ok
+	 * otherwise we might be in trouble as the buffers are
+	 * changing.
+	 */
+	if (new_cur == kring->nr_hwcur && kring->nr_hwavail == 0) {
+		/* all ok */
+		D("+++ NR_REINIT ok on %s RX[%d]", na->ifp->if_xname, n);
+	} else {
+		D("+++ NR_REINIT set on %s RX[%d]", na->ifp->if_xname, n);
+	}
+	ring->flags |= NR_REINIT;
+	na->flags |= NR_REINIT;
+	ring->avail = kring->nr_hwavail = 0; /* no data */
+	ring->cur = kring->nr_hwcur = new_cur;
+    }
+
+	slot = ring->slot;
+	/*
+	 * Check that buffer indexes are correct. If we find a
+	 * bogus value we are a bit in trouble because we cannot
+	 * recover easily. Best we can do is (probably) persistently
+	 * reset the ring.
+	 */
+	for (i = 0; i < kring->nkr_num_slots; i++) {
+		if (slot[i].buf_idx >= netmap_total_buffers) {
+			D("invalid buf_idx %d at slot %d", slot[i].buf_idx, i);
+			slot[i].buf_idx = 0; /* XXX reset */
+		}
+		/* XXX we don't really need to set the length */
+		slot[i].len = 0;
+	}
+	/* wakeup possible waiters, both on the ring and on the global
+	 * selfd. Perhaps a bit early now but the device specific
+	 * routine is locked so hopefully we won't have a race.
+	 */
+	selwakeuppri(&kring->si, PI_NET);
+	selwakeuppri(&kring[na->num_queues + 1 - n].si, PI_NET);
+	return kring->ring->slot;
+}
+
+static void
+ns_dmamap_cb(__unused void *arg, __unused bus_dma_segment_t * segs,
+	__unused int nseg, __unused int error)
+{
+}
+
+/* unload a bus_dmamap and create a new one. Used when the
+ * buffer in the slot is changed.
+ * XXX buflen is probably not needed, buffers have constant size.
+ */
+void
+netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map,
+	void *buf, bus_size_t buflen)
+{
+	bus_addr_t paddr;
+	bus_dmamap_unload(tag, map);
+	bus_dmamap_load(tag, map, buf, buflen, ns_dmamap_cb, &paddr,
+				BUS_DMA_NOWAIT);
+}
+
+void
+netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map,
+	void *buf, bus_size_t buflen)
+{
+	bus_addr_t paddr;
+	bus_dmamap_load(tag, map, buf, buflen, ns_dmamap_cb, &paddr,
+				BUS_DMA_NOWAIT);
+}
+
+/*------ netmap memory allocator -------*/
+/*
+ * Request for a chunk of memory.
+ *
+ * Memory objects are arranged into a list, hence we need to walk this
+ * list until we find an object with the needed amount of data free. 
+ * This sounds like a completely inefficient implementation, but given
+ * the fact that data allocation is done once, we can handle it
+ * flawlessly.
+ *
+ * Return NULL on failure.
+ */
+static void *
+netmap_malloc(size_t size, __unused const char *msg)
+{
+	struct netmap_mem_obj *mem_obj, *new_mem_obj;
+	void *ret = NULL;
+
+	NMA_LOCK();
+	TAILQ_FOREACH(mem_obj, &netmap_mem_d->nm_molist, nmo_next) {
+		if (mem_obj->nmo_used != 0 || mem_obj->nmo_size < size)
+			continue;
+
+		new_mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP,
+				     M_WAITOK | M_ZERO);
+		TAILQ_INSERT_BEFORE(mem_obj, new_mem_obj, nmo_next);
+
+		new_mem_obj->nmo_used = 1;
+		new_mem_obj->nmo_size = size;
+		new_mem_obj->nmo_data = mem_obj->nmo_data;
+		memset(new_mem_obj->nmo_data, 0, new_mem_obj->nmo_size);
+
+		mem_obj->nmo_size -= size;
+		mem_obj->nmo_data = (char *) mem_obj->nmo_data + size;
+		if (mem_obj->nmo_size == 0) {
+			TAILQ_REMOVE(&netmap_mem_d->nm_molist, mem_obj,
+				     nmo_next);
+			free(mem_obj, M_NETMAP);
+		}
+
+		ret = new_mem_obj->nmo_data;
+
+		break;
+	}
+	NMA_UNLOCK();
+	ND("%s: %d bytes at %p", msg, size, ret);
+
+	return (ret);
+}
+
+/*
+ * Return the memory to the allocator.
+ *
+ * While freeing a memory object, we try to merge adjacent chunks in
+ * order to reduce memory fragmentation.
+ */
+static void
+netmap_free(void *addr, const char *msg)
+{
+	size_t size;
+	struct netmap_mem_obj *cur, *prev, *next;
+
+	if (addr == NULL) {
+		D("NULL addr for %s", msg);
+		return;
+	}
+
+	NMA_LOCK();
+	TAILQ_FOREACH(cur, &netmap_mem_d->nm_molist, nmo_next) {
+		if (cur->nmo_data == addr && cur->nmo_used)
+			break;
+	}
+	if (cur == NULL) {
+		NMA_UNLOCK();
+		D("invalid addr %s %p", msg, addr);
+		return;
+	}
+
+	size = cur->nmo_size;
+	cur->nmo_used = 0;
+
+	/* merge current chunk of memory with the previous one,
+	   if present. */
+	prev = TAILQ_PREV(cur, netmap_mem_obj_h, nmo_next);
+	if (prev && prev->nmo_used == 0) {
+		TAILQ_REMOVE(&netmap_mem_d->nm_molist, cur, nmo_next);
+		prev->nmo_size += cur->nmo_size;
+		free(cur, M_NETMAP);
+		cur = prev;
+	}
+
+	/* merge with the next one */
+	next = TAILQ_NEXT(cur, nmo_next);
+	if (next && next->nmo_used == 0) {
+		TAILQ_REMOVE(&netmap_mem_d->nm_molist, next, nmo_next);
+		cur->nmo_size += next->nmo_size;
+		free(next, M_NETMAP);
+	}
+	NMA_UNLOCK();
+	ND("freed %s %d bytes at %p", msg, size, addr);
+}
+
+
+/*
+ * Initialize the memory allocator.
+ *
+ * Create the descriptor for the memory , allocate the pool of memory
+ * and initialize the list of memory objects with a single chunk
+ * containing the whole pre-allocated memory marked as free.
+ *
+ * Start with a large size, then halve as needed if we fail to
+ * allocate the block. While halving, always add one extra page
+ * because buffers 0 and 1 are used for special purposes.
+ * Return 0 on success, errno otherwise.
+ */
+static int
+netmap_memory_init(void)
+{
+	struct netmap_mem_obj *mem_obj;
+	void *buf = NULL;
+	int i, n, sz = NETMAP_MEMORY_SIZE;
+	int extra_sz = 0; // space for rings and two spare buffers
+
+	for (; !buf && sz >= 1<<20; sz >>=1) {
+		extra_sz = sz/200;
+		extra_sz = (extra_sz + 2*PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
+	        buf = contigmalloc(sz + extra_sz,
+			     M_NETMAP,
+			     M_WAITOK | M_ZERO,
+			     0, /* low address */
+			     -1UL, /* high address */
+			     PAGE_SIZE, /* alignment */
+			     0 /* boundary */
+			    );
+	} 
+	if (buf == NULL)
+		return (ENOMEM);
+	sz += extra_sz;
+	netmap_mem_d = malloc(sizeof(struct netmap_mem_d), M_NETMAP,
+			      M_WAITOK | M_ZERO);
+	mtx_init(&netmap_mem_d->nm_mtx, "netmap memory allocator lock", NULL,
+		 MTX_DEF);
+	TAILQ_INIT(&netmap_mem_d->nm_molist);
+	netmap_mem_d->nm_buffer = buf;
+	netmap_mem_d->nm_totalsize = sz;
+
+	/*
+	 * A buffer takes 2k, a slot takes 8 bytes + ring overhead,
+	 * so the ratio is 200:1. In other words, we can use 1/200 of
+	 * the memory for the rings, and the rest for the buffers,
+	 * and be sure we never run out.
+	 */
+	netmap_mem_d->nm_size = sz/200;
+	netmap_mem_d->nm_buf_start =
+		(netmap_mem_d->nm_size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
+	netmap_mem_d->nm_buf_len = sz - netmap_mem_d->nm_buf_start;
+
+	nm_buf_pool.base = netmap_mem_d->nm_buffer;
+	nm_buf_pool.base += netmap_mem_d->nm_buf_start;
+	netmap_buffer_base = nm_buf_pool.base;
+	D("netmap_buffer_base %p (offset %d)",
+		netmap_buffer_base, netmap_mem_d->nm_buf_start);
+	/* number of buffers, they all start as free */
+
+	netmap_total_buffers = nm_buf_pool.total_buffers =
+		netmap_mem_d->nm_buf_len / NETMAP_BUF_SIZE;
+	nm_buf_pool.bufsize = NETMAP_BUF_SIZE;
+
+	D("Have %d MB, use %dKB for rings, %d buffers at %p",
+		(sz >> 20), (netmap_mem_d->nm_size >> 10),
+		nm_buf_pool.total_buffers, nm_buf_pool.base);
+
+	/* allocate and initialize the bitmap. Entry 0 is considered
+	 * always busy (used as default when there are no buffers left).
+	 */
+	n = (nm_buf_pool.total_buffers + 31) / 32;
+	nm_buf_pool.bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP,
+			 M_WAITOK | M_ZERO);
+	nm_buf_pool.bitmap[0] = ~3; /* slot 0 and 1 always busy */
+	for (i = 1; i < n; i++)
+		nm_buf_pool.bitmap[i] = ~0;
+	nm_buf_pool.free = nm_buf_pool.total_buffers - 2;
+	
+	mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP,
+			 M_WAITOK | M_ZERO);
+	TAILQ_INSERT_HEAD(&netmap_mem_d->nm_molist, mem_obj, nmo_next);
+	mem_obj->nmo_used = 0;
+	mem_obj->nmo_size = netmap_mem_d->nm_size;
+	mem_obj->nmo_data = netmap_mem_d->nm_buffer;
+
+	return (0);
+}
+
+
+/*
+ * Finalize the memory allocator.
+ *
+ * Free all the memory objects contained inside the list, and deallocate
+ * the pool of memory; finally free the memory allocator descriptor.
+ */
+static void
+netmap_memory_fini(void)
+{
+	struct netmap_mem_obj *mem_obj;
+
+	while (!TAILQ_EMPTY(&netmap_mem_d->nm_molist)) {
+		mem_obj = TAILQ_FIRST(&netmap_mem_d->nm_molist);
+		TAILQ_REMOVE(&netmap_mem_d->nm_molist, mem_obj, nmo_next);
+		if (mem_obj->nmo_used == 1) {
+			printf("netmap: leaked %d bytes at %p\n",
+			       mem_obj->nmo_size,
+			       mem_obj->nmo_data);
+		}
+		free(mem_obj, M_NETMAP);
+	}
+	contigfree(netmap_mem_d->nm_buffer, netmap_mem_d->nm_totalsize, M_NETMAP);
+	// XXX mutex_destroy(nm_mtx);
+	free(netmap_mem_d, M_NETMAP);
+}
+
+
+/*
+ * Module loader.
+ *
+ * Create the /dev/netmap device and initialize all global
+ * variables.
+ *
+ * Return 0 on success, errno on failure.
+ */
+static int
+netmap_init(void)
+{
+	int error;
+
+
+	error = netmap_memory_init();
+	if (error != 0) {
+		printf("netmap: unable to initialize the memory allocator.");
+		return (error);
+	}
+	printf("netmap: loaded module with %d Mbytes\n",
+		netmap_mem_d->nm_totalsize >> 20);
+
+	netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660,
+			      "netmap");
+
+	return (0);
+}
+
+
+/*
+ * Module unloader.
+ *
+ * Free all the memory, and destroy the ``/dev/netmap`` device.
+ */
+static void
+netmap_fini(void)
+{
+	destroy_dev(netmap_dev);
+
+	netmap_memory_fini();
+
+	printf("netmap: unloaded module.\n");
+}
+
+
+/*
+ * Kernel entry point.
+ *
+ * Initialize/finalize the module and return.
+ *
+ * Return 0 on success, errno on failure.
+ */
+static int
+netmap_loader(__unused struct module *module, int event, __unused void *arg)
+{
+	int error = 0;
+
+	switch (event) {
+	case MOD_LOAD:
+		error = netmap_init();
+		break;
+
+	case MOD_UNLOAD:
+		netmap_fini();
+		break;
+
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+
+	return (error);
+}
+
+
+DEV_MODULE(netmap, netmap_loader, NULL);
diff --git a/sys/dev/netmap/netmap_kern.h b/sys/dev/netmap/netmap_kern.h
new file mode 100644
index 0000000..5434609
--- /dev/null
+++ b/sys/dev/netmap/netmap_kern.h
@@ -0,0 +1,221 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: netmap_kern.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * The header contains the definitions of constants and function
+ * prototypes used only in kernelspace.
+ */
+
+#ifndef _NET_NETMAP_KERN_H_
+#define _NET_NETMAP_KERN_H_
+
+#ifdef MALLOC_DECLARE
+MALLOC_DECLARE(M_NETMAP);
+#endif
+
+#define ND(format, ...)
+#define D(format, ...)					\
+	do {						\
+		struct timeval __xxts;			\
+		microtime(&__xxts);				\
+		printf("%03d.%06d %s [%d] " format "\n",\
+		(int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec,		\
+		__FUNCTION__, __LINE__, ##__VA_ARGS__);	\
+	} while (0)
+ 
+struct netmap_adapter;
+
+/*
+ * private, kernel view of a ring.
+ *
+ * XXX 20110627-todo
+ * The index in the NIC and netmap ring is offset by nkr_hwofs slots.
+ * This is so that, on a reset, buffers owned by userspace are not
+ * modified by the kernel. In particular:
+ * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides
+ * 	the next empty buffer as known by the hardware (next_to_check or so).
+ * TX rings: hwcur + hwofs coincides with next_to_send
+ */
+struct netmap_kring {
+	struct netmap_ring *ring;
+	u_int nr_hwcur;
+	int nr_hwavail;
+	u_int nr_kflags;
+	u_int nkr_num_slots;
+
+	u_int	nkr_hwofs;	/* offset between NIC and netmap ring */
+	struct netmap_adapter *na;	 // debugging
+	struct selinfo si; /* poll/select wait queue */
+};
+
+/*
+ * This struct is part of and extends the 'struct adapter' (or
+ * equivalent) device descriptor. It contains all fields needed to
+ * support netmap operation.
+ */
+struct netmap_adapter {
+	int refcount; /* number of user-space descriptors using this
+			 interface, which is equal to the number of
+			 struct netmap_if objs in the mapped region. */
+
+	int separate_locks; /* set if the interface suports different
+			       locks for rx, tx and core. */
+
+	u_int num_queues; /* number of tx/rx queue pairs: this is
+			   a duplicate field needed to simplify the
+			   signature of ``netmap_detach``. */
+
+	u_int num_tx_desc; /* number of descriptor in each queue */
+	u_int num_rx_desc;
+	u_int buff_size;
+
+	u_int	flags;	/* NR_REINIT */
+	/* tx_rings and rx_rings are private but allocated
+	 * as a contiguous chunk of memory. Each array has
+	 * N+1 entries, for the adapter queues and for the host queue.
+	 */
+	struct netmap_kring *tx_rings; /* array of TX rings. */
+	struct netmap_kring *rx_rings; /* array of RX rings. */
+
+	/* copy of if_qflush and if_transmit pointers, to intercept
+	 * packets from the network stack when netmap is active.
+	 * XXX probably if_qflush is not necessary.
+	 */
+	void    (*if_qflush)(struct ifnet *);
+	int     (*if_transmit)(struct ifnet *, struct mbuf *);
+
+	/* references to the ifnet and device routines, used by
+	 * the generic netmap functions.
+	 */
+	struct ifnet *ifp; /* adapter is ifp->if_softc */
+
+	int (*nm_register)(struct ifnet *, int onoff);
+	void (*nm_lock)(void *, int what, u_int ringid);
+	int (*nm_txsync)(void *, u_int ring, int lock);
+	int (*nm_rxsync)(void *, u_int ring, int lock);
+};
+
+/*
+ * The combination of "enable" (ifp->if_capabilities &IFCAP_NETMAP)
+ * and refcount gives the status of the interface, namely:
+ *
+ *	enable	refcount	Status
+ *
+ *	FALSE	0		normal operation
+ *	FALSE	!= 0		-- (impossible)
+ *	TRUE	1		netmap mode
+ *	TRUE	0		being deleted.
+ */
+
+#define NETMAP_DELETING(_na)  (  ((_na)->refcount == 0) &&	\
+	( (_na)->ifp->if_capenable & IFCAP_NETMAP) )
+
+/*
+ * parameters for (*nm_lock)(adapter, what, index)
+ */
+enum {
+	NETMAP_NO_LOCK = 0,
+	NETMAP_CORE_LOCK, NETMAP_CORE_UNLOCK,
+	NETMAP_TX_LOCK, NETMAP_TX_UNLOCK,
+	NETMAP_RX_LOCK, NETMAP_RX_UNLOCK,
+};
+
+/*
+ * The following are support routines used by individual drivers to
+ * support netmap operation.
+ *
+ * netmap_attach() initializes a struct netmap_adapter, allocating the
+ * 	struct netmap_ring's and the struct selinfo.
+ *
+ * netmap_detach() frees the memory allocated by netmap_attach().
+ *
+ * netmap_start() replaces the if_transmit routine of the interface,
+ *	and is used to intercept packets coming from the stack.
+ *
+ * netmap_load_map/netmap_reload_map are helper routines to set/reset
+ *	the dmamap for a packet buffer
+ *
+ * netmap_reset() is a helper routine to be called in the driver
+ *	when reinitializing a ring.
+ */
+int netmap_attach(struct netmap_adapter *, int);
+void netmap_detach(struct ifnet *);
+int netmap_start(struct ifnet *, struct mbuf *);
+enum txrx { NR_RX = 0, NR_TX = 1 };
+struct netmap_slot *netmap_reset(struct netmap_adapter *na,
+	enum txrx tx, int n, u_int new_cur);
+void netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map,
+        void *buf, bus_size_t buflen);
+void netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map,
+        void *buf, bus_size_t buflen);
+int netmap_ring_reinit(struct netmap_kring *);
+
+/*
+ * XXX eventually, get rid of netmap_total_buffers and netmap_buffer_base
+ * in favour of the structure
+ */
+// struct netmap_buf_pool;
+// extern struct netmap_buf_pool nm_buf_pool;
+extern u_int netmap_total_buffers;
+extern char *netmap_buffer_base;
+extern int netmap_verbose;	// XXX debugging
+enum {                                  /* verbose flags */
+	NM_VERB_ON = 1,                 /* generic verbose */
+	NM_VERB_HOST = 0x2,             /* verbose host stack */
+	NM_VERB_RXSYNC = 0x10,          /* verbose on rxsync/txsync */
+	NM_VERB_TXSYNC = 0x20,
+	NM_VERB_RXINTR = 0x100,         /* verbose on rx/tx intr (driver) */
+	NM_VERB_TXINTR = 0x200,
+	NM_VERB_NIC_RXSYNC = 0x1000,    /* verbose on rx/tx intr (driver) */
+	NM_VERB_NIC_TXSYNC = 0x2000,
+};
+
+/*
+ * return a pointer to the struct netmap adapter from the ifp
+ */
+#define	NA(_ifp)	((struct netmap_adapter *)(_ifp)->if_pspare[0])
+
+
+/*
+ * return the address of a buffer.
+ * XXX this is a special version with hardwired 2k bufs
+ * On error return netmap_buffer_base which is detected as a bad pointer.
+ */
+static inline char *
+NMB(struct netmap_slot *slot)
+{
+	uint32_t i = slot->buf_idx;
+	return (i >= netmap_total_buffers) ? netmap_buffer_base :
+#if NETMAP_BUF_SIZE == 2048
+		netmap_buffer_base + (i << 11);
+#else
+		netmap_buffer_base + (i *NETMAP_BUF_SIZE);
+#endif
+}
+
+#endif /* _NET_NETMAP_KERN_H_ */
diff --git a/sys/dev/nge/if_nge.c b/sys/dev/nge/if_nge.c
index db20ad2..3d7ad63 100644
--- a/sys/dev/nge/if_nge.c
+++ b/sys/dev/nge/if_nge.c
@@ -117,6 +117,7 @@ __FBSDID("$FreeBSD$");
 #include <net/if_vlan_var.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -138,7 +139,7 @@ MODULE_DEPEND(nge, miibus, 1, 1, 1);
 /*
  * Various supported device vendors/types and their names.
  */
-static struct nge_type nge_devs[] = {
+static const struct nge_type const nge_devs[] = {
 	{ NGE_VENDORID, NGE_DEVICEID,
 	    "National Semiconductor Gigabit Ethernet" },
 	{ 0, 0, NULL }
@@ -180,11 +181,6 @@ static void nge_eeprom_putbyte(struct nge_softc *, int);
 static void nge_eeprom_getword(struct nge_softc *, int, uint16_t *);
 static void nge_read_eeprom(struct nge_softc *, caddr_t, int, int);
 
-static void nge_mii_sync(struct nge_softc *);
-static void nge_mii_send(struct nge_softc *, uint32_t, int);
-static int nge_mii_readreg(struct nge_softc *, struct nge_mii_frame *);
-static int nge_mii_writereg(struct nge_softc *, struct nge_mii_frame *);
-
 static int nge_miibus_readreg(device_t, int, int);
 static int nge_miibus_writereg(device_t, int, int, int);
 static void nge_miibus_statchg(device_t);
@@ -200,6 +196,24 @@ static void nge_sysctl_node(struct nge_softc *);
 static int sysctl_int_range(SYSCTL_HANDLER_ARGS, int, int);
 static int sysctl_hw_nge_int_holdoff(SYSCTL_HANDLER_ARGS);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t nge_mii_bitbang_read(device_t);
+static void nge_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops nge_mii_bitbang_ops = {
+	nge_mii_bitbang_read,
+	nge_mii_bitbang_write,
+	{
+		NGE_MEAR_MII_DATA,	/* MII_BIT_MDO */
+		NGE_MEAR_MII_DATA,	/* MII_BIT_MDI */
+		NGE_MEAR_MII_CLK,	/* MII_BIT_MDC */
+		NGE_MEAR_MII_DIR,	/* MII_BIT_DIR_HOST_PHY */
+		0,			/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static device_method_t nge_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		nge_probe),
@@ -366,180 +380,42 @@ nge_read_eeprom(struct nge_softc *sc, caddr_t dest, int off, int cnt)
 }
 
 /*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
- */
-static void
-nge_mii_sync(struct nge_softc *sc)
-{
-	int i;
-
-	SIO_SET(NGE_MEAR_MII_DIR|NGE_MEAR_MII_DATA);
-
-	for (i = 0; i < 32; i++) {
-		SIO_SET(NGE_MEAR_MII_CLK);
-		DELAY(1);
-		SIO_CLR(NGE_MEAR_MII_CLK);
-		DELAY(1);
-	}
-}
-
-/*
- * Clock a series of bits through the MII.
- */
-static void
-nge_mii_send(struct nge_softc *sc, uint32_t bits, int cnt)
-{
-	int i;
-
-	SIO_CLR(NGE_MEAR_MII_CLK);
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		if (bits & i) {
-			SIO_SET(NGE_MEAR_MII_DATA);
-		} else {
-			SIO_CLR(NGE_MEAR_MII_DATA);
-		}
-		DELAY(1);
-		SIO_CLR(NGE_MEAR_MII_CLK);
-		DELAY(1);
-		SIO_SET(NGE_MEAR_MII_CLK);
-	}
-}
-
-/*
- * Read an PHY register through the MII.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static int
-nge_mii_readreg(struct nge_softc *sc, struct nge_mii_frame *frame)
+static uint32_t
+nge_mii_bitbang_read(device_t dev)
 {
-	int i, ack;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = NGE_MII_STARTDELIM;
-	frame->mii_opcode = NGE_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	CSR_WRITE_4(sc, NGE_MEAR, 0);
-
-	/*
- 	 * Turn on data xmit.
-	 */
-	SIO_SET(NGE_MEAR_MII_DIR);
-
-	nge_mii_sync(sc);
-
-	/*
-	 * Send command/address info.
-	 */
-	nge_mii_send(sc, frame->mii_stdelim, 2);
-	nge_mii_send(sc, frame->mii_opcode, 2);
-	nge_mii_send(sc, frame->mii_phyaddr, 5);
-	nge_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Idle bit */
-	SIO_CLR((NGE_MEAR_MII_CLK|NGE_MEAR_MII_DATA));
-	DELAY(1);
-	SIO_SET(NGE_MEAR_MII_CLK);
-	DELAY(1);
-
-	/* Turn off xmit. */
-	SIO_CLR(NGE_MEAR_MII_DIR);
-	/* Check for ack */
-	SIO_CLR(NGE_MEAR_MII_CLK);
-	DELAY(1);
-	ack = CSR_READ_4(sc, NGE_MEAR) & NGE_MEAR_MII_DATA;
-	SIO_SET(NGE_MEAR_MII_CLK);
-	DELAY(1);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for (i = 0; i < 16; i++) {
-			SIO_CLR(NGE_MEAR_MII_CLK);
-			DELAY(1);
-			SIO_SET(NGE_MEAR_MII_CLK);
-			DELAY(1);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		SIO_CLR(NGE_MEAR_MII_CLK);
-		DELAY(1);
-		if (!ack) {
-			if (CSR_READ_4(sc, NGE_MEAR) & NGE_MEAR_MII_DATA)
-				frame->mii_data |= i;
-			DELAY(1);
-		}
-		SIO_SET(NGE_MEAR_MII_CLK);
-		DELAY(1);
-	}
+	struct nge_softc *sc;
+	uint32_t val;
 
-fail:
+	sc = device_get_softc(dev);
 
-	SIO_CLR(NGE_MEAR_MII_CLK);
-	DELAY(1);
-	SIO_SET(NGE_MEAR_MII_CLK);
-	DELAY(1);
+	val = CSR_READ_4(sc, NGE_MEAR);
+	CSR_BARRIER_4(sc, NGE_MEAR,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	if (ack)
-		return (1);
-	return (0);
+	return (val);
 }
 
 /*
- * Write to a PHY register through the MII.
+ * Write the MII serial port for the MII bit-bang module.
  */
-static int
-nge_mii_writereg(struct nge_softc *sc, struct nge_mii_frame *frame)
+static void
+nge_mii_bitbang_write(device_t dev, uint32_t val)
 {
+	struct nge_softc *sc;
 
-	/*
-	 * Set up frame for TX.
-	 */
-
-	frame->mii_stdelim = NGE_MII_STARTDELIM;
-	frame->mii_opcode = NGE_MII_WRITEOP;
-	frame->mii_turnaround = NGE_MII_TURNAROUND;
-
-	/*
- 	 * Turn on data output.
-	 */
-	SIO_SET(NGE_MEAR_MII_DIR);
-
-	nge_mii_sync(sc);
-
-	nge_mii_send(sc, frame->mii_stdelim, 2);
-	nge_mii_send(sc, frame->mii_opcode, 2);
-	nge_mii_send(sc, frame->mii_phyaddr, 5);
-	nge_mii_send(sc, frame->mii_regaddr, 5);
-	nge_mii_send(sc, frame->mii_turnaround, 2);
-	nge_mii_send(sc, frame->mii_data, 16);
-
-	/* Idle bit. */
-	SIO_SET(NGE_MEAR_MII_CLK);
-	DELAY(1);
-	SIO_CLR(NGE_MEAR_MII_CLK);
-	DELAY(1);
-
-	/*
-	 * Turn off xmit.
-	 */
-	SIO_CLR(NGE_MEAR_MII_DIR);
+	sc = device_get_softc(dev);
 
-	return (0);
+	CSR_WRITE_4(sc, NGE_MEAR, val);
+	CSR_BARRIER_4(sc, NGE_MEAR,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
 nge_miibus_readreg(device_t dev, int phy, int reg)
 {
 	struct nge_softc *sc;
-	struct nge_mii_frame frame;
 	int rv;
 
 	sc = device_get_softc(dev);
@@ -583,20 +459,13 @@ nge_miibus_readreg(device_t dev, int phy, int reg)
 		return (CSR_READ_4(sc, reg));
 	}
 
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	nge_mii_readreg(sc, &frame);
-
-	return (frame.mii_data);
+	return (mii_bitbang_readreg(dev, &nge_mii_bitbang_ops, phy, reg));
 }
 
 static int
 nge_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	struct nge_softc *sc;
-	struct nge_mii_frame frame;
 
 	sc = device_get_softc(dev);
 	if ((sc->nge_flags & NGE_FLAG_TBI) != 0) {
@@ -633,12 +502,7 @@ nge_miibus_writereg(device_t dev, int phy, int reg, int data)
 		return (0);
 	}
 
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
-	nge_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &nge_mii_bitbang_ops, phy, reg, data);
 
 	return (0);
 }
@@ -813,7 +677,7 @@ nge_rxfilter(struct nge_softc *sc)
 	rxfilt = CSR_READ_4(sc, NGE_RXFILT_CTL);
 	rxfilt &= ~NGE_RXFILTCTL_ENABLE;
 	CSR_WRITE_4(sc, NGE_RXFILT_CTL, rxfilt);
-	CSR_BARRIER_WRITE_4(sc, NGE_RXFILT_CTL);
+	CSR_BARRIER_4(sc, NGE_RXFILT_CTL, BUS_SPACE_BARRIER_WRITE);
 
 	rxfilt &= ~(NGE_RXFILTCTL_ALLMULTI | NGE_RXFILTCTL_ALLPHYS);
 	rxfilt &= ~NGE_RXFILTCTL_BROAD;
@@ -882,7 +746,7 @@ done:
 	/* Turn the receive filter on. */
 	rxfilt |= NGE_RXFILTCTL_ENABLE;
 	CSR_WRITE_4(sc, NGE_RXFILT_CTL, rxfilt);
-	CSR_BARRIER_WRITE_4(sc, NGE_RXFILT_CTL);
+	CSR_BARRIER_4(sc, NGE_RXFILT_CTL, BUS_SPACE_BARRIER_WRITE);
 }
 
 static void
@@ -932,7 +796,7 @@ nge_reset(struct nge_softc *sc)
 static int
 nge_probe(device_t dev)
 {
-	struct nge_type *t;
+	const struct nge_type *t;
 
 	t = nge_devs;
 
@@ -2216,7 +2080,7 @@ nge_init_locked(struct nge_softc *sc)
 
 	/* Disable Rx filter prior to programming Rx filter. */
 	CSR_WRITE_4(sc, NGE_RXFILT_CTL, 0);
-	CSR_BARRIER_WRITE_4(sc, NGE_RXFILT_CTL);
+	CSR_BARRIER_4(sc, NGE_RXFILT_CTL, BUS_SPACE_BARRIER_WRITE);
 
 	mii = device_get_softc(sc->nge_miibus);
 
@@ -2704,12 +2568,12 @@ nge_wol(struct nge_softc *sc)
 		 * (i.e. Silent Rx mode.)
 		 */
 		CSR_WRITE_4(sc, NGE_RX_LISTPTR_HI, 0);
-		CSR_BARRIER_WRITE_4(sc, NGE_RX_LISTPTR_HI);
+		CSR_BARRIER_4(sc, NGE_RX_LISTPTR_HI, BUS_SPACE_BARRIER_WRITE);
 		CSR_WRITE_4(sc, NGE_RX_LISTPTR_LO, 0);
-		CSR_BARRIER_WRITE_4(sc, NGE_RX_LISTPTR_LO);
+		CSR_BARRIER_4(sc, NGE_RX_LISTPTR_LO, BUS_SPACE_BARRIER_WRITE);
 		/* Enable Rx again. */
 		NGE_SETBIT(sc, NGE_CSR, NGE_CSR_RX_ENABLE);
-		CSR_BARRIER_WRITE_4(sc, NGE_CSR);
+		CSR_BARRIER_4(sc, NGE_CSR, BUS_SPACE_BARRIER_WRITE);
 
 		/* Configure WOL events. */
 		reg = 0;
diff --git a/sys/dev/nge/if_ngereg.h b/sys/dev/nge/if_ngereg.h
index 4ad1bf9..9df0d92 100644
--- a/sys/dev/nge/if_ngereg.h
+++ b/sys/dev/nge/if_ngereg.h
@@ -611,26 +611,9 @@ struct nge_ring_data {
 struct nge_type {
 	uint16_t		nge_vid;
 	uint16_t		nge_did;
-	char			*nge_name;
+	const char		*nge_name;
 };
 
-struct nge_mii_frame {
-	uint8_t			mii_stdelim;
-	uint8_t			mii_opcode;
-	uint8_t			mii_phyaddr;
-	uint8_t			mii_regaddr;
-	uint8_t			mii_turnaround;
-	uint16_t		mii_data;
-};
-
-/*
- * MII constants
- */
-#define NGE_MII_STARTDELIM	0x01
-#define NGE_MII_READOP		0x02
-#define NGE_MII_WRITEOP		0x01
-#define NGE_MII_TURNAROUND	0x02
-
 #define NGE_JUMBO_FRAMELEN	9022
 #define NGE_JUMBO_MTU		\
 	(NGE_JUMBO_FRAMELEN - sizeof(struct ether_vlan_header) - ETHER_CRC_LEN)
@@ -691,8 +674,9 @@ struct nge_softc {
  */
 #define CSR_WRITE_4(sc, reg, val)	\
 	bus_write_4((sc)->nge_res, reg, val)
-#define CSR_BARRIER_WRITE_4(sc, reg)	\
-	bus_barrier((sc)->nge_res, reg, 4, BUS_SPACE_BARRIER_WRITE)
+
+#define CSR_BARRIER_4(sc, reg, flags)	\
+	bus_barrier((sc)->nge_res, reg, 4, flags)
 
 #define CSR_READ_4(sc, reg)		\
 	bus_read_4((sc)->nge_res, reg)
diff --git a/sys/dev/nmdm/nmdm.c b/sys/dev/nmdm/nmdm.c
index 2476eb2..ceee762 100644
--- a/sys/dev/nmdm/nmdm.c
+++ b/sys/dev/nmdm/nmdm.c
@@ -52,7 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/taskqueue.h>
 
-MALLOC_DEFINE(M_NMDM, "nullmodem", "nullmodem data structures");
+static MALLOC_DEFINE(M_NMDM, "nullmodem", "nullmodem data structures");
 
 static tsw_inwakeup_t	nmdm_outwakeup;
 static tsw_outwakeup_t	nmdm_inwakeup;
diff --git a/sys/dev/ofw/openfirm.c b/sys/dev/ofw/openfirm.c
index a8cb8f7..9ff72df 100644
--- a/sys/dev/ofw/openfirm.c
+++ b/sys/dev/ofw/openfirm.c
@@ -127,7 +127,7 @@ OF_init(void *cookie)
 	 * then statically initialize the OFW object.
 	 */
 	kobj_class_compile_static(ofw_def_impl, &ofw_kernel_kops);
-	kobj_init((kobj_t)ofw_obj, ofw_def_impl);
+	kobj_init_static((kobj_t)ofw_obj, ofw_def_impl);
 
 	rv = OFW_INIT(ofw_obj, cookie);
 
diff --git a/sys/dev/pccard/pccard.c b/sys/dev/pccard/pccard.c
index 1de571c..43bc4b3 100644
--- a/sys/dev/pccard/pccard.c
+++ b/sys/dev/pccard/pccard.c
@@ -59,7 +59,7 @@ __FBSDID("$FreeBSD$");
 #define PCCARDDEBUG
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, pccard, CTLFLAG_RD, 0, "PCCARD parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, pccard, CTLFLAG_RD, 0, "PCCARD parameters");
 
 int	pccard_debug = 0;
 TUNABLE_INT("hw.pccard.debug", &pccard_debug);
diff --git a/sys/dev/pccbb/pccbb.c b/sys/dev/pccbb/pccbb.c
index 3c60f37..b116b54 100644
--- a/sys/dev/pccbb/pccbb.c
+++ b/sys/dev/pccbb/pccbb.c
@@ -129,7 +129,7 @@ __FBSDID("$FreeBSD$");
 devclass_t cbb_devclass;
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, cbb, CTLFLAG_RD, 0, "CBB parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, cbb, CTLFLAG_RD, 0, "CBB parameters");
 
 /* There's no way to say TUNEABLE_LONG to get the right types */
 u_long cbb_start_mem = CBB_START_MEM;
diff --git a/sys/dev/pccbb/pccbb_isa.c b/sys/dev/pccbb/pccbb_isa.c
index 677967d..fb45e64 100644
--- a/sys/dev/pccbb/pccbb_isa.c
+++ b/sys/dev/pccbb/pccbb_isa.c
@@ -68,7 +68,7 @@ __FBSDID("$FreeBSD$");
  *****************************************************************************/
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, pcic, CTLFLAG_RD, 0, "PCIC parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, pcic, CTLFLAG_RD, 0, "PCIC parameters");
 
 static int isa_intr_mask = EXCA_INT_MASK_ALLOWED;
 TUNABLE_INT("hw.cbb.intr_mask", &isa_intr_mask);
diff --git a/sys/dev/ppbus/ppb_base.c b/sys/dev/ppbus/ppb_base.c
index 30c42a4..62769b0 100644
--- a/sys/dev/ppbus/ppb_base.c
+++ b/sys/dev/ppbus/ppb_base.c
@@ -236,11 +236,8 @@ ppb_unlock(device_t bus)
 void
 _ppb_assert_locked(device_t bus, const char *file, int line)
 {
-#ifdef INVARIANTS
-	struct ppb_data *ppb = DEVTOSOFTC(bus);
 
-	_mtx_assert(ppb->ppc_lock, MA_OWNED, file, line);
-#endif
+	mtx_assert_(DEVTOSOFTC(bus)->ppc_lock, MA_OWNED, file, line);
 }
 
 void
diff --git a/sys/dev/ppbus/ppbconf.c b/sys/dev/ppbus/ppbconf.c
index 9e21c67..858e5b2 100644
--- a/sys/dev/ppbus/ppbconf.c
+++ b/sys/dev/ppbus/ppbconf.c
@@ -422,20 +422,14 @@ ppbus_attach(device_t dev)
 static int
 ppbus_detach(device_t dev)
 {
-	device_t *children;
-	int error, nchildren, i;
+	int error;
 
 	error = bus_generic_detach(dev);
 	if (error)
 		return (error);
 
 	/* detach & delete all children */
-	if (!device_get_children(dev, &children, &nchildren)) {
-		for (i = 0; i < nchildren; i++)
-			if (children[i])
-				device_delete_child(dev, children[i]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
 
 	return (0);
 }
diff --git a/sys/dev/ppc/ppc.c b/sys/dev/ppc/ppc.c
index 1322a33..ef505d8 100644
--- a/sys/dev/ppc/ppc.c
+++ b/sys/dev/ppc/ppc.c
@@ -1851,20 +1851,13 @@ int
 ppc_detach(device_t dev)
 {
 	struct ppc_data *ppc = DEVTOSOFTC(dev);
-	device_t *children;
-	int nchildren, i;
 
 	if (ppc->res_irq == 0) {
 		return (ENXIO);
 	}
 
 	/* detach & delete all children */
-	if (!device_get_children(dev, &children, &nchildren)) {
-		for (i = 0; i < nchildren; i++)
-			if (children[i])
-				device_delete_child(dev, children[i]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
 
 	if (ppc->res_irq != 0) {
 		bus_teardown_intr(dev, ppc->res_irq, ppc->intr_cookie);
diff --git a/sys/dev/puc/puc.c b/sys/dev/puc/puc.c
index 9bb3ceb..7b93306 100644
--- a/sys/dev/puc/puc.c
+++ b/sys/dev/puc/puc.c
@@ -68,7 +68,7 @@ struct puc_port {
 devclass_t puc_devclass;
 const char puc_driver_name[] = "puc";
 
-MALLOC_DEFINE(M_PUC, "PUC", "PUC driver");
+static MALLOC_DEFINE(M_PUC, "PUC", "PUC driver");
 
 struct puc_bar *
 puc_get_bar(struct puc_softc *sc, int rid)
diff --git a/sys/dev/puc/pucdata.c b/sys/dev/puc/pucdata.c
index 80dff6e..94d1c2c 100644
--- a/sys/dev/puc/pucdata.c
+++ b/sys/dev/puc/pucdata.c
@@ -51,6 +51,7 @@ static puc_config_f puc_config_amc;
 static puc_config_f puc_config_diva;
 static puc_config_f puc_config_exar;
 static puc_config_f puc_config_icbook;
+static puc_config_f puc_config_moxa;
 static puc_config_f puc_config_oxford_pcie;
 static puc_config_f puc_config_quatech;
 static puc_config_f puc_config_syba;
@@ -518,12 +519,25 @@ const struct puc_cfg puc_pci_devices[] = {
 	    PUC_PORT_4S, 0x18, 0, 8,
 	},
 
+	{   0x1393, 0x1042, 0xffff, 0,
+	    "Moxa Technologies, Smartio CP-104JU/PCI",
+	    DEFAULT_RCLK * 8,
+	    PUC_PORT_4S, 0x18, 0, 8,
+	},
+
 	{   0x1393, 0x1043, 0xffff, 0,
 	    "Moxa Technologies, Smartio CP-104EL/PCIe",
 	    DEFAULT_RCLK * 8,
 	    PUC_PORT_4S, 0x18, 0, 8,
 	},
 
+	{   0x1393, 0x1045, 0xffff, 0,
+	    "Moxa Technologies, Smartio CP-104EL-A/PCIe",
+	    DEFAULT_RCLK * 8,
+	    PUC_PORT_4S, 0x14, 0, -1,
+		.config_function = puc_config_moxa
+	},
+
 	{   0x1393, 0x1120, 0xffff, 0,
 	    "Moxa Technologies, CP-112UL",
 	    DEFAULT_RCLK * 8,
@@ -850,6 +864,18 @@ const struct puc_cfg puc_pci_devices[] = {
 	    PUC_PORT_2S, 0x10, 4, 0,
 	},
 
+	{   0x14d2, 0xa007, 0xffff, 0,
+	    "Titan VScom PCIex-800H",
+	    DEFAULT_RCLK * 8,
+	    PUC_PORT_4S, 0x10, 0, 8,
+	},
+
+	{   0x14d2, 0xa008, 0xffff, 0,
+	    "Titan VScom PCIex-800H",
+	    DEFAULT_RCLK * 8,
+	    PUC_PORT_4S, 0x10, 0, 8,
+	},
+
 	{   0x14db, 0x2130, 0xffff, 0,
 	    "Avlab Technology, PCI IO 2S",
 	    DEFAULT_RCLK,
@@ -1086,6 +1112,19 @@ puc_config_icbook(struct puc_softc *sc, enum puc_cfg_cmd cmd, int port,
 }
 
 static int
+puc_config_moxa(struct puc_softc *sc, enum puc_cfg_cmd cmd, int port,
+    intptr_t *res)
+{
+	const struct puc_cfg *cfg = sc->sc_cfg;
+	
+	if (cmd == PUC_CFG_GET_OFS && cfg->device == 0x1045) {
+		*res = ((port == 3) ? 7 : port) * 0x200;
+		return 0;
+	}
+	return (ENXIO);
+}
+
+static int
 puc_config_quatech(struct puc_softc *sc, enum puc_cfg_cmd cmd, int port,
     intptr_t *res)
 {
diff --git a/sys/dev/qlxgb/README.txt b/sys/dev/qlxgb/README.txt
new file mode 100644
index 0000000..d9773cc
--- /dev/null
+++ b/sys/dev/qlxgb/README.txt
@@ -0,0 +1,99 @@
+# $FreeBSD$
+
+			README File
+	QLogic 3200 and 8200 series Single/Dual Port
+10 Gigabit Ethernet & CNA Adapter Driver for FreeBSD 7.x/8.x/9.x
+
+		QLogic Corporation.
+		All rights reserved. 
+
+
+Table of Contents
+1. Package Contents 
+2. OS Support
+3. Supported Features
+4. Using the Driver
+   4.1 Installing the driver
+   4.2 Removing the driver
+5. Driver Parameters
+6. Additional Notes
+7. Contacting Support
+
+1. Package Contents 
+ * Documentation 
+   - README (this document) version:1.0
+   - Release Notes Version:1.0
+ * Driver (if_qlxgb.ko)
+	- FreeBSD 7.x/8.x/9.x
+ * Firmware: pre-flashed on QLogic adapter;
+
+2. OS Support
+
+The Qlogic 10Gigabit Ethernet/CNA driver is compatible with the 
+following OS platforms:
+ * FreeBSD 7.x/8.x/9.x (64-bit) [Intel EM64T, AMD64]
+
+3. Supported Features
+10Gigabit Ethernet NIC/CNA driver supports following features
+
+* Large Segment Offload over TCP IPV4
+* Large Segment Offload over TCP IPV6
+* Receive Side scaling
+* TCP over IPv4 checksum offload
+* UDP over IPv4 checksum offload
+* IPV4 checksum offload
+* TCP over IPv6 checksum offload
+* UDP over IPv6 checksum offload
+* Jumbo frames
+* VLAN Tag
+
+
+4. Using the driver
+
+ 4.1 Installing the driver
+
+   - copy the driver file (if_qlxgb.ko) into some directory (say qla_driver)
+   - cd <to qla_driver>
+   - kldload -v ./if_qlxgb.ko
+
+ 4.2 Removing the driver
+ 
+  - kldunload if_qlxgb
+
+5. Parameters to set prior to installing the driver
+
+   - Add the following lines to /etc/sysctl.conf and reboot the machine prior
+     to installing the driver
+   
+	kern.ipc.nmbjumbo9=262144
+	net.inet.tcp.recvbuf_max=262144
+	net.inet.tcp.recvbuf_inc=16384
+	kern.ipc.nmbclusters=1000000
+	kern.ipc.maxsockbuf=2097152
+	net.inet.tcp.recvspace=131072
+	net.inet.tcp.sendbuf_max=262144
+	net.inet.tcp.sendspace=65536
+ 
+   - If you do not want to reboot the system please run the following commands
+
+	login or su to root
+
+	sysctl kern.ipc.nmbjumbo9=262144
+	sysctl net.inet.tcp.recvbuf_max=262144
+	sysctl net.inet.tcp.recvbuf_inc=16384
+	sysctl kern.ipc.nmbclusters=1000000
+	sysctl kern.ipc.maxsockbuf=2097152
+	sysctl net.inet.tcp.recvspace=131072
+	sysctl net.inet.tcp.sendbuf_max=262144
+	sysctl net.inet.tcp.sendspace=65536
+
+6. Contacting Support 
+Please feel free to contact your QLogic approved reseller or QLogic 
+Technical Support at any phase of integration for assistance. QLogic
+Technical Support can be reached by the following methods: 
+Web:    http://support.qlogic.com
+E-mail: support@qlogic.com
+(c) Copyright 2011. All rights reserved worldwide. QLogic, the QLogic 
+logo, and the Powered by QLogic logo are registered trademarks of
+QLogic Corporation. All other brand and product names are trademarks 
+or registered trademarks of their respective owners. 
diff --git a/sys/dev/qlxgb/qla_dbg.c b/sys/dev/qlxgb/qla_dbg.c
new file mode 100644
index 0000000..5fc6f46
--- /dev/null
+++ b/sys/dev/qlxgb/qla_dbg.c
@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * File : qla_dbg.c
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qla_os.h"
+#include "qla_reg.h"
+#include "qla_hw.h"
+#include "qla_def.h"
+#include "qla_inline.h"
+#include "qla_ver.h"
+#include "qla_glbl.h"
+#include "qla_dbg.h"
+
+
+uint32_t dbg_level = 0 ;
+/*
+ * Name: qla_dump_buf32
+ * Function: dumps a buffer as 32 bit words
+ */
+void qla_dump_buf32(qla_host_t *ha, char *msg, void *dbuf32, uint32_t len32)
+{
+        device_t dev;
+	uint32_t i = 0;
+	uint32_t *buf;
+
+        dev = ha->pci_dev;
+	buf = dbuf32;
+
+	device_printf(dev, "%s: %s dump start\n", __func__, msg);
+
+	while (len32 >= 4) {
+		device_printf(dev,"0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+			i, buf[0], buf[1], buf[2], buf[3]);
+		i += 4 * 4;
+		len32 -= 4;
+		buf += 4;
+	}
+	switch (len32) {
+	case 1:
+		device_printf(dev,"0x%08x: 0x%08x\n", i, buf[0]);
+		break;
+	case 2:
+		device_printf(dev,"0x%08x: 0x%08x 0x%08x\n", i, buf[0], buf[1]);
+		break;
+	case 3:
+		device_printf(dev,"0x%08x: 0x%08x 0x%08x 0x%08x\n",
+			i, buf[0], buf[1], buf[2]);
+		break;
+	default:
+		break;
+	}
+	device_printf(dev, "%s: %s dump end\n", __func__, msg);
+}
+
+/*
+ * Name: qla_dump_buf16
+ * Function: dumps a buffer as 16 bit words
+ */
+void qla_dump_buf16(qla_host_t *ha, char *msg, void *dbuf16, uint32_t len16)
+{
+        device_t dev;
+	uint32_t i = 0;
+	uint16_t *buf;
+
+        dev = ha->pci_dev;
+	buf = dbuf16;
+
+	device_printf(dev, "%s: %s dump start\n", __func__, msg);
+
+	while (len16 >= 8) {
+		device_printf(dev,"0x%08x: 0x%04x 0x%04x 0x%04x 0x%04x"
+			" 0x%04x 0x%04x 0x%04x 0x%04x\n", i, buf[0],
+			buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7]);
+		i += 16;
+		len16 -= 8;
+		buf += 8;
+	}
+	switch (len16) {
+	case 1:
+		device_printf(dev,"0x%08x: 0x%04x\n", i, buf[0]);
+		break;
+	case 2:
+		device_printf(dev,"0x%08x: 0x%04x 0x%04x\n", i, buf[0], buf[1]);
+		break;
+	case 3:
+		device_printf(dev,"0x%08x: 0x%04x 0x%04x 0x%04x\n",
+			i, buf[0], buf[1], buf[2]);
+		break;
+	case 4:
+		device_printf(dev,"0x%08x: 0x%04x 0x%04x 0x%04x 0x%04x\n", i,
+			buf[0], buf[1], buf[2], buf[3]);
+		break;
+	case 5:
+		device_printf(dev,"0x%08x:"
+			" 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4]);
+		break;
+	case 6:
+		device_printf(dev,"0x%08x:"
+			" 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
+		break;
+	case 7:
+		device_printf(dev,"0x%04x: 0x%04x 0x%04x 0x%04x 0x%04x"
+			" 0x%04x 0x%04x 0x%04x\n", i, buf[0], buf[1],
+			buf[2], buf[3], buf[4], buf[5], buf[6]);
+		break;
+	default:
+		break;
+	}
+	device_printf(dev, "%s: %s dump end\n", __func__, msg);
+}
+
+/*
+ * Name: qla_dump_buf8
+ * Function: dumps a buffer as bytes
+ */
+void qla_dump_buf8(qla_host_t *ha, char *msg, void *dbuf, uint32_t len)
+{
+        device_t dev;
+	uint32_t i = 0;
+	uint8_t *buf;
+
+        dev = ha->pci_dev;
+	buf = dbuf;
+
+	device_printf(dev, "%s: %s 0x%x dump start\n", __func__, msg, len);
+	
+	while (len >= 16) {
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3],
+			buf[4], buf[5], buf[6], buf[7],
+			buf[8], buf[9], buf[10], buf[11],
+			buf[12], buf[13], buf[14], buf[15]);
+		i += 16;
+		len -= 16;
+		buf += 16;
+	}
+	switch (len) {
+	case 1:
+		device_printf(dev,"0x%08x: %02x\n", i, buf[0]);
+		break;
+	case 2:
+		device_printf(dev,"0x%08x: %02x %02x\n", i, buf[0], buf[1]);
+		break;
+	case 3:
+		device_printf(dev,"0x%08x: %02x %02x %02x\n",
+			i, buf[0], buf[1], buf[2]);
+		break;
+	case 4:
+		device_printf(dev,"0x%08x: %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3]);
+		break;
+	case 5:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4]);
+		break;
+	case 6:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
+		break;
+	case 7:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6]);
+		break;
+	case 8:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7]);
+		break;
+	case 9:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8]);
+		break;
+	case 10:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8], buf[9]);
+		break;
+	case 11:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8], buf[9], buf[10]);
+		break;
+	case 12:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8], buf[9], buf[10], buf[11]);
+		break;
+	case 13:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8], buf[9], buf[10], buf[11], buf[12]);
+		break;
+	case 14:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8], buf[9], buf[10], buf[11], buf[12],
+			buf[13]);
+		break;
+	case 15:
+		device_printf(dev,"0x%08x:"
+			" %02x %02x %02x %02x %02x %02x %02x %02x"
+			" %02x %02x %02x %02x %02x %02x %02x\n", i,
+			buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
+			buf[7], buf[8], buf[9], buf[10], buf[11], buf[12],
+			buf[13], buf[14]);
+		break;
+	default:
+		break;
+	}
+	
+	device_printf(dev, "%s: %s dump end\n", __func__, msg);
+}
diff --git a/sys/dev/qlxgb/qla_dbg.h b/sys/dev/qlxgb/qla_dbg.h
new file mode 100644
index 0000000..1f0d184
--- /dev/null
+++ b/sys/dev/qlxgb/qla_dbg.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * File : qla_dbg.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#ifndef _QL_DBG_H_
+#define _QL_DBG_H_
+
+extern uint32_t dbg_level;
+
+extern void qla_dump_buf8(qla_host_t *ha, char *str, void *dbuf,
+		uint32_t len);
+extern void qla_dump_buf16(qla_host_t *ha, char *str, void *dbuf,
+		uint32_t len16);
+extern void qla_dump_buf32(qla_host_t *ha, char *str, void *dbuf,
+		uint32_t len32);
+
+
+#define DBG 1
+
+#if DBG
+
+#define QL_DPRINT1(x)	if (dbg_level & 0x0001) device_printf x
+#define QL_DPRINT2(x)	if (dbg_level & 0x0002) device_printf x
+#define QL_DPRINT4(x)	if (dbg_level & 0x0004) device_printf x
+#define QL_DPRINT8(x)	if (dbg_level & 0x0008) device_printf x
+#define QL_DPRINT10(x)	if (dbg_level & 0x0010) device_printf x
+#define QL_DPRINT20(x)	if (dbg_level & 0x0020) device_printf x
+#define QL_DPRINT40(x)	if (dbg_level & 0x0040) device_printf x
+#define QL_DPRINT80(x)	if (dbg_level & 0x0080) device_printf x
+
+#define QL_DUMP_BUFFER8(h, s, b, n) if (dbg_level & 0x08000000)\
+					qla_dump_buf8(h, s, b, n)
+#define QL_DUMP_BUFFER16(h, s, b, n) if (dbg_level & 0x08000000)\
+					qla_dump_buf16(h, s, b, n)
+#define QL_DUMP_BUFFER32(h, s, b, n) if (dbg_level & 0x08000000)\
+					qla_dump_buf32(h, s, b, n)
+
+#else
+
+#define QL_DPRINT1(x)
+#define QL_DPRINT2(x)
+#define QL_DPRINT4(x)
+#define QL_DPRINT8(x)
+#define QL_DPRINT10(x)
+#define QL_DPRINT20(x)
+#define QL_DPRINT40(x)
+#define QL_DPRINT80(x)
+
+#define QL_DUMP_BUFFER8(h, s, b, n)
+#define QL_DUMP_BUFFER16(h, s, b, n)
+#define QL_DUMP_BUFFER32(h, s, b, n)
+
+#endif
+
+#endif /* #ifndef _QL_DBG_H_ */
diff --git a/sys/dev/qlxgb/qla_def.h b/sys/dev/qlxgb/qla_def.h
new file mode 100644
index 0000000..d40d5e2
--- /dev/null
+++ b/sys/dev/qlxgb/qla_def.h
@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * File: qla_def.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#ifndef _QLA_DEF_H_
+#define _QLA_DEF_H_
+
+#define BIT_0                   (0x1 << 0)
+#define BIT_1                   (0x1 << 1)
+#define BIT_2                   (0x1 << 2)
+#define BIT_3                   (0x1 << 3)
+#define BIT_4                   (0x1 << 4)
+#define BIT_5                   (0x1 << 5)
+#define BIT_6                   (0x1 << 6)
+#define BIT_7                   (0x1 << 7)
+#define BIT_8                   (0x1 << 8)
+#define BIT_9                   (0x1 << 9)
+#define BIT_10                  (0x1 << 10)
+#define BIT_11                  (0x1 << 11)
+#define BIT_12                  (0x1 << 12)
+#define BIT_13                  (0x1 << 13)
+#define BIT_14                  (0x1 << 14)
+#define BIT_15                  (0x1 << 15)
+#define BIT_16                  (0x1 << 16)
+#define BIT_17                  (0x1 << 17)
+#define BIT_18                  (0x1 << 18)
+#define BIT_19                  (0x1 << 19)
+#define BIT_20                  (0x1 << 20)
+#define BIT_21                  (0x1 << 21)
+#define BIT_22                  (0x1 << 22)
+#define BIT_23                  (0x1 << 23)
+#define BIT_24                  (0x1 << 24)
+#define BIT_25                  (0x1 << 25)
+#define BIT_26                  (0x1 << 26)
+#define BIT_27                  (0x1 << 27)
+#define BIT_28                  (0x1 << 28)
+#define BIT_29                  (0x1 << 29)
+#define BIT_30                  (0x1 << 30)
+#define BIT_31                  (0x1 << 31)
+
+struct qla_rx_buf {
+	struct mbuf	*m_head;
+	bus_dmamap_t	map;
+	bus_addr_t      paddr;
+	uint32_t	handle;
+	void		*next;
+};
+typedef struct qla_rx_buf qla_rx_buf_t;
+
+struct qla_tx_buf {
+	struct mbuf	*m_head;
+	bus_dmamap_t	map;
+};
+typedef struct qla_tx_buf qla_tx_buf_t;
+
+#define QLA_MAX_SEGMENTS	63	/* maximum # of segs in a sg list */
+#define QLA_MAX_FRAME_SIZE	MJUM9BYTES
+#define QLA_STD_FRAME_SIZE	1514
+#define QLA_MAX_TSO_FRAME_SIZE	((64 * 1024 - 1) + 22)
+
+/* Number of MSIX/MSI Vectors required */
+#define Q8_MSI_COUNT		4
+
+struct qla_ivec {
+	struct resource		*irq;
+	void			*handle;
+	int			irq_rid;
+	void			*ha;
+	struct task		rcv_task;
+	struct taskqueue	*rcv_tq;
+};
+
+typedef struct qla_ivec qla_ivec_t;
+
+#define QLA_WATCHDOG_CALLOUT_TICKS	1
+
+/*
+ * Adapter structure contains the hardware independant information of the
+ * pci function.
+ */
+struct qla_host {
+        volatile struct {
+                volatile uint32_t
+			qla_watchdog_active  :1,
+			qla_watchdog_exit    :1,
+			qla_watchdog_pause   :1,
+			lro_init	:1,
+			stop_rcv	:1,
+			link_up		:1,
+			parent_tag	:1,
+			lock_init	:1;
+        } flags;
+
+	device_t		pci_dev;
+
+	uint8_t			pci_func;
+	uint16_t		watchdog_ticks;
+	uint8_t			resvd;
+
+        /* ioctl related */
+        struct cdev             *ioctl_dev;
+
+	/* register mapping */
+	struct resource		*pci_reg;
+	int			reg_rid;
+
+	/* interrupts */
+	struct resource         *irq;
+	int			msix_count;
+	void			*intr_handle;
+	qla_ivec_t		irq_vec[Q8_MSI_COUNT];
+	
+	/* parent dma tag */
+	bus_dma_tag_t           parent_tag;
+
+	/* interface to o.s */
+	struct ifnet		*ifp;
+
+	struct ifmedia		media;
+	uint16_t		max_frame_size;
+	uint16_t		rsrvd0;
+	int			if_flags;
+
+	/* hardware access lock */
+	struct mtx		hw_lock;
+	volatile uint32_t	hw_lock_held;
+
+	/* transmit and receive buffers */
+	qla_tx_buf_t		tx_buf[NUM_TX_DESCRIPTORS];
+	bus_dma_tag_t		tx_tag;
+	struct mtx		tx_lock;
+	struct task		tx_task;
+	struct taskqueue	*tx_tq;
+	struct callout		tx_callout;
+
+	qla_rx_buf_t		rx_buf[NUM_RX_DESCRIPTORS];
+	qla_rx_buf_t		rx_jbuf[NUM_RX_JUMBO_DESCRIPTORS];
+	bus_dma_tag_t		rx_tag;
+
+	struct mtx		rx_lock;
+	struct mtx		rxj_lock;
+
+	/* stats */
+	uint32_t		err_m_getcl;
+	uint32_t		err_m_getjcl;
+	uint32_t		err_tx_dmamap_create;
+	uint32_t		err_tx_dmamap_load;
+	uint32_t		err_tx_defrag;
+
+	uint64_t		rx_frames;
+	uint64_t		rx_bytes;
+
+	uint64_t		tx_frames;
+	uint64_t		tx_bytes;
+
+        uint32_t                fw_ver_major;
+        uint32_t                fw_ver_minor;
+        uint32_t                fw_ver_sub;
+        uint32_t                fw_ver_build;
+
+	/* hardware specific */
+	qla_hw_t		hw;
+
+	/* debug stuff */
+	volatile const char 	*qla_lock;
+	volatile const char	*qla_unlock;
+};
+typedef struct qla_host qla_host_t;
+
+/* note that align has to be a power of 2 */
+#define QL_ALIGN(size, align) (size + (align - 1)) & ~(align - 1);
+#define QL_MIN(x, y) ((x < y) ? x : y)
+
+#define QL_RUNNING(ifp) \
+		((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == \
+			IFF_DRV_RUNNING)
+
+#endif /* #ifndef _QLA_DEF_H_ */
diff --git a/sys/dev/qlxgb/qla_glbl.h b/sys/dev/qlxgb/qla_glbl.h
new file mode 100644
index 0000000..21ee99c
--- /dev/null
+++ b/sys/dev/qlxgb/qla_glbl.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_glbl.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ * Content: Contains prototypes of the exported functions from each file.
+ */
+#ifndef _QLA_GLBL_H_
+#define _QLA_GLBL_H_
+
+/*
+ * from qla_isr.c
+ */
+extern void qla_isr(void *arg);
+extern void qla_rcv(void *context, int pending);
+
+/*
+ * from qla_os.c
+ */
+extern uint32_t std_replenish;
+extern uint32_t jumbo_replenish;
+extern uint32_t rcv_pkt_thres;
+extern uint32_t rcv_pkt_thres_d;
+extern uint32_t snd_pkt_thres;
+extern uint32_t free_pkt_thres;
+
+extern int qla_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf);
+extern void qla_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf);
+extern void qla_start(struct ifnet *ifp);
+extern int qla_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp,
+	        uint32_t jumbo);
+
+
+/*
+ * from qla_flash.c
+ */
+extern int qla_flash_rd32_words(qla_host_t *ha, uint32_t addr,
+		uint32_t *val, uint32_t num);
+extern int qla_flash_rd32(qla_host_t *ha, uint32_t addr, uint32_t *val);
+
+/*
+ * from qla_hw.c
+ */
+extern int qla_get_msix_count(qla_host_t *ha);
+extern int qla_alloc_dma(qla_host_t *ha);
+extern void qla_free_dma(qla_host_t *ha);
+extern void qla_hw_add_sysctls(qla_host_t *ha);
+extern int qla_hw_send(qla_host_t *ha, bus_dma_segment_t *segs, int nsegs,
+		uint32_t *tx_idx, struct mbuf *mp);
+extern int qla_init_hw_if(qla_host_t *ha);
+extern void qla_get_hw_caps(qla_host_t *ha);
+extern void qla_hw_set_multi(qla_host_t *ha, uint8_t *mta, uint32_t mcnt,
+	uint32_t add_multi);
+extern void qla_del_hw_if(qla_host_t *ha);
+extern void qla_set_promisc(qla_host_t *ha);
+extern void qla_set_allmulti(qla_host_t *ha);
+extern void qla_reset_promisc_allmulti(qla_host_t *ha);
+extern void qla_config_ipv4_addr(qla_host_t *ha, uint32_t ipv4_addr);
+extern int qla_hw_tx_compl(qla_host_t *ha);
+extern void qla_update_link_state(qla_host_t *ha);
+extern void qla_hw_tx_done(qla_host_t *ha);
+extern int qla_config_lro(qla_host_t *ha);
+extern void qla_free_lro(qla_host_t *ha);
+extern int qla_set_max_mtu(qla_host_t *ha, uint32_t mtu, uint16_t cntxt_id);
+extern void qla_hw_stop_rcv(qla_host_t *ha);
+
+/*
+ * from qla_misc.c
+ */
+extern int qla_init_hw(qla_host_t *ha);
+extern int qla_rdwr_indreg32(qla_host_t *ha, uint32_t addr, uint32_t *val,
+		uint32_t rd);
+extern int qla_rd_flash32(qla_host_t *ha, uint32_t addr, uint32_t *data);
+
+/*
+ * from qla_ioctl.c
+ */
+extern int qla_make_cdev(qla_host_t *ha);
+extern void qla_del_cdev(qla_host_t *ha);
+extern int qla_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
+		struct thread *td);
+
+#endif /* #ifndef_QLA_GLBL_H_ */
diff --git a/sys/dev/qlxgb/qla_hw.c b/sys/dev/qlxgb/qla_hw.c
new file mode 100644
index 0000000..477eb57
--- /dev/null
+++ b/sys/dev/qlxgb/qla_hw.c
@@ -0,0 +1,1776 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File: qla_hw.c
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ * Content: Contains Hardware dependant functions
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qla_os.h"
+#include "qla_reg.h"
+#include "qla_hw.h"
+#include "qla_def.h"
+#include "qla_inline.h"
+#include "qla_ver.h"
+#include "qla_glbl.h"
+#include "qla_dbg.h"
+
+static uint32_t sysctl_num_rds_rings = 2;
+static uint32_t sysctl_num_sds_rings = 4;
+
+/*
+ * Static Functions
+ */
+
+static void qla_init_cntxt_regions(qla_host_t *ha);
+static int qla_issue_cmd(qla_host_t *ha, qla_cdrp_t *cdrp);
+static int qla_fw_cmd(qla_host_t *ha, void *fw_cmd, uint32_t size);
+static int qla_config_mac_addr(qla_host_t *ha, uint8_t *mac_addr,
+		uint16_t cntxt_id, uint32_t add_multi);
+static void qla_del_rcv_cntxt(qla_host_t *ha);
+static int qla_init_rcv_cntxt(qla_host_t *ha);
+static void qla_del_xmt_cntxt(qla_host_t *ha);
+static int qla_init_xmt_cntxt(qla_host_t *ha);
+static int qla_get_max_rds(qla_host_t *ha);
+static int qla_get_max_sds(qla_host_t *ha);
+static int qla_get_max_rules(qla_host_t *ha);
+static int qla_get_max_rcv_cntxts(qla_host_t *ha);
+static int qla_get_max_tx_cntxts(qla_host_t *ha);
+static int qla_get_max_mtu(qla_host_t *ha);
+static int qla_get_max_lro(qla_host_t *ha);
+static int qla_get_flow_control(qla_host_t *ha);
+static void qla_hw_tx_done_locked(qla_host_t *ha);
+
+int
+qla_get_msix_count(qla_host_t *ha)
+{
+	return (sysctl_num_sds_rings);
+}
+
+/*
+ * Name: qla_hw_add_sysctls
+ * Function: Add P3Plus specific sysctls
+ */
+void
+qla_hw_add_sysctls(qla_host_t *ha)
+{
+        device_t	dev;
+
+        dev = ha->pci_dev;
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "num_rds_rings", CTLFLAG_RD, &sysctl_num_rds_rings,
+		sysctl_num_rds_rings, "Number of Rcv Descriptor Rings");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "num_sds_rings", CTLFLAG_RD, &sysctl_num_sds_rings,
+		sysctl_num_sds_rings, "Number of Status Descriptor Rings");
+}
+
+/*
+ * Name: qla_free_dma
+ * Function: Frees the DMA'able memory allocated in qla_alloc_dma()
+ */
+void
+qla_free_dma(qla_host_t *ha)
+{
+	uint32_t i;
+
+        if (ha->hw.dma_buf.flags.context) {
+		qla_free_dmabuf(ha, &ha->hw.dma_buf.context);
+        	ha->hw.dma_buf.flags.context = 0;
+	}
+
+        if (ha->hw.dma_buf.flags.sds_ring) {
+		for (i = 0; i < ha->hw.num_sds_rings; i++)
+			qla_free_dmabuf(ha, &ha->hw.dma_buf.sds_ring[i]);
+        	ha->hw.dma_buf.flags.sds_ring = 0;
+	}
+
+        if (ha->hw.dma_buf.flags.rds_ring) {
+		for (i = 0; i < ha->hw.num_rds_rings; i++)
+			qla_free_dmabuf(ha, &ha->hw.dma_buf.rds_ring[i]);
+        	ha->hw.dma_buf.flags.rds_ring = 0;
+	}
+
+        if (ha->hw.dma_buf.flags.tx_ring) {
+		qla_free_dmabuf(ha, &ha->hw.dma_buf.tx_ring);
+        	ha->hw.dma_buf.flags.tx_ring = 0;
+	}
+}
+
+/*
+ * Name: qla_alloc_dma
+ * Function: Allocates DMA'able memory for Tx/Rx Rings, Tx/Rx Contexts.
+ */
+int
+qla_alloc_dma(qla_host_t *ha)
+{
+        device_t                dev;
+	uint32_t		i, j, size;
+
+        dev = ha->pci_dev;
+
+        QL_DPRINT2((dev, "%s: enter\n", __func__));
+
+	ha->hw.num_rds_rings = (uint16_t)sysctl_num_rds_rings;
+	ha->hw.num_sds_rings = (uint16_t)sysctl_num_sds_rings;
+
+	/*
+	 * Allocate Transmit Ring
+	 */
+
+	ha->hw.dma_buf.tx_ring.alignment = 8;
+	ha->hw.dma_buf.tx_ring.size =
+		(sizeof(q80_tx_cmd_t)) * NUM_TX_DESCRIPTORS;
+	
+        if (qla_alloc_dmabuf(ha, &ha->hw.dma_buf.tx_ring)) {
+                device_printf(dev, "%s: tx ring alloc failed\n", __func__);
+                goto qla_alloc_dma_exit;
+        }
+        ha->hw.dma_buf.flags.tx_ring = 1;
+
+	QL_DPRINT2((dev, "%s: tx_ring phys %p virt %p\n",
+		__func__, (void *)(ha->hw.dma_buf.tx_ring.dma_addr),
+		ha->hw.dma_buf.tx_ring.dma_b));
+	/*
+	 * Allocate Receive Descriptor Rings
+	 */
+
+	for (i = 0; i < ha->hw.num_rds_rings; i++) {
+		ha->hw.dma_buf.rds_ring[i].alignment = 8;
+
+		if (i == RDS_RING_INDEX_NORMAL) {
+			ha->hw.dma_buf.rds_ring[i].size =
+				(sizeof(q80_recv_desc_t)) * NUM_RX_DESCRIPTORS;
+		} else if (i == RDS_RING_INDEX_JUMBO) {
+			ha->hw.dma_buf.rds_ring[i].size = 
+				(sizeof(q80_recv_desc_t)) *
+					NUM_RX_JUMBO_DESCRIPTORS;
+		} else
+			break;
+	
+		if (qla_alloc_dmabuf(ha, &ha->hw.dma_buf.rds_ring[i])) {
+			QL_DPRINT4((dev, "%s: rds ring alloc failed\n",
+				__func__));
+
+			for (j = 0; j < i; j++)
+				qla_free_dmabuf(ha,
+					&ha->hw.dma_buf.rds_ring[j]);
+
+			goto qla_alloc_dma_exit;
+		}
+		QL_DPRINT4((dev, "%s: rx_ring[%d] phys %p virt %p\n",
+			__func__, i,
+			(void *)(ha->hw.dma_buf.rds_ring[i].dma_addr),
+			ha->hw.dma_buf.rds_ring[i].dma_b));
+	}
+	ha->hw.dma_buf.flags.rds_ring = 1;
+
+	/*
+	 * Allocate Status Descriptor Rings
+	 */
+
+	for (i = 0; i < ha->hw.num_sds_rings; i++) {
+		ha->hw.dma_buf.sds_ring[i].alignment = 8;
+		ha->hw.dma_buf.sds_ring[i].size =
+			(sizeof(q80_stat_desc_t)) * NUM_STATUS_DESCRIPTORS;
+
+		if (qla_alloc_dmabuf(ha, &ha->hw.dma_buf.sds_ring[i])) {
+			device_printf(dev, "%s: sds ring alloc failed\n",
+				__func__);
+
+			for (j = 0; j < i; j++)
+				qla_free_dmabuf(ha,
+					&ha->hw.dma_buf.sds_ring[j]);
+
+			goto qla_alloc_dma_exit;
+		}
+		QL_DPRINT4((dev, "%s: sds_ring[%d] phys %p virt %p\n",
+			__func__, i,
+			(void *)(ha->hw.dma_buf.sds_ring[i].dma_addr),
+			ha->hw.dma_buf.sds_ring[i].dma_b));
+	}
+	ha->hw.dma_buf.flags.sds_ring = 1;
+
+	/*
+	 * Allocate Context Area
+	 */
+	size = QL_ALIGN((sizeof (q80_tx_cntxt_req_t)), QL_BUFFER_ALIGN);
+
+	size += QL_ALIGN((sizeof (q80_tx_cntxt_rsp_t)), QL_BUFFER_ALIGN);
+
+	size += QL_ALIGN((sizeof (q80_rcv_cntxt_req_t)), QL_BUFFER_ALIGN);
+
+	size += QL_ALIGN((sizeof (q80_rcv_cntxt_rsp_t)), QL_BUFFER_ALIGN);
+
+	size += sizeof (uint32_t); /* for tx consumer index */
+
+	size = QL_ALIGN(size, PAGE_SIZE);
+	
+	ha->hw.dma_buf.context.alignment = 8;
+	ha->hw.dma_buf.context.size = size;
+	
+        if (qla_alloc_dmabuf(ha, &ha->hw.dma_buf.context)) {
+                device_printf(dev, "%s: context alloc failed\n", __func__);
+                goto qla_alloc_dma_exit;
+        }
+        ha->hw.dma_buf.flags.context = 1;
+	QL_DPRINT2((dev, "%s: context phys %p virt %p\n",
+		__func__, (void *)(ha->hw.dma_buf.context.dma_addr),
+		ha->hw.dma_buf.context.dma_b));
+
+	qla_init_cntxt_regions(ha);
+
+	return 0;
+
+qla_alloc_dma_exit:
+	qla_free_dma(ha);
+	return -1;
+}
+
+/*
+ * Name: qla_init_cntxt_regions
+ * Function: Initializes Tx/Rx Contexts.
+ */
+static void
+qla_init_cntxt_regions(qla_host_t *ha)
+{
+	qla_hw_t		*hw;
+	q80_tx_cntxt_req_t	*tx_cntxt_req;
+	q80_rcv_cntxt_req_t	*rx_cntxt_req;
+	bus_addr_t		phys_addr;
+	uint32_t		i;
+        device_t                dev;
+	uint32_t		size;
+
+        dev = ha->pci_dev;
+
+	hw = &ha->hw;
+
+	hw->tx_ring_base = hw->dma_buf.tx_ring.dma_b;
+	
+	for (i = 0; i < ha->hw.num_sds_rings; i++)
+		hw->sds[i].sds_ring_base =
+			(q80_stat_desc_t *)hw->dma_buf.sds_ring[i].dma_b;
+
+
+	phys_addr = hw->dma_buf.context.dma_addr;
+
+	memset((void *)hw->dma_buf.context.dma_b, 0,
+		ha->hw.dma_buf.context.size);
+
+	hw->tx_cntxt_req	=
+		(q80_tx_cntxt_req_t *)hw->dma_buf.context.dma_b;
+	hw->tx_cntxt_req_paddr	= phys_addr;
+
+	size = QL_ALIGN((sizeof (q80_tx_cntxt_req_t)), QL_BUFFER_ALIGN);
+
+	hw->tx_cntxt_rsp	=
+		(q80_tx_cntxt_rsp_t *)((uint8_t *)hw->tx_cntxt_req + size);
+	hw->tx_cntxt_rsp_paddr	= hw->tx_cntxt_req_paddr + size;
+
+	size = QL_ALIGN((sizeof (q80_tx_cntxt_rsp_t)), QL_BUFFER_ALIGN);
+
+	hw->rx_cntxt_req =
+		(q80_rcv_cntxt_req_t *)((uint8_t *)hw->tx_cntxt_rsp + size);
+	hw->rx_cntxt_req_paddr = hw->tx_cntxt_rsp_paddr + size;
+
+	size = QL_ALIGN((sizeof (q80_rcv_cntxt_req_t)), QL_BUFFER_ALIGN);
+
+	hw->rx_cntxt_rsp =
+		(q80_rcv_cntxt_rsp_t *)((uint8_t *)hw->rx_cntxt_req + size);
+	hw->rx_cntxt_rsp_paddr = hw->rx_cntxt_req_paddr + size;
+
+	size = QL_ALIGN((sizeof (q80_rcv_cntxt_rsp_t)), QL_BUFFER_ALIGN);
+
+	hw->tx_cons = (uint32_t *)((uint8_t *)hw->rx_cntxt_rsp + size);
+	hw->tx_cons_paddr = hw->rx_cntxt_rsp_paddr + size;
+
+	/*
+	 * Initialize the Transmit Context Request so that we don't need to
+	 * do it everytime we need to create a context
+	 */
+	tx_cntxt_req = hw->tx_cntxt_req;
+
+	tx_cntxt_req->rsp_dma_addr = qla_host_to_le64(hw->tx_cntxt_rsp_paddr);
+
+	tx_cntxt_req->cmd_cons_dma_addr = qla_host_to_le64(hw->tx_cons_paddr);
+
+	tx_cntxt_req->caps[0] = qla_host_to_le32((CNTXT_CAP0_BASEFW |
+					CNTXT_CAP0_LEGACY_MN | CNTXT_CAP0_LSO));
+	
+	tx_cntxt_req->intr_mode = qla_host_to_le32(CNTXT_INTR_MODE_SHARED);
+
+	tx_cntxt_req->phys_addr =
+		qla_host_to_le64(hw->dma_buf.tx_ring.dma_addr);
+
+	tx_cntxt_req->num_entries = qla_host_to_le32(NUM_TX_DESCRIPTORS);
+
+	/*
+	 * Initialize the Receive Context Request
+	 */
+
+	rx_cntxt_req = hw->rx_cntxt_req;
+
+	rx_cntxt_req->rx_req.rsp_dma_addr =
+		qla_host_to_le64(hw->rx_cntxt_rsp_paddr);
+
+	rx_cntxt_req->rx_req.caps[0] = qla_host_to_le32(CNTXT_CAP0_BASEFW |
+						CNTXT_CAP0_LEGACY_MN |
+						CNTXT_CAP0_JUMBO |
+						CNTXT_CAP0_LRO|
+						CNTXT_CAP0_HW_LRO);
+
+	rx_cntxt_req->rx_req.intr_mode =
+		qla_host_to_le32(CNTXT_INTR_MODE_SHARED);
+
+	rx_cntxt_req->rx_req.rds_intr_mode =
+		qla_host_to_le32(CNTXT_INTR_MODE_UNIQUE);
+
+	rx_cntxt_req->rx_req.rds_ring_offset = 0;
+	rx_cntxt_req->rx_req.sds_ring_offset = qla_host_to_le32(
+		(hw->num_rds_rings * sizeof(q80_rq_rds_ring_t)));
+	rx_cntxt_req->rx_req.num_rds_rings =
+		qla_host_to_le16(hw->num_rds_rings);
+	rx_cntxt_req->rx_req.num_sds_rings =
+		qla_host_to_le16(hw->num_sds_rings);
+
+	for (i = 0; i < hw->num_rds_rings; i++) {
+		rx_cntxt_req->rds_req[i].phys_addr =
+			qla_host_to_le64(hw->dma_buf.rds_ring[i].dma_addr);
+
+		if (i == RDS_RING_INDEX_NORMAL) {
+			rx_cntxt_req->rds_req[i].buf_size =
+				qla_host_to_le64(MCLBYTES);
+			rx_cntxt_req->rds_req[i].size =
+				qla_host_to_le32(NUM_RX_DESCRIPTORS);
+		} else {
+			rx_cntxt_req->rds_req[i].buf_size =
+				qla_host_to_le64(MJUM9BYTES);
+			rx_cntxt_req->rds_req[i].size =
+				qla_host_to_le32(NUM_RX_JUMBO_DESCRIPTORS);
+		}
+	}
+
+	for (i = 0; i < hw->num_sds_rings; i++) {
+		rx_cntxt_req->sds_req[i].phys_addr =
+			qla_host_to_le64(hw->dma_buf.sds_ring[i].dma_addr);
+		rx_cntxt_req->sds_req[i].size =
+			qla_host_to_le32(NUM_STATUS_DESCRIPTORS);
+		rx_cntxt_req->sds_req[i].msi_index = qla_host_to_le16(i);
+	}
+
+	QL_DPRINT2((ha->pci_dev, "%s: tx_cntxt_req = %p paddr %p\n",
+		__func__, hw->tx_cntxt_req, (void *)hw->tx_cntxt_req_paddr));
+	QL_DPRINT2((ha->pci_dev, "%s: tx_cntxt_rsp = %p paddr %p\n",
+		__func__, hw->tx_cntxt_rsp, (void *)hw->tx_cntxt_rsp_paddr));
+	QL_DPRINT2((ha->pci_dev, "%s: rx_cntxt_req = %p paddr %p\n",
+		__func__, hw->rx_cntxt_req, (void *)hw->rx_cntxt_req_paddr));
+	QL_DPRINT2((ha->pci_dev, "%s: rx_cntxt_rsp = %p paddr %p\n",
+		__func__, hw->rx_cntxt_rsp, (void *)hw->rx_cntxt_rsp_paddr));
+	QL_DPRINT2((ha->pci_dev, "%s: tx_cons      = %p paddr %p\n",
+		__func__, hw->tx_cons, (void *)hw->tx_cons_paddr));
+}
+
+/*
+ * Name: qla_issue_cmd
+ * Function: Issues commands on the CDRP interface and returns responses.
+ */
+static int
+qla_issue_cmd(qla_host_t *ha, qla_cdrp_t *cdrp)
+{
+	int	ret = 0;
+	uint32_t signature;
+	uint32_t count = 400; /* 4 seconds or 400 10ms intervals */
+	uint32_t data;
+	device_t dev;
+
+	dev = ha->pci_dev;
+
+	signature = 0xcafe0000 | 0x0100 | ha->pci_func;
+
+	ret = qla_sem_lock(ha, Q8_SEM5_LOCK, 0, (uint32_t)ha->pci_func);
+	
+	if (ret) {
+		device_printf(dev, "%s: SEM5_LOCK lock failed\n", __func__);
+		return (ret);
+	}
+
+	WRITE_OFFSET32(ha, Q8_NX_CDRP_SIGNATURE, signature);
+
+	WRITE_OFFSET32(ha, Q8_NX_CDRP_ARG1, (cdrp->cmd_arg1));
+	WRITE_OFFSET32(ha, Q8_NX_CDRP_ARG2, (cdrp->cmd_arg2));
+	WRITE_OFFSET32(ha, Q8_NX_CDRP_ARG3, (cdrp->cmd_arg3));
+
+	WRITE_OFFSET32(ha, Q8_NX_CDRP_CMD_RSP, cdrp->cmd);
+
+	while (count) {
+		qla_mdelay(__func__, 10);
+
+		data = READ_REG32(ha, Q8_NX_CDRP_CMD_RSP);
+
+		if ((!(data & 0x80000000)))
+			break;
+		count--;
+	}
+	if ((!count) || (data != 1))
+		ret = -1;
+
+	cdrp->rsp = READ_REG32(ha, Q8_NX_CDRP_CMD_RSP);
+	cdrp->rsp_arg1 = READ_REG32(ha, Q8_NX_CDRP_ARG1);
+	cdrp->rsp_arg2 = READ_REG32(ha, Q8_NX_CDRP_ARG2);
+	cdrp->rsp_arg3 = READ_REG32(ha, Q8_NX_CDRP_ARG3);
+
+	qla_sem_unlock(ha, Q8_SEM5_UNLOCK);
+
+	if (ret) {
+		device_printf(dev, "%s: "
+			"cmd[0x%08x] = 0x%08x\n"
+			"\tsig[0x%08x] = 0x%08x\n"
+			"\targ1[0x%08x] = 0x%08x\n"
+			"\targ2[0x%08x] = 0x%08x\n"
+			"\targ3[0x%08x] = 0x%08x\n",
+			__func__, Q8_NX_CDRP_CMD_RSP, cdrp->cmd,
+			Q8_NX_CDRP_SIGNATURE, signature,
+			Q8_NX_CDRP_ARG1, cdrp->cmd_arg1,
+			Q8_NX_CDRP_ARG2, cdrp->cmd_arg2,
+			Q8_NX_CDRP_ARG3, cdrp->cmd_arg3);
+		
+		device_printf(dev, "%s: exit (ret = 0x%x)\n"
+			"\t\t rsp = 0x%08x\n"
+			"\t\t arg1 = 0x%08x\n"
+			"\t\t arg2 = 0x%08x\n"
+			"\t\t arg3 = 0x%08x\n",
+			__func__, ret, cdrp->rsp,
+			cdrp->rsp_arg1, cdrp->rsp_arg2, cdrp->rsp_arg3);
+	}
+
+	return (ret);
+}
+
+#define QLA_TX_MIN_FREE	2
+
+/*
+ * Name: qla_fw_cmd
+ * Function: Issues firmware control commands on the Tx Ring.
+ */
+static int
+qla_fw_cmd(qla_host_t *ha, void *fw_cmd, uint32_t size)
+{
+	device_t dev;
+        q80_tx_cmd_t *tx_cmd;
+        qla_hw_t *hw = &ha->hw;
+	int count = 100;
+
+	dev = ha->pci_dev;
+
+	QLA_TX_LOCK(ha);
+
+        if (hw->txr_free <= QLA_TX_MIN_FREE) {
+		while (count--) {
+			qla_hw_tx_done_locked(ha);
+			if (hw->txr_free > QLA_TX_MIN_FREE)
+				break;
+
+			QLA_TX_UNLOCK(ha);
+			qla_mdelay(__func__, 10);
+			QLA_TX_LOCK(ha);
+		}
+        	if (hw->txr_free <= QLA_TX_MIN_FREE) {
+			QLA_TX_UNLOCK(ha);
+			device_printf(dev, "%s: xmit queue full\n", __func__);
+                	return (-1);
+		}
+        }
+        tx_cmd = &hw->tx_ring_base[hw->txr_next];
+
+        bzero((void *)tx_cmd, sizeof(q80_tx_cmd_t));
+
+	bcopy(fw_cmd, tx_cmd, size);
+
+	hw->txr_next = (hw->txr_next + 1) & (NUM_TX_DESCRIPTORS - 1);
+	hw->txr_free--;
+
+	QL_UPDATE_TX_PRODUCER_INDEX(ha, hw->txr_next);
+
+	QLA_TX_UNLOCK(ha);
+
+	return (0);
+}
+
+/*
+ * Name: qla_config_rss
+ * Function: Configure RSS for the context/interface.
+ */
+const uint64_t rss_key[] = { 0xbeac01fa6a42b73bULL, 0x8030f20c77cb2da3ULL,
+			0xae7b30b4d0ca2bcbULL, 0x43a38fb04167253dULL,
+			0x255b0ec26d5a56daULL };
+
+static int
+qla_config_rss(qla_host_t *ha, uint16_t cntxt_id)
+{
+	qla_fw_cds_config_rss_t rss_config;
+	int ret, i;
+
+	bzero(&rss_config, sizeof(qla_fw_cds_config_rss_t));
+
+	rss_config.hdr.cmd = Q8_FWCD_CNTRL_REQ;
+	rss_config.hdr.opcode = Q8_FWCD_OPCODE_CONFIG_RSS;
+	rss_config.hdr.cntxt_id = cntxt_id;
+
+	rss_config.hash_type = (Q8_FWCD_RSS_HASH_TYPE_IPV4_TCP_IP |
+					Q8_FWCD_RSS_HASH_TYPE_IPV6_TCP_IP);
+	rss_config.flags = Q8_FWCD_RSS_FLAGS_ENABLE_RSS;
+
+	rss_config.ind_tbl_mask = 0x7;
+	
+	for (i = 0; i < 5; i++)
+		rss_config.rss_key[i] = rss_key[i];
+
+	ret = qla_fw_cmd(ha, &rss_config, sizeof(qla_fw_cds_config_rss_t));
+
+	return ret;
+}
+
+/*
+ * Name: qla_config_intr_coalesce
+ * Function: Configure Interrupt Coalescing.
+ */
+static int
+qla_config_intr_coalesce(qla_host_t *ha, uint16_t cntxt_id, int tenable)
+{
+	qla_fw_cds_config_intr_coalesc_t intr_coalesce;
+	int ret;
+
+	bzero(&intr_coalesce, sizeof(qla_fw_cds_config_intr_coalesc_t));
+
+	intr_coalesce.hdr.cmd = Q8_FWCD_CNTRL_REQ;
+	intr_coalesce.hdr.opcode = Q8_FWCD_OPCODE_CONFIG_INTR_COALESCING;
+	intr_coalesce.hdr.cntxt_id = cntxt_id;
+	
+	intr_coalesce.flags = 0x04;
+	intr_coalesce.max_rcv_pkts = 256;
+	intr_coalesce.max_rcv_usecs = 3;
+	intr_coalesce.max_snd_pkts = 64;
+	intr_coalesce.max_snd_usecs = 4;
+
+	if (tenable) {
+		intr_coalesce.usecs_to = 1000; /* 1 millisecond */
+		intr_coalesce.timer_type = Q8_FWCMD_INTR_COALESC_TIMER_PERIODIC;
+		intr_coalesce.sds_ring_bitmask =
+			Q8_FWCMD_INTR_COALESC_SDS_RING_0;
+	}
+
+	ret = qla_fw_cmd(ha, &intr_coalesce,
+			sizeof(qla_fw_cds_config_intr_coalesc_t));
+
+	return ret;
+}
+
+
+/*
+ * Name: qla_config_mac_addr
+ * Function: binds a MAC address to the context/interface.
+ *	Can be unicast, multicast or broadcast.
+ */
+static int
+qla_config_mac_addr(qla_host_t *ha, uint8_t *mac_addr, uint16_t cntxt_id,
+	uint32_t add_multi)
+{
+	qla_fw_cds_config_mac_addr_t mac_config;
+	int ret;
+
+//	device_printf(ha->pci_dev,
+//		"%s: mac_addr %02x:%02x:%02x:%02x:%02x:%02x\n", __func__,
+//		mac_addr[0], mac_addr[1], mac_addr[2],
+//		mac_addr[3], mac_addr[4], mac_addr[5]);
+
+	bzero(&mac_config, sizeof(qla_fw_cds_config_mac_addr_t));
+
+	mac_config.hdr.cmd = Q8_FWCD_CNTRL_REQ;
+	mac_config.hdr.opcode = Q8_FWCD_OPCODE_CONFIG_MAC_ADDR;
+	mac_config.hdr.cntxt_id = cntxt_id;
+	
+	if (add_multi)
+		mac_config.cmd = Q8_FWCD_ADD_MAC_ADDR;
+	else
+		mac_config.cmd = Q8_FWCD_DEL_MAC_ADDR;
+	bcopy(mac_addr, mac_config.mac_addr,6); 
+
+	ret = qla_fw_cmd(ha, &mac_config, sizeof(qla_fw_cds_config_mac_addr_t));
+
+	return ret;
+}
+
+
+/*
+ * Name: qla_set_mac_rcv_mode
+ * Function: Enable/Disable AllMulticast and Promiscous Modes.
+ */
+static int
+qla_set_mac_rcv_mode(qla_host_t *ha, uint16_t cntxt_id, uint32_t mode)
+{
+	qla_set_mac_rcv_mode_t rcv_mode;
+	int ret;
+
+	bzero(&rcv_mode, sizeof(qla_set_mac_rcv_mode_t));
+
+	rcv_mode.hdr.cmd = Q8_FWCD_CNTRL_REQ;
+	rcv_mode.hdr.opcode = Q8_FWCD_OPCODE_CONFIG_MAC_RCV_MODE;
+	rcv_mode.hdr.cntxt_id = cntxt_id;
+	
+	rcv_mode.mode = mode;
+
+	ret = qla_fw_cmd(ha, &rcv_mode, sizeof(qla_set_mac_rcv_mode_t));
+
+	return ret;
+}
+
+void
+qla_set_promisc(qla_host_t *ha)
+{
+	(void)qla_set_mac_rcv_mode(ha,
+		(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id,
+		Q8_MAC_RCV_ENABLE_PROMISCUOUS);
+}
+
+void
+qla_set_allmulti(qla_host_t *ha)
+{
+	(void)qla_set_mac_rcv_mode(ha,
+		(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id,
+		Q8_MAC_RCV_ENABLE_ALLMULTI);
+}
+
+void
+qla_reset_promisc_allmulti(qla_host_t *ha)
+{
+	(void)qla_set_mac_rcv_mode(ha,
+		(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id,
+		Q8_MAC_RCV_RESET_PROMISC_ALLMULTI);
+}
+
+/*
+ * Name: qla_config_ipv4_addr
+ * Function: Configures the Destination IP Addr for LRO.
+ */
+void
+qla_config_ipv4_addr(qla_host_t *ha, uint32_t ipv4_addr)
+{
+	qla_config_ipv4_t ip_conf;
+
+	bzero(&ip_conf, sizeof(qla_config_ipv4_t));
+
+	ip_conf.hdr.cmd = Q8_FWCD_CNTRL_REQ;
+	ip_conf.hdr.opcode = Q8_FWCD_OPCODE_CONFIG_IPADDR;
+	ip_conf.hdr.cntxt_id = (ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id;
+	
+	ip_conf.cmd = (uint64_t)Q8_CONFIG_CMD_IP_ENABLE;
+	ip_conf.ipv4_addr = (uint64_t)ipv4_addr;
+
+	(void)qla_fw_cmd(ha, &ip_conf, sizeof(qla_config_ipv4_t));
+
+	return;
+}
+
+/*
+ * Name: qla_tx_tso
+ * Function: Checks if the packet to be transmitted is a candidate for
+ *	Large TCP Segment Offload. If yes, the appropriate fields in the Tx
+ *	Ring Structure are plugged in.
+ */
+static int
+qla_tx_tso(qla_host_t *ha, struct mbuf *mp, q80_tx_cmd_t *tx_cmd)
+{
+	struct ether_vlan_header *eh;
+	struct ip *ip = NULL;
+	struct tcphdr *th = NULL;
+	uint32_t ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
+	uint16_t etype, opcode, offload = 1;
+	device_t dev;
+
+	dev = ha->pci_dev;
+
+	if (mp->m_pkthdr.len <= ha->max_frame_size)
+		return (-1);
+
+	eh = mtod(mp, struct ether_vlan_header *);
+
+	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
+		ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
+		etype = ntohs(eh->evl_proto);
+	} else {
+		ehdrlen = ETHER_HDR_LEN;
+		etype = ntohs(eh->evl_encap_proto);
+	}
+
+	switch (etype) {
+		case ETHERTYPE_IP:
+			ip = (struct ip *)(mp->m_data + ehdrlen);
+			ip_hlen = ip->ip_hl << 2;
+			opcode = Q8_TX_CMD_OP_XMT_TCP_LSO;
+
+			if (ip->ip_p != IPPROTO_TCP) {
+				offload = 0;
+			} else
+				th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
+		break;
+
+		default:
+			QL_DPRINT8((dev, "%s: type!=ip\n", __func__));
+			offload = 0;
+		break;
+	}
+
+	if (!offload)
+		return (-1);
+
+	tcp_hlen = th->th_off << 2;
+
+	hdrlen = ehdrlen + ip_hlen + tcp_hlen;
+
+	if (mp->m_len < hdrlen) {
+		device_printf(dev, "%s: (mp->m_len < hdrlen)\n", __func__);
+		return (-1);
+	}
+
+	tx_cmd->flags_opcode = opcode ;
+	tx_cmd->tcp_hdr_off = ip_hlen + ehdrlen;
+	tx_cmd->ip_hdr_off = ehdrlen;
+	tx_cmd->mss = mp->m_pkthdr.tso_segsz;
+	tx_cmd->total_hdr_len = hdrlen;
+
+	/* Check for Multicast least significant bit of MSB == 1 */
+	if (eh->evl_dhost[0] & 0x01) {
+		tx_cmd->flags_opcode = Q8_TX_CMD_FLAGS_MULTICAST;
+	}
+
+	return (0);
+}
+
+/*
+ * Name: qla_tx_chksum
+ * Function: Checks if the packet to be transmitted is a candidate for
+ *	TCP/UDP Checksum offload. If yes, the appropriate fields in the Tx
+ *	Ring Structure are plugged in.
+ */
+static int
+qla_tx_chksum(qla_host_t *ha, struct mbuf *mp, q80_tx_cmd_t *tx_cmd)
+{
+	struct ether_vlan_header *eh;
+	struct ip *ip;
+	struct ip6_hdr *ip6;
+	uint32_t ehdrlen, ip_hlen;
+	uint16_t etype, opcode, offload = 1;
+	device_t dev;
+
+	dev = ha->pci_dev;
+
+	if ((mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_UDP)) == 0)
+		return (-1);
+
+	eh = mtod(mp, struct ether_vlan_header *);
+
+	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
+		ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
+		etype = ntohs(eh->evl_proto);
+	} else {
+		ehdrlen = ETHER_HDR_LEN;
+		etype = ntohs(eh->evl_encap_proto);
+	}
+
+		
+	switch (etype) {
+		case ETHERTYPE_IP:
+			ip = (struct ip *)(mp->m_data + ehdrlen);
+
+			ip_hlen = ip->ip_hl << 2;
+
+			if (mp->m_len < (ehdrlen + ip_hlen)) {
+				device_printf(dev, "%s: ipv4 mlen\n", __func__);
+				offload = 0;
+				break;
+			}
+
+			if (ip->ip_p == IPPROTO_TCP)
+				opcode = Q8_TX_CMD_OP_XMT_TCP_CHKSUM;
+			else if (ip->ip_p == IPPROTO_UDP)
+				opcode = Q8_TX_CMD_OP_XMT_UDP_CHKSUM;
+			else {
+				device_printf(dev, "%s: ipv4\n", __func__);
+				offload = 0;
+			}
+		break;
+
+		case ETHERTYPE_IPV6:
+			ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
+
+			ip_hlen = sizeof(struct ip6_hdr);
+
+			if (mp->m_len < (ehdrlen + ip_hlen)) {
+				device_printf(dev, "%s: ipv6 mlen\n", __func__);
+				offload = 0;
+				break;
+			}
+
+			if (ip6->ip6_nxt == IPPROTO_TCP)
+				opcode = Q8_TX_CMD_OP_XMT_TCP_CHKSUM_IPV6;
+			else if (ip6->ip6_nxt == IPPROTO_UDP)
+				opcode = Q8_TX_CMD_OP_XMT_UDP_CHKSUM_IPV6;
+			else {
+				device_printf(dev, "%s: ipv6\n", __func__);
+				offload = 0;
+			}
+		break;
+
+		default:
+			offload = 0;
+		break;
+	}
+	if (!offload)
+		return (-1);
+
+	tx_cmd->flags_opcode = opcode;
+
+	tx_cmd->tcp_hdr_off = ip_hlen + ehdrlen;
+
+	return (0);
+}
+
+/*
+ * Name: qla_hw_send
+ * Function: Transmits a packet. It first checks if the packet is a
+ *	candidate for Large TCP Segment Offload and then for UDP/TCP checksum
+ *	offload. If either of these creteria are not met, it is transmitted
+ *	as a regular ethernet frame.
+ */
+int
+qla_hw_send(qla_host_t *ha, bus_dma_segment_t *segs, int nsegs,
+	uint32_t *tx_idx,  struct mbuf *mp)
+{
+	struct ether_vlan_header *eh;
+	qla_hw_t *hw = &ha->hw;
+	q80_tx_cmd_t *tx_cmd, tso_cmd;
+	bus_dma_segment_t *c_seg;
+	uint32_t num_tx_cmds, hdr_len = 0;
+	uint32_t total_length = 0, bytes, tx_cmd_count = 0;
+	device_t dev;
+	int i;
+
+	dev = ha->pci_dev;
+
+	/*
+	 * Always make sure there is atleast one empty slot in the tx_ring
+	 * tx_ring is considered full when there only one entry available
+	 */
+        num_tx_cmds = (nsegs + (Q8_TX_CMD_MAX_SEGMENTS - 1)) >> 2;
+
+	total_length = mp->m_pkthdr.len;
+	if (total_length > QLA_MAX_TSO_FRAME_SIZE) {
+		device_printf(dev, "%s: total length exceeds maxlen(%d)\n",
+			__func__, total_length);
+		return (-1);
+	}
+
+	bzero((void *)&tso_cmd, sizeof(q80_tx_cmd_t));
+
+	if (qla_tx_tso(ha, mp, &tso_cmd) == 0) {
+		/* find the additional tx_cmd descriptors required */
+
+		hdr_len = tso_cmd.total_hdr_len;
+
+		bytes = sizeof(q80_tx_cmd_t) - Q8_TX_CMD_TSO_ALIGN;
+		bytes = QL_MIN(bytes, hdr_len);
+
+		num_tx_cmds++;
+		hdr_len -= bytes;
+
+		while (hdr_len) {
+			bytes = QL_MIN((sizeof(q80_tx_cmd_t)), hdr_len);
+			hdr_len -= bytes;
+			num_tx_cmds++;
+		}
+		hdr_len = tso_cmd.total_hdr_len;
+	}
+
+	if (hw->txr_free <= (num_tx_cmds + QLA_TX_MIN_FREE)) {
+		qla_hw_tx_done_locked(ha);
+		if (hw->txr_free <= (num_tx_cmds + QLA_TX_MIN_FREE)) {
+        		QL_DPRINT8((dev, "%s: (hw->txr_free <= "
+				"(num_tx_cmds + QLA_TX_MIN_FREE))\n",
+				__func__));
+			return (-1);
+		}
+	}
+
+	*tx_idx = hw->txr_next;
+
+        tx_cmd = &hw->tx_ring_base[hw->txr_next];
+
+	if (hdr_len == 0) {
+		if ((nsegs > Q8_TX_MAX_SEGMENTS) ||
+			(mp->m_pkthdr.len > ha->max_frame_size)){
+			/* TBD: copy into private buffer and send it */
+        		device_printf(dev,
+				"%s: (nsegs[%d, %d, 0x%x] > Q8_TX_MAX_SEGMENTS)\n",
+				__func__, nsegs, mp->m_pkthdr.len,
+				mp->m_pkthdr.csum_flags);
+			qla_dump_buf8(ha, "qla_hw_send: wrong pkt",
+				mtod(mp, char *), mp->m_len);
+			return (EINVAL);
+		}
+		bzero((void *)tx_cmd, sizeof(q80_tx_cmd_t));
+		if (qla_tx_chksum(ha, mp, tx_cmd) != 0) 
+        		tx_cmd->flags_opcode = Q8_TX_CMD_OP_XMT_ETHER;
+	} else {
+		bcopy(&tso_cmd, tx_cmd, sizeof(q80_tx_cmd_t));
+	}
+
+	eh = mtod(mp, struct ether_vlan_header *);
+	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
+        	tx_cmd->flags_opcode |= Q8_TX_CMD_FLAGS_VLAN_TAGGED;
+	else if (mp->m_flags & M_VLANTAG) {
+        	tx_cmd->flags_opcode |= (Q8_TX_CMD_FLAGS_VLAN_TAGGED |
+						Q8_TX_CMD_FLAGS_HW_VLAN_ID);
+		tx_cmd->vlan_tci = mp->m_pkthdr.ether_vtag;
+	}
+
+
+        tx_cmd->n_bufs = (uint8_t)nsegs;
+        tx_cmd->data_len_lo = (uint8_t)(total_length & 0xFF);
+        tx_cmd->data_len_hi = qla_host_to_le16(((uint16_t)(total_length >> 8)));
+	tx_cmd->port_cntxtid = Q8_TX_CMD_PORT_CNXTID(ha->pci_func);
+
+	c_seg = segs;
+
+	while (1) {
+		for (i = 0; ((i < Q8_TX_CMD_MAX_SEGMENTS) && nsegs); i++) {
+
+			switch (i) {
+			case 0:
+				tx_cmd->buf1_addr = c_seg->ds_addr;
+				tx_cmd->buf1_len = c_seg->ds_len;
+				break;
+
+			case 1:
+				tx_cmd->buf2_addr = c_seg->ds_addr;
+				tx_cmd->buf2_len = c_seg->ds_len;
+				break;
+
+			case 2:
+				tx_cmd->buf3_addr = c_seg->ds_addr;
+				tx_cmd->buf3_len = c_seg->ds_len;
+				break;
+
+			case 3:
+				tx_cmd->buf4_addr = c_seg->ds_addr;
+				tx_cmd->buf4_len = c_seg->ds_len;
+				break;
+			}
+
+			c_seg++;
+			nsegs--;
+		}
+
+		hw->txr_next = (hw->txr_next + 1) & (NUM_TX_DESCRIPTORS - 1);
+		tx_cmd_count++;
+
+		if (!nsegs)
+			break;
+		
+        	tx_cmd = &hw->tx_ring_base[hw->txr_next];
+		bzero((void *)tx_cmd, sizeof(q80_tx_cmd_t));
+	}
+
+	if (hdr_len) {
+		/* TSO : Copy the header in the following tx cmd descriptors */
+		uint8_t *src, *dst;
+
+		src = (uint8_t *)eh;
+
+		tx_cmd = &hw->tx_ring_base[hw->txr_next];
+		bzero((void *)tx_cmd, sizeof(q80_tx_cmd_t));
+
+		bytes = sizeof(q80_tx_cmd_t) - Q8_TX_CMD_TSO_ALIGN;
+		bytes = QL_MIN(bytes, hdr_len);
+
+		dst = (uint8_t *)tx_cmd + Q8_TX_CMD_TSO_ALIGN;
+
+		if (mp->m_flags & M_VLANTAG) {
+			/* first copy the src/dst MAC addresses */
+			bcopy(src, dst, (ETHER_ADDR_LEN * 2));
+			dst += (ETHER_ADDR_LEN * 2);
+			src += (ETHER_ADDR_LEN * 2);
+			
+			hdr_len -= (ETHER_ADDR_LEN * 2);
+
+			*((uint16_t *)dst) = htons(ETHERTYPE_VLAN);
+			dst += 2;
+			*((uint16_t *)dst) = mp->m_pkthdr.ether_vtag;
+			dst += 2;
+
+			bytes -= ((ETHER_ADDR_LEN * 2) + 4);
+
+			bcopy(src, dst, bytes);
+			src += bytes;
+			hdr_len -= bytes;
+		} else {
+			bcopy(src, dst, bytes);
+			src += bytes;
+			hdr_len -= bytes;
+		}
+
+		hw->txr_next = (hw->txr_next + 1) & (NUM_TX_DESCRIPTORS - 1);
+		tx_cmd_count++;
+		
+		while (hdr_len) {
+			tx_cmd = &hw->tx_ring_base[hw->txr_next];
+			bzero((void *)tx_cmd, sizeof(q80_tx_cmd_t));
+
+			bytes = QL_MIN((sizeof(q80_tx_cmd_t)), hdr_len);
+
+			bcopy(src, tx_cmd, bytes);
+			src += bytes;
+			hdr_len -= bytes;
+			hw->txr_next =
+				(hw->txr_next + 1) & (NUM_TX_DESCRIPTORS - 1);
+			tx_cmd_count++;
+		}
+	}
+
+	hw->txr_free = hw->txr_free - tx_cmd_count;
+
+	QL_UPDATE_TX_PRODUCER_INDEX(ha, hw->txr_next);
+       	QL_DPRINT8((dev, "%s: return\n", __func__));
+	return (0);
+}
+
+/*
+ * Name: qla_del_hw_if
+ * Function: Destroys the hardware specific entities corresponding to an
+ *	Ethernet Interface
+ */
+void
+qla_del_hw_if(qla_host_t *ha)
+{
+	int	i;
+
+	for (i = 0; i < ha->hw.num_sds_rings; i++)
+		QL_DISABLE_INTERRUPTS(ha, i);
+	
+	qla_del_rcv_cntxt(ha);
+	qla_del_xmt_cntxt(ha);
+	
+	ha->hw.flags.lro = 0;
+}
+
+/*
+ * Name: qla_init_hw_if
+ * Function: Creates the hardware specific entities corresponding to an
+ *	Ethernet Interface - Transmit and Receive Contexts. Sets the MAC Address
+ *	corresponding to the interface. Enables LRO if allowed.
+ */
+int
+qla_init_hw_if(qla_host_t *ha)
+{
+	device_t	dev;
+	int		i;
+	uint8_t		bcast_mac[6];
+
+	qla_get_hw_caps(ha);
+
+	dev = ha->pci_dev;
+
+	for (i = 0; i < ha->hw.num_sds_rings; i++) {
+		bzero(ha->hw.dma_buf.sds_ring[i].dma_b,
+			ha->hw.dma_buf.sds_ring[i].size);
+	}
+	/*
+	 * Create Receive Context
+	 */
+	if (qla_init_rcv_cntxt(ha)) {
+		return (-1);
+	}
+
+	ha->hw.rx_next = NUM_RX_DESCRIPTORS - 2;
+	ha->hw.rxj_next = NUM_RX_JUMBO_DESCRIPTORS - 2;
+	ha->hw.rx_in = ha->hw.rxj_in = 0;
+
+	/* Update the RDS Producer Indices */
+	QL_UPDATE_RDS_PRODUCER_INDEX(ha, 0, ha->hw.rx_next);
+	QL_UPDATE_RDS_PRODUCER_INDEX(ha, 1, ha->hw.rxj_next);
+
+	/*
+	 * Create Transmit Context
+	 */
+	if (qla_init_xmt_cntxt(ha)) {
+		qla_del_rcv_cntxt(ha);
+		return (-1);
+	}
+
+	qla_config_mac_addr(ha, ha->hw.mac_addr,
+		(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id, 1);
+
+	bcast_mac[0] = 0xFF; bcast_mac[1] = 0xFF; bcast_mac[2] = 0xFF;
+	bcast_mac[3] = 0xFF; bcast_mac[4] = 0xFF; bcast_mac[5] = 0xFF;
+	qla_config_mac_addr(ha, bcast_mac,
+		(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id, 1);
+
+	qla_config_rss(ha, (ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id);
+
+	qla_config_intr_coalesce(ha, (ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id, 0);
+
+	for (i = 0; i < ha->hw.num_sds_rings; i++)
+		QL_ENABLE_INTERRUPTS(ha, i);
+
+	return (0);
+}
+
+/*
+ * Name: qla_init_rcv_cntxt
+ * Function: Creates the Receive Context.
+ */
+static int
+qla_init_rcv_cntxt(qla_host_t *ha)
+{
+	device_t		dev;
+	qla_cdrp_t		cdrp;
+	q80_rcv_cntxt_rsp_t	*rsp;
+	q80_stat_desc_t		*sdesc;
+	bus_addr_t		phys_addr;
+	int			i, j;
+        qla_hw_t		*hw = &ha->hw;
+
+	dev = ha->pci_dev;
+
+	/*
+	 * Create Receive Context
+	 */
+
+	for (i = 0; i < hw->num_sds_rings; i++) {
+		sdesc = (q80_stat_desc_t *)&hw->sds[i].sds_ring_base[0];
+		for (j = 0; j < NUM_STATUS_DESCRIPTORS; j++) {
+			sdesc->data[0] =
+				Q8_STAT_DESC_SET_OWNER(Q8_STAT_DESC_OWNER_FW);
+		}
+	}
+
+	phys_addr = ha->hw.rx_cntxt_req_paddr;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_CREATE_RX_CNTXT;
+	cdrp.cmd_arg1 = (uint32_t)(phys_addr >> 32);
+	cdrp.cmd_arg2 = (uint32_t)(phys_addr);
+	cdrp.cmd_arg3 = (uint32_t)(sizeof (q80_rcv_cntxt_req_t));
+	
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_CREATE_RX_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		rsp = ha->hw.rx_cntxt_rsp;
+
+		QL_DPRINT2((dev, "%s: rcv cntxt successful"
+			" rds_ring_offset = 0x%08x"
+			" sds_ring_offset = 0x%08x"
+			" cntxt_state = 0x%08x"
+			" funcs_per_port = 0x%08x"
+			" num_rds_rings = 0x%04x"
+			" num_sds_rings = 0x%04x"
+			" cntxt_id = 0x%04x"
+			" phys_port = 0x%02x"
+			" virt_port = 0x%02x\n",
+			__func__,
+			rsp->rx_rsp.rds_ring_offset,
+			rsp->rx_rsp.sds_ring_offset,
+			rsp->rx_rsp.cntxt_state,
+			rsp->rx_rsp.funcs_per_port,
+			rsp->rx_rsp.num_rds_rings,
+			rsp->rx_rsp.num_sds_rings,
+			rsp->rx_rsp.cntxt_id,
+			rsp->rx_rsp.phys_port,
+			rsp->rx_rsp.virt_port));
+
+		for (i = 0; i < ha->hw.num_rds_rings; i++) {
+			QL_DPRINT2((dev,
+				"%s: rcv cntxt rds[%i].producer_reg = 0x%08x\n",
+				__func__, i, rsp->rds_rsp[i].producer_reg));
+		}
+		for (i = 0; i < ha->hw.num_sds_rings; i++) {
+			QL_DPRINT2((dev,
+				"%s: rcv cntxt sds[%i].consumer_reg = 0x%08x"
+				" sds[%i].intr_mask_reg = 0x%08x\n",
+				__func__, i, rsp->sds_rsp[i].consumer_reg,
+				i, rsp->sds_rsp[i].intr_mask_reg));
+		}
+	}
+	ha->hw.flags.init_rx_cnxt = 1;
+	return (0);
+}
+
+/*
+ * Name: qla_del_rcv_cntxt
+ * Function: Destroys the Receive Context.
+ */
+void
+qla_del_rcv_cntxt(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev = ha->pci_dev;
+
+	if (!ha->hw.flags.init_rx_cnxt)
+		return;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_DESTROY_RX_CNTXT;
+	cdrp.cmd_arg1 = (uint32_t) (ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_DESTROY_RX_CNTXT failed\n",
+			__func__);
+	}
+	ha->hw.flags.init_rx_cnxt = 0;
+}
+
+/*
+ * Name: qla_init_xmt_cntxt
+ * Function: Creates the Transmit Context.
+ */
+static int
+qla_init_xmt_cntxt(qla_host_t *ha)
+{
+	bus_addr_t		phys_addr;
+	device_t		dev;
+	q80_tx_cntxt_rsp_t	*tx_rsp;
+	qla_cdrp_t		cdrp;
+        qla_hw_t		*hw = &ha->hw;
+
+	dev = ha->pci_dev;
+
+	/*
+	 * Create Transmit Context
+	 */
+	phys_addr = ha->hw.tx_cntxt_req_paddr;
+	tx_rsp = ha->hw.tx_cntxt_rsp;
+
+	hw->txr_comp = hw->txr_next = 0;
+	*(hw->tx_cons) = 0;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_CREATE_TX_CNTXT;
+	cdrp.cmd_arg1 = (uint32_t)(phys_addr >> 32);
+	cdrp.cmd_arg2 = (uint32_t)(phys_addr);
+	cdrp.cmd_arg3 = (uint32_t)(sizeof (q80_tx_cntxt_req_t));
+	
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_CREATE_TX_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		ha->hw.tx_prod_reg = tx_rsp->producer_reg;
+
+		QL_DPRINT2((dev, "%s: tx cntxt successful"
+			" cntxt_state = 0x%08x "
+			" cntxt_id = 0x%04x "
+			" phys_port_id = 0x%02x "
+			" virt_port_id = 0x%02x "
+			" producer_reg = 0x%08x "
+			" intr_mask_reg = 0x%08x\n",
+			__func__, tx_rsp->cntxt_state, tx_rsp->cntxt_id,
+			tx_rsp->phys_port_id, tx_rsp->virt_port_id,
+			tx_rsp->producer_reg, tx_rsp->intr_mask_reg));
+	}
+	ha->hw.txr_free = NUM_TX_DESCRIPTORS;
+
+	ha->hw.flags.init_tx_cnxt = 1;
+	return (0);
+}
+
+/*
+ * Name: qla_del_xmt_cntxt
+ * Function: Destroys the Transmit Context.
+ */
+static void
+qla_del_xmt_cntxt(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev = ha->pci_dev;
+
+	if (!ha->hw.flags.init_tx_cnxt)
+		return;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_DESTROY_TX_CNTXT;
+	cdrp.cmd_arg1 = (uint32_t) (ha->hw.tx_cntxt_rsp)->cntxt_id;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_DESTROY_TX_CNTXT failed\n",
+			__func__);
+	}
+	ha->hw.flags.init_tx_cnxt = 0;
+}
+
+/*
+ * Name: qla_get_max_rds
+ * Function: Returns the maximum number of Receive Descriptor Rings per context.
+ */
+static int
+qla_get_max_rds(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_RDS_PER_CNTXT;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_RDS_PER_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		ha->hw.max_rds_per_cntxt = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_rds_per_context 0x%08x\n",
+			__func__, ha->hw.max_rds_per_cntxt));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_max_sds
+ * Function: Returns the maximum number of Status Descriptor Rings per context.
+ */
+static int
+qla_get_max_sds(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_SDS_PER_CNTXT;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_RDS_PER_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		ha->hw.max_sds_per_cntxt = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_sds_per_context 0x%08x\n",
+			__func__, ha->hw.max_sds_per_cntxt));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_max_rules
+ * Function: Returns the maximum number of Rules per context.
+ */
+static int
+qla_get_max_rules(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_RULES_PER_CNTXT;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_RULES_PER_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		ha->hw.max_rules_per_cntxt = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_rules_per_cntxt 0x%08x\n",
+			__func__, ha->hw.max_rules_per_cntxt));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_max_rcv_cntxts
+ * Function: Returns the maximum number of Receive Contexts supported.
+ */
+static int
+qla_get_max_rcv_cntxts(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_RX_CNTXT;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_RX_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		ha->hw.max_rcv_cntxts = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_rcv_cntxts 0x%08x\n",
+			__func__, ha->hw.max_rcv_cntxts));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_max_tx_cntxts
+ * Function: Returns the maximum number of Transmit Contexts supported.
+ */
+static int
+qla_get_max_tx_cntxts(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_TX_CNTXT;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_TX_CNTXT failed\n",
+			__func__);
+		return (-1);
+	} else {
+		ha->hw.max_xmt_cntxts = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_xmt_cntxts 0x%08x\n",
+			__func__, ha->hw.max_xmt_cntxts));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_max_mtu
+ * Function: Returns the MTU supported for a context.
+ */
+static int
+qla_get_max_mtu(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_MTU;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_MTU failed\n", __func__);
+		return (-1);
+	} else {
+		ha->hw.max_mtu = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_mtu 0x%08x\n", __func__,
+			ha->hw.max_mtu));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_set_max_mtu
+ * Function:
+ *	Sets the maximum transfer unit size for the specified rcv context.
+ */
+int
+qla_set_max_mtu(qla_host_t *ha, uint32_t mtu, uint16_t cntxt_id)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_SET_MTU;
+	cdrp.cmd_arg1 = (uint32_t)cntxt_id;
+	cdrp.cmd_arg2 = mtu;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_MTU failed\n", __func__);
+		return (-1);
+	} else {
+		ha->hw.max_mtu = cdrp.rsp_arg1;
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_max_lro
+ * Function: Returns the maximum number of TCP Connection which can be supported
+ *	with LRO.
+ */
+static int
+qla_get_max_lro(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_RD_MAX_LRO;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_RD_MAX_LRO failed\n", __func__);
+		return (-1);
+	} else {
+		ha->hw.max_lro = cdrp.rsp_arg1;
+		QL_DPRINT2((dev, "%s: max_lro 0x%08x\n", __func__,
+			ha->hw.max_lro));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_flow_control
+ * Function: Returns the Receive/Transmit Flow Control (PAUSE) settings for
+ *	PCI function.
+ */
+static int
+qla_get_flow_control(qla_host_t *ha)
+{
+	qla_cdrp_t	cdrp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	bzero(&cdrp, sizeof(qla_cdrp_t));
+
+	cdrp.cmd = Q8_CMD_GET_FLOW_CNTRL;
+
+	if (qla_issue_cmd(ha, &cdrp)) {
+		device_printf(dev, "%s: Q8_CMD_GET_FLOW_CNTRL failed\n",
+			__func__);
+		return (-1);
+	} else {
+		QL_DPRINT2((dev, "%s: flow control 0x%08x\n", __func__,
+			cdrp.rsp_arg1));
+	}
+	return 0;
+}
+
+/*
+ * Name: qla_get_flow_control
+ * Function: Retrieves hardware capabilities
+ */
+void
+qla_get_hw_caps(qla_host_t *ha)
+{
+	//qla_read_mac_addr(ha);
+	qla_get_max_rds(ha);
+	qla_get_max_sds(ha);
+	qla_get_max_rules(ha);
+	qla_get_max_rcv_cntxts(ha);
+	qla_get_max_tx_cntxts(ha);
+	qla_get_max_mtu(ha);
+	qla_get_max_lro(ha);
+	qla_get_flow_control(ha);
+	return;
+}
+
+/*
+ * Name: qla_hw_set_multi
+ * Function: Sets the Multicast Addresses provided the host O.S into the
+ *	hardware (for the given interface)
+ */
+void
+qla_hw_set_multi(qla_host_t *ha, uint8_t *mta, uint32_t mcnt,
+	uint32_t add_multi)
+{
+	q80_rcv_cntxt_rsp_t	*rsp;
+	int i;
+
+	rsp = ha->hw.rx_cntxt_rsp;
+	for (i = 0; i < mcnt; i++) {
+		qla_config_mac_addr(ha, mta, rsp->rx_rsp.cntxt_id, add_multi);
+		mta += Q8_MAC_ADDR_LEN;
+	}
+	return;
+}
+
+/*
+ * Name: qla_hw_tx_done_locked
+ * Function: Handle Transmit Completions
+ */
+static void
+qla_hw_tx_done_locked(qla_host_t *ha)
+{
+	qla_tx_buf_t *txb;
+        qla_hw_t *hw = &ha->hw;
+	uint32_t comp_idx, comp_count = 0;
+
+	/* retrieve index of last entry in tx ring completed */
+	comp_idx = qla_le32_to_host(*(hw->tx_cons));
+
+	while (comp_idx != hw->txr_comp) {
+
+		txb = &ha->tx_buf[hw->txr_comp];
+
+		hw->txr_comp++;
+		if (hw->txr_comp == NUM_TX_DESCRIPTORS)
+			hw->txr_comp = 0;
+
+		comp_count++;
+
+		if (txb->m_head) {
+			bus_dmamap_sync(ha->tx_tag, txb->map,
+				BUS_DMASYNC_POSTWRITE);
+			bus_dmamap_unload(ha->tx_tag, txb->map);
+			bus_dmamap_destroy(ha->tx_tag, txb->map);
+			m_freem(txb->m_head);
+
+			txb->map = (bus_dmamap_t)0;
+			txb->m_head = NULL;
+		}
+	}
+
+	hw->txr_free += comp_count;
+
+       	QL_DPRINT8((ha->pci_dev, "%s: return [c,f, p, pn][%d, %d, %d, %d]\n", __func__,
+		hw->txr_comp, hw->txr_free, hw->txr_next, READ_REG32(ha, (ha->hw.tx_prod_reg + 0x1b2000))));
+
+	return;
+}
+
+/*
+ * Name: qla_hw_tx_done
+ * Function: Handle Transmit Completions
+ */
+void
+qla_hw_tx_done(qla_host_t *ha)
+{
+	if (!mtx_trylock(&ha->tx_lock)) {
+       		QL_DPRINT8((ha->pci_dev,
+			"%s: !mtx_trylock(&ha->tx_lock)\n", __func__));
+		return;
+	}
+	qla_hw_tx_done_locked(ha);
+
+	if (ha->hw.txr_free > free_pkt_thres)
+		ha->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+
+	mtx_unlock(&ha->tx_lock);
+	return;
+}
+
+void
+qla_update_link_state(qla_host_t *ha)
+{
+	uint32_t link_state;
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+		ha->hw.flags.link_up = 0;
+		return;
+	}
+	link_state = READ_REG32(ha, Q8_LINK_STATE);
+
+	if (ha->pci_func == 0) 
+		ha->hw.flags.link_up = (((link_state & 0xF) == 1)? 1 : 0);
+	else
+		ha->hw.flags.link_up = ((((link_state >> 4)& 0xF) == 1)? 1 : 0);
+}
+
+int
+qla_config_lro(qla_host_t *ha)
+{
+	int i;
+        qla_hw_t *hw = &ha->hw;
+	struct lro_ctrl *lro;
+
+	for (i = 0; i < hw->num_sds_rings; i++) {
+		lro = &hw->sds[i].lro;
+		if (tcp_lro_init(lro)) {
+			device_printf(ha->pci_dev, "%s: tcp_lro_init failed\n",
+				__func__);
+			return (-1);
+		}
+		lro->ifp = ha->ifp;
+	}
+	ha->flags.lro_init = 1;
+
+	QL_DPRINT2((ha->pci_dev, "%s: LRO initialized\n", __func__));
+	return (0);
+}
+
+void
+qla_free_lro(qla_host_t *ha)
+{
+	int i;
+        qla_hw_t *hw = &ha->hw;
+	struct lro_ctrl *lro;
+
+	if (!ha->flags.lro_init)
+		return;
+
+	for (i = 0; i < hw->num_sds_rings; i++) {
+		lro = &hw->sds[i].lro;
+		tcp_lro_free(lro);
+	}
+	ha->flags.lro_init = 0;
+}
+
+void
+qla_hw_stop_rcv(qla_host_t *ha)
+{
+	int i, done, count = 100;
+
+	while (count--) {
+		done = 1;
+		for (i = 0; i < ha->hw.num_sds_rings; i++) {
+			if (ha->hw.sds[i].rcv_active)
+				done = 0;
+		}
+		if (done)
+			break;
+		else 
+			qla_mdelay(__func__, 10);
+	}
+}
+
diff --git a/sys/dev/qlxgb/qla_hw.h b/sys/dev/qlxgb/qla_hw.h
new file mode 100644
index 0000000..46780be
--- /dev/null
+++ b/sys/dev/qlxgb/qla_hw.h
@@ -0,0 +1,831 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_hw.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+#ifndef _QLA_HW_H_
+#define _QLA_HW_H_
+
+#define Q8_MAX_NUM_MULTICAST_ADDRS	128
+#define Q8_MAC_ADDR_LEN			6
+
+/*
+ * Firmware Interface
+ */
+
+/*
+ * Command Response Interface - Commands
+ */
+typedef struct qla_cdrp {
+	uint32_t cmd;
+	uint32_t cmd_arg1;
+	uint32_t cmd_arg2;
+	uint32_t cmd_arg3;
+	uint32_t rsp;
+	uint32_t rsp_arg1;
+	uint32_t rsp_arg2;
+	uint32_t rsp_arg3;
+} qla_cdrp_t;
+ 
+#define Q8_CMD_RD_MAX_RDS_PER_CNTXT	0x80000002
+#define Q8_CMD_RD_MAX_SDS_PER_CNTXT	0x80000003
+#define Q8_CMD_RD_MAX_RULES_PER_CNTXT	0x80000004
+#define Q8_CMD_RD_MAX_RX_CNTXT		0x80000005
+#define Q8_CMD_RD_MAX_TX_CNTXT		0x80000006
+#define Q8_CMD_CREATE_RX_CNTXT		0x80000007
+#define Q8_CMD_DESTROY_RX_CNTXT		0x80000008
+#define Q8_CMD_CREATE_TX_CNTXT		0x80000009
+#define Q8_CMD_DESTROY_TX_CNTXT		0x8000000A
+#define Q8_CMD_SETUP_STATS		0x8000000E
+#define Q8_CMD_GET_STATS		0x8000000F
+#define Q8_CMD_DELETE_STATS		0x80000010
+#define Q8_CMD_GEN_INT			0x80000011
+#define Q8_CMD_SET_MTU			0x80000012
+#define Q8_CMD_GET_FLOW_CNTRL		0x80000016
+#define Q8_CMD_SET_FLOW_CNTRL		0x80000017
+#define Q8_CMD_RD_MAX_MTU		0x80000018
+#define Q8_CMD_RD_MAX_LRO		0x80000019
+
+/*
+ * Command Response Interface - Response
+ */
+#define Q8_RSP_SUCCESS			0x00000000
+#define Q8_RSP_NO_HOST_MEM		0x00000001
+#define Q8_RSP_NO_HOST_RSRC		0x00000002
+#define Q8_RSP_NO_CARD_CRB		0x00000003
+#define Q8_RSP_NO_CARD_MEM		0x00000004
+#define Q8_RSP_NO_CARD_RSRC		0x00000005
+#define Q8_RSP_INVALID_ARGS		0x00000006
+#define Q8_RSP_INVALID_ACTION		0x00000007
+#define Q8_RSP_INVALID_STATE		0x00000008
+#define Q8_RSP_NOT_SUPPORTED		0x00000009
+#define Q8_RSP_NOT_PERMITTED		0x0000000A
+#define Q8_RSP_NOT_READY		0x0000000B
+#define Q8_RSP_DOES_NOT_EXIST		0x0000000C
+#define Q8_RSP_ALREADY_EXISTS		0x0000000D
+#define Q8_RSP_BAD_SIGNATURE		0x0000000E
+#define Q8_RSP_CMD_NOT_IMPLEMENTED	0x0000000F
+#define Q8_RSP_CMD_INVALID		0x00000010
+#define Q8_RSP_TIMEOUT			0x00000011
+
+
+/*
+ * Transmit Related Definitions
+ */
+
+/*
+ * Transmit Context - Q8_CMD_CREATE_TX_CNTXT Command Configuration Data
+ */
+
+typedef struct _q80_tx_cntxt_req {
+	uint64_t rsp_dma_addr;		/* rsp from firmware is DMA'ed here */
+	uint64_t cmd_cons_dma_addr;
+	uint64_t rsrvd0;
+
+	uint32_t caps[4];		/* capabilities  - bit vector*/
+#define CNTXT_CAP0_BASEFW		0x0001
+#define CNTXT_CAP0_LEGACY_MN		0x0004
+#define CNTXT_CAP0_LSO			0x0040
+
+	uint32_t intr_mode;		/* Interrupt Mode */
+#define CNTXT_INTR_MODE_UNIQUE	0x0000
+#define CNTXT_INTR_MODE_SHARED	0x0001
+
+	uint64_t rsrvd1;
+	uint16_t msi_index;
+	uint16_t rsrvd2;
+	uint64_t phys_addr;		/* physical address of transmit ring
+					 * in system memory */
+	uint32_t num_entries;		/* number of entries in transmit ring */
+	uint8_t rsrvd3[128];
+} __packed q80_tx_cntxt_req_t; /* 188 bytes total */
+	
+
+/*
+ * Transmit Context - Response from Firmware to Q8_CMD_CREATE_TX_CNTXT
+ */
+
+typedef struct _q80_tx_cntxt_rsp {
+	uint32_t cntxt_state;	/* starting state */
+#define CNTXT_STATE_ALLOCATED_NOT_ACTIVE	0x0001
+#define CNTXT_STATE_ACTIVE			0x0002
+#define CNTXT_STATE_QUIESCED			0x0004
+
+	uint16_t cntxt_id;	/* handle for context */
+	uint8_t phys_port_id;	/* physical id of port */
+	uint8_t virt_port_id;	/* virtual or logical id of port */
+	uint32_t producer_reg;	/* producer register for transmit ring */
+	uint32_t intr_mask_reg;	/* interrupt mask register */
+	uint8_t rsrvd[128];
+} __packed q80_tx_cntxt_rsp_t; /* 144 bytes */
+
+/*
+ * Transmit Command Descriptor
+ * These commands are issued on the Transmit Ring associated with a Transmit
+ * context
+ */
+typedef struct _q80_tx_cmd {
+	uint8_t		tcp_hdr_off;	/* TCP Header Offset */
+	uint8_t		ip_hdr_off;	/* IP Header Offset */
+	uint16_t	flags_opcode;	/* Bits 0-6: flags; 7-12: opcode */
+
+	/* flags field */
+#define Q8_TX_CMD_FLAGS_MULTICAST	0x01
+#define Q8_TX_CMD_FLAGS_LSO_TSO		0x02
+#define Q8_TX_CMD_FLAGS_VLAN_TAGGED	0x10
+#define Q8_TX_CMD_FLAGS_HW_VLAN_ID	0x40
+
+	/* opcode field */
+#define Q8_TX_CMD_OP_XMT_UDP_CHKSUM_IPV6	(0xC << 7)
+#define Q8_TX_CMD_OP_XMT_TCP_CHKSUM_IPV6	(0xB << 7)
+#define Q8_TX_CMD_OP_XMT_TCP_LSO_IPV6		(0x6 << 7)
+#define Q8_TX_CMD_OP_XMT_TCP_LSO		(0x5 << 7)
+#define Q8_TX_CMD_OP_XMT_UDP_CHKSUM		(0x3 << 7)
+#define Q8_TX_CMD_OP_XMT_TCP_CHKSUM		(0x2 << 7)
+#define Q8_TX_CMD_OP_XMT_ETHER			(0x1 << 7)
+
+	uint8_t		n_bufs;		/* # of data segs in data buffer */
+	uint8_t		data_len_lo;	/* data length lower 8 bits */
+	uint16_t	data_len_hi;	/* data length upper 16 bits */
+
+	uint64_t	buf2_addr;	/* buffer 2 address */
+
+	uint16_t	rsrvd0;
+	uint16_t	mss;		/* MSS for this packet */
+	uint8_t		port_cntxtid;	/* Bits 7-4: ContextId; 3-0: reserved */
+
+#define Q8_TX_CMD_PORT_CNXTID(c_id) ((c_id & 0xF) << 4)
+
+	uint8_t		total_hdr_len;	/* MAC+IP+TCP Header Length for LSO */
+	uint16_t	rsrvd1;
+
+	uint64_t	buf3_addr;	/* buffer 3 address */
+	uint64_t	buf1_addr;	/* buffer 1 address */
+
+	uint16_t	buf1_len;	/* length of buffer 1 */
+	uint16_t	buf2_len;	/* length of buffer 2 */
+	uint16_t	buf3_len;	/* length of buffer 3 */
+	uint16_t	buf4_len;	/* length of buffer 4 */
+
+	uint64_t	buf4_addr;	/* buffer 4 address */
+
+	uint32_t	rsrvd2;
+	uint16_t	rsrvd3;
+	uint16_t	vlan_tci;	/* VLAN TCI when hw tagging is enabled*/
+
+} __packed q80_tx_cmd_t; /* 64 bytes */
+
+#define Q8_TX_CMD_MAX_SEGMENTS	4
+#define Q8_TX_CMD_TSO_ALIGN	2
+#define Q8_TX_MAX_SEGMENTS	14
+
+
+/*
+ * Receive Related Definitions
+ */
+/*
+ * Receive Context - Q8_CMD_CREATE_RX_CNTXT Command Configuration Data
+ */
+
+typedef struct _q80_rq_sds_ring {
+	uint64_t phys_addr; /* physical addr of status ring in system memory */
+	uint32_t size; /* number of entries in status ring */
+	uint16_t msi_index;
+	uint16_t rsrvd;
+} __packed q80_rq_sds_ring_t; /* 16 bytes */
+
+typedef struct _q80_rq_rds_ring {
+	uint64_t phys_addr;	/* physical addr of rcv ring in system memory */
+	uint64_t buf_size;	/* packet buffer size */
+	uint32_t size;		/* number of entries in ring */
+	uint32_t rsrvd;
+} __packed q80_rq_rds_ring_t; /* 24 bytes */
+
+typedef struct _q80_rq_rcv_cntxt {
+	uint64_t rsp_dma_addr;	/* rsp from firmware is DMA'ed here */
+	uint32_t caps[4];	/* bit vector */
+#define CNTXT_CAP0_JUMBO		0x0080 /* Contiguous Jumbo buffers*/
+#define CNTXT_CAP0_LRO			0x0100
+#define CNTXT_CAP0_HW_LRO		0x0800 /* HW LRO */
+
+	uint32_t intr_mode;	/* same as q80_tx_cntxt_req_t */
+	uint32_t rds_intr_mode; /* same as q80_tx_cntxt_req_t */
+
+	uint32_t rds_ring_offset; /* rds configuration relative to data[0] */
+	uint32_t sds_ring_offset; /* sds configuration relative to data[0] */
+
+	uint16_t num_rds_rings;
+	uint16_t num_sds_rings;
+
+	uint8_t rsrvd1[132];
+} __packed q80_rq_rcv_cntxt_t; /* 176 bytes header + rds + sds ring rqsts */
+
+/*
+ * Receive Context - Response from Firmware to Q8_CMD_CREATE_RX_CNTXT
+ */
+
+typedef struct _q80_rsp_rds_ring {
+	uint32_t producer_reg;
+	uint32_t rsrvd;
+} __packed q80_rsp_rds_ring_t; /* 8 bytes */
+
+typedef struct _q80_rsp_sds_ring {
+	uint32_t consumer_reg;
+	uint32_t intr_mask_reg;
+} __packed q80_rsp_sds_ring_t; /* 8 bytes */
+
+typedef struct _q80_rsp_rcv_cntxt {
+	uint32_t rds_ring_offset; /* rds configuration relative to data[0] */
+	uint32_t sds_ring_offset; /* sds configuration relative to data[0] */
+
+	uint32_t cntxt_state; /* starting state */
+	uint32_t funcs_per_port; /* number of PCI functions sharing each port */
+
+	uint16_t num_rds_rings;
+	uint16_t num_sds_rings;
+
+	uint16_t cntxt_id; /* handle for context */
+
+	uint8_t phys_port; /* physical id of port */
+	uint8_t virt_port; /* virtual or logical id of port */
+
+	uint8_t rsrvd[128];
+	uint8_t data[0];
+} __packed q80_rsp_rcv_cntxt_t; /* 152 bytes header + rds + sds ring rspncs */
+
+
+/*
+ * Note:
+ *	Transmit Context
+ *	188 (rq) + 144 (rsp) = 332 bytes are required
+ *	
+ *	Receive Context
+ *	1 RDS and 1 SDS rings: (16+24+176)+(8+8+152) = 384 bytes
+ *
+ *	3 RDS and 4 SDS rings: (((16+24)*3)+176) + (((8+8)*4)+152) =
+ *				= 296 + 216 = 512 bytes
+ *	Clearly this within the minimum PAGE size of most O.S platforms
+ *	(typically 4Kbytes). Hence it is simpler to simply allocate one PAGE
+ *	and then carve out space for each context. It is also a good idea to
+ * 	to throw in the shadown register for the consumer index of the transmit
+ *	ring in this PAGE.
+ */
+
+/*
+ * Receive Descriptor corresponding to each entry in the receive ring
+ */
+typedef struct _q80_rcv_desc {
+	uint16_t handle;
+	uint16_t rsrvd;
+	uint32_t buf_size; /* buffer size in bytes */
+	uint64_t buf_addr; /* physical address of buffer */
+} __packed q80_recv_desc_t;
+
+/*
+ * Status Descriptor corresponding to each entry in the Status ring
+ */
+typedef struct _q80_stat_desc {
+	uint64_t data[2];
+} __packed q80_stat_desc_t;
+
+/*
+ * definitions for data[0] field of Status Descriptor
+ */
+#define Q8_STAT_DESC_OWNER(data)		((data >> 56) & 0x3)
+#define		Q8_STAT_DESC_OWNER_HOST		0x1
+#define		Q8_STAT_DESC_OWNER_FW		0x2
+
+#define Q8_STAT_DESC_OWNER_MASK			(((uint64_t)0x3) << 56)
+#define Q8_STAT_DESC_SET_OWNER(owner)	(uint64_t)(((uint64_t)owner) << 56)
+
+#define Q8_STAT_DESC_OPCODE(data)		((data >> 58) & 0x003F)
+#define		Q8_STAT_DESC_OPCODE_SYN_OFFLOAD		0x03
+#define		Q8_STAT_DESC_OPCODE_RCV_PKT		0x04
+#define		Q8_STAT_DESC_OPCODE_CTRL_MSG		0x05
+#define		Q8_STAT_DESC_OPCODE_LRO_PKT		0x12
+
+/*
+ * definitions for data[0] field of Status Descriptor for standard frames
+ * status descriptor opcode equals 0x04
+ */
+#define Q8_STAT_DESC_PORT(data)			((data) & 0x000F)
+#define Q8_STAT_DESC_STATUS(data)		((data >> 4) & 0x000F)
+#define		Q8_STAT_DESC_STATUS_NO_CHKSUM		0x01
+#define		Q8_STAT_DESC_STATUS_CHKSUM_OK		0x02
+#define		Q8_STAT_DESC_STATUS_CHKSUM_ERR		0x03
+
+#define Q8_STAT_DESC_TYPE(data)			((data >> 8) & 0x000F)
+#define Q8_STAT_DESC_TOTAL_LENGTH(data)		((data >> 12) & 0xFFFF)
+#define Q8_STAT_DESC_HANDLE(data)		((data >> 28) & 0xFFFF)
+#define Q8_STAT_DESC_PROTOCOL(data)		((data >> 44) & 0x000F)
+#define Q8_STAT_DESC_L2_OFFSET(data)		((data >> 48) & 0x001F)
+#define Q8_STAT_DESC_COUNT(data)		((data >> 53) & 0x0007)
+
+/*
+ * definitions for data[0-1] fields of Status Descriptor for LRO
+ * status descriptor opcode equals 0x05
+ */
+/* definitions for data[0] field */
+#define Q8_LRO_STAT_DESC_HANDLE(data)		((data) & 0xFFFF)
+#define Q8_LRO_STAT_DESC_PAYLOAD_LENGTH(data)	((data >> 16) & 0xFFFF)
+#define Q8_LRO_STAT_DESC_L2_OFFSET(data)	((data >> 32) & 0xFF)
+#define Q8_LRO_STAT_DESC_L4_OFFSET(data)	((data >> 40) & 0xFF)
+#define Q8_LRO_STAT_DESC_TS_PRESENT(data)	((data >> 48) & 0x1)
+#define Q8_LRO_STAT_DESC_TYPE(data)		((data >> 49) & 0x7)
+#define Q8_LRO_STAT_DESC_PUSH_BIT(data)		((data >> 52) & 0x1)
+
+/* definitions for data[1] field */
+#define Q8_LRO_STAT_DESC_SEQ_NUM(data)		(uint32_t)(data)
+
+/** Driver Related Definitions Begin **/
+
+#define MAX_RDS_RINGS           2 /* Max# of Receive Descriptor Rings */
+#define MAX_SDS_RINGS           4 /* Max# of Status Descriptor Rings */
+#define TX_SMALL_PKT_SIZE	128 /* size in bytes of small packets */
+
+/* The number of descriptors should be a power of 2 */
+#define NUM_TX_DESCRIPTORS		2048
+#define NUM_RX_DESCRIPTORS		8192
+//#define NUM_RX_JUMBO_DESCRIPTORS	1024
+#define NUM_RX_JUMBO_DESCRIPTORS	2048
+//#define NUM_STATUS_DESCRIPTORS		8192
+#define NUM_STATUS_DESCRIPTORS		2048
+
+typedef struct _q80_rcv_cntxt_req {
+	q80_rq_rcv_cntxt_t	rx_req;
+	q80_rq_rds_ring_t	rds_req[MAX_RDS_RINGS];
+	q80_rq_sds_ring_t	sds_req[MAX_SDS_RINGS];
+} __packed q80_rcv_cntxt_req_t;
+
+typedef struct _q80_rcv_cntxt_rsp {
+	q80_rsp_rcv_cntxt_t	rx_rsp;
+	q80_rsp_rds_ring_t	rds_rsp[MAX_RDS_RINGS];
+	q80_rsp_sds_ring_t	sds_rsp[MAX_SDS_RINGS];
+} __packed q80_rcv_cntxt_rsp_t;
+
+/*
+ * structure describing various dma buffers
+ */
+#define RDS_RING_INDEX_NORMAL	0
+#define RDS_RING_INDEX_JUMBO	1
+
+typedef struct qla_dmabuf {
+        volatile struct {
+                uint32_t        tx_ring		:1,
+                                rds_ring	:1,
+                                sds_ring	:1,
+                                context		:1;
+        } flags;
+
+        qla_dma_t               tx_ring;
+        qla_dma_t               rds_ring[MAX_RDS_RINGS];
+        qla_dma_t               sds_ring[MAX_SDS_RINGS];
+        qla_dma_t               context;
+} qla_dmabuf_t;
+
+/** Driver Related Definitions End **/
+
+/*
+ * Firmware Control Descriptor
+ */
+typedef struct _qla_fw_cds_hdr {
+	uint64_t cmd; 
+#define Q8_FWCD_CNTRL_REQ	(0x13 << 23)
+	uint8_t	opcode;
+	uint8_t cookie;
+	uint16_t cntxt_id;
+	uint8_t response;
+#define Q8_FW_CDS_HDR_COMPLETION	0x1
+	uint16_t rsrvd;
+	uint8_t sub_opcode;
+} __packed qla_fw_cds_hdr_t;
+
+/*
+ * definitions for opcode in qla_fw_cds_hdr_t
+ */
+#define Q8_FWCD_OPCODE_CONFIG_RSS		0x01
+#define Q8_FWCD_OPCODE_CONFIG_RSS_TABLE		0x02
+#define Q8_FWCD_OPCODE_CONFIG_INTR_COALESCING	0x03
+#define Q8_FWCD_OPCODE_CONFIG_LED		0x04
+#define Q8_FWCD_OPCODE_CONFIG_MAC_ADDR		0x06
+#define Q8_FWCD_OPCODE_LRO_FLOW			0x07
+#define Q8_FWCD_OPCODE_GET_SNMP_STATS		0x08
+#define Q8_FWCD_OPCODE_CONFIG_MAC_RCV_MODE	0x0C
+#define Q8_FWCD_OPCODE_STATISTICS		0x10
+#define Q8_FWCD_OPCODE_CONFIG_IPADDR		0x12
+#define Q8_FWCD_OPCODE_CONFIG_LOOPBACK		0x13
+#define Q8_FWCD_OPCODE_LINK_EVENT_REQ		0x15
+#define Q8_FWCD_OPCODE_CONFIG_BRIDGING		0x17
+#define Q8_FWCD_OPCODE_CONFIG_LRO		0x18
+
+/*
+ * Configure RSS
+ */
+typedef struct _qla_fw_cds_config_rss {
+	qla_fw_cds_hdr_t	hdr;
+	uint8_t			hash_type;
+#define Q8_FWCD_RSS_HASH_TYPE_IPV4_TCP		(0x2 << 4)
+#define Q8_FWCD_RSS_HASH_TYPE_IPV4_IP		(0x1 << 4)
+#define Q8_FWCD_RSS_HASH_TYPE_IPV4_TCP_IP	(0x3 << 4)
+#define Q8_FWCD_RSS_HASH_TYPE_IPV6_TCP		(0x2 << 6)
+#define Q8_FWCD_RSS_HASH_TYPE_IPV6_IP		(0x1 << 6)
+#define Q8_FWCD_RSS_HASH_TYPE_IPV6_TCP_IP	(0x3 << 6)
+
+	uint8_t			flags;
+#define Q8_FWCD_RSS_FLAGS_ENABLE_RSS		0x1
+#define Q8_FWCD_RSS_FLAGS_USE_IND_TABLE		0x2
+	uint8_t			rsrvd[4];
+	uint16_t		ind_tbl_mask;
+	uint64_t		rss_key[5];
+} __packed qla_fw_cds_config_rss_t;
+
+/*
+ * Configure RSS Table
+ */
+typedef struct _qla_fw_cds_config_rss_table {
+	qla_fw_cds_hdr_t	hdr;
+	uint64_t		index;
+	uint8_t			table[40];
+} __packed qla_fw_cds_config_rss_table_t;
+
+/*
+ * Configure Interrupt Coalescing
+ */
+typedef struct _qla_fw_cds_config_intr_coalesc {
+	qla_fw_cds_hdr_t	hdr;
+	uint16_t		rsrvd0;
+	uint16_t		rsrvd1;
+	uint16_t		flags;
+	uint16_t		rsrvd2;
+	uint64_t		rsrvd3;
+	uint16_t		max_rcv_pkts;
+	uint16_t		max_rcv_usecs;
+	uint16_t		max_snd_pkts;
+	uint16_t		max_snd_usecs;
+	uint64_t		rsrvd4;
+	uint64_t		rsrvd5;
+	uint32_t		usecs_to;
+	uint8_t			timer_type;
+#define Q8_FWCMD_INTR_COALESC_TIMER_NONE	0x00
+#define Q8_FWCMD_INTR_COALESC_TIMER_ONCE	0x01
+#define Q8_FWCMD_INTR_COALESC_TIMER_PERIODIC	0x02
+
+	uint8_t			sds_ring_bitmask;
+#define Q8_FWCMD_INTR_COALESC_SDS_RING_0	0x01
+#define Q8_FWCMD_INTR_COALESC_SDS_RING_1	0x02
+#define Q8_FWCMD_INTR_COALESC_SDS_RING_2	0x04
+#define Q8_FWCMD_INTR_COALESC_SDS_RING_3	0x08
+
+	uint16_t		rsrvd6;
+} __packed qla_fw_cds_config_intr_coalesc_t;
+
+/*
+ * Configure LED Parameters
+ */
+typedef struct _qla_fw_cds_config_led {
+	qla_fw_cds_hdr_t	hdr;
+	uint32_t		cntxt_id;
+	uint32_t		blink_rate;
+	uint32_t		blink_state;
+	uint32_t		rsrvd;
+} __packed qla_fw_cds_config_led_t;
+
+/*
+ * Configure MAC Address
+ */
+typedef struct _qla_fw_cds_config_mac_addr {
+	qla_fw_cds_hdr_t	hdr;
+	uint8_t			cmd;
+#define Q8_FWCD_ADD_MAC_ADDR	0x1
+#define Q8_FWCD_DEL_MAC_ADDR	0x2
+	uint8_t			rsrvd;
+	uint8_t			mac_addr[6];
+} __packed qla_fw_cds_config_mac_addr_t;
+
+/*
+ * Configure Add/Delete LRO
+ */
+typedef struct _qla_fw_cds_config_lro {
+	qla_fw_cds_hdr_t	hdr;
+	uint32_t		dst_ip_addr;
+	uint32_t		src_ip_addr;
+	uint16_t		dst_tcp_port;
+	uint16_t		src_tcp_port;
+	uint8_t			ipv6;
+	uint8_t			time_stamp;
+	uint16_t		rsrvd;
+	uint32_t		rss_hash;
+	uint32_t		host_handle;
+} __packed qla_fw_cds_config_lro_t;
+
+/*
+ * Get SNMP Statistics
+ */
+typedef struct _qla_fw_cds_get_snmp {
+	qla_fw_cds_hdr_t	hdr;
+	uint64_t		phys_addr;
+	uint16_t		size;
+	uint16_t		cntxt_id;
+	uint32_t		rsrvd;
+} __packed qla_fw_cds_get_snmp_t;
+
+typedef struct _qla_snmp_stats {
+	uint64_t		jabber_state;
+	uint64_t		false_carrier;
+	uint64_t		rsrvd;
+	uint64_t		mac_cntrl;
+	uint64_t		align_errors;
+	uint64_t		chksum_errors;
+	uint64_t		oversize_frames;
+	uint64_t		tx_errors;
+	uint64_t		mac_rcv_errors;
+	uint64_t		phy_rcv_errors;
+	uint64_t		rcv_pause;
+	uint64_t		tx_pause;
+} __packed qla_snmp_stats_t;
+
+/*
+ * Enable Link Event Requests
+ */
+typedef struct _qla_link_event_req {
+	qla_fw_cds_hdr_t	hdr;
+	uint8_t			enable;
+	uint8_t			get_clnk_params;
+	uint8_t			pad[6];
+} __packed qla_link_event_req_t;
+
+
+/*
+ * Set MAC Receive Mode
+ */
+typedef struct _qla_set_mac_rcv_mode {
+	qla_fw_cds_hdr_t	hdr;
+
+	uint32_t		mode;
+#define Q8_MAC_RCV_RESET_PROMISC_ALLMULTI	0x00
+#define Q8_MAC_RCV_ENABLE_PROMISCUOUS		0x01
+#define Q8_MAC_RCV_ENABLE_ALLMULTI		0x02
+
+	uint8_t			pad[4];
+} __packed qla_set_mac_rcv_mode_t;
+
+/*
+ * Configure IP Address
+ */
+typedef struct _qla_config_ipv4 {
+	qla_fw_cds_hdr_t	hdr;
+
+	uint64_t		cmd;
+#define Q8_CONFIG_CMD_IP_ENABLE		0x02
+#define Q8_CONFIG_CMD_IP_DISABLE	0x03
+
+	uint64_t		ipv4_addr;
+} __packed qla_config_ipv4_t;
+
+/*
+ * Configure LRO
+ */
+typedef struct _qla_config_lro {
+	qla_fw_cds_hdr_t	hdr;
+
+	uint64_t		cmd;
+#define Q8_CONFIG_LRO_ENABLE		0x08
+} __packed qla_config_lro_t;
+
+
+/*
+ * Control Messages Received on SDS Ring
+ */
+/* Header */
+typedef struct _qla_cntrl_msg_hdr {
+	uint16_t rsrvd0;
+	uint16_t err_code;
+	uint8_t  rsp_type;
+	uint8_t  comp_id;
+	uint16_t tag;
+#define Q8_CTRL_MSG_TAG_DESC_COUNT_MASK		(0x7 << 5)
+#define Q8_CTRL_MSG_TAG_OWNER_MASK		(0x3 << 8)
+#define Q8_CTRL_MSG_TAG_OPCODE_MASK		(0x3F << 10)
+} __packed qla_cntrl_msg_hdr_t;
+
+/*
+ * definitions for rsp_type in qla_cntrl_msg_hdr_t
+ */
+#define Q8_CTRL_CONFIG_MAC_RSP			0x85
+#define Q8_CTRL_LRO_FLOW_DELETE_RSP		0x86
+#define Q8_CTRL_LRO_FLOW_ADD_FAILURE_RSP	0x87
+#define Q8_CTRL_GET_SNMP_STATS_RSP		0x88
+#define Q8_CTRL_GET_NETWORK_STATS_RSP		0x8C
+#define Q8_CTRL_LINK_EVENT_NOTIFICATION		0x8D
+
+/*
+ * Configure MAC Response
+ */
+typedef struct _qla_config_mac_rsp {
+	uint32_t		rval;
+	uint32_t		rsrvd;
+} __packed qla_config_mac_rsp_t;
+
+/*
+ * LRO Flow Response (can be LRO Flow Delete and LRO Flow Add Failure)
+ */
+typedef struct _qla_lro_flow_rsp {
+	uint32_t		handle;
+	uint32_t		rss_hash;
+	uint32_t		dst_ip;
+	uint32_t		src_ip;
+	uint16_t		dst_tcp_port;
+	uint16_t		src_tcp_port;
+	uint8_t			ipv6;
+	uint8_t			rsrvd0;
+	uint16_t		rsrvd1;
+} __packed qla_lro_flow_rsp_t;
+
+/*
+ * Get SNMP Statistics Response
+ */
+typedef struct _qla_get_snmp_stats_rsp {
+	uint64_t		rsrvd;
+} __packed qla_get_snmp_stats_rsp_t;
+
+/*
+ * Get Network Statistics Response
+ */
+typedef struct _qla_get_net_stats_rsp {
+	uint64_t		rsrvd;
+} __packed qla_get_net_stats_rsp_t;
+
+/*
+ * Link Event Notification
+ */
+typedef struct _qla_link_event {
+	uint32_t		cable_oui;
+	uint16_t		cable_length;
+
+	uint16_t		link_speed;
+#define Q8_LE_SPEED_MASK	0xFFF
+#define Q8_LE_SPEED_10GBPS	0x710
+#define Q8_LE_SPEED_1GBPS	0x3E8
+#define Q8_LE_SPEED_100MBPS	0x064
+#define Q8_LE_SPEED_10MBPS	0x00A
+
+	uint8_t			link_up;/* 0 = down; else up */
+
+	uint8_t			mod_info;
+#define Q8_LE_MI_MODULE_NOT_PRESENT		0x01
+#define Q8_LE_MI_UNKNOWN_OPTICAL_MODULE		0x02
+#define Q8_LE_MI_SR_LR_OPTICAL_MODULE		0x03
+#define Q8_LE_MI_LRM_OPTICAL_MODULE		0x04
+#define Q8_LE_MI_SFP_1G_MODULE			0x05
+#define Q8_LE_MI_UNSUPPORTED_TWINAX		0x06
+#define Q8_LE_MI_UNSUPPORTED_TWINAX_LENGTH	0x07
+#define Q8_LE_MI_SUPPORTED_TWINAX		0x08
+
+	uint8_t			fduplex; /* 1 = full duplex; 0 = half duplex */
+	uint8_t			autoneg; /* 1 = autoneg enable; 0 = disabled */ 
+	uint32_t		rsrvd;
+} __packed qla_link_event_t;
+
+typedef struct _qla_sds {
+	q80_stat_desc_t *sds_ring_base; /* start of sds ring */
+	uint32_t	sdsr_next; /* next entry in SDS ring to process */
+	struct lro_ctrl	lro;
+	void		*rxb_free;
+	uint32_t	rx_free;
+	void		*rxjb_free;
+	uint32_t	rxj_free;
+	volatile uint32_t rcv_active;
+} qla_sds_t;
+
+/*
+ * struct for storing hardware specific information for a given interface
+ */
+typedef struct _qla_hw {
+	struct {
+		uint32_t
+			lro		:1,
+			init_tx_cnxt	:1,
+			init_rx_cnxt	:1,
+			fduplex		:1,
+			autoneg		:1,
+			link_up		:1;
+	} flags;
+
+	uint16_t	link_speed;
+	uint16_t	cable_length;
+	uint16_t	cable_oui;
+	uint8_t		mod_info;
+	uint8_t		rsrvd;
+
+	uint32_t	max_rds_per_cntxt;
+	uint32_t	max_sds_per_cntxt;
+	uint32_t	max_rules_per_cntxt;
+	uint32_t	max_rcv_cntxts;
+	uint32_t	max_xmt_cntxts;
+	uint32_t	max_mtu;
+	uint32_t	max_lro;
+
+	uint8_t		mac_addr[ETHER_ADDR_LEN];
+
+	uint16_t	num_rds_rings;
+	uint16_t	num_sds_rings;
+
+        qla_dmabuf_t	dma_buf;
+	
+	/* Transmit Side */
+
+	q80_tx_cmd_t	*tx_ring_base;
+
+	q80_tx_cntxt_req_t *tx_cntxt_req; /* TX Context Request */
+	bus_addr_t	tx_cntxt_req_paddr;
+
+	q80_tx_cntxt_rsp_t *tx_cntxt_rsp; /* TX Context Response */
+	bus_addr_t	tx_cntxt_rsp_paddr;
+
+	uint32_t	*tx_cons; /* tx consumer shadow reg */
+	bus_addr_t	tx_cons_paddr;
+
+	volatile uint32_t txr_free; /* # of free entries in tx ring */
+	volatile uint32_t txr_next; /* # next available tx ring entry */
+	volatile uint32_t txr_comp; /* index of last tx entry completed */
+
+	uint32_t	tx_prod_reg;
+
+	/* Receive Side */
+	volatile uint32_t rx_next; /* next standard rcv ring to arm fw */
+	volatile int32_t  rxj_next; /* next jumbo rcv ring to arm fw */
+
+	volatile int32_t  rx_in; /* next standard rcv ring to add mbufs */
+	volatile int32_t  rxj_in; /* next jumbo rcv ring to add mbufs */
+
+	q80_rcv_cntxt_req_t *rx_cntxt_req; /* Rcv Context Request */
+	bus_addr_t	rx_cntxt_req_paddr;
+	q80_rcv_cntxt_rsp_t *rx_cntxt_rsp; /* Rcv Context Response */
+	bus_addr_t	rx_cntxt_rsp_paddr;
+	
+	qla_sds_t	sds[MAX_SDS_RINGS]; 
+} qla_hw_t;
+
+#define QL_UPDATE_RDS_PRODUCER_INDEX(ha, i, val) \
+	WRITE_REG32(ha, ((ha->hw.rx_cntxt_rsp)->rds_rsp[i].producer_reg +\
+		0x1b2000), val)
+
+#define QL_UPDATE_TX_PRODUCER_INDEX(ha, val) \
+	WRITE_REG32(ha, (ha->hw.tx_prod_reg + 0x1b2000), val)
+
+#define QL_UPDATE_SDS_CONSUMER_INDEX(ha, i, val) \
+	WRITE_REG32(ha, ((ha->hw.rx_cntxt_rsp)->sds_rsp[i].consumer_reg +\
+		0x1b2000), val)
+
+#define QL_CLEAR_INTERRUPTS(ha) \
+	if (ha->pci_func == 0) {\
+		WRITE_REG32(ha, Q8_INT_TARGET_STATUS_F0, 0xFFFFFFFF);\
+	} else {\
+		WRITE_REG32(ha, Q8_INT_TARGET_STATUS_F1, 0xFFFFFFFF);\
+	}\
+
+#define QL_ENABLE_INTERRUPTS(ha, sds_index) \
+	{\
+		q80_rsp_sds_ring_t *rsp_sds;\
+		rsp_sds = &((ha->hw.rx_cntxt_rsp)->sds_rsp[sds_index]);\
+		WRITE_REG32(ha, (rsp_sds->intr_mask_reg + 0x1b2000), 0x1);\
+	}
+
+#define QL_DISABLE_INTERRUPTS(ha, sds_index) \
+	{\
+		q80_rsp_sds_ring_t *rsp_sds;\
+		rsp_sds = &((ha->hw.rx_cntxt_rsp)->sds_rsp[sds_index]);\
+		WRITE_REG32(ha, (rsp_sds->intr_mask_reg + 0x1b2000), 0x0);\
+	}
+
+
+#define QL_BUFFER_ALIGN                16
+
+#endif /* #ifndef _QLA_HW_H_ */
diff --git a/sys/dev/qlxgb/qla_inline.h b/sys/dev/qlxgb/qla_inline.h
new file mode 100644
index 0000000..6a6be5f
--- /dev/null
+++ b/sys/dev/qlxgb/qla_inline.h
@@ -0,0 +1,229 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_inline.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+#ifndef _QLA_INLINE_H_
+#define _QLA_INLINE_H_
+
+/*
+ * Function: qla_hw_reset
+ */
+static __inline void qla_hw_reset(qla_host_t *ha)
+{
+        WRITE_OFFSET32(ha, Q8_ASIC_RESET, 0xFFFFFFFF);
+}
+
+#define QL8_SEMLOCK_TIMEOUT	1000/* QLA8020 Semaphore Lock Timeout 10ms */
+
+
+/*
+ * Inline functions for hardware semaphores
+ */
+
+/*
+ * Name:	qla_sem_lock
+ * Function:	Locks one of the semaphore registers (semaphore 2,3,5 & 7)
+ *		If the id_reg is valid, then id_val is written into it.
+ *		This is for debugging purpose
+ * Returns:	0 on success; otherwise its failed.
+ */
+static __inline int
+qla_sem_lock(qla_host_t *ha, uint32_t sem_reg, uint32_t id_reg, uint32_t id_val)
+{
+	int count = QL8_SEMLOCK_TIMEOUT;
+
+	while (count) {
+		if ((READ_REG32(ha, sem_reg) & SEM_LOCK_BIT))
+			break;
+		count--;
+
+		if (!count)
+			return(-1);
+		qla_mdelay(__func__, 10);
+	}
+	if (id_reg)
+		WRITE_OFFSET32(ha, id_reg, id_val);
+
+	return(0);
+}
+
+/*
+ * Name:	qla_sem_unlock
+ * Function:	Unlocks the semaphore registers (semaphore 2,3,5 & 7)
+ *		previously locked by qla_sem_lock()
+ */
+static __inline void
+qla_sem_unlock(qla_host_t *ha, uint32_t sem_reg)
+{
+	READ_REG32(ha, sem_reg);
+}
+
+static __inline int
+qla_get_ifq_snd_maxlen(qla_host_t *ha)
+{
+	return((NUM_TX_DESCRIPTORS - 1));
+}
+
+static __inline uint32_t
+qla_get_optics(qla_host_t *ha)
+{
+	uint32_t link_speed;
+
+	link_speed = READ_REG32(ha, Q8_LINK_SPEED_0);
+	if (ha->pci_func == 0)
+		link_speed = link_speed & 0xFF;
+	else
+		link_speed = (link_speed >> 8) & 0xFF;
+
+	switch (link_speed) {
+	case 0x1:
+		link_speed = IFM_100_FX;
+		break;
+
+	case 0x10:
+		link_speed = IFM_1000_SX;
+		break;
+
+	default:
+		link_speed = (IFM_10G_LR | IFM_10G_SR);
+		break;
+	}
+
+	return(link_speed);
+}
+
+static __inline uint8_t *
+qla_get_mac_addr(qla_host_t *ha)
+{
+	return (ha->hw.mac_addr);
+}
+
+static __inline void
+qla_read_mac_addr(qla_host_t *ha)
+{
+	uint32_t mac_crb_addr;
+	uint32_t mac_lo;
+	uint32_t mac_hi;
+	uint8_t	*macp;
+
+	mac_crb_addr = Q8_CRB_MAC_BLOCK_START +
+		(((ha->pci_func >> 1) * 3) << 2) + ((ha->pci_func & 0x01) << 2);
+
+	mac_lo = READ_REG32(ha, mac_crb_addr);
+	mac_hi = READ_REG32(ha, (mac_crb_addr + 0x4));
+
+	if (ha->pci_func & 0x01) {
+		mac_lo = mac_lo >> 16;
+
+		macp = (uint8_t *)&mac_lo;
+
+		ha->hw.mac_addr[5] = macp[0];
+		ha->hw.mac_addr[4] = macp[1];
+
+		macp = (uint8_t *)&mac_hi;
+
+		ha->hw.mac_addr[3] = macp[0];
+		ha->hw.mac_addr[2] = macp[1];
+		ha->hw.mac_addr[1] = macp[2];
+		ha->hw.mac_addr[0] = macp[3];
+	} else {
+		macp = (uint8_t *)&mac_lo;
+
+		ha->hw.mac_addr[5] = macp[0];
+		ha->hw.mac_addr[4] = macp[1];
+		ha->hw.mac_addr[3] = macp[2];
+		ha->hw.mac_addr[2] = macp[3];
+
+		macp = (uint8_t *)&mac_hi;
+
+		ha->hw.mac_addr[1] = macp[0];
+		ha->hw.mac_addr[0] = macp[1];
+	}
+	return;
+}
+
+static __inline void
+qla_set_hw_rcv_desc(qla_host_t *ha, uint32_t ridx, uint32_t index,
+	uint32_t handle, bus_addr_t paddr, uint32_t buf_size)
+{
+	q80_recv_desc_t *rcv_desc;
+
+	rcv_desc = (q80_recv_desc_t *)ha->hw.dma_buf.rds_ring[ridx].dma_b;
+
+	rcv_desc += index;
+
+	rcv_desc->handle = (uint16_t)handle;
+	rcv_desc->buf_size = buf_size;
+	rcv_desc->buf_addr = paddr;
+
+	return;
+}
+
+static __inline void
+qla_init_hw_rcv_descriptors(qla_host_t *ha, uint32_t ridx)
+{
+	if (ridx == RDS_RING_INDEX_NORMAL)
+		bzero((void *)ha->hw.dma_buf.rds_ring[ridx].dma_b,
+			(sizeof(q80_recv_desc_t) * NUM_RX_DESCRIPTORS));
+	else if (ridx == RDS_RING_INDEX_JUMBO)
+		bzero((void *)ha->hw.dma_buf.rds_ring[ridx].dma_b,
+			(sizeof(q80_recv_desc_t) * NUM_RX_JUMBO_DESCRIPTORS));
+	else
+		QL_ASSERT(0, ("%s: invalid rds index [%d]\n", __func__, ridx));
+}
+
+static __inline void
+qla_lock(qla_host_t *ha, const char *str)
+{
+	while (1) {
+		mtx_lock(&ha->hw_lock);
+		if (!ha->hw_lock_held) {
+			ha->hw_lock_held = 1;
+			ha->qla_lock = str;
+			mtx_unlock(&ha->hw_lock);
+			break;
+		}
+		mtx_unlock(&ha->hw_lock);
+		qla_mdelay(__func__, 1);
+	}
+	return;
+}
+
+static __inline void
+qla_unlock(qla_host_t *ha, const char *str)
+{
+	mtx_lock(&ha->hw_lock);
+	ha->hw_lock_held = 0;
+	ha->qla_unlock = str;
+	mtx_unlock(&ha->hw_lock);
+}
+
+#endif /* #ifndef _QLA_INLINE_H_ */
diff --git a/sys/dev/qlxgb/qla_ioctl.c b/sys/dev/qlxgb/qla_ioctl.c
new file mode 100644
index 0000000..1e9557a
--- /dev/null
+++ b/sys/dev/qlxgb/qla_ioctl.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * File: qla_ioctl.c
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qla_os.h"
+#include "qla_reg.h"
+#include "qla_hw.h"
+#include "qla_def.h"
+#include "qla_reg.h"
+#include "qla_inline.h"
+#include "qla_glbl.h"
+#include "qla_ioctl.h"
+
+static struct cdevsw qla_cdevsw = {
+	.d_version = D_VERSION,
+	.d_ioctl = qla_eioctl,
+	.d_name = "qlcnic",
+};
+
+int
+qla_make_cdev(qla_host_t *ha)
+{
+        ha->ioctl_dev = make_dev(&qla_cdevsw,
+				ha->ifp->if_dunit,
+                                UID_ROOT,
+                                GID_WHEEL,
+                                0600,
+                                "%s",
+                                if_name(ha->ifp));
+
+	if (ha->ioctl_dev == NULL)
+		return (-1);
+
+        ha->ioctl_dev->si_drv1 = ha;
+
+	return (0);
+}
+
+void
+qla_del_cdev(qla_host_t *ha)
+{
+	if (ha->ioctl_dev != NULL)
+		destroy_dev(ha->ioctl_dev);
+	return;
+}
+
+int
+qla_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
+        struct thread *td)
+{
+        qla_host_t *ha;
+        int rval = 0;
+        qla_reg_val_t *rv;
+        qla_rd_flash_t *rdf;
+
+        if ((ha = (qla_host_t *)dev->si_drv1) == NULL)
+                return ENXIO;
+
+        switch(cmd) {
+
+        case QLA_RDWR_REG:
+
+                rv = (qla_reg_val_t *)data;
+
+                if (rv->direct) {
+                        if (rv->rd) {
+                                rv->val = READ_OFFSET32(ha, rv->reg);
+                        } else {
+                                WRITE_OFFSET32(ha, rv->reg, rv->val);
+                        }
+                } else {
+                        if ((rval = qla_rdwr_indreg32(ha, rv->reg, &rv->val,
+                                rv->rd)))
+                                rval = ENXIO;
+                }
+                break;
+
+        case QLA_RD_FLASH:
+                rdf = (qla_rd_flash_t *)data;
+                if ((rval = qla_rd_flash32(ha, rdf->off, &rdf->data)))
+                        rval = ENXIO;
+                break;
+        default:
+                break;
+        }
+
+        return rval;
+}
+
diff --git a/sys/dev/qlxgb/qla_ioctl.h b/sys/dev/qlxgb/qla_ioctl.h
new file mode 100644
index 0000000..160c46c
--- /dev/null
+++ b/sys/dev/qlxgb/qla_ioctl.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_ioctl.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#ifndef _QLA_IOCTL_H_
+#define _QLA_IOCTL_H_
+
+#include <sys/ioccom.h>
+
+struct qla_reg_val {
+        uint16_t rd;
+        uint16_t direct;
+        uint32_t reg;
+        uint32_t val;
+};
+typedef struct qla_reg_val qla_reg_val_t;
+
+struct qla_rd_flash {
+        uint32_t off;
+        uint32_t data;
+};
+typedef struct qla_rd_flash qla_rd_flash_t;
+
+
+/*
+ * Read/Write Register
+ */
+#define QLA_RDWR_REG                    _IOWR('q', 1, qla_reg_val_t)
+
+/*
+ * Read Flash
+ */
+#define QLA_RD_FLASH                    _IOWR('q', 2, qla_rd_flash_t)
+
+#endif /* #ifndef _QLA_IOCTL_H_ */
diff --git a/sys/dev/qlxgb/qla_isr.c b/sys/dev/qlxgb/qla_isr.c
new file mode 100644
index 0000000..382d565
--- /dev/null
+++ b/sys/dev/qlxgb/qla_isr.c
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File: qla_isr.c
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qla_os.h"
+#include "qla_reg.h"
+#include "qla_hw.h"
+#include "qla_def.h"
+#include "qla_inline.h"
+#include "qla_ver.h"
+#include "qla_glbl.h"
+#include "qla_dbg.h"
+
+static void qla_replenish_normal_rx(qla_host_t *ha, qla_sds_t *sdsp);
+static void qla_replenish_jumbo_rx(qla_host_t *ha, qla_sds_t *sdsp);
+
+/*
+ * Name: qla_rx_intr
+ * Function: Handles normal ethernet frames received
+ */
+static void
+qla_rx_intr(qla_host_t *ha, uint64_t data, uint32_t sds_idx,
+	struct lro_ctrl *lro)
+{
+	uint32_t idx, length, status, ring;
+	qla_rx_buf_t *rxb;
+	struct mbuf *mp;
+	struct ifnet *ifp = ha->ifp;
+	qla_sds_t *sdsp;
+	struct ether_vlan_header *eh;
+	
+	sdsp = &ha->hw.sds[sds_idx];
+	
+	ring = (uint32_t)Q8_STAT_DESC_TYPE(data);
+	idx = (uint32_t)Q8_STAT_DESC_HANDLE(data);
+	length = (uint32_t)Q8_STAT_DESC_TOTAL_LENGTH(data);
+	status = (uint32_t)Q8_STAT_DESC_STATUS(data);
+
+	if (ring == 0) {
+		if ((idx >= NUM_RX_DESCRIPTORS) || (length > MCLBYTES)) {
+			device_printf(ha->pci_dev, "%s: ring[%d] index[0x%08x]"
+				" len[0x%08x] invalid\n",
+				__func__, ring, idx, length);
+			return;
+		}
+	} else {
+		if ((idx >= NUM_RX_JUMBO_DESCRIPTORS)||(length > MJUM9BYTES)) {
+			device_printf(ha->pci_dev, "%s: ring[%d] index[0x%08x]"
+				" len[0x%08x] invalid\n",
+				__func__, ring, idx, length);
+			return;
+		}
+	}
+
+	if (ring == 0)
+		rxb = &ha->rx_buf[idx];
+	else 
+		rxb = &ha->rx_jbuf[idx];
+
+	QL_ASSERT((rxb != NULL),\
+		("%s: [r, i, sds_idx]=[%d, 0x%x, %d] rxb != NULL\n",\
+		 __func__, ring, idx, sds_idx));
+
+	mp = rxb->m_head;
+
+	QL_ASSERT((mp != NULL),\
+		("%s: [r,i,rxb, sds_idx]=[%d, 0x%x, %p, %d] mp != NULL\n",\
+		 __func__, ring, idx, rxb, sds_idx));
+
+	bus_dmamap_sync(ha->rx_tag, rxb->map, BUS_DMASYNC_POSTREAD);
+
+	if (ring == 0) {
+		rxb->m_head = NULL;
+		rxb->next = sdsp->rxb_free;
+		sdsp->rxb_free = rxb;
+		sdsp->rx_free++;
+	} else {
+		rxb->m_head = NULL;
+		rxb->next = sdsp->rxjb_free;
+		sdsp->rxjb_free = rxb;
+		sdsp->rxj_free++;
+	}
+	
+	mp->m_len = length;
+	mp->m_pkthdr.len = length;
+	mp->m_pkthdr.rcvif = ifp;
+	
+	eh = mtod(mp, struct ether_vlan_header *);
+
+	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
+		uint32_t *data = (uint32_t *)eh;
+
+		mp->m_pkthdr.ether_vtag = ntohs(eh->evl_tag);
+		mp->m_flags |= M_VLANTAG;
+
+		*(data + 3) = *(data + 2);
+		*(data + 2) = *(data + 1);
+		*(data + 1) = *data;
+
+		m_adj(mp, ETHER_VLAN_ENCAP_LEN);
+	}
+
+	if (status == Q8_STAT_DESC_STATUS_CHKSUM_OK) {
+		mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
+	} else {
+		mp->m_pkthdr.csum_flags = 0;
+	}
+
+	if (lro->lro_cnt && (tcp_lro_rx(lro, mp, 0) == 0)) {
+		/* LRO packet has been successfuly queued */
+	} else {
+		(*ifp->if_input)(ifp, mp);
+	}
+
+	if (sdsp->rx_free > std_replenish)
+		qla_replenish_normal_rx(ha, sdsp);
+
+	if (sdsp->rxj_free > jumbo_replenish)
+		qla_replenish_jumbo_rx(ha, sdsp);
+
+	return;
+}
+
+static void
+qla_replenish_jumbo_rx(qla_host_t *ha, qla_sds_t *sdsp)
+{
+	qla_rx_buf_t *rxb;
+	int count = jumbo_replenish;
+	uint32_t rxj_next;
+
+	if (!mtx_trylock(&ha->rxj_lock))
+		return;
+
+	rxj_next = ha->hw.rxj_next;
+
+	while (count--) {
+		rxb = sdsp->rxjb_free;
+
+		if (rxb == NULL)
+			break;
+
+		sdsp->rxjb_free = rxb->next;
+		sdsp->rxj_free--;
+
+
+		if (qla_get_mbuf(ha, rxb, NULL, RDS_RING_INDEX_JUMBO) == 0) {
+			qla_set_hw_rcv_desc(ha, RDS_RING_INDEX_JUMBO,
+				ha->hw.rxj_in, rxb->handle, rxb->paddr,
+				(rxb->m_head)->m_pkthdr.len);
+			ha->hw.rxj_in++;
+			if (ha->hw.rxj_in == NUM_RX_JUMBO_DESCRIPTORS)
+				ha->hw.rxj_in = 0;
+			ha->hw.rxj_next++;
+			if (ha->hw.rxj_next == NUM_RX_JUMBO_DESCRIPTORS)
+				ha->hw.rxj_next = 0;
+		} else {
+			device_printf(ha->pci_dev,
+				"%s: qla_get_mbuf [1,(%d),(%d)] failed\n",
+				__func__, ha->hw.rxj_in, rxb->handle);
+
+			rxb->m_head = NULL;
+			rxb->next = sdsp->rxjb_free;
+			sdsp->rxjb_free = rxb;
+			sdsp->rxj_free++;
+
+			break;
+		}
+	}
+
+	if (rxj_next != ha->hw.rxj_next) {
+		QL_UPDATE_RDS_PRODUCER_INDEX(ha, 1, ha->hw.rxj_next);
+	}
+	mtx_unlock(&ha->rxj_lock);
+}
+
+static void
+qla_replenish_normal_rx(qla_host_t *ha, qla_sds_t *sdsp)
+{
+	qla_rx_buf_t *rxb;
+	int count = std_replenish;
+	uint32_t rx_next;
+
+	if (!mtx_trylock(&ha->rx_lock))
+		return;
+
+	rx_next = ha->hw.rx_next;
+
+	while (count--) {
+		rxb = sdsp->rxb_free;
+
+		if (rxb == NULL)
+			break;
+
+		sdsp->rxb_free = rxb->next;
+		sdsp->rx_free--;
+
+		if (qla_get_mbuf(ha, rxb, NULL, RDS_RING_INDEX_NORMAL) == 0) {
+			qla_set_hw_rcv_desc(ha, RDS_RING_INDEX_NORMAL,
+				ha->hw.rx_in, rxb->handle, rxb->paddr,
+				(rxb->m_head)->m_pkthdr.len);
+			ha->hw.rx_in++;
+			if (ha->hw.rx_in == NUM_RX_DESCRIPTORS)
+				ha->hw.rx_in = 0;
+			ha->hw.rx_next++;
+			if (ha->hw.rx_next == NUM_RX_DESCRIPTORS)
+				ha->hw.rx_next = 0;
+		} else {
+			device_printf(ha->pci_dev,
+				"%s: qla_get_mbuf [0,(%d),(%d)] failed\n",
+				__func__, ha->hw.rx_in, rxb->handle);
+
+			rxb->m_head = NULL;
+			rxb->next = sdsp->rxb_free;
+			sdsp->rxb_free = rxb;
+			sdsp->rx_free++;
+
+			break;
+		}
+	}
+
+	if (rx_next != ha->hw.rx_next) {
+		QL_UPDATE_RDS_PRODUCER_INDEX(ha, 0, ha->hw.rx_next);
+	}
+	mtx_unlock(&ha->rx_lock);
+}
+
+/*
+ * Name: qla_isr
+ * Function: Main Interrupt Service Routine
+ */
+static uint32_t
+qla_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count)
+{
+	device_t dev;
+	qla_hw_t *hw;
+	uint32_t comp_idx, desc_count;
+	q80_stat_desc_t *sdesc;
+	struct lro_ctrl *lro;
+	struct lro_entry *queued;
+	uint32_t ret = 0;
+
+	dev = ha->pci_dev;
+	hw = &ha->hw;
+
+	hw->sds[sds_idx].rcv_active = 1;
+	if (ha->flags.stop_rcv) {
+		hw->sds[sds_idx].rcv_active = 0;
+		return 0;
+	}
+
+	QL_DPRINT2((dev, "%s: [%d]enter\n", __func__, sds_idx));
+
+	/*
+	 * receive interrupts
+	 */
+	comp_idx = hw->sds[sds_idx].sdsr_next;
+	lro = &hw->sds[sds_idx].lro;
+
+	while (count--) {
+
+		sdesc = (q80_stat_desc_t *)
+				&hw->sds[sds_idx].sds_ring_base[comp_idx];
+
+		if (Q8_STAT_DESC_OWNER((sdesc->data[0])) !=
+			Q8_STAT_DESC_OWNER_HOST) {
+			QL_DPRINT2((dev, "%s:  data %p sdsr_next 0x%08x\n",
+				__func__, (void *)sdesc->data[0], comp_idx));
+			break;
+		}
+
+		desc_count = Q8_STAT_DESC_COUNT((sdesc->data[0]));
+
+		switch (Q8_STAT_DESC_OPCODE((sdesc->data[0]))) {
+
+		case Q8_STAT_DESC_OPCODE_RCV_PKT:
+		case Q8_STAT_DESC_OPCODE_SYN_OFFLOAD:
+			qla_rx_intr(ha, (sdesc->data[0]), sds_idx, lro);
+			
+			break;
+
+		default:
+			device_printf(dev, "%s: default 0x%llx!\n", __func__,
+					(long long unsigned int)sdesc->data[0]);
+			break;
+		}
+
+		while (desc_count--) {
+			sdesc->data[0] =
+				Q8_STAT_DESC_SET_OWNER(Q8_STAT_DESC_OWNER_FW);
+			comp_idx = (comp_idx + 1) & (NUM_STATUS_DESCRIPTORS-1);
+			sdesc = (q80_stat_desc_t *)
+				&hw->sds[sds_idx].sds_ring_base[comp_idx];
+		}
+	}
+
+	while((!SLIST_EMPTY(&lro->lro_active))) {
+		queued = SLIST_FIRST(&lro->lro_active);
+		SLIST_REMOVE_HEAD(&lro->lro_active, next);
+		tcp_lro_flush(lro, queued);
+	}
+
+	if (hw->sds[sds_idx].sdsr_next != comp_idx) {
+		QL_UPDATE_SDS_CONSUMER_INDEX(ha, sds_idx, comp_idx);
+	}
+	hw->sds[sds_idx].sdsr_next = comp_idx;
+
+	sdesc = (q80_stat_desc_t *)&hw->sds[sds_idx].sds_ring_base[comp_idx];
+	if ((sds_idx == 0) && (Q8_STAT_DESC_OWNER((sdesc->data[0])) ==
+					Q8_STAT_DESC_OWNER_HOST)) {
+		ret = -1;
+	}
+
+	hw->sds[sds_idx].rcv_active = 0;
+	return (ret);
+}
+
+void
+qla_isr(void *arg)
+{
+	qla_ivec_t *ivec = arg;
+	qla_host_t *ha;
+	uint32_t sds_idx;
+	uint32_t ret;
+
+	ha = ivec->ha;
+	sds_idx = ivec->irq_rid - 1;
+
+	if (sds_idx >= ha->hw.num_sds_rings) {
+		device_printf(ha->pci_dev, "%s: bogus sds_idx 0x%x\n", __func__,
+			sds_idx);
+		
+		return;
+	}
+
+	if (sds_idx == 0)
+		taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+
+	ret = qla_rcv_isr(ha, sds_idx, rcv_pkt_thres);
+
+	if (sds_idx == 0)
+		taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+
+	if (ret) {
+		taskqueue_enqueue(ha->irq_vec[sds_idx].rcv_tq,
+			&ha->irq_vec[sds_idx].rcv_task);
+	} else {
+		QL_ENABLE_INTERRUPTS(ha, sds_idx);
+	}
+}
+
+void
+qla_rcv(void *context, int pending)
+{
+	qla_ivec_t *ivec = context;
+	qla_host_t *ha;
+	device_t dev;
+	qla_hw_t *hw;
+	uint32_t sds_idx;
+	uint32_t ret;
+	struct ifnet *ifp;
+
+	ha = ivec->ha;
+	dev = ha->pci_dev;
+	hw = &ha->hw;
+	sds_idx = ivec->irq_rid - 1;
+	ifp = ha->ifp;
+
+	do {
+		if (sds_idx == 0) {
+			if (qla_le32_to_host(*(hw->tx_cons)) != hw->txr_comp) {
+				taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+			} else if ((ifp->if_snd.ifq_head != NULL) &&
+					QL_RUNNING(ifp)) {
+				taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+			}
+		}
+		ret = qla_rcv_isr(ha, sds_idx, rcv_pkt_thres_d);
+	} while (ret);
+
+	if (sds_idx == 0)
+		taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+
+	QL_ENABLE_INTERRUPTS(ha, sds_idx);
+}
+
diff --git a/sys/dev/qlxgb/qla_misc.c b/sys/dev/qlxgb/qla_misc.c
new file mode 100644
index 0000000..c616d4f
--- /dev/null
+++ b/sys/dev/qlxgb/qla_misc.c
@@ -0,0 +1,624 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * File : qla_misc.c
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qla_os.h"
+#include "qla_reg.h"
+#include "qla_hw.h"
+#include "qla_def.h"
+#include "qla_reg.h"
+#include "qla_inline.h"
+#include "qla_glbl.h"
+#include "qla_dbg.h"
+
+/*
+ * structure encapsulating the value to read/write to offchip memory
+ */
+typedef struct _offchip_mem_val {
+        uint32_t data_lo;
+        uint32_t data_hi;
+        uint32_t data_ulo;
+        uint32_t data_uhi;
+} offchip_mem_val_t;
+
+#define Q8_ADDR_UNDEFINED		0xFFFFFFFF
+
+/*
+ * The index to this table is Bits 20-27 of the indirect register address
+ */ 
+static uint32_t indirect_to_base_map[] =
+	{
+		Q8_ADDR_UNDEFINED,	/* 0x00 */
+		0x77300000,		/* 0x01 */
+		0x29500000,		/* 0x02 */
+		0x2A500000,		/* 0x03 */
+		Q8_ADDR_UNDEFINED,	/* 0x04 */
+		0x0D000000,		/* 0x05 */
+		0x1B100000,		/* 0x06 */
+		0x0E600000,		/* 0x07 */
+		0x0E000000,		/* 0x08 */
+		0x0E100000,		/* 0x09 */
+		0x0E200000,		/* 0x0A */
+		0x0E300000,		/* 0x0B */
+		0x42000000,		/* 0x0C */
+		0x41700000,		/* 0x0D */
+		0x42100000,		/* 0x0E */
+		0x34B00000,		/* 0x0F */
+		0x40500000,		/* 0x10 */
+		0x34000000,		/* 0x11 */
+		0x34100000,		/* 0x12 */
+		0x34200000,		/* 0x13 */
+		0x34300000,		/* 0x14 */
+		0x34500000,		/* 0x15 */
+		0x34400000,		/* 0x16 */
+		0x3C000000,		/* 0x17 */
+		0x3C100000,		/* 0x18 */
+		0x3C200000,		/* 0x19 */
+		0x3C300000,		/* 0x1A */
+		Q8_ADDR_UNDEFINED,	/* 0x1B */
+		0x3C400000,		/* 0x1C */
+		0x41000000,		/* 0x1D */
+		Q8_ADDR_UNDEFINED,	/* 0x1E */
+		0x0D100000,		/* 0x1F */
+		Q8_ADDR_UNDEFINED,	/* 0x20 */
+		0x77300000,		/* 0x21 */
+		0x41600000,		/* 0x22 */
+		Q8_ADDR_UNDEFINED,	/* 0x23 */
+		Q8_ADDR_UNDEFINED,	/* 0x24 */
+		Q8_ADDR_UNDEFINED,	/* 0x25 */
+		Q8_ADDR_UNDEFINED,	/* 0x26 */
+		Q8_ADDR_UNDEFINED,	/* 0x27 */
+		0x41700000,		/* 0x28 */
+		Q8_ADDR_UNDEFINED,	/* 0x29 */
+		0x08900000,		/* 0x2A */
+		0x70A00000,		/* 0x2B */
+		0x70B00000,		/* 0x2C */
+		0x70C00000,		/* 0x2D */
+		0x08D00000,		/* 0x2E */
+		0x08E00000,		/* 0x2F */
+		0x70F00000,		/* 0x30 */
+		0x40500000,		/* 0x31 */
+		0x42000000,		/* 0x32 */
+		0x42100000,		/* 0x33 */
+		Q8_ADDR_UNDEFINED,	/* 0x34 */
+		0x08800000,		/* 0x35 */
+		0x09100000,		/* 0x36 */
+		0x71200000,		/* 0x37 */
+		0x40600000,		/* 0x38 */
+		Q8_ADDR_UNDEFINED,	/* 0x39 */
+		0x71800000,		/* 0x3A */
+		0x19900000,		/* 0x3B */
+		0x1A900000,		/* 0x3C */
+		Q8_ADDR_UNDEFINED,	/* 0x3D */
+		0x34600000,		/* 0x3E */
+		Q8_ADDR_UNDEFINED,	/* 0x3F */
+	};
+
+/*
+ * Address Translation Table for CRB to offsets from PCI BAR0
+ */
+typedef struct _crb_to_pci {
+	uint32_t crb_addr;
+	uint32_t pci_addr;
+} crb_to_pci_t;
+
+static crb_to_pci_t crbinit_to_pciaddr[] = {
+	{(0x088 << 20), (0x035 << 20)},
+	{(0x089 << 20), (0x02A << 20)},
+	{(0x08D << 20), (0x02E << 20)},
+	{(0x08E << 20), (0x02F << 20)},
+	{(0x0C6 << 20), (0x023 << 20)},
+	{(0x0C7 << 20), (0x024 << 20)},
+	{(0x0C8 << 20), (0x025 << 20)},
+	{(0x0D0 << 20), (0x005 << 20)},
+	{(0x0D1 << 20), (0x01F << 20)},
+	{(0x0E0 << 20), (0x008 << 20)},
+	{(0x0E1 << 20), (0x009 << 20)},
+	{(0x0E2 << 20), (0x00A << 20)},
+	{(0x0E3 << 20), (0x00B << 20)},
+	{(0x0E6 << 20), (0x007 << 20)},
+	{(0x199 << 20), (0x03B << 20)},
+	{(0x1B1 << 20), (0x006 << 20)},
+	{(0x295 << 20), (0x002 << 20)},
+	{(0x29A << 20), (0x000 << 20)},
+	{(0x2A5 << 20), (0x003 << 20)},
+	{(0x340 << 20), (0x011 << 20)},
+	{(0x341 << 20), (0x012 << 20)},
+	{(0x342 << 20), (0x013 << 20)},
+	{(0x343 << 20), (0x014 << 20)},
+	{(0x344 << 20), (0x016 << 20)},
+	{(0x345 << 20), (0x015 << 20)},
+	{(0x3C0 << 20), (0x017 << 20)},
+	{(0x3C1 << 20), (0x018 << 20)},
+	{(0x3C2 << 20), (0x019 << 20)},
+	{(0x3C3 << 20), (0x01A << 20)},
+	{(0x3C4 << 20), (0x01C << 20)},
+	{(0x3C5 << 20), (0x01B << 20)},
+	{(0x405 << 20), (0x031 << 20)},
+	{(0x406 << 20), (0x038 << 20)},
+	{(0x410 << 20), (0x01D << 20)},
+	{(0x416 << 20), (0x022 << 20)},
+	{(0x417 << 20), (0x028 << 20)},
+	{(0x420 << 20), (0x032 << 20)},
+	{(0x421 << 20), (0x033 << 20)},
+	{(0x700 << 20), (0x00C << 20)},
+	{(0x701 << 20), (0x00D << 20)},
+	{(0x702 << 20), (0x00E << 20)},
+	{(0x703 << 20), (0x00F << 20)},
+	{(0x704 << 20), (0x010 << 20)},
+	{(0x70A << 20), (0x02B << 20)},
+	{(0x70B << 20), (0x02C << 20)},
+	{(0x70C << 20), (0x02D << 20)},
+	{(0x70F << 20), (0x030 << 20)},
+	{(0x718 << 20), (0x03A << 20)},
+	{(0x758 << 20), (0x026 << 20)},
+	{(0x759 << 20), (0x027 << 20)},
+	{(0x773 << 20), (0x001 << 20)}
+};
+ 
+#define Q8_INVALID_ADDRESS	(-1)
+#define Q8_ADDR_MASK		(0xFFF << 20)
+
+typedef struct _addr_val {
+	uint32_t addr;
+	uint32_t value;
+	uint32_t pci_addr;
+	uint32_t ind_addr;
+} addr_val_t;
+
+/*
+ * Name: qla_rdwr_indreg32
+ * Function: Read/Write an Indirect Register
+ */
+int
+qla_rdwr_indreg32(qla_host_t *ha, uint32_t addr, uint32_t *val, uint32_t rd)
+{
+	uint32_t offset;
+	int count = 100;
+
+	offset = (addr & 0xFFF00000) >> 20;
+
+	if (offset > 0x3F) {
+		device_printf(ha->pci_dev, "%s: invalid addr 0x%08x\n",
+			__func__, addr);
+		return -1;
+	}
+
+	offset = indirect_to_base_map[offset];
+	if (offset == Q8_ADDR_UNDEFINED) {
+		device_printf(ha->pci_dev, "%s: undefined map 0x%08x\n",
+			__func__, addr);
+		return -1;
+	}
+
+	offset = offset | (addr & 0x000F0000);
+
+	if (qla_sem_lock(ha, Q8_SEM7_LOCK, 0, 0)) {
+		device_printf(ha->pci_dev, "%s: SEM7_LOCK failed\n", __func__);
+		return (-1);
+	}
+
+	WRITE_OFFSET32(ha, Q8_CRB_WINDOW_2M, offset);
+
+	while (offset != (READ_OFFSET32(ha, Q8_CRB_WINDOW_2M))) {
+		count--;
+		if (!count) {
+			qla_sem_unlock(ha, Q8_SEM7_UNLOCK);
+			return -1;
+		}
+
+		qla_mdelay(__func__, 1);
+	}
+
+	if (rd) {
+		*val = READ_OFFSET32(ha, ((addr & 0xFFFF) | 0x1E0000));
+	} else {
+		WRITE_OFFSET32(ha, ((addr & 0xFFFF) | 0x1E0000), *val);
+	} 
+
+	qla_sem_unlock(ha, Q8_SEM7_UNLOCK);
+	return 0;
+}
+
+/*
+ * Name: qla_rdwr_offchip_mem
+ * Function: Read/Write OffChip Memory
+ */
+static int
+qla_rdwr_offchip_mem(qla_host_t *ha, uint64_t addr, offchip_mem_val_t *val,
+	uint32_t rd)
+{
+	uint32_t count = 100;
+	uint32_t data;
+
+	WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_ADDR_LO, (uint32_t)addr);
+	WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_ADDR_HI, (uint32_t)(addr >> 32));
+
+	if (!rd) {
+		WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_WRDATA_LO, val->data_lo);
+		WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_WRDATA_HI, val->data_hi);
+		WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_WRDATA_ULO, val->data_ulo);
+		WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_WRDATA_UHI, val->data_uhi);
+		WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_CTRL, 0x07); /* Write */
+	} else {
+		WRITE_OFFSET32(ha, Q8_MIU_TEST_AGT_CTRL, 0x03); /* Read */
+	}
+
+	while (count--) {
+		data = READ_OFFSET32(ha, Q8_MIU_TEST_AGT_CTRL);
+		if (!(data & BIT_3)) {
+			if (rd) {
+				val->data_lo = READ_OFFSET32(ha, \
+						Q8_MIU_TEST_AGT_RDDATA_LO);
+				val->data_hi = READ_OFFSET32(ha, \
+						Q8_MIU_TEST_AGT_RDDATA_HI);
+				val->data_ulo = READ_OFFSET32(ha, \
+						Q8_MIU_TEST_AGT_RDDATA_ULO);
+				val->data_uhi = READ_OFFSET32(ha, \
+						Q8_MIU_TEST_AGT_RDDATA_UHI);
+			}
+			return 0;
+		} else 
+			qla_mdelay(__func__, 1);
+	}
+	
+	device_printf(ha->pci_dev, "%s: failed[0x%08x]\n", __func__, data);
+	return (-1);
+}
+
+/*
+ * Name: qla_rd_flash32
+ * Function: Read Flash Memory
+ */
+int
+qla_rd_flash32(qla_host_t *ha, uint32_t addr, uint32_t *data)
+{
+	uint32_t val;
+	uint32_t count = 100;
+
+	if (qla_sem_lock(ha, Q8_SEM2_LOCK, 0, 0)) {
+		device_printf(ha->pci_dev, "%s: SEM2_LOCK failed\n", __func__);
+		return (-1);
+	}
+	WRITE_OFFSET32(ha, Q8_ROM_LOCKID, 0xa5a5a5a5);
+
+	val = addr;
+	qla_rdwr_indreg32(ha, Q8_ROM_ADDRESS, &val, 0);
+	val = 0;
+	qla_rdwr_indreg32(ha, Q8_ROM_DUMMY_BYTE_COUNT, &val, 0);
+	val = 3;
+	qla_rdwr_indreg32(ha, Q8_ROM_ADDR_BYTE_COUNT, &val, 0);
+
+	QLA_USEC_DELAY(100);
+
+	val = ROM_OPCODE_FAST_RD;
+	qla_rdwr_indreg32(ha, Q8_ROM_INSTR_OPCODE, &val, 0);
+
+	while (!((val = READ_OFFSET32(ha, Q8_ROM_STATUS)) & BIT_1)) {
+		count--;
+		if (!count) {
+			qla_sem_unlock(ha, Q8_SEM7_UNLOCK);
+			return -1;
+		}
+	}
+
+	val = 0;
+	qla_rdwr_indreg32(ha, Q8_ROM_DUMMY_BYTE_COUNT, &val, 0);
+	qla_rdwr_indreg32(ha, Q8_ROM_ADDR_BYTE_COUNT, &val, 0);
+
+	QLA_USEC_DELAY(100);
+
+	qla_rdwr_indreg32(ha, Q8_ROM_RD_DATA, data, 1);
+
+	qla_sem_unlock(ha, Q8_SEM2_UNLOCK);
+	return 0;
+}
+
+/*
+ * Name: qla_int_to_pci_addr_map
+ * Function: Convert's Internal(CRB) Address to Indirect Address
+ */
+static uint32_t
+qla_int_to_pci_addr_map(qla_host_t *ha, uint32_t int_addr)
+{
+	uint32_t crb_to_pci_table_size, i;
+	uint32_t addr;
+
+	crb_to_pci_table_size = sizeof(crbinit_to_pciaddr)/sizeof(crb_to_pci_t);
+	addr = int_addr & Q8_ADDR_MASK;
+
+	for (i = 0; i < crb_to_pci_table_size; i++) {
+		if (crbinit_to_pciaddr[i].crb_addr == addr) {
+			addr = (int_addr & ~Q8_ADDR_MASK) |
+					crbinit_to_pciaddr[i].pci_addr;
+			return (addr);
+		}
+	}
+	return (Q8_INVALID_ADDRESS);
+}
+
+/*
+ * Name: qla_filter_pci_addr
+ * Function: Filter's out Indirect Addresses which are not writeable
+ */
+static uint32_t
+qla_filter_pci_addr(qla_host_t *ha, uint32_t addr)
+{
+	if ((addr == Q8_INVALID_ADDRESS) ||
+		(addr == 0x00112040) ||
+		(addr == 0x00112048) ||
+		((addr & 0xFFFF0FFF) == 0x001100C4) ||
+		((addr & 0xFFFF0FFF) == 0x001100C8) ||
+		((addr & 0x0FF00000) == 0x00200000) ||
+		(addr == 0x022021FC) ||
+		(addr == 0x0330001C) ||
+		(addr == 0x03300024) ||
+		(addr == 0x033000A8) ||
+		(addr == 0x033000C8) ||
+		(addr == 0x033000BC) ||
+		((addr & 0x0FF00000) == 0x03A00000) ||
+		(addr == 0x03B0001C))
+		return (Q8_INVALID_ADDRESS);
+	else
+		return (addr); 
+}
+
+/*
+ * Name: qla_crb_init
+ * Function: CRB Initialization - first step in the initialization after reset
+ *	Essentially reads the address/value pairs from address = 0x00 and
+ *	writes the value into address in the addr/value pair.
+ */
+static int
+qla_crb_init(qla_host_t *ha)
+{
+	uint32_t val, sig;
+	uint32_t offset, count, i;
+	addr_val_t *addr_val_map, *avmap;
+
+	qla_rd_flash32(ha, 0, &sig);
+	QL_DPRINT2((ha->pci_dev, "%s: val[0] = 0x%08x\n", __func__, val));
+
+	qla_rd_flash32(ha, 4, &val);
+	QL_DPRINT2((ha->pci_dev, "%s: val[4] = 0x%08x\n", __func__, val));
+
+	count = val >> 16;
+	offset = val & 0xFFFF;
+	offset = offset << 2;
+
+	QL_DPRINT2((ha->pci_dev, "%s: [sig,val]=[0x%08x, 0x%08x] %d pairs\n",
+		__func__, sig, val, count));
+
+	addr_val_map = avmap = malloc((sizeof(addr_val_t) * count),
+					M_QLA8XXXBUF, M_NOWAIT);
+
+	if (addr_val_map == NULL) {
+		device_printf(ha->pci_dev, "%s: malloc failed\n", __func__);
+		return (-1);
+	}
+	memset(avmap, 0, (sizeof(addr_val_t) * count));
+
+	count = count << 1;
+	for (i = 0; i < count; ) {
+		qla_rd_flash32(ha, (offset + (i * 4)), &avmap->value);
+		i++;
+		qla_rd_flash32(ha, (offset + (i * 4)), &avmap->addr);
+		i++;
+
+		avmap->pci_addr = qla_int_to_pci_addr_map(ha, avmap->addr);
+		avmap->ind_addr = qla_filter_pci_addr(ha, avmap->pci_addr);
+
+		QL_DPRINT2((ha->pci_dev,
+			"%s: [0x%02x][0x%08x:0x%08x:0x%08x] 0x%08x\n",
+			__func__, (i >> 1), avmap->addr, avmap->pci_addr,
+			avmap->ind_addr, avmap->value));
+
+		if (avmap->ind_addr != Q8_INVALID_ADDRESS) {
+			qla_rdwr_indreg32(ha, avmap->ind_addr, &avmap->value,0);
+			qla_mdelay(__func__, 1);
+		}
+		avmap++;
+	}
+
+	free (addr_val_map, M_QLA8XXXBUF);
+	return (0);
+}
+
+/*
+ * Name: qla_init_peg_regs
+ * Function: Protocol Engine Register Initialization
+ */
+static void
+qla_init_peg_regs(qla_host_t *ha)
+{
+	WRITE_OFFSET32(ha, Q8_PEG_D_RESET1, 0x001E);
+	WRITE_OFFSET32(ha, Q8_PEG_D_RESET2, 0x0008);
+	WRITE_OFFSET32(ha, Q8_PEG_I_RESET, 0x0008);
+	WRITE_OFFSET32(ha, Q8_PEG_0_CLR1, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_0_CLR2, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_1_CLR1, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_1_CLR2, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_2_CLR1, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_2_CLR2, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_3_CLR1, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_3_CLR2, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_4_CLR1, 0x0000);
+	WRITE_OFFSET32(ha, Q8_PEG_4_CLR2, 0x0000);
+}
+
+/*
+ * Name: qla_load_fw_from_flash
+ * Function: Reads the Bootloader from Flash and Loads into Offchip Memory
+ */
+static void
+qla_load_fw_from_flash(qla_host_t *ha)
+{
+	uint64_t mem_off	= 0x10000;
+	uint32_t flash_off	= 0x10000;
+	uint32_t count;
+	offchip_mem_val_t val;
+
+
+	/* only bootloader needs to be loaded into memory */
+	for (count = 0; count < 0x20000 ; ) {
+		qla_rd_flash32(ha, flash_off, &val.data_lo);
+		count = count + 4;
+		flash_off = flash_off + 4;
+
+		qla_rd_flash32(ha, flash_off, &val.data_hi);
+		count = count + 4;
+		flash_off = flash_off + 4;
+
+		qla_rd_flash32(ha, flash_off, &val.data_ulo);
+		count = count + 4;
+		flash_off = flash_off + 4;
+
+		qla_rd_flash32(ha, flash_off, &val.data_uhi);
+		count = count + 4;
+		flash_off = flash_off + 4;
+
+		qla_rdwr_offchip_mem(ha, mem_off, &val, 0);
+
+		mem_off = mem_off + 16;
+	}
+	return;
+}
+
+/*
+ * Name: qla_init_from_flash
+ * Function: Performs Initialization which consists of the following sequence
+ *	- reset
+ *	- CRB Init
+ *	- Peg Init
+ *	- Read the Bootloader from Flash and Load into Offchip Memory
+ *	- Kick start the bootloader which loads the rest of the firmware
+ *		and performs the remaining steps in the initialization process.
+ */
+static int
+qla_init_from_flash(qla_host_t *ha)
+{
+	uint32_t delay = 300;
+	uint32_t data;
+
+	qla_hw_reset(ha);
+	qla_mdelay(__func__, 100);
+
+	qla_crb_init(ha);
+	qla_mdelay(__func__, 10);
+
+	qla_init_peg_regs(ha);
+	qla_mdelay(__func__, 10);
+
+	qla_load_fw_from_flash(ha);
+	
+	WRITE_OFFSET32(ha, Q8_CMDPEG_STATE, 0x00000000);
+	WRITE_OFFSET32(ha, Q8_PEG_0_RESET, 0x00001020);
+	WRITE_OFFSET32(ha, Q8_ASIC_RESET, 0x0080001E);
+	qla_mdelay(__func__, 100);
+
+	do {
+		data = READ_OFFSET32(ha, Q8_CMDPEG_STATE);
+
+		QL_DPRINT2((ha->pci_dev, "%s: func[%d] cmdpegstate 0x%08x\n",
+				__func__, ha->pci_func, data));
+		if (data == CMDPEG_PHAN_INIT_COMPLETE) {
+			QL_DPRINT2((ha->pci_dev,
+				"%s: func[%d] init complete\n",
+				__func__, ha->pci_func));
+			return(0);
+		}
+		qla_mdelay(__func__, 100);
+	} while (delay--);
+
+	device_printf(ha->pci_dev,
+		"%s: func[%d] Q8_PEG_HALT_STATUS1[0x%08x] STATUS2[0x%08x]"
+		" HEARTBEAT[0x%08x] RCVPEG_STATE[0x%08x]"
+		" CMDPEG_STATE[0x%08x]\n",
+		__func__, ha->pci_func,
+		(READ_OFFSET32(ha, Q8_PEG_HALT_STATUS1)),
+		(READ_OFFSET32(ha, Q8_PEG_HALT_STATUS2)),
+		(READ_OFFSET32(ha, Q8_FIRMWARE_HEARTBEAT)),
+		(READ_OFFSET32(ha, Q8_RCVPEG_STATE)), data);
+	
+	return (-1);
+}
+
+/*
+ * Name: qla_init_hw
+ * Function: Initializes P3+ hardware.
+ */
+int
+qla_init_hw(qla_host_t *ha)
+{
+        device_t dev;
+        int ret = 0;
+        uint32_t val, delay = 300;
+
+        dev = ha->pci_dev;
+
+        QL_DPRINT1((dev, "%s: enter\n", __func__));
+
+	qla_mdelay(__func__, 100);
+
+	if (ha->pci_func & 0x1) {
+        	while ((ha->pci_func & 0x1) && delay--) {
+			val = READ_OFFSET32(ha, Q8_CMDPEG_STATE);
+
+			if (val == CMDPEG_PHAN_INIT_COMPLETE) {
+				QL_DPRINT2((dev,
+					"%s: func = %d init complete\n",
+					__func__, ha->pci_func));
+				qla_mdelay(__func__, 100);
+				goto qla_init_exit;
+			}
+			qla_mdelay(__func__, 100);
+		}
+		return (-1);
+	}
+
+	val = READ_OFFSET32(ha, Q8_CMDPEG_STATE);
+
+	if (val != CMDPEG_PHAN_INIT_COMPLETE) {
+        	ret = qla_init_from_flash(ha);
+		qla_mdelay(__func__, 100);
+	}
+
+qla_init_exit:
+        ha->fw_ver_major = READ_OFFSET32(ha, Q8_FW_VER_MAJOR);
+        ha->fw_ver_minor = READ_OFFSET32(ha, Q8_FW_VER_MINOR);
+        ha->fw_ver_sub = READ_OFFSET32(ha, Q8_FW_VER_SUB);
+        ha->fw_ver_build = READ_OFFSET32(ha, Q8_FW_VER_BUILD);
+
+        return (ret);
+}
+
diff --git a/sys/dev/qlxgb/qla_os.c b/sys/dev/qlxgb/qla_os.c
new file mode 100644
index 0000000..1fc30f5
--- /dev/null
+++ b/sys/dev/qlxgb/qla_os.c
@@ -0,0 +1,1481 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File: qla_os.c
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qla_os.h"
+#include "qla_reg.h"
+#include "qla_hw.h"
+#include "qla_def.h"
+#include "qla_inline.h"
+#include "qla_ver.h"
+#include "qla_glbl.h"
+#include "qla_dbg.h"
+
+/*
+ * Some PCI Configuration Space Related Defines
+ */
+
+#ifndef PCI_VENDOR_QLOGIC
+#define PCI_VENDOR_QLOGIC	0x1077
+#endif
+
+#ifndef PCI_PRODUCT_QLOGIC_ISP8020
+#define PCI_PRODUCT_QLOGIC_ISP8020	0x8020
+#endif
+
+#define PCI_QLOGIC_ISP8020 \
+	((PCI_PRODUCT_QLOGIC_ISP8020 << 16) | PCI_VENDOR_QLOGIC)
+
+/*
+ * static functions
+ */
+static int qla_alloc_parent_dma_tag(qla_host_t *ha);
+static void qla_free_parent_dma_tag(qla_host_t *ha);
+static int qla_alloc_xmt_bufs(qla_host_t *ha);
+static void qla_free_xmt_bufs(qla_host_t *ha);
+static int qla_alloc_rcv_bufs(qla_host_t *ha);
+static void qla_free_rcv_bufs(qla_host_t *ha);
+
+static void qla_init_ifnet(device_t dev, qla_host_t *ha);
+static int qla_sysctl_get_stats(SYSCTL_HANDLER_ARGS);
+static void qla_release(qla_host_t *ha);
+static void qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
+		int error);
+static void qla_stop(qla_host_t *ha);
+static int qla_send(qla_host_t *ha, struct mbuf **m_headp);
+static void qla_tx_done(void *context, int pending);
+
+/*
+ * Hooks to the Operating Systems
+ */
+static int qla_pci_probe (device_t);
+static int qla_pci_attach (device_t);
+static int qla_pci_detach (device_t);
+
+static void qla_init(void *arg);
+static int qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
+static int qla_media_change(struct ifnet *ifp);
+static void qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr);
+
+static device_method_t qla_pci_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe, qla_pci_probe),
+	DEVMETHOD(device_attach, qla_pci_attach),
+	DEVMETHOD(device_detach, qla_pci_detach),
+	{ 0, 0 }
+};
+
+static driver_t qla_pci_driver = {
+	"ql", qla_pci_methods, sizeof (qla_host_t),
+};
+
+static devclass_t qla80xx_devclass;
+
+DRIVER_MODULE(qla80xx, pci, qla_pci_driver, qla80xx_devclass, 0, 0);
+
+MODULE_DEPEND(qla80xx, pci, 1, 1, 1);
+MODULE_DEPEND(qla80xx, ether, 1, 1, 1);
+
+MALLOC_DEFINE(M_QLA8XXXBUF, "qla80xxbuf", "Buffers for qla80xx driver");
+
+uint32_t std_replenish = 8;
+uint32_t jumbo_replenish = 2;
+uint32_t rcv_pkt_thres = 128;
+uint32_t rcv_pkt_thres_d = 32;
+uint32_t snd_pkt_thres = 16;
+uint32_t free_pkt_thres = (NUM_TX_DESCRIPTORS / 2);
+
+static char dev_str[64];
+
+/*
+ * Name:	qla_pci_probe
+ * Function:	Validate the PCI device to be a QLA80XX device
+ */
+static int
+qla_pci_probe(device_t dev)
+{
+        switch ((pci_get_device(dev) << 16) | (pci_get_vendor(dev))) {
+        case PCI_QLOGIC_ISP8020:
+		snprintf(dev_str, sizeof(dev_str), "%s v%d.%d.%d",
+			"Qlogic ISP 80xx PCI CNA Adapter-Ethernet Function",
+			QLA_VERSION_MAJOR, QLA_VERSION_MINOR,
+			QLA_VERSION_BUILD);
+                device_set_desc(dev, dev_str);
+                break;
+        default:
+                return (ENXIO);
+        }
+
+        if (bootverbose)
+                printf("%s: %s\n ", __func__, dev_str);
+
+        return (BUS_PROBE_DEFAULT);
+}
+
+static void
+qla_add_sysctls(qla_host_t *ha)
+{
+        device_t dev = ha->pci_dev;
+
+        SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RD,
+                (void *)ha, 0,
+                qla_sysctl_get_stats, "I", "Statistics");
+
+	dbg_level = 0;
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "debug", CTLFLAG_RW,
+                &dbg_level, dbg_level, "Debug Level");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "std_replenish", CTLFLAG_RW,
+                &std_replenish, std_replenish,
+                "Threshold for Replenishing Standard Frames");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "jumbo_replenish", CTLFLAG_RW,
+                &jumbo_replenish, jumbo_replenish,
+                "Threshold for Replenishing Jumbo Frames");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "rcv_pkt_thres",  CTLFLAG_RW,
+                &rcv_pkt_thres, rcv_pkt_thres,
+                "Threshold for # of rcv pkts to trigger indication isr");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "rcv_pkt_thres_d",  CTLFLAG_RW,
+                &rcv_pkt_thres_d, rcv_pkt_thres_d,
+                "Threshold for # of rcv pkts to trigger indication defered");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "snd_pkt_thres",  CTLFLAG_RW,
+                &snd_pkt_thres, snd_pkt_thres,
+                "Threshold for # of snd packets");
+
+        SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
+                SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+                OID_AUTO, "free_pkt_thres",  CTLFLAG_RW,
+                &free_pkt_thres, free_pkt_thres,
+                "Threshold for # of packets to free at a time");
+
+        return;
+}
+
+static void
+qla_watchdog(void *arg)
+{
+	qla_host_t *ha = arg;
+	qla_hw_t *hw;
+	struct ifnet *ifp;
+
+	hw = &ha->hw;
+	ifp = ha->ifp;
+
+        if (ha->flags.qla_watchdog_exit)
+		return;
+
+	if (!ha->flags.qla_watchdog_pause) {
+		if (qla_le32_to_host(*(hw->tx_cons)) != hw->txr_comp) {
+			taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+		} else if ((ifp->if_snd.ifq_head != NULL) && QL_RUNNING(ifp)) {
+			taskqueue_enqueue(ha->tx_tq, &ha->tx_task);
+		}
+	}
+	ha->watchdog_ticks = ha->watchdog_ticks++ % 1000;
+	callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
+		qla_watchdog, ha);
+}
+
+/*
+ * Name:	qla_pci_attach
+ * Function:	attaches the device to the operating system
+ */
+static int
+qla_pci_attach(device_t dev)
+{
+	qla_host_t *ha = NULL;
+	uint32_t rsrc_len, i;
+
+	QL_DPRINT2((dev, "%s: enter\n", __func__));
+
+        if ((ha = device_get_softc(dev)) == NULL) {
+                device_printf(dev, "cannot get softc\n");
+                return (ENOMEM);
+        }
+
+        memset(ha, 0, sizeof (qla_host_t));
+
+        if (pci_get_device(dev) != PCI_PRODUCT_QLOGIC_ISP8020) {
+                device_printf(dev, "device is not ISP8020\n");
+                return (ENXIO);
+	}
+
+        ha->pci_func = pci_get_function(dev);
+
+        ha->pci_dev = dev;
+
+	pci_enable_busmaster(dev);
+
+	ha->reg_rid = PCIR_BAR(0);
+	ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid,
+				RF_ACTIVE);
+
+        if (ha->pci_reg == NULL) {
+                device_printf(dev, "unable to map any ports\n");
+                goto qla_pci_attach_err;
+        }
+
+	rsrc_len = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
+					ha->reg_rid);
+
+	mtx_init(&ha->hw_lock, "qla80xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
+	mtx_init(&ha->tx_lock, "qla80xx_tx_lock", MTX_NETWORK_LOCK, MTX_DEF);
+	mtx_init(&ha->rx_lock, "qla80xx_rx_lock", MTX_NETWORK_LOCK, MTX_DEF);
+	mtx_init(&ha->rxj_lock, "qla80xx_rxj_lock", MTX_NETWORK_LOCK, MTX_DEF);
+	ha->flags.lock_init = 1;
+
+	ha->msix_count = pci_msix_count(dev);
+
+	if (ha->msix_count < qla_get_msix_count(ha)) {
+		device_printf(dev, "%s: msix_count[%d] not enough\n", __func__,
+			ha->msix_count);
+		goto qla_pci_attach_err;
+	}
+
+	QL_DPRINT2((dev, "%s: ha %p irq %p pci_func 0x%x rsrc_count 0x%08x"
+		" msix_count 0x%x pci_reg %p\n", __func__, ha,
+		ha->irq, ha->pci_func, rsrc_len, ha->msix_count, ha->pci_reg));
+
+	ha->msix_count = qla_get_msix_count(ha);
+
+	if (pci_alloc_msix(dev, &ha->msix_count)) {
+		device_printf(dev, "%s: pci_alloc_msi[%d] failed\n", __func__,
+			ha->msix_count);
+		ha->msix_count = 0;
+		goto qla_pci_attach_err;
+	}
+
+	TASK_INIT(&ha->tx_task, 0, qla_tx_done, ha);
+	ha->tx_tq = taskqueue_create_fast("qla_txq", M_NOWAIT,
+			taskqueue_thread_enqueue, &ha->tx_tq);
+	taskqueue_start_threads(&ha->tx_tq, 1, PI_NET, "%s txq",
+		device_get_nameunit(ha->pci_dev));
+
+        for (i = 0; i < ha->msix_count; i++) {
+                ha->irq_vec[i].irq_rid = i+1;
+                ha->irq_vec[i].ha = ha;
+
+                ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+                                        &ha->irq_vec[i].irq_rid,
+                                        (RF_ACTIVE | RF_SHAREABLE));
+
+                if (ha->irq_vec[i].irq == NULL) {
+                        device_printf(dev, "could not allocate interrupt\n");
+                        goto qla_pci_attach_err;
+                }
+
+                if (bus_setup_intr(dev, ha->irq_vec[i].irq,
+                        (INTR_TYPE_NET | INTR_MPSAFE),
+                        NULL, qla_isr, &ha->irq_vec[i],
+                        &ha->irq_vec[i].handle)) {
+                        device_printf(dev, "could not setup interrupt\n");
+                        goto qla_pci_attach_err;
+                }
+
+		TASK_INIT(&ha->irq_vec[i].rcv_task, 0, qla_rcv,\
+			&ha->irq_vec[i]);
+
+		ha->irq_vec[i].rcv_tq = taskqueue_create_fast("qla_rcvq",
+			M_NOWAIT, taskqueue_thread_enqueue,
+			&ha->irq_vec[i].rcv_tq);
+
+		taskqueue_start_threads(&ha->irq_vec[i].rcv_tq, 1, PI_NET,
+			"%s rcvq",
+			device_get_nameunit(ha->pci_dev));
+        }
+
+	qla_add_sysctls(ha);
+
+	/* add hardware specific sysctls */
+	qla_hw_add_sysctls(ha);
+
+	/* initialize hardware */
+	if (qla_init_hw(ha)) {
+		device_printf(dev, "%s: qla_init_hw failed\n", __func__);
+		goto qla_pci_attach_err;
+	}
+
+	device_printf(dev, "%s: firmware[%d.%d.%d.%d]\n", __func__,
+		ha->fw_ver_major, ha->fw_ver_minor, ha->fw_ver_sub,
+		ha->fw_ver_build);
+
+	//qla_get_hw_caps(ha);
+	qla_read_mac_addr(ha);
+
+	/* allocate parent dma tag */
+	if (qla_alloc_parent_dma_tag(ha)) {
+		device_printf(dev, "%s: qla_alloc_parent_dma_tag failed\n",
+			__func__);
+		goto qla_pci_attach_err;
+	}
+
+	/* alloc all dma buffers */
+	if (qla_alloc_dma(ha)) {
+		device_printf(dev, "%s: qla_alloc_dma failed\n", __func__);
+		goto qla_pci_attach_err;
+	}
+
+	/* create the o.s ethernet interface */
+	qla_init_ifnet(dev, ha);
+
+	ha->flags.qla_watchdog_active = 1;
+	ha->flags.qla_watchdog_pause = 1;
+	
+	callout_init(&ha->tx_callout, TRUE);
+
+	/* create ioctl device interface */
+	if (qla_make_cdev(ha)) {
+		device_printf(dev, "%s: qla_make_cdev failed\n", __func__);
+		goto qla_pci_attach_err;
+	}
+
+	callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS,
+		qla_watchdog, ha);
+
+	QL_DPRINT2((dev, "%s: exit 0\n", __func__));
+        return (0);
+
+qla_pci_attach_err:
+
+	qla_release(ha);
+
+	QL_DPRINT2((dev, "%s: exit ENXIO\n", __func__));
+        return (ENXIO);
+}
+
+/*
+ * Name:	qla_pci_detach
+ * Function:	Unhooks the device from the operating system
+ */
+static int
+qla_pci_detach(device_t dev)
+{
+	qla_host_t *ha = NULL;
+	struct ifnet *ifp;
+	int i;
+
+	QL_DPRINT2((dev, "%s: enter\n", __func__));
+
+        if ((ha = device_get_softc(dev)) == NULL) {
+                device_printf(dev, "cannot get softc\n");
+                return (ENOMEM);
+        }
+
+	ifp = ha->ifp;
+
+	QLA_LOCK(ha, __func__);
+	qla_stop(ha);
+	QLA_UNLOCK(ha, __func__);
+
+	if (ha->tx_tq) {
+		taskqueue_drain(ha->tx_tq, &ha->tx_task);
+		taskqueue_free(ha->tx_tq);
+	}
+
+        for (i = 0; i < ha->msix_count; i++) {
+		taskqueue_drain(ha->irq_vec[i].rcv_tq,
+			&ha->irq_vec[i].rcv_task);
+		taskqueue_free(ha->irq_vec[i].rcv_tq);
+	}
+
+	qla_release(ha);
+
+	QL_DPRINT2((dev, "%s: exit\n", __func__));
+
+        return (0);
+}
+
+/*
+ * SYSCTL Related Callbacks
+ */
+static int
+qla_sysctl_get_stats(SYSCTL_HANDLER_ARGS)
+{
+	int err, ret = 0;
+	qla_host_t *ha;
+
+	err = sysctl_handle_int(oidp, &ret, 0, req);
+
+	if (err)
+		return (err);
+
+	ha = (qla_host_t *)arg1;
+	//qla_get_stats(ha);
+	QL_DPRINT2((ha->pci_dev, "%s: called ret %d\n", __func__, ret));
+	return (err);
+}
+
+
+/*
+ * Name:	qla_release
+ * Function:	Releases the resources allocated for the device
+ */
+static void
+qla_release(qla_host_t *ha)
+{
+	device_t dev;
+	int i;
+
+	dev = ha->pci_dev;
+
+	qla_del_cdev(ha);
+
+	if (ha->flags.qla_watchdog_active)
+		ha->flags.qla_watchdog_exit = 1;
+
+	callout_stop(&ha->tx_callout);
+	qla_mdelay(__func__, 100);
+
+	if (ha->ifp != NULL)
+		ether_ifdetach(ha->ifp);
+
+	qla_free_dma(ha); 
+	qla_free_parent_dma_tag(ha);
+
+	for (i = 0; i < ha->msix_count; i++) {
+		if (ha->irq_vec[i].handle)
+			(void)bus_teardown_intr(dev, ha->irq_vec[i].irq,
+				ha->irq_vec[i].handle);
+		if (ha->irq_vec[i].irq)
+			(void) bus_release_resource(dev, SYS_RES_IRQ,
+				ha->irq_vec[i].irq_rid,
+				ha->irq_vec[i].irq);
+	}
+	if (ha->msix_count)
+		pci_release_msi(dev);
+
+	if (ha->flags.lock_init) {
+		mtx_destroy(&ha->tx_lock);
+		mtx_destroy(&ha->rx_lock);
+		mtx_destroy(&ha->rxj_lock);
+		mtx_destroy(&ha->hw_lock);
+	}
+
+        if (ha->pci_reg)
+                (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
+				ha->pci_reg);
+}
+
+/*
+ * DMA Related Functions
+ */
+
+static void
+qla_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+        *((bus_addr_t *)arg) = 0;
+
+        if (error) {
+                printf("%s: bus_dmamap_load failed (%d)\n", __func__, error);
+                return;
+	}
+
+        QL_ASSERT((nsegs == 1), ("%s: %d segments returned!", __func__, nsegs));
+
+        *((bus_addr_t *)arg) = segs[0].ds_addr;
+
+	return;
+}
+
+int
+qla_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
+{
+        int             ret = 0;
+        device_t        dev;
+        bus_addr_t      b_addr;
+
+        dev = ha->pci_dev;
+
+        QL_DPRINT2((dev, "%s: enter\n", __func__));
+
+        ret = bus_dma_tag_create(
+                        ha->parent_tag,/* parent */
+                        dma_buf->alignment,
+                        ((bus_size_t)(1ULL << 32)),/* boundary */
+                        BUS_SPACE_MAXADDR,      /* lowaddr */
+                        BUS_SPACE_MAXADDR,      /* highaddr */
+                        NULL, NULL,             /* filter, filterarg */
+                        dma_buf->size,          /* maxsize */
+                        1,                      /* nsegments */
+                        dma_buf->size,          /* maxsegsize */
+                        0,                      /* flags */
+                        NULL, NULL,             /* lockfunc, lockarg */
+                        &dma_buf->dma_tag);
+
+        if (ret) {
+                device_printf(dev, "%s: could not create dma tag\n", __func__);
+                goto qla_alloc_dmabuf_exit;
+        }
+        ret = bus_dmamem_alloc(dma_buf->dma_tag,
+                        (void **)&dma_buf->dma_b,
+                        (BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT),
+                        &dma_buf->dma_map);
+        if (ret) {
+                bus_dma_tag_destroy(dma_buf->dma_tag);
+                device_printf(dev, "%s: bus_dmamem_alloc failed\n", __func__);
+                goto qla_alloc_dmabuf_exit;
+        }
+
+        ret = bus_dmamap_load(dma_buf->dma_tag,
+                        dma_buf->dma_map,
+                        dma_buf->dma_b,
+                        dma_buf->size,
+                        qla_dmamap_callback,
+                        &b_addr, BUS_DMA_NOWAIT);
+
+        if (ret || !b_addr) {
+                bus_dma_tag_destroy(dma_buf->dma_tag);
+                bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b,
+                        dma_buf->dma_map);
+                ret = -1;
+                goto qla_alloc_dmabuf_exit;
+        }
+
+        dma_buf->dma_addr = b_addr;
+
+qla_alloc_dmabuf_exit:
+        QL_DPRINT2((dev, "%s: exit ret 0x%08x tag %p map %p b %p sz 0x%x\n",
+                __func__, ret, (void *)dma_buf->dma_tag,
+                (void *)dma_buf->dma_map, (void *)dma_buf->dma_b,
+		dma_buf->size));
+
+        return ret;
+}
+
+void
+qla_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf)
+{
+        bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map);
+        bus_dma_tag_destroy(dma_buf->dma_tag);
+}
+
+static int
+qla_alloc_parent_dma_tag(qla_host_t *ha)
+{
+	int		ret;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+        /*
+         * Allocate parent DMA Tag
+         */
+        ret = bus_dma_tag_create(
+                        bus_get_dma_tag(dev),   /* parent */
+                        1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */
+                        BUS_SPACE_MAXADDR,      /* lowaddr */
+                        BUS_SPACE_MAXADDR,      /* highaddr */
+                        NULL, NULL,             /* filter, filterarg */
+                        BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
+                        0,                      /* nsegments */
+                        BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
+                        0,                      /* flags */
+                        NULL, NULL,             /* lockfunc, lockarg */
+                        &ha->parent_tag);
+
+        if (ret) {
+                device_printf(dev, "%s: could not create parent dma tag\n",
+                        __func__);
+		return (-1);
+        }
+
+        ha->flags.parent_tag = 1;
+	
+	return (0);
+}
+
+static void
+qla_free_parent_dma_tag(qla_host_t *ha)
+{
+        if (ha->flags.parent_tag) {
+                bus_dma_tag_destroy(ha->parent_tag);
+                ha->flags.parent_tag = 0;
+        }
+}
+
+/*
+ * Name: qla_init_ifnet
+ * Function: Creates the Network Device Interface and Registers it with the O.S
+ */
+
+static void
+qla_init_ifnet(device_t dev, qla_host_t *ha)
+{
+	struct ifnet *ifp;
+
+	QL_DPRINT2((dev, "%s: enter\n", __func__));
+
+	ifp = ha->ifp = if_alloc(IFT_ETHER);
+
+	if (ifp == NULL)
+		panic("%s: cannot if_alloc()\n", device_get_nameunit(dev));
+
+	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
+
+	ifp->if_mtu = ETHERMTU;
+	ifp->if_baudrate = (1 * 1000 * 1000 *1000);
+	ifp->if_init = qla_init;
+	ifp->if_softc = ha;
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_ioctl = qla_ioctl;
+	ifp->if_start = qla_start;
+
+	IFQ_SET_MAXLEN(&ifp->if_snd, qla_get_ifq_snd_maxlen(ha));
+	ifp->if_snd.ifq_drv_maxlen = qla_get_ifq_snd_maxlen(ha);
+	IFQ_SET_READY(&ifp->if_snd);
+
+	ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
+
+	ether_ifattach(ifp, qla_get_mac_addr(ha));
+
+	ifp->if_capabilities = IFCAP_HWCSUM |
+				IFCAP_TSO4 |
+				IFCAP_TSO6 |
+				IFCAP_JUMBO_MTU;
+
+	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
+
+#if defined(__FreeBSD_version) && (__FreeBSD_version < 900002)
+	ifp->if_timer = 0;
+	ifp->if_watchdog = NULL;
+#endif /* #if defined(__FreeBSD_version) && (__FreeBSD_version < 900002) */
+
+	ifp->if_capenable = ifp->if_capabilities;
+
+	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
+
+	ifmedia_init(&ha->media, IFM_IMASK, qla_media_change, qla_media_status);
+
+	ifmedia_add(&ha->media, (IFM_ETHER | qla_get_optics(ha) | IFM_FDX), 0,
+		NULL);
+	ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL);
+
+	ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
+
+	QL_DPRINT2((dev, "%s: exit\n", __func__));
+
+	return;
+}
+
+static void
+qla_init_locked(qla_host_t *ha)
+{
+	struct ifnet *ifp = ha->ifp;
+
+	qla_stop(ha);
+
+	if (qla_alloc_xmt_bufs(ha) != 0) 
+		return;
+
+	if (qla_alloc_rcv_bufs(ha) != 0)
+		return;
+
+	if (qla_config_lro(ha))
+		return;
+
+	bcopy(IF_LLADDR(ha->ifp), ha->hw.mac_addr, ETHER_ADDR_LEN);
+
+	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
+
+	ha->flags.stop_rcv = 0;
+	if (qla_init_hw_if(ha) == 0) {
+		ifp = ha->ifp;
+		ifp->if_drv_flags |= IFF_DRV_RUNNING;
+		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+		ha->flags.qla_watchdog_pause = 0;
+	}
+
+	return;
+}
+
+static void
+qla_init(void *arg)
+{
+	qla_host_t *ha;
+
+	ha = (qla_host_t *)arg;
+
+	QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
+
+	QLA_LOCK(ha, __func__);
+	qla_init_locked(ha);
+	QLA_UNLOCK(ha, __func__);
+
+	QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
+}
+
+static void
+qla_set_multi(qla_host_t *ha, uint32_t add_multi)
+{
+	uint8_t mta[Q8_MAX_NUM_MULTICAST_ADDRS * Q8_MAC_ADDR_LEN];
+	struct ifmultiaddr *ifma;
+	int mcnt = 0;
+	struct ifnet *ifp = ha->ifp;
+
+	IF_ADDR_LOCK(ifp);
+
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+
+		if (mcnt == Q8_MAX_NUM_MULTICAST_ADDRS)
+			break;
+
+		bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
+			&mta[mcnt * Q8_MAC_ADDR_LEN], Q8_MAC_ADDR_LEN);
+
+		mcnt++;
+	}
+
+	IF_ADDR_UNLOCK(ifp);
+
+	qla_hw_set_multi(ha, mta, mcnt, add_multi);
+
+	return;
+}
+
+static int
+qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+	int ret = 0;
+	struct ifreq *ifr = (struct ifreq *)data;
+	struct ifaddr *ifa = (struct ifaddr *)data;
+	qla_host_t *ha;
+
+	ha = (qla_host_t *)ifp->if_softc;
+
+	switch (cmd) {
+	case SIOCSIFADDR:
+		QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFADDR (0x%lx)\n",
+			__func__, cmd));
+
+		if (ifa->ifa_addr->sa_family == AF_INET) {
+			ifp->if_flags |= IFF_UP;
+			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+				QLA_LOCK(ha, __func__);
+				qla_init_locked(ha);
+				QLA_UNLOCK(ha, __func__);
+			}
+		QL_DPRINT4((ha->pci_dev,
+			"%s: SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
+			__func__, cmd, ntohl(IA_SIN(ifa)->sin_addr.s_addr)));
+
+			arp_ifinit(ifp, ifa);
+			if (ntohl(IA_SIN(ifa)->sin_addr.s_addr) != INADDR_ANY) {
+				qla_config_ipv4_addr(ha,
+					(IA_SIN(ifa)->sin_addr.s_addr));
+			}
+		} else {
+			ether_ioctl(ifp, cmd, data);
+		}
+		break;
+
+	case SIOCSIFMTU:
+		QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFMTU (0x%lx)\n",
+			__func__, cmd));
+
+		if (ifr->ifr_mtu > QLA_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
+			ret = EINVAL;
+		} else {
+			QLA_LOCK(ha, __func__);
+			ifp->if_mtu = ifr->ifr_mtu;
+			ha->max_frame_size =
+				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
+			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+				ret = qla_set_max_mtu(ha, ha->max_frame_size,
+					(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id);
+			}
+			QLA_UNLOCK(ha, __func__);
+
+			if (ret)
+				ret = EINVAL;
+		}
+
+		break;
+
+	case SIOCSIFFLAGS:
+		QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFFLAGS (0x%lx)\n",
+			__func__, cmd));
+
+		if (ifp->if_flags & IFF_UP) {
+			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+				if ((ifp->if_flags ^ ha->if_flags) &
+					IFF_PROMISC) {
+					qla_set_promisc(ha);
+				} else if ((ifp->if_flags ^ ha->if_flags) &
+					IFF_ALLMULTI) {
+					qla_set_allmulti(ha);
+				}
+			} else {
+				QLA_LOCK(ha, __func__);
+				qla_init_locked(ha);
+				ha->max_frame_size = ifp->if_mtu +
+					ETHER_HDR_LEN + ETHER_CRC_LEN;
+				ret = qla_set_max_mtu(ha, ha->max_frame_size,
+					(ha->hw.rx_cntxt_rsp)->rx_rsp.cntxt_id);
+				QLA_UNLOCK(ha, __func__);
+			}
+		} else {
+			QLA_LOCK(ha, __func__);
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+				qla_stop(ha);
+			ha->if_flags = ifp->if_flags;
+			QLA_UNLOCK(ha, __func__);
+		}
+		break;
+
+	case SIOCADDMULTI:
+		QL_DPRINT4((ha->pci_dev,
+			"%s: %s (0x%lx)\n", __func__, "SIOCADDMULTI", cmd));
+
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			qla_set_multi(ha, 1);
+		}
+		break;
+
+	case SIOCDELMULTI:
+		QL_DPRINT4((ha->pci_dev,
+			"%s: %s (0x%lx)\n", __func__, "SIOCDELMULTI", cmd));
+
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			qla_set_multi(ha, 0);
+		}
+		break;
+
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		QL_DPRINT4((ha->pci_dev,
+			"%s: SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n",
+			__func__, cmd));
+		ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
+		break;
+
+	case SIOCSIFCAP:
+	{
+		int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+
+		QL_DPRINT4((ha->pci_dev, "%s: SIOCSIFCAP (0x%lx)\n",
+			__func__, cmd));
+
+		if (mask & IFCAP_HWCSUM)
+			ifp->if_capenable ^= IFCAP_HWCSUM;
+		if (mask & IFCAP_TSO4)
+			ifp->if_capenable ^= IFCAP_TSO4;
+		if (mask & IFCAP_TSO6)
+			ifp->if_capenable ^= IFCAP_TSO6;
+		if (mask & IFCAP_VLAN_HWTAGGING)
+			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
+
+		if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
+			qla_init(ha);
+
+		VLAN_CAPABILITIES(ifp);
+		break;
+	}
+
+	default:
+		QL_DPRINT4((ha->pci_dev, "%s: default (0x%lx)\n",
+			__func__, cmd));
+		ret = ether_ioctl(ifp, cmd, data);
+		break;
+	}
+
+	return (ret);
+}
+
+static int
+qla_media_change(struct ifnet *ifp)
+{
+	qla_host_t *ha;
+	struct ifmedia *ifm;
+	int ret = 0;
+
+	ha = (qla_host_t *)ifp->if_softc;
+
+	QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
+
+	ifm = &ha->media;
+
+	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
+		ret = EINVAL;
+
+	QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
+
+	return (ret);
+}
+
+static void
+qla_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	qla_host_t *ha;
+
+	ha = (qla_host_t *)ifp->if_softc;
+
+	QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
+
+	ifmr->ifm_status = IFM_AVALID;
+	ifmr->ifm_active = IFM_ETHER;
+	
+	qla_update_link_state(ha);
+	if (ha->hw.flags.link_up) {
+		ifmr->ifm_status |= IFM_ACTIVE;
+		ifmr->ifm_active |= (IFM_FDX | qla_get_optics(ha));
+	}
+
+	QL_DPRINT2((ha->pci_dev, "%s: exit (%s)\n", __func__,\
+		(ha->hw.flags.link_up ? "link_up" : "link_down")));
+
+	return;
+}
+
+void
+qla_start(struct ifnet *ifp)
+{
+	struct mbuf    *m_head;
+	qla_host_t *ha = (qla_host_t *)ifp->if_softc;
+
+	QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__));
+
+	if (!mtx_trylock(&ha->tx_lock)) {
+		QL_DPRINT8((ha->pci_dev,
+			"%s: mtx_trylock(&ha->tx_lock) failed\n", __func__));
+		return;
+	}
+
+	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 
+		IFF_DRV_RUNNING) {
+		QL_DPRINT8((ha->pci_dev, "%s: !IFF_DRV_RUNNING\n", __func__));
+		QLA_TX_UNLOCK(ha);
+		return;
+	}
+
+	if (!ha->watchdog_ticks)
+		qla_update_link_state(ha);
+
+	if (!ha->hw.flags.link_up) {
+		QL_DPRINT8((ha->pci_dev, "%s: link down\n", __func__));
+		QLA_TX_UNLOCK(ha);
+		return;
+	}
+
+	while (ifp->if_snd.ifq_head != NULL) {
+		IF_DEQUEUE(&ifp->if_snd, m_head);
+
+		if (m_head == NULL) {
+			QL_DPRINT8((ha->pci_dev, "%s: m_head == NULL\n",
+				__func__));
+			break;
+		}
+
+		if (qla_send(ha, &m_head)) {
+			if (m_head == NULL)
+				break;
+			QL_DPRINT8((ha->pci_dev, "%s: PREPEND\n", __func__));
+			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+			IF_PREPEND(&ifp->if_snd, m_head);
+			break;
+		}
+		/* Send a copy of the frame to the BPF listener */
+		ETHER_BPF_MTAP(ifp, m_head);
+	}
+	QLA_TX_UNLOCK(ha);
+	QL_DPRINT8((ha->pci_dev, "%s: exit\n", __func__));
+	return;
+}
+
+static int
+qla_send(qla_host_t *ha, struct mbuf **m_headp)
+{
+	bus_dma_segment_t	segs[QLA_MAX_SEGMENTS];
+	bus_dmamap_t		map;
+	int			nsegs;
+	int			ret = -1;
+	uint32_t		tx_idx;
+	struct mbuf *m_head = *m_headp;
+
+	QL_DPRINT8((ha->pci_dev, "%s: enter\n", __func__));
+
+	if ((ret = bus_dmamap_create(ha->tx_tag, BUS_DMA_NOWAIT, &map))) {
+		ha->err_tx_dmamap_create++;
+		device_printf(ha->pci_dev,
+			"%s: bus_dmamap_create failed[%d, %d]\n",
+			__func__, ret, m_head->m_pkthdr.len);
+		return (ret);
+	}
+
+	ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
+			BUS_DMA_NOWAIT);
+
+	if ((ret == EFBIG) ||
+		((nsegs > Q8_TX_MAX_SEGMENTS) &&
+		 (((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0) ||
+			(m_head->m_pkthdr.len <= ha->max_frame_size)))) {
+
+		struct mbuf *m;
+
+		QL_DPRINT8((ha->pci_dev, "%s: EFBIG [%d]\n", __func__,
+			m_head->m_pkthdr.len));
+
+		m = m_defrag(m_head, M_DONTWAIT);
+		if (m == NULL) {
+			ha->err_tx_defrag++;
+			m_freem(m_head);
+			*m_headp = NULL;
+			device_printf(ha->pci_dev,
+				"%s: m_defrag() = NULL [%d]\n",
+				__func__, ret);
+			return (ENOBUFS);
+		}
+		m_head = m;
+
+		if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head,
+					segs, &nsegs, BUS_DMA_NOWAIT))) {
+
+			ha->err_tx_dmamap_load++;
+
+			device_printf(ha->pci_dev,
+				"%s: bus_dmamap_load_mbuf_sg failed0[%d, %d]\n",
+				__func__, ret, m_head->m_pkthdr.len);
+
+			bus_dmamap_destroy(ha->tx_tag, map);
+			if (ret != ENOMEM) {
+				m_freem(m_head);
+				*m_headp = NULL;
+			}
+			return (ret);
+		}
+	} else if (ret) {
+		ha->err_tx_dmamap_load++;
+
+		device_printf(ha->pci_dev,
+			"%s: bus_dmamap_load_mbuf_sg failed1[%d, %d]\n",
+			__func__, ret, m_head->m_pkthdr.len);
+
+		bus_dmamap_destroy(ha->tx_tag, map);
+
+		if (ret != ENOMEM) {
+			m_freem(m_head);
+			*m_headp = NULL;
+		}
+		return (ret);
+	}
+
+	QL_ASSERT((nsegs != 0), ("qla_send: empty packet"));
+
+	bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE);
+
+	if (!(ret = qla_hw_send(ha, segs, nsegs, &tx_idx, m_head))) {
+		ha->tx_buf[tx_idx].m_head = m_head;
+		ha->tx_buf[tx_idx].map = map;
+	} else {
+		if (ret == EINVAL) {
+			m_freem(m_head);
+			*m_headp = NULL;
+		}
+	}
+
+	QL_DPRINT8((ha->pci_dev, "%s: exit\n", __func__));
+	return (ret);
+}
+
+static void
+qla_stop(qla_host_t *ha)
+{
+	struct ifnet *ifp = ha->ifp;
+	device_t	dev;
+
+	dev = ha->pci_dev;
+
+	ha->flags.qla_watchdog_pause = 1;
+	qla_mdelay(__func__, 100);
+
+	ha->flags.stop_rcv = 1;
+	qla_hw_stop_rcv(ha);
+
+	qla_del_hw_if(ha);
+
+	qla_free_lro(ha);
+
+	qla_free_xmt_bufs(ha);
+	qla_free_rcv_bufs(ha);
+
+	ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
+
+	return;
+}
+
+/*
+ * Buffer Management Functions for Transmit and Receive Rings
+ */
+static int
+qla_alloc_xmt_bufs(qla_host_t *ha)
+{
+	if (bus_dma_tag_create(NULL,    /* parent */
+		1, 0,    /* alignment, bounds */
+		BUS_SPACE_MAXADDR,       /* lowaddr */
+		BUS_SPACE_MAXADDR,       /* highaddr */
+		NULL, NULL,      /* filter, filterarg */
+		QLA_MAX_TSO_FRAME_SIZE,     /* maxsize */
+		QLA_MAX_SEGMENTS,        /* nsegments */
+		PAGE_SIZE,        /* maxsegsize */
+		BUS_DMA_ALLOCNOW,        /* flags */
+		NULL,    /* lockfunc */
+		NULL,    /* lockfuncarg */
+		&ha->tx_tag)) {
+		device_printf(ha->pci_dev, "%s: tx_tag alloc failed\n",
+			__func__);
+		return (ENOMEM);
+	}
+	bzero((void *)ha->tx_buf, (sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
+
+	return 0;
+}
+
+/*
+ * Release mbuf after it sent on the wire
+ */
+static void
+qla_clear_tx_buf(qla_host_t *ha, qla_tx_buf_t *txb)
+{
+	QL_DPRINT2((ha->pci_dev, "%s: enter\n", __func__));
+
+	if (txb->m_head) {
+
+		bus_dmamap_unload(ha->tx_tag, txb->map);
+		bus_dmamap_destroy(ha->tx_tag, txb->map);
+
+		m_freem(txb->m_head);
+		txb->m_head = NULL;
+	}
+
+	QL_DPRINT2((ha->pci_dev, "%s: exit\n", __func__));
+}
+
+static void
+qla_free_xmt_bufs(qla_host_t *ha)
+{
+	int		i;
+
+	for (i = 0; i < NUM_TX_DESCRIPTORS; i++)
+		qla_clear_tx_buf(ha, &ha->tx_buf[i]);
+
+	if (ha->tx_tag != NULL) {
+		bus_dma_tag_destroy(ha->tx_tag);
+		ha->tx_tag = NULL;
+	}
+	bzero((void *)ha->tx_buf, (sizeof(qla_tx_buf_t) * NUM_TX_DESCRIPTORS));
+
+	return;
+}
+
+
+static int
+qla_alloc_rcv_bufs(qla_host_t *ha)
+{
+	int		i, j, ret = 0;
+	qla_rx_buf_t	*rxb;
+
+	if (bus_dma_tag_create(NULL,    /* parent */
+			1, 0,    /* alignment, bounds */
+			BUS_SPACE_MAXADDR,       /* lowaddr */
+			BUS_SPACE_MAXADDR,       /* highaddr */
+			NULL, NULL,      /* filter, filterarg */
+			MJUM9BYTES,     /* maxsize */
+			1,        /* nsegments */
+			MJUM9BYTES,        /* maxsegsize */
+			BUS_DMA_ALLOCNOW,        /* flags */
+			NULL,    /* lockfunc */
+			NULL,    /* lockfuncarg */
+			&ha->rx_tag)) {
+
+		device_printf(ha->pci_dev, "%s: rx_tag alloc failed\n",
+			__func__);
+
+		return (ENOMEM);
+	}
+
+	bzero((void *)ha->rx_buf, (sizeof(qla_rx_buf_t) * NUM_RX_DESCRIPTORS));
+	bzero((void *)ha->rx_jbuf,
+		(sizeof(qla_rx_buf_t) * NUM_RX_JUMBO_DESCRIPTORS));
+
+	for (i = 0; i < MAX_SDS_RINGS; i++) {
+		ha->hw.sds[i].sdsr_next = 0;
+		ha->hw.sds[i].rxb_free = NULL;
+		ha->hw.sds[i].rx_free = 0;
+		ha->hw.sds[i].rxjb_free = NULL;
+		ha->hw.sds[i].rxj_free = 0;
+	}
+
+	for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
+
+		rxb = &ha->rx_buf[i];
+
+		ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT, &rxb->map);
+
+		if (ret) {
+			device_printf(ha->pci_dev,
+				"%s: dmamap[%d] failed\n", __func__, i);
+
+			for (j = 0; j < i; j++) {
+				bus_dmamap_destroy(ha->rx_tag,
+					ha->rx_buf[j].map);
+			}
+			goto qla_alloc_rcv_bufs_failed;
+		}
+	}
+
+	qla_init_hw_rcv_descriptors(ha, RDS_RING_INDEX_NORMAL);
+
+	for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
+		rxb = &ha->rx_buf[i];
+		rxb->handle = i;
+		if (!(ret = qla_get_mbuf(ha, rxb, NULL, 0))) {
+			/*
+		 	 * set the physical address in the corresponding
+			 * descriptor entry in the receive ring/queue for the
+			 * hba 
+			 */
+			qla_set_hw_rcv_desc(ha, RDS_RING_INDEX_NORMAL, i,
+				rxb->handle, rxb->paddr,
+				(rxb->m_head)->m_pkthdr.len);
+		} else {
+			device_printf(ha->pci_dev,
+				"%s: qla_get_mbuf [standard(%d)] failed\n",
+				__func__, i);
+			bus_dmamap_destroy(ha->rx_tag, rxb->map);
+			goto qla_alloc_rcv_bufs_failed;
+		}
+	}
+
+
+	for (i = 0; i < NUM_RX_JUMBO_DESCRIPTORS; i++) {
+
+		rxb = &ha->rx_jbuf[i];
+
+		ret = bus_dmamap_create(ha->rx_tag, BUS_DMA_NOWAIT, &rxb->map);
+
+		if (ret) {
+			device_printf(ha->pci_dev,
+				"%s: dmamap[%d] failed\n", __func__, i);
+
+			for (j = 0; j < i; j++) {
+				bus_dmamap_destroy(ha->rx_tag,
+					ha->rx_jbuf[j].map);
+			}
+			goto qla_alloc_rcv_bufs_failed;
+		}
+	}
+
+	qla_init_hw_rcv_descriptors(ha, RDS_RING_INDEX_JUMBO);
+
+	for (i = 0; i < NUM_RX_JUMBO_DESCRIPTORS; i++) {
+		rxb = &ha->rx_jbuf[i];
+		rxb->handle = i;
+		if (!(ret = qla_get_mbuf(ha, rxb, NULL, 1))) {
+			/*
+		 	 * set the physical address in the corresponding
+			 * descriptor entry in the receive ring/queue for the
+			 * hba 
+			 */
+			qla_set_hw_rcv_desc(ha, RDS_RING_INDEX_JUMBO, i,
+				rxb->handle, rxb->paddr,
+				(rxb->m_head)->m_pkthdr.len);
+		} else {
+			device_printf(ha->pci_dev,
+				"%s: qla_get_mbuf [jumbo(%d)] failed\n",
+				__func__, i);
+			bus_dmamap_destroy(ha->rx_tag, rxb->map);
+			goto qla_alloc_rcv_bufs_failed;
+		}
+	}
+
+	return (0);
+
+qla_alloc_rcv_bufs_failed:
+	qla_free_rcv_bufs(ha);
+	return (ret);
+}
+
+static void
+qla_free_rcv_bufs(qla_host_t *ha)
+{
+	int		i;
+	qla_rx_buf_t	*rxb;
+
+	for (i = 0; i < NUM_RX_DESCRIPTORS; i++) {
+		rxb = &ha->rx_buf[i];
+		if (rxb->m_head != NULL) {
+			bus_dmamap_unload(ha->rx_tag, rxb->map);
+			bus_dmamap_destroy(ha->rx_tag, rxb->map);
+			m_freem(rxb->m_head);
+			rxb->m_head = NULL;
+		}
+	}
+
+	for (i = 0; i < NUM_RX_JUMBO_DESCRIPTORS; i++) {
+		rxb = &ha->rx_jbuf[i];
+		if (rxb->m_head != NULL) {
+			bus_dmamap_unload(ha->rx_tag, rxb->map);
+			bus_dmamap_destroy(ha->rx_tag, rxb->map);
+			m_freem(rxb->m_head);
+			rxb->m_head = NULL;
+		}
+	}
+
+	if (ha->rx_tag != NULL) {
+		bus_dma_tag_destroy(ha->rx_tag);
+		ha->rx_tag = NULL;
+	}
+
+	bzero((void *)ha->rx_buf, (sizeof(qla_rx_buf_t) * NUM_RX_DESCRIPTORS));
+	bzero((void *)ha->rx_jbuf,
+		(sizeof(qla_rx_buf_t) * NUM_RX_JUMBO_DESCRIPTORS));
+
+	for (i = 0; i < MAX_SDS_RINGS; i++) {
+		ha->hw.sds[i].sdsr_next = 0;
+		ha->hw.sds[i].rxb_free = NULL;
+		ha->hw.sds[i].rx_free = 0;
+		ha->hw.sds[i].rxjb_free = NULL;
+		ha->hw.sds[i].rxj_free = 0;
+	}
+
+	return;
+}
+
+int
+qla_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp,
+	uint32_t jumbo)
+{
+	register struct mbuf *mp = nmp;
+	struct ifnet   *ifp;
+	int             ret = 0;
+	uint32_t	offset;
+
+	QL_DPRINT2((ha->pci_dev, "%s: jumbo(0x%x) enter\n", __func__, jumbo));
+
+	ifp = ha->ifp;
+
+	if (mp == NULL) {
+
+		if (!jumbo) {
+			mp = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+
+			if (mp == NULL) {
+				ha->err_m_getcl++;
+				ret = ENOBUFS;
+				device_printf(ha->pci_dev,
+					"%s: m_getcl failed\n", __func__);
+				goto exit_qla_get_mbuf;
+			}
+			mp->m_len = mp->m_pkthdr.len = MCLBYTES;
+		} else {
+			mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR,
+				MJUM9BYTES);
+			if (mp == NULL) {
+				ha->err_m_getjcl++;
+				ret = ENOBUFS;
+				device_printf(ha->pci_dev,
+					"%s: m_getjcl failed\n", __func__);
+				goto exit_qla_get_mbuf;
+			}
+			mp->m_len = mp->m_pkthdr.len = MJUM9BYTES;
+		}
+	} else {
+		if (!jumbo)
+			mp->m_len = mp->m_pkthdr.len = MCLBYTES;
+		else
+			mp->m_len = mp->m_pkthdr.len = MJUM9BYTES;
+
+		mp->m_data = mp->m_ext.ext_buf;
+		mp->m_next = NULL;
+	}
+
+
+	offset = (uint32_t)((unsigned long long)mp->m_data & 0x7ULL);
+	if (offset) {
+		offset = 8 - offset;
+		m_adj(mp, offset);
+	}
+
+	/*
+	 * Using memory from the mbuf cluster pool, invoke the bus_dma
+	 * machinery to arrange the memory mapping.
+	 */
+	ret = bus_dmamap_load(ha->rx_tag, rxb->map,
+				mtod(mp, void *), mp->m_len,
+				qla_dmamap_callback, &rxb->paddr,
+				BUS_DMA_NOWAIT);
+	if (ret || !rxb->paddr) {
+		m_free(mp);
+		rxb->m_head = NULL;
+		device_printf(ha->pci_dev,
+			"%s: bus_dmamap_load failed\n", __func__);
+                ret = -1;
+		goto exit_qla_get_mbuf;
+	}
+	rxb->m_head = mp;
+	bus_dmamap_sync(ha->rx_tag, rxb->map, BUS_DMASYNC_PREREAD);
+
+exit_qla_get_mbuf:
+	QL_DPRINT2((ha->pci_dev, "%s: exit ret = 0x%08x\n", __func__, ret));
+	return (ret);
+}
+
+static void
+qla_tx_done(void *context, int pending)
+{
+	qla_host_t *ha = context;
+
+	qla_hw_tx_done(ha);
+	qla_start(ha->ifp);
+}
+
diff --git a/sys/dev/qlxgb/qla_os.h b/sys/dev/qlxgb/qla_os.h
new file mode 100644
index 0000000..955be5d
--- /dev/null
+++ b/sys/dev/qlxgb/qla_os.h
@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_os.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#ifndef _QLA_OS_H_
+#define _QLA_OS_H_
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/mbuf.h>
+#include <sys/protosw.h>
+#include <sys/socket.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/kernel.h>
+#include <sys/sockio.h>
+#include <sys/types.h>
+#include <machine/atomic.h>
+#include <sys/conf.h>
+
+#if __FreeBSD_version < 700112
+#error FreeBSD Version not supported - use version >= 700112
+#endif
+
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <net/ethernet.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/bpf.h>
+#include <net/if_types.h>
+#include <net/if_vlan_var.h>
+
+#include <netinet/in_systm.h>
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <netinet/in_var.h>
+#include <netinet/tcp_lro.h>
+
+#include <sys/bus.h>
+#include <machine/bus.h>
+#include <sys/rman.h>
+#include <machine/resource.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <sys/mutex.h>
+#include <sys/condvar.h>
+#include <sys/proc.h>
+#include <sys/sysctl.h>
+#include <sys/endian.h>
+#include <sys/taskqueue.h>
+#include <sys/pcpu.h>
+
+#include <sys/unistd.h>
+#include <sys/kthread.h>
+
+#define QLA_USEC_DELAY(usec)	DELAY(usec)
+
+static __inline int qla_ms_to_hz(int ms)
+{
+	int qla_hz;
+
+	struct timeval t;
+
+	t.tv_sec = ms / 1000;
+	t.tv_usec = (ms % 1000) * 1000;
+
+	qla_hz = tvtohz(&t);
+
+	if (qla_hz < 0)
+		qla_hz = 0x7fffffff;
+	if (!qla_hz)
+		qla_hz = 1;
+
+	return (qla_hz);
+}
+
+static __inline int qla_sec_to_hz(int sec)
+{
+	struct timeval t;
+
+	t.tv_sec = sec;
+	t.tv_usec = 0;
+
+	return (tvtohz(&t));
+}
+
+
+#define qla_host_to_le16(x)	htole16(x)
+#define qla_host_to_le32(x)	htole32(x)
+#define qla_host_to_le64(x)	htole64(x)
+#define qla_host_to_be16(x)	htobe16(x)
+#define qla_host_to_be32(x)	htobe32(x)
+#define qla_host_to_be64(x)	htobe64(x)
+
+#define qla_le16_to_host(x)	le16toh(x)
+#define qla_le32_to_host(x)	le32toh(x)
+#define qla_le64_to_host(x)	le64toh(x)
+#define qla_be16_to_host(x)	be16toh(x)
+#define qla_be32_to_host(x)	be32toh(x)
+#define qla_be64_to_host(x)	be64toh(x)
+
+MALLOC_DECLARE(M_QLA8XXXBUF);
+
+#define qla_mdelay(fn, msecs)	\
+	{\
+		if (cold) \
+			DELAY((msecs * 1000)); \
+		else  \
+			pause(fn, qla_ms_to_hz(msecs)); \
+	}
+	
+/*
+ * Locks
+ */
+#define QLA_LOCK(ha, str) qla_lock(ha, str);
+#define QLA_UNLOCK(ha, str) qla_unlock(ha, str)
+ 
+#define QLA_TX_LOCK(ha)		mtx_lock(&ha->tx_lock);
+#define QLA_TX_UNLOCK(ha)	mtx_unlock(&ha->tx_lock);
+
+#define QLA_RX_LOCK(ha)		mtx_lock(&ha->rx_lock);
+#define QLA_RX_UNLOCK(ha)	mtx_unlock(&ha->rx_lock);
+
+#define QLA_RXJ_LOCK(ha)	mtx_lock(&ha->rxj_lock);
+#define QLA_RXJ_UNLOCK(ha)	mtx_unlock(&ha->rxj_lock);
+
+/*
+ * structure encapsulating a DMA buffer
+ */
+struct qla_dma {
+        bus_size_t              alignment;
+        uint32_t                size;
+        void                    *dma_b;
+        bus_addr_t              dma_addr;
+        bus_dmamap_t            dma_map;
+        bus_dma_tag_t           dma_tag;
+};
+typedef struct qla_dma qla_dma_t;
+
+#define QL_ASSERT(x, y) if (!x) panic y
+
+#endif /* #ifndef _QLA_OS_H_ */
diff --git a/sys/dev/qlxgb/qla_reg.h b/sys/dev/qlxgb/qla_reg.h
new file mode 100644
index 0000000..2f190f3
--- /dev/null
+++ b/sys/dev/qlxgb/qla_reg.h
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_reg.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#ifndef _QLA_REG_H_
+#define _QLA_REG_H_
+
+/*
+ * Begin Definitions for QLA82xx Registers
+ */
+
+/*
+ * Register offsets for QLA8022
+ */
+
+/******************************
+ * PCIe Registers
+ ******************************/
+#define Q8_CRB_WINDOW_2M		0x130060
+
+#define Q8_INT_VECTOR			0x130100
+#define Q8_INT_MASK			0x130104
+
+#define Q8_INT_TARGET_STATUS_F0		0x130118
+#define Q8_INT_TARGET_MASK_F0		0x130128
+#define Q8_INT_TARGET_STATUS_F1		0x130160
+#define Q8_INT_TARGET_MASK_F1		0x130170
+#define Q8_INT_TARGET_STATUS_F2		0x130164
+#define Q8_INT_TARGET_MASK_F2		0x130174
+#define Q8_INT_TARGET_STATUS_F3		0x130168
+#define Q8_INT_TARGET_MASK_F3		0x130178
+#define Q8_INT_TARGET_STATUS_F4		0x130360
+#define Q8_INT_TARGET_MASK_F4		0x130370
+#define Q8_INT_TARGET_STATUS_F5		0x130364
+#define Q8_INT_TARGET_MASK_F5		0x130374
+#define Q8_INT_TARGET_STATUS_F6		0x130368
+#define Q8_INT_TARGET_MASK_F6		0x130378
+#define Q8_INT_TARGET_STATUS_F7		0x13036C
+#define Q8_INT_TARGET_MASK_F7		0x13037C
+
+#define Q8_SEM2_LOCK			0x13C010
+#define Q8_SEM2_UNLOCK			0x13C014
+#define Q8_SEM3_LOCK			0x13C018
+#define Q8_SEM3_UNLOCK			0x13C01C
+#define Q8_SEM5_LOCK			0x13C028
+#define Q8_SEM5_UNLOCK			0x13C02C
+#define Q8_SEM7_LOCK			0x13C038
+#define Q8_SEM7_UNLOCK			0x13C03C
+
+/* Valid bit for a SEM<N>_LOCK registers */
+#define SEM_LOCK_BIT			0x00000001
+
+
+#define Q8_ROM_LOCKID			0x1B2100
+
+/*******************************
+ * Firmware Interface Registers
+ *******************************/
+#define Q8_FW_VER_MAJOR			0x1B2150
+#define Q8_FW_VER_MINOR			0x1B2154
+#define Q8_FW_VER_SUB			0x1B2158
+#define Q8_FW_VER_BUILD			0x1B2168
+
+#define Q8_CMDPEG_STATE			0x1B2250
+#define Q8_RCVPEG_STATE			0x1B233C
+/*
+ * definitions for Q8_CMDPEG_STATE
+ */
+#define CMDPEG_PHAN_INIT_COMPLETE	0xFF01
+
+#define Q8_ROM_STATUS			0x1A0004
+/*
+ * definitions for Q8_ROM_STATUS
+ * bit definitions for Q8_UNM_ROMUSB_GLB_STATUS 
+ * 31:3 Reserved; Rest as below
+ */
+#define	ROM_STATUS_RDY			0x0004
+#define	ROM_STATUS_DONE			0x0002
+#define	ROM_STATUS_AUTO_ROM_SHDW	0x0001
+
+#define Q8_ASIC_RESET			0x1A0008
+/*
+ * definitions for Q8_ASIC_RESET
+ */
+#define ASIC_RESET_RST_XDMA		0x00800000 /* Reset XDMA */
+#define ASIC_RESET_PEG_ICACHE		0x00000020 /* Reset PEG_ICACHE */
+#define ASIC_RESET_PEG_DCACHE		0x00000010 /* Reset PEG_DCACHE */
+#define ASIC_RESET_PEG_3		0x00000008 /* Reset PEG_3 */
+#define ASIC_RESET_PEG_2		0x00000004 /* Reset PEG_2 */
+#define ASIC_RESET_PEG_1		0x00000002 /* Reset PEG_1 */
+#define ASIC_RESET_PEG_0		0x00000001 /* Reset PEG_0 */
+
+#define Q8_COLD_BOOT			0x1B21FC
+/*
+ * definitions for Q8_COLD_BOOT
+ */
+#define COLD_BOOT_VALUE		0x12345678
+
+
+#define Q8_MIU_TEST_AGT_CTRL		0x180090
+#define Q8_MIU_TEST_AGT_ADDR_LO		0x180094
+#define Q8_MIU_TEST_AGT_ADDR_HI		0x180098
+#define Q8_MIU_TEST_AGT_WRDATA_LO	0x1800A0
+#define Q8_MIU_TEST_AGT_WRDATA_HI	0x1800A4
+#define Q8_MIU_TEST_AGT_RDDATA_LO	0x1800A8
+#define Q8_MIU_TEST_AGT_RDDATA_HI	0x1800AC
+#define Q8_MIU_TEST_AGT_WRDATA_ULO	0x1800B0
+#define Q8_MIU_TEST_AGT_WRDATA_UHI	0x1800B4
+#define Q8_MIU_TEST_AGT_RDDATA_ULO	0x1800B8
+#define Q8_MIU_TEST_AGT_RDDATA_UHI	0x1800BC
+
+#define Q8_PEG_0_RESET			0x160018
+#define Q8_PEG_0_CLR1			0x160008
+#define Q8_PEG_0_CLR2			0x16000C
+#define Q8_PEG_1_CLR1			0x161008
+#define Q8_PEG_1_CLR2			0x16100C
+#define Q8_PEG_2_CLR1			0x162008
+#define Q8_PEG_2_CLR2			0x16200C
+#define Q8_PEG_3_CLR1			0x163008
+#define Q8_PEG_3_CLR2			0x16300C
+#define Q8_PEG_4_CLR1			0x164008
+#define Q8_PEG_4_CLR2			0x16400C
+#define Q8_PEG_D_RESET1			0x1650EC
+#define Q8_PEG_D_RESET2			0x16504C
+#define Q8_PEG_HALT_STATUS1		0x1B20A8
+#define Q8_PEG_HALT_STATUS2		0x1B20AC
+#define Q8_FIRMWARE_HEARTBEAT		0x1B20B0
+#define Q8_PEG_I_RESET			0x16604C
+
+#define Q8_CRB_MAC_BLOCK_START		0x1B21C0
+
+/***************************************************
+ * Flash ROM Access Registers ( Indirect Registers )
+ ***************************************************/
+
+#define Q8_ROM_INSTR_OPCODE		0x03310004
+/*
+ * bit definitions for Q8_ROM_INSTR_OPCODE 
+ * 31:8 Reserved; Rest Below
+ */
+#define ROM_OPCODE_WR_STATUS_REG	0x01
+#define ROM_OPCODE_PROG_PAGE		0x02
+#define ROM_OPCODE_RD_BYTE		0x03
+#define ROM_OPCODE_WR_DISABLE		0x04
+#define ROM_OPCODE_RD_STATUS_REG	0x05
+#define ROM_OPCODE_WR_ENABLE		0x06
+#define ROM_OPCODE_FAST_RD		0x0B
+#define ROM_OPCODE_REL_DEEP_PWR_DWN	0xAB
+#define ROM_OPCODE_BULK_ERASE		0xC7
+#define ROM_OPCODE_DEEP_PWR_DWN		0xC9
+#define ROM_OPCODE_SECTOR_ERASE		0xD8
+
+#define Q8_ROM_ADDRESS			0x03310008
+/*
+ * bit definitions for Q8_ROM_ADDRESS 
+ * 31:24 Reserved;
+ * 23:0  Physical ROM Address in bytes
+ */
+
+#define Q8_ROM_ADDR_BYTE_COUNT		0x03310010
+/*
+ * bit definitions for Q8_ROM_ADDR_BYTE_COUNT 
+ * 31:2 Reserved;
+ * 1:0  max address bytes for ROM Interface
+ */
+ 
+#define Q8_ROM_DUMMY_BYTE_COUNT		0x03310014
+/*
+ * bit definitions for Q8_ROM_DUMMY_BYTE_COUNT 
+ * 31:2 Reserved;
+ * 1:0 dummy bytes for ROM Instructions
+ */
+
+#define Q8_ROM_RD_DATA			0x03310018
+
+#define Q8_NX_CDRP_CMD_RSP		0x1B2218
+#define Q8_NX_CDRP_ARG1			0x1B221C
+#define Q8_NX_CDRP_ARG2			0x1B2220
+#define Q8_NX_CDRP_ARG3			0x1B2224
+#define Q8_NX_CDRP_SIGNATURE		0x1B2228
+
+#define Q8_LINK_STATE			0x1B2298
+#define Q8_LINK_SPEED_0			0x1B22E8
+/*
+ * Macros for reading and writing registers
+ */
+
+#if defined(__i386__) || defined(__amd64__)
+#define Q8_MB()    __asm volatile("mfence" ::: "memory")
+#define Q8_WMB()   __asm volatile("sfence" ::: "memory")
+#define Q8_RMB()   __asm volatile("lfence" ::: "memory")
+#else
+#define Q8_MB()
+#define Q8_WMB()
+#define Q8_RMB()
+#endif
+
+#define READ_REG32(ha, reg) bus_read_4((ha->pci_reg), reg)
+#define READ_OFFSET32(ha, off) READ_REG32(ha, off)
+
+#define WRITE_REG32(ha, reg, val) \
+	{\
+		bus_write_4((ha->pci_reg), reg, val);\
+		bus_read_4((ha->pci_reg), reg);\
+	}
+
+#define WRITE_REG32_MB(ha, reg, val) \
+	{\
+		Q8_WMB();\
+		bus_write_4((ha->pci_reg), reg, val);\
+	}
+
+#define WRITE_OFFSET32(ha, off, val)\
+		{\
+			bus_write_4((ha->pci_reg), off, val);\
+			bus_read_4((ha->pci_reg), off);\
+		}
+
+#endif /* #ifndef _QLA_REG_H_ */
diff --git a/sys/dev/qlxgb/qla_ver.h b/sys/dev/qlxgb/qla_ver.h
new file mode 100644
index 0000000..8c33ff4
--- /dev/null
+++ b/sys/dev/qlxgb/qla_ver.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2010-2011 Qlogic Corporation
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * File: qla_ver.h
+ * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+ */
+
+#ifndef _QLA_VER_H_
+#define _QLA_VER_H_
+
+#define QLA_VERSION_MAJOR 	1
+#define QLA_VERSION_MINOR	1
+#define QLA_VERSION_BUILD	30
+
+#endif /* #ifndef _QLA_VER_H_ */
diff --git a/sys/dev/quicc/quicc_core.c b/sys/dev/quicc/quicc_core.c
index 21cfdb3..532bb8e 100644
--- a/sys/dev/quicc/quicc_core.c
+++ b/sys/dev/quicc/quicc_core.c
@@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");
 devclass_t quicc_devclass;
 char quicc_driver_name[] = "quicc";
 
-MALLOC_DEFINE(M_QUICC, "QUICC", "QUICC driver");
+static MALLOC_DEFINE(M_QUICC, "QUICC", "QUICC driver");
 
 struct quicc_device {
 	struct rman	*qd_rman;
diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index 704276e..3ade04c 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -171,7 +171,7 @@ TUNABLE_INT("hw.re.prefer_iomap", &prefer_iomap);
 /*
  * Various supported device vendors/types and their names.
  */
-static struct rl_type re_devs[] = {
+static const struct rl_type const re_devs[] = {
 	{ DLINK_VENDORID, DLINK_DEVICEID_528T, 0,
 	    "D-Link DGE-528(T) Gigabit Ethernet Adapter" },
 	{ DLINK_VENDORID, DLINK_DEVICEID_530T_REVC, 0,
@@ -181,7 +181,7 @@ static struct rl_type re_devs[] = {
 	{ RT_VENDORID, RT_DEVICEID_8101E, 0,
 	    "RealTek 810xE PCIe 10/100baseTX" },
 	{ RT_VENDORID, RT_DEVICEID_8168, 0,
-	    "RealTek 8168/8111 B/C/CP/D/DP/E PCIe Gigabit Ethernet" },
+	    "RealTek 8168/8111 B/C/CP/D/DP/E/F PCIe Gigabit Ethernet" },
 	{ RT_VENDORID, RT_DEVICEID_8169, 0,
 	    "RealTek 8169/8169S/8169SB(L)/8110S/8110SB(L) Gigabit Ethernet" },
 	{ RT_VENDORID, RT_DEVICEID_8169SC, 0,
@@ -194,8 +194,8 @@ static struct rl_type re_devs[] = {
 	    "US Robotics 997902 (RTL8169S) Gigabit Ethernet" }
 };
 
-static struct rl_hwrev re_hwrevs[] = {
-	{ RL_HWREV_8139, RL_8139,  "", RL_MTU },
+static const struct rl_hwrev const re_hwrevs[] = {
+	{ RL_HWREV_8139, RL_8139, "", RL_MTU },
 	{ RL_HWREV_8139A, RL_8139, "A", RL_MTU },
 	{ RL_HWREV_8139AG, RL_8139, "A-G", RL_MTU },
 	{ RL_HWREV_8139B, RL_8139, "B", RL_MTU },
@@ -220,7 +220,9 @@ static struct rl_hwrev re_hwrevs[] = {
 	{ RL_HWREV_8102EL_SPIN1, RL_8169, "8102EL", RL_MTU },
 	{ RL_HWREV_8103E, RL_8169, "8103E", RL_MTU },
 	{ RL_HWREV_8401E, RL_8169, "8401E", RL_MTU },
+	{ RL_HWREV_8402, RL_8169, "8402", RL_MTU },
 	{ RL_HWREV_8105E, RL_8169, "8105E", RL_MTU },
+	{ RL_HWREV_8105E_SPIN1, RL_8169, "8105E", RL_MTU },
 	{ RL_HWREV_8168B_SPIN2, RL_8169, "8168", RL_JUMBO_MTU },
 	{ RL_HWREV_8168B_SPIN3, RL_8169, "8168", RL_JUMBO_MTU },
 	{ RL_HWREV_8168C, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
@@ -230,6 +232,8 @@ static struct rl_hwrev re_hwrevs[] = {
 	{ RL_HWREV_8168DP, RL_8169, "8168DP/8111DP", RL_JUMBO_MTU_9K },
 	{ RL_HWREV_8168E, RL_8169, "8168E/8111E", RL_JUMBO_MTU_9K},
 	{ RL_HWREV_8168E_VL, RL_8169, "8168E/8111E-VL", RL_JUMBO_MTU_6K},
+	{ RL_HWREV_8168F, RL_8169, "8168F/8111F", RL_JUMBO_MTU_9K},
+	{ RL_HWREV_8411, RL_8169, "8411", RL_JUMBO_MTU_9K},
 	{ 0, 0, NULL, 0 }
 };
 
@@ -868,7 +872,7 @@ re_diag(struct rl_softc *sc)
 		device_printf(sc->rl_dev, "expected TX data: %6D/%6D/0x%x\n",
 		    dst, ":", src, ":", ETHERTYPE_IP);
 		device_printf(sc->rl_dev, "received RX data: %6D/%6D/0x%x\n",
-		    eh->ether_dhost, ":",  eh->ether_shost, ":",
+		    eh->ether_dhost, ":", eh->ether_shost, ":",
 		    ntohs(eh->ether_type));
 		device_printf(sc->rl_dev, "You may have a defective 32-bit "
 		    "NIC plugged into a 64-bit PCI slot.\n");
@@ -903,7 +907,7 @@ done:
 static int
 re_probe(device_t dev)
 {
-	struct rl_type		*t;
+	const struct rl_type	*t;
 	uint16_t		devid, vendor;
 	uint16_t		revid, sdevid;
 	int			i;
@@ -1183,7 +1187,8 @@ re_attach(device_t dev)
 	u_int16_t		as[ETHER_ADDR_LEN / 2];
 	struct rl_softc		*sc;
 	struct ifnet		*ifp;
-	struct rl_hwrev		*hw_rev;
+	const struct rl_hwrev	*hw_rev;
+	u_int32_t		cap, ctl;
 	int			hwrev;
 	u_int16_t		devid, re_did = 0;
 	int			error = 0, i, phy, rid;
@@ -1239,8 +1244,10 @@ re_attach(device_t dev)
 
 	msic = pci_msi_count(dev);
 	msixc = pci_msix_count(dev);
-	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0)
+	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
 		sc->rl_flags |= RL_FLAG_PCIE;
+		sc->rl_expcap = reg;
+	}
 	if (bootverbose) {
 		device_printf(dev, "MSI count : %d\n", msic);
 		device_printf(dev, "MSI-X count : %d\n", msixc);
@@ -1332,6 +1339,23 @@ re_attach(device_t dev)
 		CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
 	}
 
+	/* Disable ASPM L0S/L1. */
+	if (sc->rl_expcap != 0) {
+		cap = pci_read_config(dev, sc->rl_expcap +
+		    PCIR_EXPRESS_LINK_CAP, 2);
+		if ((cap & PCIM_LINK_CAP_ASPM) != 0) {
+			ctl = pci_read_config(dev, sc->rl_expcap +
+			    PCIR_EXPRESS_LINK_CTL, 2);
+			if ((ctl & 0x0003) != 0) {
+				ctl &= ~0x0003;
+				pci_write_config(dev, sc->rl_expcap +
+				    PCIR_EXPRESS_LINK_CTL, ctl, 2);
+				device_printf(dev, "ASPM disabled\n");
+			}
+		} else
+			device_printf(dev, "no ASPM capability\n");
+	}
+
 	hw_rev = re_hwrevs;
 	hwrev = CSR_READ_4(sc, RL_TXCFG);
 	switch (hwrev & 0x70000000) {
@@ -1381,7 +1405,9 @@ re_attach(device_t dev)
 		    RL_FLAG_AUTOPAD | RL_FLAG_MACSLEEP;
 		break;
 	case RL_HWREV_8401E:
+	case RL_HWREV_8402:
 	case RL_HWREV_8105E:
+	case RL_HWREV_8105E_SPIN1:
 		sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
 		    RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
 		    RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD;
@@ -1413,6 +1439,8 @@ re_attach(device_t dev)
 		    RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2;
 		break;
 	case RL_HWREV_8168E_VL:
+	case RL_HWREV_8168F:
+	case RL_HWREV_8411:
 		sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
 		    RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
 		    RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2;
@@ -3308,6 +3336,7 @@ re_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 			}
 		}
 #endif /* DEVICE_POLLING */
+		RL_LOCK(sc);
 		if ((mask & IFCAP_TXCSUM) != 0 &&
 		    (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
 			ifp->if_capenable ^= IFCAP_TXCSUM;
@@ -3366,8 +3395,9 @@ re_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 		}
 		if (reinit && ifp->if_drv_flags & IFF_DRV_RUNNING) {
 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
-			re_init(sc);
+			re_init_locked(sc);
 		}
+		RL_UNLOCK(sc);
 		VLAN_CAPABILITIES(ifp);
 	    }
 		break;
diff --git a/sys/dev/rndtest/rndtest.c b/sys/dev/rndtest/rndtest.c
index 15ddccee..4e42aa8 100644
--- a/sys/dev/rndtest/rndtest.c
+++ b/sys/dev/rndtest/rndtest.c
@@ -70,8 +70,8 @@ static const struct rndtest_testfunc {
 
 #define	RNDTEST_NTESTS	(sizeof(rndtest_funcs)/sizeof(rndtest_funcs[0]))
 
-SYSCTL_NODE(_kern, OID_AUTO, rndtest, CTLFLAG_RD, 0, "RNG test parameters");
-
+static SYSCTL_NODE(_kern, OID_AUTO, rndtest, CTLFLAG_RD, 0,
+	    "RNG test parameters");
 static	int rndtest_retest = 120;		/* interval in seconds */
 SYSCTL_INT(_kern_rndtest, OID_AUTO, retest, CTLFLAG_RW, &rndtest_retest,
 	    0, "retest interval (seconds)");
diff --git a/sys/dev/rt/if_rt.c b/sys/dev/rt/if_rt.c
index 22105dc..edaf95e 100644
--- a/sys/dev/rt/if_rt.c
+++ b/sys/dev/rt/if_rt.c
@@ -136,7 +136,7 @@ static int	rt_miibus_writereg(device_t, int, int, int);
 static int	rt_ifmedia_upd(struct ifnet *);
 static void	rt_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 
-SYSCTL_NODE(_hw, OID_AUTO, rt, CTLFLAG_RD, 0, "RT driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, rt, CTLFLAG_RD, 0, "RT driver parameters");
 #ifdef IF_RT_DEBUG
 static int rt_debug = 0;
 SYSCTL_INT(_hw_rt, OID_AUTO, debug, CTLFLAG_RW, &rt_debug, 0,
diff --git a/sys/dev/safe/safe.c b/sys/dev/safe/safe.c
index 18ef5e5..00c00ea 100644
--- a/sys/dev/safe/safe.c
+++ b/sys/dev/safe/safe.c
@@ -147,7 +147,8 @@ static	void safe_totalreset(struct safe_softc *);
 
 static	int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
 
-SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD, 0, "SafeNet driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD, 0,
+    "SafeNet driver parameters");
 
 #ifdef SAFE_DEBUG
 static	void safe_dump_dmastatus(struct safe_softc *, const char *);
diff --git a/sys/dev/scc/scc_core.c b/sys/dev/scc/scc_core.c
index 76388ad..4e12244 100644
--- a/sys/dev/scc/scc_core.c
+++ b/sys/dev/scc/scc_core.c
@@ -48,7 +48,7 @@ __FBSDID("$FreeBSD$");
 devclass_t scc_devclass;
 char scc_driver_name[] = "scc";
 
-MALLOC_DEFINE(M_SCC, "SCC", "SCC driver");
+static MALLOC_DEFINE(M_SCC, "SCC", "SCC driver");
 
 static int
 scc_bfe_intr(void *arg)
diff --git a/sys/dev/sdhci/sdhci.c b/sys/dev/sdhci/sdhci.c
index 24cba57..e071155 100644
--- a/sys/dev/sdhci/sdhci.c
+++ b/sys/dev/sdhci/sdhci.c
@@ -154,7 +154,7 @@ struct sdhci_softc {
 	struct sdhci_slot slots[6];
 };
 
-SYSCTL_NODE(_hw, OID_AUTO, sdhci, CTLFLAG_RD, 0, "sdhci driver");
+static SYSCTL_NODE(_hw, OID_AUTO, sdhci, CTLFLAG_RD, 0, "sdhci driver");
 
 int	sdhci_debug;
 TUNABLE_INT("hw.sdhci.debug", &sdhci_debug);
diff --git a/sys/dev/sfxge/common/efsys.h b/sys/dev/sfxge/common/efsys.h
new file mode 100644
index 0000000..3e83d24
--- /dev/null
+++ b/sys/dev/sfxge/common/efsys.h
@@ -0,0 +1,834 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_SYS_EFSYS_H
+#define	_SYS_EFSYS_H
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/rwlock.h>
+#include <sys/sdt.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+#include <machine/endian.h>
+
+#define	EFSYS_HAS_UINT64 1
+#define	EFSYS_USE_UINT64 0
+#if _BYTE_ORDER == _BIG_ENDIAN
+#define EFSYS_IS_BIG_ENDIAN 1
+#define EFSYS_IS_LITTLE_ENDIAN 0
+#elif _BYTE_ORDER == _LITTLE_ENDIAN
+#define EFSYS_IS_BIG_ENDIAN 0
+#define EFSYS_IS_LITTLE_ENDIAN 1
+#endif
+#include "efx_types.h"
+
+/* Common code requires this */
+#if __FreeBSD_version < 800068
+#define memmove(d, s, l) bcopy(s, d, l)
+#endif
+	
+/* FreeBSD equivalents of Solaris things */
+#ifndef _NOTE
+#define _NOTE(s)
+#endif
+
+#ifndef B_FALSE
+#define B_FALSE FALSE
+#endif
+#ifndef B_TRUE
+#define B_TRUE TRUE
+#endif
+
+#ifndef IS_P2ALIGNED
+#define	IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
+#endif
+
+#ifndef P2ROUNDUP
+#define P2ROUNDUP(x, align)             (-(-(x) & -(align)))
+#endif
+
+#ifndef IS2P
+#define ISP2(x)         (((x) & ((x) - 1)) == 0)
+#endif
+
+#define ENOTACTIVE EINVAL
+
+/* Memory type to use on FreeBSD */
+MALLOC_DECLARE(M_SFXGE);
+
+/* Machine dependend prefetch wrappers */
+#if defined(__i386__) || defined(__amd64__)
+static __inline void
+prefetch_read_many(void *addr)
+{
+
+	__asm__(
+	    "prefetcht0 (%0)"
+	    :
+	    : "r" (addr));
+}
+
+static __inline void
+prefetch_read_once(void *addr)
+{
+
+	__asm__(
+	    "prefetchnta (%0)"
+	    :
+	    : "r" (addr));
+}
+#elif defined(__sparc64__)
+static __inline void
+prefetch_read_many(void *addr)
+{
+
+	__asm__(
+	    "prefetch [%0], 0"
+	    :
+	    : "r" (addr));
+}
+
+static __inline void
+prefetch_read_once(void *addr)
+{
+
+	__asm__(
+	    "prefetch [%0], 1"
+	    :
+	    : "r" (addr));
+}
+#else
+static __inline void
+prefetch_read_many(void *addr)
+{
+
+}
+
+static __inline void
+prefetch_read_once(void *addr)
+{
+
+}
+#endif
+
+#if defined(__i386__) || defined(__amd64__)
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#endif
+static __inline void
+sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
+    struct mbuf *m, bus_dma_segment_t *seg)
+{
+#if defined(__i386__) || defined(__amd64__)
+	seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
+	seg->ds_len = m->m_len;
+#else
+	int nsegstmp;
+
+	bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
+#endif
+}
+
+/* Modifiers used for DOS builds */
+#define	__cs
+#define	__far
+
+/* Modifiers used for Windows builds */
+#define	__in
+#define	__in_opt
+#define	__in_ecount(_n)
+#define	__in_ecount_opt(_n)
+#define	__in_bcount(_n)
+#define	__in_bcount_opt(_n)
+
+#define	__out
+#define	__out_opt
+#define	__out_ecount(_n)
+#define	__out_ecount_opt(_n)
+#define	__out_bcount(_n)
+#define	__out_bcount_opt(_n)
+
+#define	__deref_out
+
+#define	__inout
+#define	__inout_opt
+#define	__inout_ecount(_n)
+#define	__inout_ecount_opt(_n)
+#define	__inout_bcount(_n)
+#define	__inout_bcount_opt(_n)
+#define	__inout_bcount_full_opt(_n)
+
+#define	__deref_out_bcount_opt(n)
+
+#define	__checkReturn
+
+#define	__drv_when(_p, _c)
+
+/* Code inclusion options */
+
+
+#define	EFSYS_OPT_NAMES 1
+
+#define	EFSYS_OPT_FALCON 0
+#define	EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
+#define	EFSYS_OPT_SIENA 1
+#ifdef DEBUG
+#define	EFSYS_OPT_CHECK_REG 1
+#else
+#define	EFSYS_OPT_CHECK_REG 0
+#endif
+
+#define	EFSYS_OPT_MCDI 1
+
+#define	EFSYS_OPT_MAC_FALCON_GMAC 0
+#define	EFSYS_OPT_MAC_FALCON_XMAC 0
+#define	EFSYS_OPT_MAC_STATS 1
+
+#define	EFSYS_OPT_LOOPBACK 0
+
+#define	EFSYS_OPT_MON_NULL 0
+#define	EFSYS_OPT_MON_LM87 0
+#define	EFSYS_OPT_MON_MAX6647 0
+#define	EFSYS_OPT_MON_SIENA 0
+#define	EFSYS_OPT_MON_STATS 0
+
+#define	EFSYS_OPT_PHY_NULL 0
+#define	EFSYS_OPT_PHY_QT2022C2 0
+#define	EFSYS_OPT_PHY_SFX7101 0
+#define	EFSYS_OPT_PHY_TXC43128 0
+#define	EFSYS_OPT_PHY_PM8358 0
+#define	EFSYS_OPT_PHY_SFT9001 0
+#define	EFSYS_OPT_PHY_QT2025C 0
+#define	EFSYS_OPT_PHY_STATS 1
+#define	EFSYS_OPT_PHY_PROPS 0
+#define	EFSYS_OPT_PHY_BIST 1
+#define	EFSYS_OPT_PHY_LED_CONTROL 1
+#define EFSYS_OPT_PHY_FLAGS 0
+
+#define	EFSYS_OPT_VPD 1
+#define	EFSYS_OPT_NVRAM 1
+#define	EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
+#define	EFSYS_OPT_NVRAM_SFT9001	0
+#define	EFSYS_OPT_NVRAM_SFX7101	0
+#define	EFSYS_OPT_BOOTCFG 0
+
+#define	EFSYS_OPT_PCIE_TUNE 0
+#define	EFSYS_OPT_DIAG 0
+#define	EFSYS_OPT_WOL 1
+#define	EFSYS_OPT_RX_SCALE 1
+#define	EFSYS_OPT_QSTATS 1
+#define EFSYS_OPT_FILTER 0
+#define EFSYS_OPT_RX_SCATTER 0
+#define	EFSYS_OPT_RX_HDR_SPLIT 0
+
+#define	EFSYS_OPT_EV_PREFETCH 0
+
+#define	EFSYS_OPT_DECODE_INTR_FATAL 1
+
+/* ID */
+
+typedef struct __efsys_identifier_s	efsys_identifier_t;
+
+/* PROBE */
+
+#ifndef KDTRACE_HOOKS
+
+#define EFSYS_PROBE(_name)
+
+#define	EFSYS_PROBE1(_name, _type1, _arg1)
+
+#define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
+
+#define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3)
+
+#define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4)
+
+#define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
+
+#define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6)
+
+#define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6, _type7, _arg7)
+
+#else /* KDTRACE_HOOKS */
+
+#define	EFSYS_PROBE(_name)						\
+	DTRACE_PROBE(_name)
+
+#define	EFSYS_PROBE1(_name, _type1, _arg1)				\
+	DTRACE_PROBE1(_name, _type1, _arg1)
+
+#define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)		\
+	DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
+
+#define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3)						\
+	DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3)
+
+#define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4)				\
+	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4)
+
+#ifdef DTRACE_PROBE5
+#define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
+	DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
+#else
+#define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
+	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4)
+#endif
+
+#ifdef DTRACE_PROBE6
+#define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6)						\
+	DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6)
+#else
+#define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6)						\
+	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
+#endif
+
+#ifdef DTRACE_PROBE7
+#define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6, _type7, _arg7)				\
+	DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6, _type7, _arg7)
+#else
+#define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6, _type7, _arg7)				\
+	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
+	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
+	    _type6, _arg6)
+#endif
+
+#endif /* KDTRACE_HOOKS */
+
+/* DMA */
+
+typedef uint64_t		efsys_dma_addr_t;
+
+typedef struct efsys_mem_s {
+	bus_dma_tag_t		esm_tag;
+	bus_dmamap_t		esm_map;
+	caddr_t			esm_base;
+	efsys_dma_addr_t	esm_addr;
+	size_t			esm_size;
+} efsys_mem_t;
+
+
+#define	EFSYS_MEM_ZERO(_esmp, _size)					\
+	do {								\
+		(void) memset((_esmp)->esm_base, 0, (_size));		\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_READD(_esmp, _offset, _edp)				\
+	do {								\
+		uint32_t *addr;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		addr = (void *)((_esmp)->esm_base + (_offset));		\
+									\
+		(_edp)->ed_u32[0] = *addr;				\
+									\
+		EFSYS_PROBE2(mem_readd, unsigned int, (_offset),	\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_READQ(_esmp, _offset, _eqp)				\
+	do {								\
+		uint32_t *addr;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		addr = (void *)((_esmp)->esm_base + (_offset));		\
+									\
+		(_eqp)->eq_u32[0] = *addr++;				\
+		(_eqp)->eq_u32[1] = *addr;				\
+									\
+		EFSYS_PROBE3(mem_readq, unsigned int, (_offset),	\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_READO(_esmp, _offset, _eop)				\
+	do {								\
+		uint32_t *addr;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		addr = (void *)((_esmp)->esm_base + (_offset));		\
+									\
+		(_eop)->eo_u32[0] = *addr++;				\
+		(_eop)->eo_u32[1] = *addr++;				\
+		(_eop)->eo_u32[2] = *addr++;				\
+		(_eop)->eo_u32[3] = *addr;				\
+									\
+		EFSYS_PROBE5(mem_reado, unsigned int, (_offset),	\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_WRITED(_esmp, _offset, _edp)				\
+	do {								\
+		uint32_t *addr;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		EFSYS_PROBE2(mem_writed, unsigned int, (_offset),	\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+									\
+		addr = (void *)((_esmp)->esm_base + (_offset));		\
+									\
+		*addr = (_edp)->ed_u32[0];				\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)				\
+	do {								\
+		uint32_t *addr;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),	\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+									\
+		addr = (void *)((_esmp)->esm_base + (_offset));		\
+									\
+		*addr++ = (_eqp)->eq_u32[0];				\
+		*addr   = (_eqp)->eq_u32[1];				\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_WRITEO(_esmp, _offset, _eop)				\
+	do {								\
+		uint32_t *addr;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),	\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+									\
+		addr = (void *)((_esmp)->esm_base + (_offset));		\
+									\
+		*addr++ = (_eop)->eo_u32[0];				\
+		*addr++ = (_eop)->eo_u32[1];				\
+		*addr++ = (_eop)->eo_u32[2];				\
+		*addr   = (_eop)->eo_u32[3];				\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_MEM_ADDR(_esmp)						\
+	((_esmp)->esm_addr)
+
+/* BAR */
+
+typedef struct efsys_bar_s {
+	struct mtx		esb_lock;
+	bus_space_tag_t		esb_tag;
+	bus_space_handle_t	esb_handle;
+	int			esb_rid;
+	struct resource		*esb_res;
+} efsys_bar_t;
+
+#define	EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_lock(&((_esbp)->esb_lock));			\
+									\
+		(_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset));			\
+									\
+		EFSYS_PROBE2(bar_readd, unsigned int, (_offset),	\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_unlock(&((_esbp)->esb_lock));		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_BAR_READQ(_esbp, _offset, _eqp)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		mtx_lock(&((_esbp)->esb_lock));				\
+									\
+		(_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset));			\
+		(_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset+4));			\
+									\
+		EFSYS_PROBE3(bar_readq, unsigned int, (_offset),	\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+									\
+		mtx_unlock(&((_esbp)->esb_lock));			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_lock(&((_esbp)->esb_lock));			\
+									\
+		(_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset));			\
+		(_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset+4));			\
+		(_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset+8));			\
+		(_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag,	\
+		    (_esbp)->esb_handle, (_offset+12));			\
+									\
+		EFSYS_PROBE5(bar_reado, unsigned int, (_offset),	\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_unlock(&((_esbp)->esb_lock));		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_lock(&((_esbp)->esb_lock));			\
+									\
+		EFSYS_PROBE2(bar_writed, unsigned int, (_offset),	\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+									\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), (_edp)->ed_u32[0]);			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_unlock(&((_esbp)->esb_lock));		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		mtx_lock(&((_esbp)->esb_lock));				\
+									\
+		EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),	\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+									\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), (_eqp)->eq_u32[0]);			\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset+4), (_eqp)->eq_u32[1]);			\
+									\
+		mtx_unlock(&((_esbp)->esb_lock));			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION)				\
+		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
+		    ("not power of 2 aligned"));			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_lock(&((_esbp)->esb_lock));			\
+									\
+		EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),	\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+									\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), (_eop)->eo_u32[0]);			\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset+4), (_eop)->eo_u32[1]);			\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset+8), (_eop)->eo_u32[2]);			\
+		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset+12), (_eop)->eo_u32[3]);			\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+		if (_lock)						\
+			mtx_unlock(&((_esbp)->esb_lock));		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* SPIN */
+
+#define	EFSYS_SPIN(_us)							\
+	do {								\
+		DELAY(_us);						\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_SLEEP	EFSYS_SPIN
+
+/* BARRIERS */
+
+/* Strict ordering guaranteed by devacc.devacc_attr_dataorder */
+#define	EFSYS_MEM_READ_BARRIER()
+#define	EFSYS_PIO_WRITE_BARRIER()
+
+/* TIMESTAMP */
+
+typedef	clock_t	efsys_timestamp_t;
+
+#define	EFSYS_TIMESTAMP(_usp)						\
+	do {								\
+		clock_t now;						\
+									\
+		now = ticks;						\
+		*(_usp) = now * hz / 1000000;				\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* KMEM */
+
+#define	EFSYS_KMEM_ALLOC(_esip, _size, _p)				\
+	do {								\
+		(_esip) = (_esip);					\
+		(_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO);	\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_KMEM_FREE(_esip, _size, _p)				\
+	do {								\
+		(void) (_esip);						\
+		(void) (_size);						\
+		free((_p), M_SFXGE);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* LOCK */
+
+typedef struct mtx	efsys_lock_t;
+
+#define	EFSYS_LOCK_MAGIC	0x000010c4
+
+#define	EFSYS_LOCK(_lockp, _state)					\
+	do {								\
+		mtx_lock(_lockp);					\
+		(_state) = EFSYS_LOCK_MAGIC;				\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_UNLOCK(_lockp, _state)					\
+	do {								\
+		if ((_state) != EFSYS_LOCK_MAGIC)			\
+			KASSERT(B_FALSE, ("not locked"));		\
+		mtx_unlock(_lockp);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* PREEMPT */
+
+#define	EFSYS_PREEMPT_DISABLE(_state)					\
+	do {								\
+		(_state) = (_state);					\
+		critical_enter();					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_PREEMPT_ENABLE(_state)					\
+	do {								\
+		(_state) = (_state);					\
+		critical_exit(_state);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* STAT */
+
+typedef uint64_t		efsys_stat_t;
+
+#define	EFSYS_STAT_INCR(_knp, _delta) 					\
+	do {								\
+		*(_knp) += (_delta);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_STAT_DECR(_knp, _delta) 					\
+	do {								\
+		*(_knp) -= (_delta);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_STAT_SET(_knp, _val)					\
+	do {								\
+		*(_knp) = (_val);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_STAT_SET_QWORD(_knp, _valp)				\
+	do {								\
+		*(_knp) = le64toh((_valp)->eq_u64[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_STAT_SET_DWORD(_knp, _valp)				\
+	do {								\
+		*(_knp) = le32toh((_valp)->ed_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_STAT_INCR_QWORD(_knp, _valp)				\
+	do {								\
+		*(_knp) += le64toh((_valp)->eq_u64[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFSYS_STAT_SUBR_QWORD(_knp, _valp)				\
+	do {								\
+		*(_knp) -= le64toh((_valp)->eq_u64[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* ERR */
+
+extern void	sfxge_err(efsys_identifier_t *, unsigned int,
+		    uint32_t, uint32_t);
+
+#if EFSYS_OPT_DECODE_INTR_FATAL
+#define	EFSYS_ERR(_esip, _code, _dword0, _dword1)			\
+	do {								\
+		sfxge_err((_esip), (_code), (_dword0), (_dword1));	\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+#endif
+
+/* ASSERT */
+
+#define	EFSYS_ASSERT(_exp) do {						\
+	if (!(_exp))							\
+		panic(#_exp);						\
+	} while (0)
+
+#define EFSYS_ASSERT3(_x, _op, _y, _t) do {				\
+	const _t __x = (_t)(_x);					\
+	const _t __y = (_t)(_y);					\
+	if (!(__x _op __y))						\
+	        panic("assertion failed at %s:%u", __FILE__, __LINE__);	\
+	} while(0)
+
+#define EFSYS_ASSERT3U(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uint64_t)
+#define EFSYS_ASSERT3S(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, int64_t)
+#define EFSYS_ASSERT3P(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_EFSYS_H */
diff --git a/sys/dev/sfxge/common/efx.h b/sys/dev/sfxge/common/efx.h
new file mode 100644
index 0000000..18c248b
--- /dev/null
+++ b/sys/dev/sfxge/common/efx.h
@@ -0,0 +1,1893 @@
+/*-
+ * Copyright 2006-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_SYS_EFX_H
+#define	_SYS_EFX_H
+
+#include "efsys.h"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#define	EFX_STATIC_ASSERT(_cond) ((void)sizeof(char[(_cond) ? 1 : -1]))
+
+#define	EFX_ARRAY_SIZE(_array) (sizeof(_array) / sizeof((_array)[0]))
+
+#ifndef EFSYS_MEM_IS_NULL
+#define	EFSYS_MEM_IS_NULL(_esmp) ((_esmp)->esm_base == NULL)
+#endif
+
+typedef enum efx_family_e {
+	EFX_FAMILY_INVALID,
+	EFX_FAMILY_FALCON,
+	EFX_FAMILY_SIENA,
+	EFX_FAMILY_NTYPES
+} efx_family_t;
+
+extern	__checkReturn	int
+efx_family(
+	__in		uint16_t venid,
+	__in		uint16_t devid,
+	__out		efx_family_t *efp);
+
+extern	__checkReturn	int
+efx_infer_family(
+	__in		efsys_bar_t *esbp,
+	__out		efx_family_t *efp);
+
+#define	EFX_PCI_VENID_SFC		0x1924
+#define	EFX_PCI_DEVID_FALCON		0x0710
+#define	EFX_PCI_DEVID_BETHPAGE		0x0803
+#define	EFX_PCI_DEVID_SIENA		0x0813
+#define	EFX_PCI_DEVID_SIENA_F1_UNINIT	0x0810
+
+#define	EFX_MEM_BAR	2
+
+/* Error codes */
+
+enum {
+	EFX_ERR_INVALID,
+	EFX_ERR_SRAM_OOB,
+	EFX_ERR_BUFID_DC_OOB,
+	EFX_ERR_MEM_PERR,
+	EFX_ERR_RBUF_OWN,
+	EFX_ERR_TBUF_OWN,
+	EFX_ERR_RDESQ_OWN,
+	EFX_ERR_TDESQ_OWN,
+	EFX_ERR_EVQ_OWN,
+	EFX_ERR_EVFF_OFLO,
+	EFX_ERR_ILL_ADDR,
+	EFX_ERR_SRAM_PERR,
+	EFX_ERR_NCODES
+};
+
+/* NIC */
+
+typedef struct efx_nic_s	efx_nic_t;
+
+extern	__checkReturn	int
+efx_nic_create(
+	__in		efx_family_t family,
+	__in		efsys_identifier_t *esip,
+	__in		efsys_bar_t *esbp,
+	__in		efsys_lock_t *eslp,
+	__deref_out	efx_nic_t **enpp);
+
+extern	__checkReturn	int
+efx_nic_probe(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_PCIE_TUNE
+
+extern	__checkReturn	int
+efx_nic_pcie_tune(
+	__in		efx_nic_t *enp,
+	unsigned int	nlanes);
+
+extern	__checkReturn	int
+efx_nic_pcie_extended_sync(
+	__in		efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_PCIE_TUNE */
+
+extern 	__checkReturn	int
+efx_nic_init(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+efx_nic_reset(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_DIAG
+
+extern	__checkReturn	int
+efx_nic_register_test(
+	__in		efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+extern		void
+efx_nic_fini(
+	__in		efx_nic_t *enp);
+
+extern		void
+efx_nic_unprobe(
+	__in		efx_nic_t *enp);
+
+extern 		void
+efx_nic_destroy(
+	__in	efx_nic_t *enp);
+
+#if EFSYS_OPT_MCDI
+
+typedef struct efx_mcdi_req_s efx_mcdi_req_t;
+
+typedef enum efx_mcdi_exception_e {
+	EFX_MCDI_EXCEPTION_MC_REBOOT,
+	EFX_MCDI_EXCEPTION_MC_BADASSERT,
+} efx_mcdi_exception_t;
+
+typedef struct efx_mcdi_transport_s {
+	void		*emt_context;
+	void		(*emt_execute)(void *, efx_mcdi_req_t *);
+	void		(*emt_ev_cpl)(void *);
+	void		(*emt_exception)(void *, efx_mcdi_exception_t);
+} efx_mcdi_transport_t;
+
+extern	__checkReturn	int
+efx_mcdi_init(
+	__in		efx_nic_t *enp,
+	__in		const efx_mcdi_transport_t *mtp);
+
+extern	__checkReturn	int
+efx_mcdi_reboot(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_mcdi_request_start(
+	__in		efx_nic_t *enp,
+	__in		efx_mcdi_req_t *emrp,
+	__in		boolean_t ev_cpl);
+
+extern	__checkReturn	boolean_t
+efx_mcdi_request_poll(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	boolean_t
+efx_mcdi_request_abort(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_mcdi_fini(
+	__in		efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_MCDI */
+
+/* INTR */
+
+#define	EFX_NINTR_FALCON 64
+#define	EFX_NINTR_SIENA 1024
+
+typedef enum efx_intr_type_e {
+	EFX_INTR_INVALID = 0,
+	EFX_INTR_LINE,
+	EFX_INTR_MESSAGE,
+	EFX_INTR_NTYPES
+} efx_intr_type_t;
+
+#define	EFX_INTR_SIZE	(sizeof (efx_oword_t))
+
+extern	__checkReturn	int
+efx_intr_init(
+	__in		efx_nic_t *enp,
+	__in		efx_intr_type_t type,
+	__in		efsys_mem_t *esmp);
+
+extern 			void
+efx_intr_enable(
+	__in		efx_nic_t *enp);
+
+extern 			void
+efx_intr_disable(
+	__in		efx_nic_t *enp);
+
+extern 			void
+efx_intr_disable_unlocked(
+	__in		efx_nic_t *enp);
+
+#define	EFX_INTR_NEVQS	32
+
+extern __checkReturn	int
+efx_intr_trigger(
+	__in		efx_nic_t *enp,
+	__in		unsigned int level);
+
+extern			void
+efx_intr_status_line(
+	__in		efx_nic_t *enp,
+	__out		boolean_t *fatalp,
+	__out		uint32_t *maskp);
+
+extern			void
+efx_intr_status_message(
+	__in		efx_nic_t *enp,
+	__in		unsigned int message,
+	__out		boolean_t *fatalp);
+
+extern			void
+efx_intr_fatal(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_intr_fini(
+	__in		efx_nic_t *enp);
+
+/* MAC */
+
+#if EFSYS_OPT_MAC_STATS
+
+/* START MKCONFIG GENERATED EfxHeaderMacBlock bb8d39428b6fdcf5 */
+typedef enum efx_mac_stat_e {
+	EFX_MAC_RX_OCTETS,
+	EFX_MAC_RX_PKTS,
+	EFX_MAC_RX_UNICST_PKTS,
+	EFX_MAC_RX_MULTICST_PKTS,
+	EFX_MAC_RX_BRDCST_PKTS,
+	EFX_MAC_RX_PAUSE_PKTS,
+	EFX_MAC_RX_LE_64_PKTS,
+	EFX_MAC_RX_65_TO_127_PKTS,
+	EFX_MAC_RX_128_TO_255_PKTS,
+	EFX_MAC_RX_256_TO_511_PKTS,
+	EFX_MAC_RX_512_TO_1023_PKTS,
+	EFX_MAC_RX_1024_TO_15XX_PKTS,
+	EFX_MAC_RX_GE_15XX_PKTS,
+	EFX_MAC_RX_ERRORS,
+	EFX_MAC_RX_FCS_ERRORS,
+	EFX_MAC_RX_DROP_EVENTS,
+	EFX_MAC_RX_FALSE_CARRIER_ERRORS,
+	EFX_MAC_RX_SYMBOL_ERRORS,
+	EFX_MAC_RX_ALIGN_ERRORS,
+	EFX_MAC_RX_INTERNAL_ERRORS,
+	EFX_MAC_RX_JABBER_PKTS,
+	EFX_MAC_RX_LANE0_CHAR_ERR,
+	EFX_MAC_RX_LANE1_CHAR_ERR,
+	EFX_MAC_RX_LANE2_CHAR_ERR,
+	EFX_MAC_RX_LANE3_CHAR_ERR,
+	EFX_MAC_RX_LANE0_DISP_ERR,
+	EFX_MAC_RX_LANE1_DISP_ERR,
+	EFX_MAC_RX_LANE2_DISP_ERR,
+	EFX_MAC_RX_LANE3_DISP_ERR,
+	EFX_MAC_RX_MATCH_FAULT,
+	EFX_MAC_RX_NODESC_DROP_CNT,
+	EFX_MAC_TX_OCTETS,
+	EFX_MAC_TX_PKTS,
+	EFX_MAC_TX_UNICST_PKTS,
+	EFX_MAC_TX_MULTICST_PKTS,
+	EFX_MAC_TX_BRDCST_PKTS,
+	EFX_MAC_TX_PAUSE_PKTS,
+	EFX_MAC_TX_LE_64_PKTS,
+	EFX_MAC_TX_65_TO_127_PKTS,
+	EFX_MAC_TX_128_TO_255_PKTS,
+	EFX_MAC_TX_256_TO_511_PKTS,
+	EFX_MAC_TX_512_TO_1023_PKTS,
+	EFX_MAC_TX_1024_TO_15XX_PKTS,
+	EFX_MAC_TX_GE_15XX_PKTS,
+	EFX_MAC_TX_ERRORS,
+	EFX_MAC_TX_SGL_COL_PKTS,
+	EFX_MAC_TX_MULT_COL_PKTS,
+	EFX_MAC_TX_EX_COL_PKTS,
+	EFX_MAC_TX_LATE_COL_PKTS,
+	EFX_MAC_TX_DEF_PKTS,
+	EFX_MAC_TX_EX_DEF_PKTS,
+	EFX_MAC_NSTATS
+} efx_mac_stat_t;
+
+/* END MKCONFIG GENERATED EfxHeaderMacBlock */
+
+#endif	/* EFSYS_OPT_MAC_STATS */
+
+typedef enum efx_link_mode_e {
+	EFX_LINK_UNKNOWN = 0,
+	EFX_LINK_DOWN,
+	EFX_LINK_10HDX,
+	EFX_LINK_10FDX,
+	EFX_LINK_100HDX,
+	EFX_LINK_100FDX,
+	EFX_LINK_1000HDX,
+	EFX_LINK_1000FDX,
+	EFX_LINK_10000FDX,
+	EFX_LINK_NMODES
+} efx_link_mode_t;
+
+#define	EFX_MAC_SDU_MAX	9202
+
+#define	EFX_MAC_PDU(_sdu) 				\
+	P2ROUNDUP(((_sdu)				\
+		    + /* EtherII */ 14			\
+		    + /* VLAN */ 4			\
+		    + /* CRC */ 4			\
+		    + /* bug16011 */ 16),		\
+		    (1 << 3))
+
+#define	EFX_MAC_PDU_MIN	60
+#define	EFX_MAC_PDU_MAX	EFX_MAC_PDU(EFX_MAC_SDU_MAX)
+
+extern	__checkReturn	int
+efx_mac_pdu_set(
+	__in		efx_nic_t *enp,
+	__in		size_t pdu);
+
+extern	__checkReturn	int
+efx_mac_addr_set(
+	__in		efx_nic_t *enp,
+	__in		uint8_t *addr);
+
+extern	__checkReturn	int
+efx_mac_filter_set(
+	__in		efx_nic_t *enp,
+	__in		boolean_t unicst,
+	__in		boolean_t brdcst);
+
+extern	__checkReturn	int
+efx_mac_drain(
+	__in		efx_nic_t *enp,
+	__in		boolean_t enabled);
+
+extern	__checkReturn	int
+efx_mac_up(
+	__in		efx_nic_t *enp,
+	__out		boolean_t *mac_upp);
+
+#define	EFX_FCNTL_RESPOND	0x00000001
+#define	EFX_FCNTL_GENERATE	0x00000002
+
+extern	__checkReturn	int
+efx_mac_fcntl_set(
+	__in		efx_nic_t *enp,
+	__in		unsigned int fcntl,
+	__in		boolean_t autoneg);
+
+extern			void
+efx_mac_fcntl_get(
+	__in		efx_nic_t *enp,
+	__out		unsigned int *fcntl_wantedp,
+	__out		unsigned int *fcntl_linkp);
+
+#define	EFX_MAC_HASH_BITS	(1 << 8)
+
+extern	__checkReturn			int
+efx_mac_hash_set(
+	__in				efx_nic_t *enp,
+	__in_ecount(EFX_MAC_HASH_BITS)	unsigned int const *bucket);
+
+#if EFSYS_OPT_MAC_STATS
+
+#if EFSYS_OPT_NAMES
+
+extern	__checkReturn			const char __cs *
+efx_mac_stat_name(
+	__in				efx_nic_t *enp,
+	__in				unsigned int id);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+#define	EFX_MAC_STATS_SIZE 0x400
+
+/*
+ * Upload mac statistics supported by the hardware into the given buffer.
+ *
+ * The reference buffer must be at least %EFX_MAC_STATS_SIZE bytes,
+ * and page aligned.
+ *
+ * The hardware will only DMA statistics that it understands (of course).
+ * Drivers should not make any assumptions about which statistics are
+ * supported, especially when the statistics are generated by firmware.
+ *
+ * Thus, drivers should zero this buffer before use, so that not-understood
+ * statistics read back as zero.
+ */
+extern	__checkReturn			int
+efx_mac_stats_upload(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp);
+
+extern	__checkReturn			int
+efx_mac_stats_periodic(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__in				uint16_t period_ms,
+	__in				boolean_t events);
+
+extern	__checkReturn			int
+efx_mac_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__inout_ecount(EFX_MAC_NSTATS)	efsys_stat_t *stat,
+	__out_opt			uint32_t *generationp);
+
+#endif	/* EFSYS_OPT_MAC_STATS */
+
+/* MON */
+
+typedef enum efx_mon_type_e {
+	EFX_MON_INVALID = 0,
+	EFX_MON_NULL,
+	EFX_MON_LM87,
+	EFX_MON_MAX6647,
+	EFX_MON_SFC90X0,
+	EFX_MON_NTYPES
+} efx_mon_type_t;
+
+#if EFSYS_OPT_NAMES
+
+extern		const char __cs *
+efx_mon_name(
+	__in	efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+extern	__checkReturn	int
+efx_mon_init(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_MON_STATS
+
+#define	EFX_MON_STATS_SIZE 0x100
+
+/* START MKCONFIG GENERATED MonitorHeaderStatsBlock 16a14e61aa4f8d80 */
+typedef enum efx_mon_stat_e {
+	EFX_MON_STAT_2_5V,
+	EFX_MON_STAT_VCCP1,
+	EFX_MON_STAT_VCC,
+	EFX_MON_STAT_5V,
+	EFX_MON_STAT_12V,
+	EFX_MON_STAT_VCCP2,
+	EFX_MON_STAT_EXT_TEMP,
+	EFX_MON_STAT_INT_TEMP,
+	EFX_MON_STAT_AIN1,
+	EFX_MON_STAT_AIN2,
+	EFX_MON_STAT_INT_COOLING,
+	EFX_MON_STAT_EXT_COOLING,
+	EFX_MON_STAT_1V,
+	EFX_MON_STAT_1_2V,
+	EFX_MON_STAT_1_8V,
+	EFX_MON_STAT_3_3V,
+	EFX_MON_NSTATS
+} efx_mon_stat_t;
+
+/* END MKCONFIG GENERATED MonitorHeaderStatsBlock */
+
+typedef enum efx_mon_stat_state_e {
+	EFX_MON_STAT_STATE_OK = 0,
+	EFX_MON_STAT_STATE_WARNING = 1,
+	EFX_MON_STAT_STATE_FATAL = 2,
+	EFX_MON_STAT_STATE_BROKEN = 3,
+} efx_mon_stat_state_t;
+
+typedef struct efx_mon_stat_value_t {
+	uint16_t	emsv_value;
+	uint16_t	emsv_state;
+} efx_mon_stat_value_t;
+
+#if EFSYS_OPT_NAMES
+
+extern					const char __cs *
+efx_mon_stat_name(
+	__in				efx_nic_t *enp,
+	__in				efx_mon_stat_t id);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+extern	__checkReturn			int
+efx_mon_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values);
+
+#endif	/* EFSYS_OPT_MON_STATS */
+
+extern		void
+efx_mon_fini(
+	__in	efx_nic_t *enp);
+
+/* PHY */
+
+#define	PMA_PMD_MMD	1
+#define	PCS_MMD		3
+#define	PHY_XS_MMD	4
+#define	DTE_XS_MMD	5
+#define	AN_MMD		7
+#define	CL22EXT_MMD	29
+
+#define	MAXMMD		((1 << 5) - 1)
+
+/* PHY types */
+#define	EFX_PHY_NULL		0x0
+#define	EFX_PHY_TXC43128	0x1
+#define	EFX_PHY_SFX7101		0x3
+#define	EFX_PHY_QT2022C2	0x4
+#define	EFX_PHY_SFT9001A	0x8
+#define	EFX_PHY_QT2025C		0x9
+#define	EFX_PHY_SFT9001B	0xa
+#define	EFX_PHY_QLX111V		0xc
+
+extern	__checkReturn	int
+efx_phy_verify(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_PHY_LED_CONTROL
+
+typedef enum efx_phy_led_mode_e {
+	EFX_PHY_LED_DEFAULT = 0,
+	EFX_PHY_LED_OFF,
+	EFX_PHY_LED_ON,
+	EFX_PHY_LED_FLASH,
+	EFX_PHY_LED_NMODES
+} efx_phy_led_mode_t;
+
+extern	__checkReturn	int
+efx_phy_led_set(
+	__in	efx_nic_t *enp,
+	__in	efx_phy_led_mode_t mode);
+
+#endif	/* EFSYS_OPT_PHY_LED_CONTROL */
+
+extern	__checkReturn	int
+efx_port_init(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_LOOPBACK
+
+typedef enum efx_loopback_type_e {
+	EFX_LOOPBACK_OFF = 0,
+	EFX_LOOPBACK_DATA = 1,
+	EFX_LOOPBACK_GMAC = 2,
+	EFX_LOOPBACK_XGMII = 3,
+	EFX_LOOPBACK_XGXS = 4,
+	EFX_LOOPBACK_XAUI = 5,
+	EFX_LOOPBACK_GMII = 6,
+	EFX_LOOPBACK_SGMII = 7,
+	EFX_LOOPBACK_XGBR = 8,
+	EFX_LOOPBACK_XFI = 9,
+	EFX_LOOPBACK_XAUI_FAR = 10,
+	EFX_LOOPBACK_GMII_FAR = 11,
+	EFX_LOOPBACK_SGMII_FAR = 12,
+	EFX_LOOPBACK_XFI_FAR = 13,
+	EFX_LOOPBACK_GPHY = 14,
+	EFX_LOOPBACK_PHY_XS = 15,
+	EFX_LOOPBACK_PCS = 16,
+	EFX_LOOPBACK_PMA_PMD = 17,
+	EFX_LOOPBACK_NTYPES
+} efx_loopback_type_t;
+
+#define	EFX_LOOPBACK_MAC_MASK			\
+	((1 << EFX_LOOPBACK_DATA) |		\
+	    (1 << EFX_LOOPBACK_GMAC) | 		\
+	    (1 << EFX_LOOPBACK_XGMII) | 	\
+	    (1 << EFX_LOOPBACK_XGXS) | 		\
+	    (1 << EFX_LOOPBACK_XAUI) |		\
+	    (1 << EFX_LOOPBACK_GMII) |		\
+	    (1 << EFX_LOOPBACK_SGMII) |		\
+	    (1 << EFX_LOOPBACK_XGBR) |		\
+	    (1 << EFX_LOOPBACK_XFI) |		\
+	    (1 << EFX_LOOPBACK_XAUI_FAR) |	\
+	    (1 << EFX_LOOPBACK_GMII_FAR) |	\
+	    (1 << EFX_LOOPBACK_SGMII_FAR) |	\
+	    (1 << EFX_LOOPBACK_XFI_FAR))
+
+#define EFX_LOOPBACK_MASK			\
+	((1 << EFX_LOOPBACK_NTYPES) - 1)
+
+extern	__checkReturn	int
+efx_port_loopback_set(
+	__in	efx_nic_t *enp,
+	__in	efx_link_mode_t link_mode,
+	__in	efx_loopback_type_t type);
+
+#if EFSYS_OPT_NAMES
+
+extern	__checkReturn	const char __cs *
+efx_loopback_type_name(
+	__in		efx_nic_t *enp,
+	__in		efx_loopback_type_t type);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+#endif	/* EFSYS_OPT_LOOPBACK */
+
+extern	__checkReturn	int
+efx_port_poll(
+	__in		efx_nic_t *enp,
+	__out		efx_link_mode_t	*link_modep);
+
+extern 		void
+efx_port_fini(
+	__in	efx_nic_t *enp);
+
+typedef enum efx_phy_cap_type_e {
+	EFX_PHY_CAP_INVALID = 0,
+	EFX_PHY_CAP_10HDX,
+	EFX_PHY_CAP_10FDX,
+	EFX_PHY_CAP_100HDX,
+	EFX_PHY_CAP_100FDX,
+	EFX_PHY_CAP_1000HDX,
+	EFX_PHY_CAP_1000FDX,
+	EFX_PHY_CAP_10000FDX,
+	EFX_PHY_CAP_PAUSE,
+	EFX_PHY_CAP_ASYM,
+	EFX_PHY_CAP_AN,
+	EFX_PHY_CAP_NTYPES
+} efx_phy_cap_type_t;
+
+
+#define	EFX_PHY_CAP_CURRENT	0x00000000
+#define	EFX_PHY_CAP_DEFAULT	0x00000001
+#define	EFX_PHY_CAP_PERM	0x00000002
+
+extern		void
+efx_phy_adv_cap_get(
+	__in		efx_nic_t *enp,
+	__in            uint32_t flag,
+	__out		uint32_t *maskp);
+
+extern	__checkReturn	int
+efx_phy_adv_cap_set(
+	__in		efx_nic_t *enp,
+	__in		uint32_t mask);
+
+extern			void
+efx_phy_lp_cap_get(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *maskp);
+
+extern	__checkReturn	int
+efx_phy_oui_get(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *ouip);
+
+typedef enum efx_phy_media_type_e {
+	EFX_PHY_MEDIA_INVALID = 0,
+	EFX_PHY_MEDIA_XAUI,
+	EFX_PHY_MEDIA_CX4,
+	EFX_PHY_MEDIA_KX4,
+	EFX_PHY_MEDIA_XFP,
+	EFX_PHY_MEDIA_SFP_PLUS,
+	EFX_PHY_MEDIA_BASE_T,
+	EFX_PHY_MEDIA_NTYPES
+} efx_phy_media_type_t;
+
+/* Get the type of medium currently used.  If the board has ports for
+ * modules, a module is present, and we recognise the media type of
+ * the module, then this will be the media type of the module.
+ * Otherwise it will be the media type of the port.
+ */
+extern			void
+efx_phy_media_type_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_media_type_t *typep);
+
+#if EFSYS_OPT_PHY_STATS
+
+/* START MKCONFIG GENERATED PhyHeaderStatsBlock 30ed56ad501f8e36 */
+typedef enum efx_phy_stat_e {
+	EFX_PHY_STAT_OUI,
+	EFX_PHY_STAT_PMA_PMD_LINK_UP,
+	EFX_PHY_STAT_PMA_PMD_RX_FAULT,
+	EFX_PHY_STAT_PMA_PMD_TX_FAULT,
+	EFX_PHY_STAT_PMA_PMD_REV_A,
+	EFX_PHY_STAT_PMA_PMD_REV_B,
+	EFX_PHY_STAT_PMA_PMD_REV_C,
+	EFX_PHY_STAT_PMA_PMD_REV_D,
+	EFX_PHY_STAT_PCS_LINK_UP,
+	EFX_PHY_STAT_PCS_RX_FAULT,
+	EFX_PHY_STAT_PCS_TX_FAULT,
+	EFX_PHY_STAT_PCS_BER,
+	EFX_PHY_STAT_PCS_BLOCK_ERRORS,
+	EFX_PHY_STAT_PHY_XS_LINK_UP,
+	EFX_PHY_STAT_PHY_XS_RX_FAULT,
+	EFX_PHY_STAT_PHY_XS_TX_FAULT,
+	EFX_PHY_STAT_PHY_XS_ALIGN,
+	EFX_PHY_STAT_PHY_XS_SYNC_A,
+	EFX_PHY_STAT_PHY_XS_SYNC_B,
+	EFX_PHY_STAT_PHY_XS_SYNC_C,
+	EFX_PHY_STAT_PHY_XS_SYNC_D,
+	EFX_PHY_STAT_AN_LINK_UP,
+	EFX_PHY_STAT_AN_MASTER,
+	EFX_PHY_STAT_AN_LOCAL_RX_OK,
+	EFX_PHY_STAT_AN_REMOTE_RX_OK,
+	EFX_PHY_STAT_CL22EXT_LINK_UP,
+	EFX_PHY_STAT_SNR_A,
+	EFX_PHY_STAT_SNR_B,
+	EFX_PHY_STAT_SNR_C,
+	EFX_PHY_STAT_SNR_D,
+	EFX_PHY_STAT_PMA_PMD_SIGNAL_A,
+	EFX_PHY_STAT_PMA_PMD_SIGNAL_B,
+	EFX_PHY_STAT_PMA_PMD_SIGNAL_C,
+	EFX_PHY_STAT_PMA_PMD_SIGNAL_D,
+	EFX_PHY_STAT_AN_COMPLETE,
+	EFX_PHY_STAT_PMA_PMD_REV_MAJOR,
+	EFX_PHY_STAT_PMA_PMD_REV_MINOR,
+	EFX_PHY_STAT_PMA_PMD_REV_MICRO,
+	EFX_PHY_STAT_PCS_FW_VERSION_0,
+	EFX_PHY_STAT_PCS_FW_VERSION_1,
+	EFX_PHY_STAT_PCS_FW_VERSION_2,
+	EFX_PHY_STAT_PCS_FW_VERSION_3,
+	EFX_PHY_STAT_PCS_FW_BUILD_YY,
+	EFX_PHY_STAT_PCS_FW_BUILD_MM,
+	EFX_PHY_STAT_PCS_FW_BUILD_DD,
+	EFX_PHY_STAT_PCS_OP_MODE,
+	EFX_PHY_NSTATS
+} efx_phy_stat_t;
+
+/* END MKCONFIG GENERATED PhyHeaderStatsBlock */
+
+#if EFSYS_OPT_NAMES
+
+extern					const char __cs *
+efx_phy_stat_name(
+	__in				efx_nic_t *enp,
+	__in				efx_phy_stat_t stat);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+#define	EFX_PHY_STATS_SIZE 0x100
+
+extern	__checkReturn			int
+efx_phy_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_PHY_NSTATS)	uint32_t *stat);
+
+#endif	/* EFSYS_OPT_PHY_STATS */
+
+#if EFSYS_OPT_PHY_PROPS
+
+#if EFSYS_OPT_NAMES
+
+extern		const char __cs *
+efx_phy_prop_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+#define	EFX_PHY_PROP_DEFAULT	0x00000001
+
+extern	__checkReturn	int
+efx_phy_prop_get(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t flags,
+	__out		uint32_t *valp);
+
+extern	__checkReturn	int
+efx_phy_prop_set(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t val);
+
+#endif	/* EFSYS_OPT_PHY_PROPS */
+
+#if EFSYS_OPT_PHY_BIST
+
+typedef enum efx_phy_bist_type_e {
+	EFX_PHY_BIST_TYPE_UNKNOWN,
+	EFX_PHY_BIST_TYPE_NORMAL,
+	EFX_PHY_BIST_TYPE_CABLE_SHORT,
+	EFX_PHY_BIST_TYPE_CABLE_LONG,
+	EFX_PHY_BIST_TYPE_NTYPES,
+} efx_phy_bist_type_t;
+
+typedef enum efx_phy_bist_result_e {
+	EFX_PHY_BIST_RESULT_UNKNOWN,
+	EFX_PHY_BIST_RESULT_RUNNING,
+	EFX_PHY_BIST_RESULT_PASSED,
+	EFX_PHY_BIST_RESULT_FAILED,
+} efx_phy_bist_result_t;
+
+typedef enum efx_phy_cable_status_e {
+	EFX_PHY_CABLE_STATUS_OK,
+	EFX_PHY_CABLE_STATUS_INVALID,
+	EFX_PHY_CABLE_STATUS_OPEN,
+	EFX_PHY_CABLE_STATUS_INTRAPAIRSHORT,
+	EFX_PHY_CABLE_STATUS_INTERPAIRSHORT,
+	EFX_PHY_CABLE_STATUS_BUSY,
+} efx_phy_cable_status_t;
+
+typedef enum efx_phy_bist_value_e {
+	EFX_PHY_BIST_CABLE_LENGTH_A,
+	EFX_PHY_BIST_CABLE_LENGTH_B,
+	EFX_PHY_BIST_CABLE_LENGTH_C,
+	EFX_PHY_BIST_CABLE_LENGTH_D,
+	EFX_PHY_BIST_CABLE_STATUS_A,
+	EFX_PHY_BIST_CABLE_STATUS_B,
+	EFX_PHY_BIST_CABLE_STATUS_C,
+	EFX_PHY_BIST_CABLE_STATUS_D,
+	EFX_PHY_BIST_FAULT_CODE,
+	EFX_PHY_BIST_NVALUES,
+} efx_phy_bist_value_t;
+
+extern	__checkReturn		int
+efx_phy_bist_start(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type);
+
+extern	__checkReturn		int
+efx_phy_bist_poll(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type,
+	__out			efx_phy_bist_result_t *resultp,
+	__out_opt		uint32_t *value_maskp,
+	__out_ecount_opt(count)	unsigned long *valuesp,
+	__in			size_t count);
+
+extern				void
+efx_phy_bist_stop(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type);
+
+#endif	/* EFSYS_OPT_PHY_BIST */
+
+#define	EFX_FEATURE_IPV6		0x00000001
+#define	EFX_FEATURE_LFSR_HASH_INSERT	0x00000002
+#define	EFX_FEATURE_LINK_EVENTS		0x00000004
+#define	EFX_FEATURE_PERIODIC_MAC_STATS	0x00000008
+#define	EFX_FEATURE_WOL			0x00000010
+#define	EFX_FEATURE_MCDI		0x00000020
+#define	EFX_FEATURE_LOOKAHEAD_SPLIT	0x00000040
+#define	EFX_FEATURE_MAC_HEADER_FILTERS	0x00000080
+
+typedef struct efx_nic_cfg_s {
+	uint32_t		enc_board_type;
+	uint32_t		enc_phy_type;
+#if EFSYS_OPT_NAMES
+	char			enc_phy_name[21];
+#endif
+	char			enc_phy_revision[21];
+	efx_mon_type_t		enc_mon_type;
+#if EFSYS_OPT_MON_STATS
+	uint32_t		enc_mon_stat_mask;
+#endif
+	unsigned int		enc_features;
+	uint8_t			enc_mac_addr[6];
+	uint8_t			enc_port;
+	uint32_t		enc_evq_limit;
+	uint32_t		enc_txq_limit;
+	uint32_t		enc_rxq_limit;
+	uint32_t		enc_buftbl_limit;
+	uint32_t		enc_evq_moderation_max;
+#if EFSYS_OPT_LOOPBACK
+	uint32_t		enc_loopback_types[EFX_LINK_NMODES];
+#endif	/* EFSYS_OPT_LOOPBACK */
+#if EFSYS_OPT_PHY_FLAGS
+	uint32_t		enc_phy_flags_mask;
+#endif	/* EFSYS_OPT_PHY_FLAGS */
+#if EFSYS_OPT_PHY_LED_CONTROL
+	uint32_t		enc_led_mask;
+#endif	/* EFSYS_OPT_PHY_LED_CONTROL */
+#if EFSYS_OPT_PHY_STATS
+	uint64_t		enc_phy_stat_mask;
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+	unsigned int		enc_phy_nprops;
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_SIENA
+	uint8_t			enc_siena_channel;
+#if EFSYS_OPT_PHY_STATS
+	uint32_t		enc_siena_phy_stat_mask;
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_MON_STATS
+	uint32_t		enc_siena_mon_stat_mask;
+#endif	/* EFSYS_OPT_MON_STATS */
+#endif	/* EFSYS_OPT_SIENA */
+#if EFSYS_OPT_PHY_BIST
+	uint32_t		enc_bist_mask;
+#endif	/* EFSYS_OPT_PHY_BIST */
+} efx_nic_cfg_t;
+
+extern			const efx_nic_cfg_t *
+efx_nic_cfg_get(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_VPD
+
+typedef enum efx_vpd_tag_e {
+	EFX_VPD_ID = 0x02,
+	EFX_VPD_END = 0x0f,
+	EFX_VPD_RO = 0x10,
+	EFX_VPD_RW = 0x11,
+} efx_vpd_tag_t;
+
+typedef uint16_t efx_vpd_keyword_t;
+
+typedef struct efx_vpd_value_s {
+	efx_vpd_tag_t		evv_tag;
+	efx_vpd_keyword_t	evv_keyword;
+	uint8_t			evv_length;
+	uint8_t			evv_value[0x100];
+} efx_vpd_value_t;
+
+
+#define	EFX_VPD_KEYWORD(x, y) ((x) | ((y) << 8))
+
+extern	__checkReturn		int
+efx_vpd_init(
+	__in			efx_nic_t *enp);
+
+extern	__checkReturn		int
+efx_vpd_size(
+	__in			efx_nic_t *enp,
+	__out			size_t *sizep);
+
+extern	__checkReturn		int
+efx_vpd_read(
+	__in			efx_nic_t *enp,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+efx_vpd_verify(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern  __checkReturn		int
+efx_vpd_reinit(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+efx_vpd_get(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__inout			efx_vpd_value_t *evvp);
+
+extern	__checkReturn		int
+efx_vpd_set(
+	__in			efx_nic_t *enp,
+	__inout_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_value_t *evvp);
+
+extern	__checkReturn		int
+efx_vpd_next(
+	__in			efx_nic_t *enp,
+	__inout_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			efx_vpd_value_t *evvp,
+	__inout			unsigned int *contp);
+
+extern __checkReturn		int
+efx_vpd_write(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern				void
+efx_vpd_fini(
+	__in			efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_VPD */
+
+/* NVRAM */
+
+#if EFSYS_OPT_NVRAM
+
+typedef enum efx_nvram_type_e {
+	EFX_NVRAM_INVALID = 0,
+	EFX_NVRAM_BOOTROM,
+	EFX_NVRAM_BOOTROM_CFG,
+	EFX_NVRAM_MC_FIRMWARE,
+	EFX_NVRAM_MC_GOLDEN,
+	EFX_NVRAM_PHY,
+	EFX_NVRAM_NULLPHY,
+	EFX_NVRAM_NTYPES,
+} efx_nvram_type_t;
+
+extern	__checkReturn		int
+efx_nvram_init(
+	__in			efx_nic_t *enp);
+
+#if EFSYS_OPT_DIAG
+
+extern	__checkReturn		int
+efx_nvram_test(
+	__in			efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+extern	__checkReturn		int
+efx_nvram_size(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			size_t *sizep);
+
+extern	__checkReturn		int
+efx_nvram_rw_start(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out_opt		size_t *pref_chunkp);
+
+extern				void
+efx_nvram_rw_finish(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type);
+
+extern	__checkReturn		int
+efx_nvram_get_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint32_t *subtypep,
+	__out_ecount(4)		uint16_t version[4]);
+
+extern	__checkReturn		int
+efx_nvram_read_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+efx_nvram_set_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint16_t version[4]);
+
+extern	 __checkReturn		int
+efx_nvram_erase(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type);
+
+extern	__checkReturn		int
+efx_nvram_write_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern				void
+efx_nvram_fini(
+	__in			efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_NVRAM */
+
+#if EFSYS_OPT_BOOTCFG
+
+extern				int
+efx_bootcfg_read(
+	__in			efx_nic_t *enp,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern				int
+efx_bootcfg_write(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+#endif	/* EFSYS_OPT_BOOTCFG */
+
+#if EFSYS_OPT_WOL
+
+typedef enum efx_wol_type_e {
+	EFX_WOL_TYPE_INVALID,
+	EFX_WOL_TYPE_MAGIC,
+	EFX_WOL_TYPE_BITMAP,
+	EFX_WOL_TYPE_LINK,
+	EFX_WOL_NTYPES,
+} efx_wol_type_t;
+
+typedef enum efx_lightsout_offload_type_e {
+	EFX_LIGHTSOUT_OFFLOAD_TYPE_INVALID,
+	EFX_LIGHTSOUT_OFFLOAD_TYPE_ARP,
+	EFX_LIGHTSOUT_OFFLOAD_TYPE_NS,
+} efx_lightsout_offload_type_t;
+
+#define	EFX_WOL_BITMAP_MASK_SIZE    (48)
+#define	EFX_WOL_BITMAP_VALUE_SIZE   (128)
+
+typedef union efx_wol_param_u {
+	struct {
+		uint8_t mac_addr[6];
+	} ewp_magic;
+	struct {
+		uint8_t mask[EFX_WOL_BITMAP_MASK_SIZE];   /* 1 bit per byte */
+		uint8_t value[EFX_WOL_BITMAP_VALUE_SIZE]; /* value to match */
+		uint8_t value_len;
+	} ewp_bitmap;
+} efx_wol_param_t;
+
+typedef union efx_lightsout_offload_param_u {
+	struct {
+		uint8_t mac_addr[6];
+		uint32_t ip;
+	} elop_arp;
+	struct {
+		uint8_t mac_addr[6];
+		uint32_t solicited_node[4];
+		uint32_t ip[4];
+	} elop_ns;
+} efx_lightsout_offload_param_t;
+
+extern	__checkReturn	int
+efx_wol_init(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+efx_wol_filter_clear(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+efx_wol_filter_add(
+	__in		efx_nic_t *enp,
+	__in		efx_wol_type_t type,
+	__in		efx_wol_param_t *paramp,
+	__out		uint32_t *filter_idp);
+
+extern	__checkReturn	int
+efx_wol_filter_remove(
+	__in		efx_nic_t *enp,
+	__in		uint32_t filter_id);
+
+extern	__checkReturn	int
+efx_lightsout_offload_add(
+	__in		efx_nic_t *enp,
+	__in		efx_lightsout_offload_type_t type,
+	__in		efx_lightsout_offload_param_t *paramp,
+	__out		uint32_t *filter_idp);
+
+extern	__checkReturn	int
+efx_lightsout_offload_remove(
+	__in		efx_nic_t *enp,
+	__in		efx_lightsout_offload_type_t type,
+	__in		uint32_t filter_id);
+
+extern			void
+efx_wol_fini(
+	__in		efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_WOL */
+
+#if EFSYS_OPT_DIAG
+
+typedef enum efx_pattern_type_t {
+	EFX_PATTERN_BYTE_INCREMENT = 0,
+	EFX_PATTERN_ALL_THE_SAME,
+	EFX_PATTERN_BIT_ALTERNATE,
+	EFX_PATTERN_BYTE_ALTERNATE,
+	EFX_PATTERN_BYTE_CHANGING,
+	EFX_PATTERN_BIT_SWEEP,
+	EFX_PATTERN_NTYPES
+} efx_pattern_type_t;
+
+typedef 		void
+(*efx_sram_pattern_fn_t)(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp);
+
+extern	__checkReturn	int
+efx_sram_test(
+	__in		efx_nic_t *enp,
+	__in		efx_pattern_type_t type);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+extern	__checkReturn	int
+efx_sram_buf_tbl_set(
+	__in		efx_nic_t *enp,
+	__in		uint32_t id,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n);
+
+extern		void
+efx_sram_buf_tbl_clear(
+	__in	efx_nic_t *enp,
+	__in	uint32_t id,
+	__in	size_t n);
+
+#define	EFX_BUF_TBL_SIZE	0x20000
+
+#define	EFX_BUF_SIZE		4096
+
+/* EV */
+
+typedef struct efx_evq_s	efx_evq_t;
+
+#if EFSYS_OPT_QSTATS
+
+/* START MKCONFIG GENERATED EfxHeaderEventQueueBlock d5614a5d669c8ca3 */
+typedef enum efx_ev_qstat_e {
+	EV_ALL,
+	EV_RX,
+	EV_RX_OK,
+	EV_RX_RECOVERY,
+	EV_RX_FRM_TRUNC,
+	EV_RX_TOBE_DISC,
+	EV_RX_PAUSE_FRM_ERR,
+	EV_RX_BUF_OWNER_ID_ERR,
+	EV_RX_IPV4_HDR_CHKSUM_ERR,
+	EV_RX_TCP_UDP_CHKSUM_ERR,
+	EV_RX_ETH_CRC_ERR,
+	EV_RX_IP_FRAG_ERR,
+	EV_RX_MCAST_PKT,
+	EV_RX_MCAST_HASH_MATCH,
+	EV_RX_TCP_IPV4,
+	EV_RX_TCP_IPV6,
+	EV_RX_UDP_IPV4,
+	EV_RX_UDP_IPV6,
+	EV_RX_OTHER_IPV4,
+	EV_RX_OTHER_IPV6,
+	EV_RX_NON_IP,
+	EV_RX_OVERRUN,
+	EV_TX,
+	EV_TX_WQ_FF_FULL,
+	EV_TX_PKT_ERR,
+	EV_TX_PKT_TOO_BIG,
+	EV_TX_UNEXPECTED,
+	EV_GLOBAL,
+	EV_GLOBAL_PHY,
+	EV_GLOBAL_MNT,
+	EV_GLOBAL_RX_RECOVERY,
+	EV_DRIVER,
+	EV_DRIVER_SRM_UPD_DONE,
+	EV_DRIVER_TX_DESCQ_FLS_DONE,
+	EV_DRIVER_RX_DESCQ_FLS_DONE,
+	EV_DRIVER_RX_DESCQ_FLS_FAILED,
+	EV_DRIVER_RX_DSC_ERROR,
+	EV_DRIVER_TX_DSC_ERROR,
+	EV_DRV_GEN,
+	EV_MCDI_RESPONSE,
+	EV_NQSTATS
+} efx_ev_qstat_t;
+
+/* END MKCONFIG GENERATED EfxHeaderEventQueueBlock */
+
+#endif	/* EFSYS_OPT_QSTATS */
+
+extern	__checkReturn	int
+efx_ev_init(
+	__in		efx_nic_t *enp);
+
+extern		void
+efx_ev_fini(
+	__in		efx_nic_t *enp);
+
+#define	EFX_MASK(_max, _min)	(-((_max) << 1) ^ -(_min))
+
+#define	EFX_EVQ_MAXNEVS		32768
+#define	EFX_EVQ_MINNEVS		512
+
+#define	EFX_EVQ_NEVS_MASK	EFX_MASK(EFX_EVQ_MAXNEVS, EFX_EVQ_MINNEVS)
+
+#define	EFX_EVQ_SIZE(_nevs)	((_nevs) * sizeof (efx_qword_t))
+#define	EFX_EVQ_NBUFS(_nevs)	(EFX_EVQ_SIZE(_nevs) / EFX_BUF_SIZE)
+
+extern	__checkReturn	int
+efx_ev_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__deref_out	efx_evq_t **eepp);
+
+extern		void
+efx_ev_qpost(
+	__in		efx_evq_t *eep,
+	__in		uint16_t data);
+
+typedef __checkReturn	boolean_t
+(*efx_initialized_ev_t)(
+	__in_opt	void *arg);
+
+#define	EFX_PKT_UNICAST		0x0004
+#define	EFX_PKT_START		0x0008
+
+#define	EFX_PKT_VLAN_TAGGED	0x0010
+#define	EFX_CKSUM_TCPUDP	0x0020
+#define	EFX_CKSUM_IPV4		0x0040
+#define	EFX_PKT_CONT		0x0080
+
+#define	EFX_CHECK_VLAN		0x0100
+#define	EFX_PKT_TCP		0x0200
+#define	EFX_PKT_UDP		0x0400
+#define	EFX_PKT_IPV4		0x0800
+
+#define	EFX_PKT_IPV6		0x1000
+#define	EFX_ADDR_MISMATCH	0x4000
+#define	EFX_DISCARD		0x8000
+
+#define	EFX_EV_RX_NLABELS	32
+#define	EFX_EV_TX_NLABELS	32
+
+typedef	__checkReturn	boolean_t
+(*efx_rx_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label,
+	__in		uint32_t id,
+	__in		uint32_t size,
+	__in		uint16_t flags);
+
+typedef	__checkReturn	boolean_t
+(*efx_tx_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label,
+	__in		uint32_t id);
+
+#define	EFX_EXCEPTION_RX_RECOVERY	0x00000001
+#define	EFX_EXCEPTION_RX_DSC_ERROR	0x00000002
+#define	EFX_EXCEPTION_TX_DSC_ERROR	0x00000003
+#define	EFX_EXCEPTION_UNKNOWN_SENSOREVT	0x00000004
+#define	EFX_EXCEPTION_FWALERT_SRAM	0x00000005
+#define	EFX_EXCEPTION_UNKNOWN_FWALERT	0x00000006
+
+typedef	__checkReturn	boolean_t
+(*efx_exception_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label,
+	__in		uint32_t data);
+
+typedef	__checkReturn	boolean_t
+(*efx_rxq_flush_done_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label);
+
+typedef	__checkReturn	boolean_t
+(*efx_rxq_flush_failed_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label);
+
+typedef	__checkReturn	boolean_t
+(*efx_txq_flush_done_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label);
+
+typedef	__checkReturn	boolean_t
+(*efx_software_ev_t)(
+	__in_opt	void *arg,
+	__in		uint16_t magic);
+
+typedef	__checkReturn	boolean_t
+(*efx_sram_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t code);
+
+#define	EFX_SRAM_CLEAR		0
+#define	EFX_SRAM_UPDATE		1
+#define	EFX_SRAM_ILLEGAL_CLEAR	2
+
+typedef	__checkReturn	boolean_t
+(*efx_wake_up_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label);
+
+typedef	__checkReturn	boolean_t
+(*efx_timer_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label);
+
+typedef __checkReturn	boolean_t
+(*efx_link_change_ev_t)(
+	__in_opt	void *arg,
+	__in		efx_link_mode_t	link_mode);
+
+#if EFSYS_OPT_MON_STATS
+
+typedef __checkReturn	boolean_t
+(*efx_monitor_ev_t)(
+	__in_opt	void *arg,
+	__in		efx_mon_stat_t id,
+	__in		efx_mon_stat_value_t value);
+
+#endif	/* EFSYS_OPT_MON_STATS */
+
+#if EFSYS_OPT_MAC_STATS
+
+typedef __checkReturn	boolean_t
+(*efx_mac_stats_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t generation
+	);
+
+#endif	/* EFSYS_OPT_MAC_STATS */
+
+typedef struct efx_ev_callbacks_s {
+	efx_initialized_ev_t		eec_initialized;
+	efx_rx_ev_t			eec_rx;
+	efx_tx_ev_t			eec_tx;
+	efx_exception_ev_t		eec_exception;
+	efx_rxq_flush_done_ev_t		eec_rxq_flush_done;
+	efx_rxq_flush_failed_ev_t	eec_rxq_flush_failed;
+	efx_txq_flush_done_ev_t		eec_txq_flush_done;
+	efx_software_ev_t		eec_software;
+	efx_sram_ev_t			eec_sram;
+	efx_wake_up_ev_t		eec_wake_up;
+	efx_timer_ev_t			eec_timer;
+	efx_link_change_ev_t		eec_link_change;
+#if EFSYS_OPT_MON_STATS
+	efx_monitor_ev_t		eec_monitor;
+#endif	/* EFSYS_OPT_MON_STATS */
+#if EFSYS_OPT_MAC_STATS
+	efx_mac_stats_ev_t		eec_mac_stats;
+#endif	/* EFSYS_OPT_MON_STATS */
+} efx_ev_callbacks_t;
+
+extern	__checkReturn	boolean_t
+efx_ev_qpending(
+	__in		efx_evq_t *eep,
+	__in		unsigned int count);
+
+#if EFSYS_OPT_EV_PREFETCH
+
+extern			void
+efx_ev_qprefetch(
+	__in		efx_evq_t *eep,
+	__in		unsigned int count);
+
+#endif	/* EFSYS_OPT_EV_PREFETCH */
+
+extern			void
+efx_ev_qpoll(
+	__in		efx_evq_t *eep,
+	__inout		unsigned int *countp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+
+extern	__checkReturn	int
+efx_ev_qmoderate(
+	__in		efx_evq_t *eep,
+	__in		unsigned int us);
+
+extern	__checkReturn	int
+efx_ev_qprime(
+	__in		efx_evq_t *eep,
+	__in		unsigned int count);
+
+#if EFSYS_OPT_QSTATS
+
+#if EFSYS_OPT_NAMES
+
+extern		const char __cs *
+efx_ev_qstat_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+extern					void
+efx_ev_qstats_update(
+	__in				efx_evq_t *eep,
+	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
+
+#endif	/* EFSYS_OPT_QSTATS */
+
+extern		void
+efx_ev_qdestroy(
+	__in	efx_evq_t *eep);
+
+/* RX */
+
+typedef struct efx_rxq_s	efx_rxq_t;
+
+extern	__checkReturn	int
+efx_rx_init(
+	__in		efx_nic_t *enp);
+
+extern		void
+efx_rx_fini(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_RX_HDR_SPLIT
+	__checkReturn	int
+efx_rx_hdr_split_enable(
+	__in		efx_nic_t *enp,
+	__in		unsigned int hdr_buf_size,
+	__in		unsigned int pld_buf_size);
+
+#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
+
+#if EFSYS_OPT_RX_SCATTER
+	__checkReturn	int
+efx_rx_scatter_enable(
+	__in		efx_nic_t *enp,
+	__in		unsigned int buf_size);
+#endif	/* EFSYS_OPT_RX_SCATTER */
+
+#if EFSYS_OPT_RX_SCALE
+
+typedef enum efx_rx_hash_alg_e {
+	EFX_RX_HASHALG_LFSR = 0,
+	EFX_RX_HASHALG_TOEPLITZ
+} efx_rx_hash_alg_t;
+
+typedef enum efx_rx_hash_type_e {
+	EFX_RX_HASH_IPV4 = 0,
+	EFX_RX_HASH_TCPIPV4,
+	EFX_RX_HASH_IPV6,
+	EFX_RX_HASH_TCPIPV6,
+} efx_rx_hash_type_t;
+
+#define	EFX_RSS_TBL_SIZE	128	/* Rows in RX indirection table */
+#define	EFX_MAXRSS	    	64	/* RX indirection entry range */
+#define	EFX_MAXRSS_LEGACY   	16 	/* See bug16611 and bug17213 */
+
+extern	__checkReturn	int
+efx_rx_scale_mode_set(
+	__in	efx_nic_t *enp,
+	__in	efx_rx_hash_alg_t alg,
+	__in	efx_rx_hash_type_t type,
+	__in	boolean_t insert);
+
+extern	__checkReturn	int
+efx_rx_scale_tbl_set(
+	__in		efx_nic_t *enp,
+	__in_ecount(n)	unsigned int *table,
+	__in		size_t n);
+
+extern	__checkReturn	int
+efx_rx_scale_toeplitz_ipv4_key_set(
+	__in		efx_nic_t *enp,
+	__in_ecount(n)	uint8_t *key,
+	__in		size_t n);
+
+extern	__checkReturn	int
+efx_rx_scale_toeplitz_ipv6_key_set(
+	__in		efx_nic_t *enp,
+	__in_ecount(n)	uint8_t *key,
+	__in		size_t n);
+
+/*
+ * The prefix is a byte array of one of the forms:
+ *
+ *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
+ * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.TT.TT.TT.TT
+ * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.LL.LL
+ *
+ * where:
+ *
+ * TT.TT.TT.TT is a 32-bit Toeplitz hash
+ * LL.LL is a 16-bit LFSR hash
+ *
+ * Hash values are in network (big-endian) byte order.
+ */
+
+#define	EFX_RX_PREFIX_SIZE	16
+
+#define	EFX_RX_HASH_VALUE(_func, _buffer)				\
+	(((_func) == EFX_RX_HASHALG_LFSR) ?				\
+		((uint16_t)(((_buffer)[14] << 8) | (_buffer)[15])) :	\
+		((uint32_t)(((_buffer)[12] << 24) |			\
+		    ((_buffer)[13] << 16) |				\
+		    ((_buffer)[14] << 8) |				\
+		    (_buffer)[15])))
+
+#define	EFX_RX_HASH_SIZE(_func)						\
+	(((_func) == EFX_RX_HASHALG_LFSR) ?				\
+		sizeof (uint16_t) : 					\
+		sizeof (uint32_t))
+
+#endif	/* EFSYS_OPT_RX_SCALE */
+
+#define	EFX_RXQ_MAXNDESCS	4096
+#define	EFX_RXQ_MINNDESCS	512
+
+#define	EFX_RXQ_NDESCS_MASK	EFX_MASK(EFX_RXQ_MAXNDESCS, EFX_RXQ_MINNDESCS)
+
+#define	EFX_RXQ_SIZE(_ndescs)	((_ndescs) * sizeof (efx_qword_t))
+#define	EFX_RXQ_NBUFS(_ndescs)	(EFX_RXQ_SIZE(_ndescs) / EFX_BUF_SIZE)
+#define	EFX_RXQ_LIMIT(_ndescs)  ((_ndescs) - 16)
+
+typedef enum efx_rxq_type_e {
+	EFX_RXQ_TYPE_DEFAULT,
+	EFX_RXQ_TYPE_SPLIT_HEADER,
+	EFX_RXQ_TYPE_SPLIT_PAYLOAD,
+	EFX_RXQ_TYPE_SCATTER,
+	EFX_RXQ_NTYPES
+} efx_rxq_type_t;
+
+extern	__checkReturn	int
+efx_rx_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		unsigned int label,
+	__in		efx_rxq_type_t type,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__in		efx_evq_t *eep,
+	__deref_out	efx_rxq_t **erpp);
+
+typedef struct efx_buffer_s {
+	efsys_dma_addr_t	eb_addr;
+	size_t			eb_size;
+	boolean_t		eb_eop;
+} efx_buffer_t;
+
+extern			void
+efx_rx_qpost(
+	__in		efx_rxq_t *erp,
+	__in_ecount(n)	efsys_dma_addr_t *addrp,
+	__in		size_t size,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__in		unsigned int added);
+
+extern		void
+efx_rx_qpush(
+	__in	efx_rxq_t *erp,
+	__in	unsigned int added);
+
+extern		void
+efx_rx_qflush(
+	__in	efx_rxq_t *erp);
+
+extern		void
+efx_rx_qenable(
+	__in	efx_rxq_t *erp);
+
+extern		void
+efx_rx_qdestroy(
+	__in	efx_rxq_t *erp);
+
+/* TX */
+
+typedef struct efx_txq_s	efx_txq_t;
+
+#if EFSYS_OPT_QSTATS
+
+/* START MKCONFIG GENERATED EfxHeaderTransmitQueueBlock 536c5fa5014944bf */
+typedef enum efx_tx_qstat_e {
+	TX_POST,
+	TX_UNALIGNED_SPLIT,
+	TX_NQSTATS
+} efx_tx_qstat_t;
+
+/* END MKCONFIG GENERATED EfxHeaderTransmitQueueBlock */
+
+#endif	/* EFSYS_OPT_QSTATS */
+
+extern	__checkReturn	int
+efx_tx_init(
+	__in		efx_nic_t *enp);
+
+extern		void
+efx_tx_fini(
+	__in	efx_nic_t *enp);
+
+#define	EFX_TXQ_MAXNDESCS	4096
+#define	EFX_TXQ_MINNDESCS	512
+
+#define	EFX_TXQ_NDESCS_MASK	EFX_MASK(EFX_TXQ_MAXNDESCS, EFX_TXQ_MINNDESCS)
+
+#define	EFX_TXQ_SIZE(_ndescs)	((_ndescs) * sizeof (efx_qword_t))
+#define	EFX_TXQ_NBUFS(_ndescs)	(EFX_TXQ_SIZE(_ndescs) / EFX_BUF_SIZE)
+#define	EFX_TXQ_LIMIT(_ndescs)  ((_ndescs) - 16)
+
+extern	__checkReturn	int
+efx_tx_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		unsigned int label,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__in		uint16_t flags,
+	__in		efx_evq_t *eep,
+	__deref_out	efx_txq_t **etpp);
+
+extern	__checkReturn	int
+efx_tx_qpost(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_buffer_t *eb,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp);
+
+extern		void
+efx_tx_qpush(
+	__in	efx_txq_t *etp,
+	__in	unsigned int added);
+
+extern		void
+efx_tx_qflush(
+	__in	efx_txq_t *etp);
+
+extern		void
+efx_tx_qenable(
+	__in	efx_txq_t *etp);
+
+#if EFSYS_OPT_QSTATS
+
+#if EFSYS_OPT_NAMES
+
+extern		const char __cs *
+efx_tx_qstat_name(
+	__in	efx_nic_t *etp,
+	__in	unsigned int id);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+extern					void
+efx_tx_qstats_update(
+	__in				efx_txq_t *etp,
+	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat);
+
+#endif	/* EFSYS_OPT_QSTATS */
+
+extern		void
+efx_tx_qdestroy(
+	__in	efx_txq_t *etp);
+
+
+/* FILTER */
+
+#if EFSYS_OPT_FILTER
+
+typedef enum efx_filter_flag_e {
+	EFX_FILTER_FLAG_RX_RSS = 0x01,		/* use RSS to spread across
+						 * multiple queues */
+	EFX_FILTER_FLAG_RX_SCATTER = 0x02,	/* enable RX scatter */
+	EFX_FILTER_FLAG_RX_OVERRIDE_IP = 0x04,	/* MAC filter overrides
+						 * any matching IP filter */
+} efx_filter_flag_t;
+
+typedef struct efx_filter_spec_s {
+	uint8_t		efs_type;
+	uint8_t		efs_flags;
+	uint16_t	efs_dmaq_id;
+	uint32_t	efs_dword[3];
+} efx_filter_spec_t;
+
+extern	__checkReturn	int
+efx_filter_init(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_filter_fini(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+efx_rx_filter_insert(
+	__in		efx_rxq_t *erp,
+	__inout		efx_filter_spec_t *spec);
+
+extern	__checkReturn	int
+efx_rx_filter_remove(
+	__in		efx_rxq_t *erp,
+	__inout		efx_filter_spec_t *spec);
+
+			void
+efx_filter_restore(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_filter_spec_rx_ipv4_tcp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_tcp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_tcp);
+
+extern			void
+efx_filter_spec_rx_ipv4_tcp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_tcp);
+
+extern			void
+efx_filter_spec_rx_ipv4_udp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_udp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_udp);
+
+extern			void
+efx_filter_spec_rx_ipv4_udp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_udp);
+
+extern			void
+efx_filter_spec_rx_mac_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint16_t vlan_id,
+	__in		uint8_t *dest_mac);
+
+extern			void
+efx_filter_spec_rx_mac_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint8_t *dest_mac);
+
+
+extern	__checkReturn	int
+efx_tx_filter_insert(
+	__in		efx_txq_t *etp,
+	__inout		efx_filter_spec_t *spec);
+
+extern	__checkReturn	int
+efx_tx_filter_remove(
+	__in		efx_txq_t *etp,
+	__inout		efx_filter_spec_t *spec);
+
+extern			void
+efx_filter_spec_tx_ipv4_tcp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_tcp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_tcp);
+
+extern			void
+efx_filter_spec_tx_ipv4_tcp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_tcp);
+
+extern			void
+efx_filter_spec_tx_ipv4_udp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_udp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_udp);
+
+extern			void
+efx_filter_spec_tx_ipv4_udp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_udp);
+
+extern			void
+efx_filter_spec_tx_mac_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint16_t vlan_id,
+	__in		uint8_t *src_mac);
+
+extern			void
+efx_filter_spec_tx_mac_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint8_t *src_mac);
+
+#endif	/* EFSYS_OPT_FILTER */
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_EFX_H */
diff --git a/sys/dev/sfxge/common/efx_bootcfg.c b/sys/dev/sfxge/common/efx_bootcfg.c
new file mode 100644
index 0000000..1a6a79c
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_bootcfg.c
@@ -0,0 +1,342 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_BOOTCFG
+
+/*
+ * Maximum size of BOOTCFG block across all nics as understood by SFCgPXE.
+ * A multiple of 0x100 so trailing 0xff characters don't contrinbute to the
+ * checksum.
+ */
+#define	BOOTCFG_MAX_SIZE 0x1000
+
+#define	DHCP_END (uint8_t)0xff
+#define	DHCP_PAD (uint8_t)0
+
+static	__checkReturn		uint8_t
+efx_bootcfg_csum(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	_NOTE(ARGUNUSED(enp))
+
+	unsigned int pos;
+	uint8_t checksum = 0;
+
+	for (pos = 0; pos < size; pos++)
+		checksum += data[pos];
+	return (checksum);
+}
+
+static	__checkReturn		int
+efx_bootcfg_verify(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			size_t *usedp)
+{
+	size_t offset = 0;
+	size_t used = 0;
+	int rc;
+
+	/* Start parsing tags immediatly after the checksum */
+	for (offset = 1; offset < size; ) {
+		uint8_t tag;
+		uint8_t length;
+
+		/* Consume tag */
+		tag = data[offset];
+		if (tag == DHCP_END) {
+			offset++;
+			used = offset;
+			break;
+		}
+		if (tag == DHCP_PAD) {
+			offset++;
+			continue;
+		}
+
+		/* Consume length */
+		if (offset + 1 >= size) {
+			rc = ENOSPC;
+			goto fail1;
+		}
+		length = data[offset + 1];
+
+		/* Consume *length */
+		if (offset + 1 + length >= size) {
+			rc = ENOSPC;
+			goto fail2;
+		}
+
+		offset += 2 + length;
+		used = offset;
+	}
+
+	/* Checksum the entire sector, including bytes after any DHCP_END */
+	if (efx_bootcfg_csum(enp, data, size) != 0) {
+		rc = EINVAL;
+		goto fail3;
+	}
+
+	if (usedp != NULL)
+		*usedp = used;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				int
+efx_bootcfg_read(
+	__in			efx_nic_t *enp,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	uint8_t *payload = NULL;
+	size_t used_bytes;
+	size_t sector_length;
+	int rc;
+
+	rc = efx_nvram_size(enp, EFX_NVRAM_BOOTROM_CFG, &sector_length);
+	if (rc != 0)
+		goto fail1;
+
+	/*
+	 * We need to read the entire BOOTCFG area to ensure we read all the
+	 * tags, because legacy bootcfg sectors are not guaranteed to end with
+	 * a DHCP_END character. If the user hasn't supplied a sufficiently
+	 * large buffer then use our own buffer.
+	 */
+	if (sector_length > BOOTCFG_MAX_SIZE)
+		sector_length = BOOTCFG_MAX_SIZE;
+	if (sector_length > size) {
+		EFSYS_KMEM_ALLOC(enp->en_esip, sector_length, payload);
+		if (payload == NULL) {
+			rc = ENOMEM;
+			goto fail2;
+		}
+	} else
+		payload = (uint8_t *)data;
+
+	if ((rc = efx_nvram_rw_start(enp, EFX_NVRAM_BOOTROM_CFG, NULL)) != 0)
+		goto fail3;
+
+	rc = efx_nvram_read_chunk(enp, EFX_NVRAM_BOOTROM_CFG, 0,
+				    (caddr_t)payload, sector_length);
+
+	efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG);
+
+	if (rc != 0)
+		goto fail4;
+
+	/* Verify that the area is correctly formatted and checksummed */
+	rc = efx_bootcfg_verify(enp, (caddr_t)payload, sector_length,
+				    &used_bytes);
+	if (rc != 0 || used_bytes == 0) {
+		payload[0] = (uint8_t)~DHCP_END;
+		payload[1] = DHCP_END;
+		used_bytes = 2;
+	}
+
+	EFSYS_ASSERT(used_bytes >= 2);	/* checksum and DHCP_END */
+	EFSYS_ASSERT(used_bytes <= sector_length);
+
+	/*
+	 * Legacy bootcfg sectors don't terminate with a DHCP_END character.
+	 * Modify the returned payload so it does. BOOTCFG_MAX_SIZE is by
+	 * definition large enough for any valid (per-port) bootcfg sector,
+	 * so reinitialise the sector if there isn't room for the character.
+	 */
+	if (payload[used_bytes - 1] != DHCP_END) {
+		if (used_bytes + 1 > sector_length) {
+			payload[0] = 0;
+			used_bytes = 1;
+		}
+
+		payload[used_bytes] = DHCP_END;
+		++used_bytes;
+	}
+
+	/*
+	 * Verify that the user supplied buffer is large enough for the
+	 * entire used bootcfg area, then copy into the user supplied buffer.
+	 */
+	if (used_bytes > size) {
+		rc = ENOSPC;
+		goto fail5;
+	}
+	if (sector_length > size) {
+		memcpy(data, payload, used_bytes);
+		EFSYS_KMEM_FREE(enp->en_esip, sector_length, payload);
+	}
+
+	/* Zero out the unused portion of the user buffer */
+	if (used_bytes < size)
+		(void) memset(data + used_bytes, 0, size - used_bytes);
+
+	/*
+	 * The checksum includes trailing data after any DHCP_END character,
+	 * which we've just modified (by truncation or appending DHCP_END).
+	 */
+	data[0] -= efx_bootcfg_csum(enp, data, size);
+
+	return (0);
+
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+
+	if (sector_length > size)
+		EFSYS_KMEM_FREE(enp->en_esip, sector_length, payload);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				int
+efx_bootcfg_write(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	uint8_t *chunk;
+	uint8_t checksum;
+	size_t sector_length;
+	size_t chunk_length;
+	size_t used_bytes;
+	size_t offset;
+	size_t remaining;
+	int rc;
+
+	rc = efx_nvram_size(enp, EFX_NVRAM_BOOTROM_CFG, &sector_length);
+	if (rc != 0)
+		goto fail1;
+
+	if (sector_length > BOOTCFG_MAX_SIZE)
+		sector_length = BOOTCFG_MAX_SIZE;
+
+	if ((rc = efx_bootcfg_verify(enp, data, size, &used_bytes)) != 0)
+		goto fail2;
+
+	/* The caller *must* terminate their block with a DHCP_END character */
+	EFSYS_ASSERT(used_bytes >= 2);		/* checksum and DHCP_END */
+	if ((uint8_t)data[used_bytes - 1] != DHCP_END) {
+		rc = ENOENT;
+		goto fail3;
+	}
+
+	/* Check that the hardware has support for this much data */
+	if (used_bytes > MIN(sector_length, BOOTCFG_MAX_SIZE)) {
+		rc = ENOSPC;
+		goto fail4;
+	}
+
+	rc = efx_nvram_rw_start(enp, EFX_NVRAM_BOOTROM_CFG, &chunk_length);
+	if (rc != 0)
+		goto fail5;
+
+	EFSYS_KMEM_ALLOC(enp->en_esip, chunk_length, chunk);
+	if (chunk == NULL) {
+		rc = ENOMEM;
+		goto fail6;
+	}
+
+	if ((rc = efx_nvram_erase(enp, EFX_NVRAM_BOOTROM_CFG)) != 0)
+		goto fail7;
+
+	/*
+	 * Write the entire sector_length bytes of data in chunks. Zero out
+	 * all data following the DHCP_END, and adjust the checksum
+	 */
+	checksum = efx_bootcfg_csum(enp, data, used_bytes);
+	for (offset = 0; offset < sector_length; offset += remaining) {
+		remaining = MIN(chunk_length, sector_length - offset);
+
+		/* Fill chunk */
+		(void) memset(chunk, 0x0, chunk_length);
+		if (offset < used_bytes)
+			memcpy(chunk, data + offset,
+			    MIN(remaining, used_bytes - offset));
+
+		/* Adjust checksum */
+		if (offset == 0)
+			chunk[0] -= checksum;
+
+		if ((rc = efx_nvram_write_chunk(enp, EFX_NVRAM_BOOTROM_CFG,
+			    offset, (caddr_t)chunk, remaining)) != 0)
+			goto fail8;
+	}
+
+	efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG);
+
+	EFSYS_KMEM_FREE(enp->en_esip, chunk_length, chunk);
+
+	return (0);
+
+fail8:
+	EFSYS_PROBE(fail8);
+fail7:
+	EFSYS_PROBE(fail7);
+
+	EFSYS_KMEM_FREE(enp->en_esip, chunk_length, chunk);
+fail6:
+	EFSYS_PROBE(fail6);
+
+	efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_BOOTCFG */
diff --git a/sys/dev/sfxge/common/efx_ev.c b/sys/dev/sfxge/common/efx_ev.c
new file mode 100644
index 0000000..2203bc1
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_ev.c
@@ -0,0 +1,1112 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_QSTATS
+#define	EFX_EV_QSTAT_INCR(_eep, _stat)					\
+	do {								\
+		(_eep)->ee_stat[_stat]++;				\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+#else
+#define	EFX_EV_QSTAT_INCR(_eep, _stat)
+#endif
+
+	__checkReturn	int
+efx_ev_init(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	if (enp->en_mod_flags & EFX_MOD_EV) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
+
+	/*
+	 * Program the event queue for receive and transmit queue
+	 * flush events.
+	 */
+	EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
+
+	enp->en_mod_flags |= EFX_MOD_EV;
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static  __checkReturn   boolean_t
+efx_ev_rx_not_ok(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		uint32_t label,
+	__in		uint32_t id,
+	__inout		uint16_t *flagsp)
+{
+	boolean_t ignore = B_FALSE;
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
+		EFSYS_PROBE(tobe_disc);
+		/* Assume this is a unicast address mismatch, unless below
+		 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
+		 * EV_RX_PAUSE_FRM_ERR is set.
+		 */
+		(*flagsp) |= EFX_ADDR_MISMATCH;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
+		EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
+		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
+		(*flagsp) |= EFX_DISCARD;
+
+#if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
+		/* Lookout for payload queue ran dry errors and ignore them.
+		 *
+		 * Sadly for the header/data split cases, the descriptor
+		 * pointer in this event refers to the header queue and
+		 * therefore cannot be easily detected as duplicate.
+		 * So we drop these and rely on the receive processing seeing
+		 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
+		 * the partially received packet.
+		 */
+		if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
+		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
+		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
+			ignore = B_TRUE;
+#endif	/* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
+		EFSYS_PROBE(crc_err);
+		(*flagsp) &= ~EFX_ADDR_MISMATCH;
+		(*flagsp) |= EFX_DISCARD;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
+		EFSYS_PROBE(pause_frm_err);
+		(*flagsp) &= ~EFX_ADDR_MISMATCH;
+		(*flagsp) |= EFX_DISCARD;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
+		EFSYS_PROBE(owner_id_err);
+		(*flagsp) |= EFX_DISCARD;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
+		EFSYS_PROBE(ipv4_err);
+		(*flagsp) &= ~EFX_CKSUM_IPV4;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
+		EFSYS_PROBE(udp_chk_err);
+		(*flagsp) &= ~EFX_CKSUM_TCPUDP;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
+
+		/*
+		 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
+		 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
+		 * condition.
+		 */
+		(*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
+	}
+
+	return (ignore);
+}
+
+static	__checkReturn	boolean_t
+efx_ev_rx(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	uint32_t id;
+	uint32_t size;
+	uint32_t label;
+	boolean_t ok;
+#if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
+	boolean_t sop;
+	boolean_t jumbo_cont;
+#endif	/* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
+	uint32_t hdr_type;
+	boolean_t is_v6;
+	uint16_t flags;
+	boolean_t ignore;
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_RX);
+
+	/* Basic packet information */
+	id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
+	size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
+	label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
+	ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
+
+#if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
+	sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
+	jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
+#endif	/* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
+
+	hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
+
+	is_v6 = (enp->en_family != EFX_FAMILY_FALCON &&
+		    EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
+
+	/*
+	 * If packet is marked as OK and packet type is TCP/IP or
+	 * UDP/IP or other IP, then we can rely on the hardware checksums.
+	 */
+	switch (hdr_type) {
+	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
+		flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
+		if (is_v6) {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
+			flags |= EFX_PKT_IPV6;
+		} else {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
+			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
+		}
+		break;
+
+	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
+		flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
+		if (is_v6) {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
+			flags |= EFX_PKT_IPV6;
+		} else {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
+			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
+		}
+		break;
+
+	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
+		if (is_v6) {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
+			flags = EFX_PKT_IPV6;
+		} else {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
+			flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
+		}
+		break;
+
+	case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
+		EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
+		flags = 0;
+		break;
+
+	default:
+		EFSYS_ASSERT(B_FALSE);
+		flags = 0;
+		break;
+	}
+
+#if EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT
+	/* Report scatter and header/lookahead split buffer flags */
+	if (sop)
+		flags |= EFX_PKT_START;
+	if (jumbo_cont)
+		flags |= EFX_PKT_CONT;
+#endif	/* EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT */
+
+	/* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
+	if (!ok) {
+		ignore = efx_ev_rx_not_ok(eep, eqp, label, id, &flags);
+		if (ignore) {
+			EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
+			    uint32_t, size, uint16_t, flags);
+
+			return (B_FALSE);
+		}
+	}
+
+	/* If we're not discarding the packet then it is ok */
+	if (~flags & EFX_DISCARD)
+		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
+
+	/* Detect multicast packets that didn't match the filter */
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
+
+		if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
+			EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
+		} else {
+			EFSYS_PROBE(mcast_mismatch);
+			flags |= EFX_ADDR_MISMATCH;
+		}
+	} else {
+		flags |= EFX_PKT_UNICAST;
+	}
+
+	/*
+	 * The packet parser in Siena can abort parsing packets under
+	 * certain error conditions, setting the PKT_NOT_PARSED bit
+	 * (which clears PKT_OK). If this is set, then don't trust
+	 * the PKT_TYPE field.
+	 */
+	if (enp->en_family != EFX_FAMILY_FALCON && !ok) {
+		uint32_t parse_err;
+
+		parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
+		if (parse_err != 0)
+			flags |= EFX_CHECK_VLAN;
+	}
+
+	if (~flags & EFX_CHECK_VLAN) {
+		uint32_t pkt_type;
+
+		pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
+		if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
+			flags |= EFX_PKT_VLAN_TAGGED;
+	}
+
+	EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
+	    uint32_t, size, uint16_t, flags);
+
+	EFSYS_ASSERT(eecp->eec_rx != NULL);
+	should_abort = eecp->eec_rx(arg, label, id, size, flags);
+
+	return (should_abort);
+}
+
+static	__checkReturn	boolean_t
+efx_ev_tx(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	uint32_t id;
+	uint32_t label;
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_TX);
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
+	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
+	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
+	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
+
+		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
+		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
+
+		EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
+
+		EFSYS_ASSERT(eecp->eec_tx != NULL);
+		should_abort = eecp->eec_tx(arg, label, id);
+
+		return (should_abort);
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
+		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
+			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
+			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
+		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
+		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
+
+	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
+		EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
+
+	EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
+	return (B_FALSE);
+}
+
+static	__checkReturn	boolean_t
+efx_ev_global(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	efx_port_t *epp = &(enp->en_port);
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
+	should_abort = B_FALSE;
+
+	/* Check for a link management event */
+	if (EFX_QWORD_FIELD(*eqp, FSF_BZ_GLB_EV_XG_MNT_INTR) != 0) {
+		EFX_EV_QSTAT_INCR(eep, EV_GLOBAL_MNT);
+
+		EFSYS_PROBE(xg_mgt);
+
+		epp->ep_mac_poll_needed = B_TRUE;
+	}
+
+	return (should_abort);
+}
+
+static	__checkReturn	boolean_t
+efx_ev_driver(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
+	should_abort = B_FALSE;
+
+	switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
+	case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
+		uint32_t label;
+
+		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
+
+		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
+
+		EFSYS_PROBE1(tx_descq_fls_done, uint32_t, label);
+
+		EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
+		should_abort = eecp->eec_txq_flush_done(arg, label);
+
+		break;
+	}
+	case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
+		uint32_t label;
+		uint32_t failed;
+
+		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
+		failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
+
+		EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
+		EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
+
+		if (failed) {
+			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
+
+			EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, label);
+
+			should_abort = eecp->eec_rxq_flush_failed(arg, label);
+		} else {
+			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
+
+			EFSYS_PROBE1(rx_descq_fls_done, uint32_t, label);
+
+			should_abort = eecp->eec_rxq_flush_done(arg, label);
+		}
+
+		break;
+	}
+	case FSE_AZ_EVQ_INIT_DONE_EV:
+		EFSYS_ASSERT(eecp->eec_initialized != NULL);
+		should_abort = eecp->eec_initialized(arg);
+
+		break;
+
+	case FSE_AZ_EVQ_NOT_EN_EV:
+		EFSYS_PROBE(evq_not_en);
+		break;
+
+	case FSE_AZ_SRM_UPD_DONE_EV: {
+		uint32_t code;
+
+		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
+
+		code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
+
+		EFSYS_ASSERT(eecp->eec_sram != NULL);
+		should_abort = eecp->eec_sram(arg, code);
+
+		break;
+	}
+	case FSE_AZ_WAKE_UP_EV: {
+		uint32_t id;
+
+		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
+
+		EFSYS_ASSERT(eecp->eec_wake_up != NULL);
+		should_abort = eecp->eec_wake_up(arg, id);
+
+		break;
+	}
+	case FSE_AZ_TX_PKT_NON_TCP_UDP:
+		EFSYS_PROBE(tx_pkt_non_tcp_udp);
+		break;
+
+	case FSE_AZ_TIMER_EV: {
+		uint32_t id;
+
+		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
+
+		EFSYS_ASSERT(eecp->eec_timer != NULL);
+		should_abort = eecp->eec_timer(arg, id);
+
+		break;
+	}
+	case FSE_AZ_RX_DSC_ERROR_EV:
+		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
+
+		EFSYS_PROBE(rx_dsc_error);
+
+		EFSYS_ASSERT(eecp->eec_exception != NULL);
+		should_abort = eecp->eec_exception(arg,
+			EFX_EXCEPTION_RX_DSC_ERROR, 0);
+
+		break;
+
+	case FSE_AZ_TX_DSC_ERROR_EV:
+		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
+
+		EFSYS_PROBE(tx_dsc_error);
+
+		EFSYS_ASSERT(eecp->eec_exception != NULL);
+		should_abort = eecp->eec_exception(arg,
+			EFX_EXCEPTION_TX_DSC_ERROR, 0);
+
+		break;
+
+	default:
+		break;
+	}
+
+	return (should_abort);
+}
+
+static	__checkReturn	boolean_t
+efx_ev_drv_gen(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	uint32_t data;
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
+
+	data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
+	if (data >= ((uint32_t)1 << 16)) {
+		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
+			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
+			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
+		return (B_TRUE);
+	}
+
+	EFSYS_ASSERT(eecp->eec_software != NULL);
+	should_abort = eecp->eec_software(arg, (uint16_t)data);
+
+	return (should_abort);
+}
+
+#if EFSYS_OPT_MCDI
+
+static	__checkReturn	boolean_t
+efx_ev_mcdi(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	unsigned code;
+	boolean_t should_abort = B_FALSE;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+
+	if (enp->en_family != EFX_FAMILY_SIENA)
+		goto out;
+
+	EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
+
+	code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
+	switch (code) {
+	case MCDI_EVENT_CODE_BADSSERT:
+		efx_mcdi_ev_death(enp, EINTR);
+		break;
+
+	case MCDI_EVENT_CODE_CMDDONE:
+		efx_mcdi_ev_cpl(enp,
+				MCDI_EV_FIELD(*eqp, CMDDONE_SEQ),
+				MCDI_EV_FIELD(*eqp, CMDDONE_DATALEN),
+				MCDI_EV_FIELD(*eqp, CMDDONE_ERRNO));
+		break;
+
+	case MCDI_EVENT_CODE_LINKCHANGE: {
+		efx_link_mode_t link_mode;
+
+		siena_phy_link_ev(enp, eqp, &link_mode);
+		should_abort = eecp->eec_link_change(arg, link_mode);
+		break;
+	}
+	case MCDI_EVENT_CODE_SENSOREVT: {
+#if EFSYS_OPT_MON_STATS
+		efx_mon_stat_t id;
+		efx_mon_stat_value_t value;
+		int rc;
+
+		if ((rc = siena_mon_ev(enp, eqp, &id, &value)) == 0)
+			should_abort = eecp->eec_monitor(arg, id, value);
+		else if (rc == ENOTSUP) {
+			should_abort = eecp->eec_exception(arg,
+				EFX_EXCEPTION_UNKNOWN_SENSOREVT,
+				MCDI_EV_FIELD(eqp, DATA));
+		} else
+			EFSYS_ASSERT(rc == ENODEV);	/* Wrong port */
+#else
+		should_abort = B_FALSE;
+#endif
+		break;
+	}
+	case MCDI_EVENT_CODE_SCHEDERR:
+		/* Informational only */
+		break;
+
+	case MCDI_EVENT_CODE_REBOOT:
+		efx_mcdi_ev_death(enp, EIO);
+		break;
+
+	case MCDI_EVENT_CODE_MAC_STATS_DMA:
+#if EFSYS_OPT_MAC_STATS
+		if (eecp->eec_mac_stats != NULL) {
+			eecp->eec_mac_stats(arg,
+			    MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
+		}
+#endif
+		break;
+
+	case MCDI_EVENT_CODE_FWALERT: {
+		uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
+
+		if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
+			should_abort = eecp->eec_exception(arg,
+				EFX_EXCEPTION_FWALERT_SRAM,
+				MCDI_EV_FIELD(eqp, FWALERT_DATA));
+		else
+			should_abort = eecp->eec_exception(arg,
+				EFX_EXCEPTION_UNKNOWN_FWALERT,
+				MCDI_EV_FIELD(eqp, DATA));
+		break;
+	}
+
+	default:
+		EFSYS_PROBE1(mc_pcol_error, int, code);
+		break;
+	}
+
+out:
+	return (should_abort);
+}
+
+#endif	/* EFSYS_OPT_SIENA */
+
+	__checkReturn	int
+efx_ev_qprime(
+	__in		efx_evq_t *eep,
+	__in		unsigned int count)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	uint32_t rptr;
+	efx_dword_t dword;
+	int rc;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	rptr = count & eep->ee_mask;
+
+	EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
+
+	EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
+			    &dword, B_FALSE);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	boolean_t
+efx_ev_qpending(
+	__in		efx_evq_t *eep,
+	__in		unsigned int count)
+{
+	size_t offset;
+	efx_qword_t qword;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
+	EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
+
+	return (EFX_QWORD_FIELD(qword, EFX_DWORD_0) != 0xffffffff &&
+		EFX_QWORD_FIELD(qword, EFX_DWORD_1) != 0xffffffff);
+}
+
+#if EFSYS_OPT_EV_PREFETCH
+
+			void
+efx_ev_qprefetch(
+	__in		efx_evq_t *eep,
+	__in		unsigned int count)
+{
+	unsigned int offset;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
+	EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
+}
+
+#endif	/* EFSYS_OPT_EV_PREFETCH */
+
+#define	EFX_EV_BATCH	8
+
+#define	EFX_EV_PRESENT(_qword)						\
+	(EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&	\
+	EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
+
+			void
+efx_ev_qpoll(
+	__in		efx_evq_t *eep,
+	__inout		unsigned int *countp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_qword_t ev[EFX_EV_BATCH];
+	unsigned int batch;
+	unsigned int total;
+	unsigned int count;
+	unsigned int index;
+	size_t offset;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+	EFSYS_ASSERT(countp != NULL);
+	EFSYS_ASSERT(eecp != NULL);
+
+	count = *countp;
+	do {
+		/* Read up until the end of the batch period */
+		batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
+		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
+		for (total = 0; total < batch; ++total) {
+			EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
+
+			if (!EFX_EV_PRESENT(ev[total]))
+				break;
+
+			EFSYS_PROBE3(event, unsigned int, eep->ee_index,
+			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
+			    uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
+
+			offset += sizeof (efx_qword_t);
+		}
+
+#if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
+		/*
+		 * Prefetch the next batch when we get within PREFETCH_PERIOD
+		 * of a completed batch. If the batch is smaller, then prefetch
+		 * immediately.
+		 */
+		if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
+			EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
+#endif	/* EFSYS_OPT_EV_PREFETCH */
+
+		/* Process the batch of events */
+		for (index = 0; index < total; ++index) {
+			boolean_t should_abort;
+			uint32_t code;
+			efx_ev_handler_t handler;
+
+#if EFSYS_OPT_EV_PREFETCH
+			/* Prefetch if we've now reached the batch period */
+			if (total == batch &&
+			    index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
+				offset = (count + batch) & eep->ee_mask;
+				offset *= sizeof (efx_qword_t);
+
+				EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
+			}
+#endif	/* EFSYS_OPT_EV_PREFETCH */
+
+			EFX_EV_QSTAT_INCR(eep, EV_ALL);
+
+			code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
+			handler = eep->ee_handler[code];
+			EFSYS_ASSERT(handler != NULL);
+			should_abort = handler(eep, &(ev[index]), eecp, arg);
+			if (should_abort) {
+				/* Ignore subsequent events */
+				total = index + 1;
+				break;
+			}
+		}
+
+		/*
+		 * Now that the hardware has most likely moved onto dma'ing
+		 * into the next cache line, clear the processed events. Take
+		 * care to only clear out events that we've processed
+		 */
+		EFX_SET_QWORD(ev[0]);
+		offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
+		for (index = 0; index < total; ++index) {
+			EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
+			offset += sizeof (efx_qword_t);
+		}
+
+		count += total;
+
+	} while (total == batch);
+
+	*countp = count;
+}
+
+		void
+efx_ev_qpost(
+	__in	efx_evq_t *eep,
+	__in	uint16_t data)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	efx_qword_t ev;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
+	    FSF_AZ_EV_DATA_DW0, (uint32_t)data);
+
+	EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
+	    EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
+	    EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
+
+	EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
+}
+
+	__checkReturn	int
+efx_ev_qmoderate(
+	__in		efx_evq_t *eep,
+	__in		unsigned int us)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	unsigned int locked;
+	efx_dword_t dword;
+	int rc;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	if (us > enp->en_nic_cfg.enc_evq_moderation_max) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	/* If the value is zero then disable the timer */
+	if (us == 0) {
+		if (enp->en_family == EFX_FAMILY_FALCON)
+			EFX_POPULATE_DWORD_2(dword,
+			    FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_DIS,
+			    FRF_AB_TC_TIMER_VAL, 0);
+		else
+			EFX_POPULATE_DWORD_2(dword,
+			    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
+			    FRF_CZ_TC_TIMER_VAL, 0);
+	} else {
+		uint32_t timer_val;
+
+		/* Calculate the timer value in quanta */
+		us -= (us % EFX_EV_TIMER_QUANTUM);
+		if (us < EFX_EV_TIMER_QUANTUM)
+			us = EFX_EV_TIMER_QUANTUM;
+
+		timer_val = us / EFX_EV_TIMER_QUANTUM;
+
+		/* Moderation value is base 0 so we need to deduct 1 */
+		if (enp->en_family == EFX_FAMILY_FALCON)
+			EFX_POPULATE_DWORD_2(dword,
+			    FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_INT_HLDOFF,
+			    FRF_AB_TIMER_VAL, timer_val - 1);
+		else
+			EFX_POPULATE_DWORD_2(dword,
+			    FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
+			    FRF_CZ_TC_TIMER_VAL, timer_val - 1);
+	}
+
+	locked = (eep->ee_index == 0) ? 1 : 0;
+
+	EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
+	    eep->ee_index, &dword, locked);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_ev_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__deref_out	efx_evq_t **eepp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	uint32_t size;
+	efx_evq_t *eep;
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
+
+	EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
+
+	if (!ISP2(n) || !(n & EFX_EVQ_NEVS_MASK)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+	if (index >= encp->enc_evq_limit) {
+		rc = EINVAL;
+		goto fail2;
+	}
+#if EFSYS_OPT_RX_SCALE
+	if (enp->en_intr.ei_type == EFX_INTR_LINE &&
+	    index >= EFX_MAXRSS_LEGACY) {
+		rc = EINVAL;
+		goto fail3;
+	}
+#endif
+	for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
+	    size++)
+		if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
+			break;
+	if (id + (1 << size) >= encp->enc_buftbl_limit) {
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	/* Allocate an EVQ object */
+	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
+	if (eep == NULL) {
+		rc = ENOMEM;
+		goto fail5;
+	}
+
+	eep->ee_magic = EFX_EVQ_MAGIC;
+	eep->ee_enp = enp;
+	eep->ee_index = index;
+	eep->ee_mask = n - 1;
+	eep->ee_esmp = esmp;
+
+	/* Set up the handler table */
+	eep->ee_handler[FSE_AZ_EV_CODE_RX_EV] = efx_ev_rx;
+	eep->ee_handler[FSE_AZ_EV_CODE_TX_EV] = efx_ev_tx;
+	eep->ee_handler[FSE_AZ_EV_CODE_DRIVER_EV] = efx_ev_driver;
+	eep->ee_handler[FSE_AZ_EV_CODE_GLOBAL_EV] = efx_ev_global;
+	eep->ee_handler[FSE_AZ_EV_CODE_DRV_GEN_EV] = efx_ev_drv_gen;
+#if EFSYS_OPT_MCDI
+	eep->ee_handler[FSE_AZ_EV_CODE_MCDI_EVRESPONSE] = efx_ev_mcdi;
+#endif	/* EFSYS_OPT_SIENA */
+
+	/* Set up the new event queue */
+	if (enp->en_family != EFX_FAMILY_FALCON) {
+		EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1);
+		EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword);
+	}
+
+	EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
+	    FRF_AZ_EVQ_BUF_BASE_ID, id);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword);
+
+	enp->en_ev_qcount++;
+	*eepp = eep;
+	return (0);
+
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+#if EFSYS_OPT_RX_SCALE
+fail3:
+	EFSYS_PROBE(fail3);
+#endif
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_NAMES
+/* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock 67e9bdcd920059bd */
+static const char 	__cs * __cs __efx_ev_qstat_name[] = {
+	"all",
+	"rx",
+	"rx_ok",
+	"rx_recovery",
+	"rx_frm_trunc",
+	"rx_tobe_disc",
+	"rx_pause_frm_err",
+	"rx_buf_owner_id_err",
+	"rx_ipv4_hdr_chksum_err",
+	"rx_tcp_udp_chksum_err",
+	"rx_eth_crc_err",
+	"rx_ip_frag_err",
+	"rx_mcast_pkt",
+	"rx_mcast_hash_match",
+	"rx_tcp_ipv4",
+	"rx_tcp_ipv6",
+	"rx_udp_ipv4",
+	"rx_udp_ipv6",
+	"rx_other_ipv4",
+	"rx_other_ipv6",
+	"rx_non_ip",
+	"rx_overrun",
+	"tx",
+	"tx_wq_ff_full",
+	"tx_pkt_err",
+	"tx_pkt_too_big",
+	"tx_unexpected",
+	"global",
+	"global_phy",
+	"global_mnt",
+	"global_rx_recovery",
+	"driver",
+	"driver_srm_upd_done",
+	"driver_tx_descq_fls_done",
+	"driver_rx_descq_fls_done",
+	"driver_rx_descq_fls_failed",
+	"driver_rx_dsc_error",
+	"driver_tx_dsc_error",
+	"drv_gen",
+	"mcdi_response",
+};
+/* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
+
+		const char __cs *
+efx_ev_qstat_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(id, <, EV_NQSTATS);
+
+	return (__efx_ev_qstat_name[id]);
+}
+#endif	/* EFSYS_OPT_NAMES */
+
+#if EFSYS_OPT_QSTATS
+					void
+efx_ev_qstats_update(
+	__in				efx_evq_t *eep,
+	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
+{
+	unsigned int id;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	for (id = 0; id < EV_NQSTATS; id++) {
+		efsys_stat_t *essp = &stat[id];
+
+		EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
+		eep->ee_stat[id] = 0;
+	}
+}
+#endif	/* EFSYS_OPT_QSTATS */
+
+		void
+efx_ev_qdestroy(
+	__in	efx_evq_t *eep)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+
+	EFSYS_ASSERT(enp->en_ev_qcount != 0);
+	--enp->en_ev_qcount;
+
+	/* Purge event queue */
+	EFX_ZERO_OWORD(oword);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
+	    eep->ee_index, &oword);
+
+	if (enp->en_family != EFX_FAMILY_FALCON) {
+		EFX_ZERO_OWORD(oword);
+		EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL,
+		    eep->ee_index, &oword);
+	}
+
+	/* Free the EVQ object */
+	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
+}
+
+		void
+efx_ev_fini(
+	__in	efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
+	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
+
+	enp->en_mod_flags &= ~EFX_MOD_EV;
+}
diff --git a/sys/dev/sfxge/common/efx_filter.c b/sys/dev/sfxge/common/efx_filter.c
new file mode 100644
index 0000000..e915e9c
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_filter.c
@@ -0,0 +1,1017 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+
+#if EFSYS_OPT_FILTER
+
+/* "Fudge factors" - difference between programmed value and actual depth.
+ * Due to pipelined implementation we need to program H/W with a value that
+ * is larger than the hop limit we want.
+ */
+#define FILTER_CTL_SRCH_FUDGE_WILD 3
+#define FILTER_CTL_SRCH_FUDGE_FULL 1
+
+/* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
+ * We also need to avoid infinite loops in efx_filter_search() when the
+ * table is full.
+ */
+#define FILTER_CTL_SRCH_MAX 200
+
+/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
+ * key derived from the n-tuple. */
+static			uint16_t
+efx_filter_tbl_hash(
+	__in 		uint32_t key)
+{
+	uint16_t tmp;
+
+	/* First 16 rounds */
+	tmp = 0x1fff ^ (uint16_t)(key >> 16);
+	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
+	tmp = tmp ^ tmp >> 9;
+
+	/* Last 16 rounds */
+	tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
+	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
+	tmp = tmp ^ tmp >> 9;
+
+	return (tmp);
+}
+
+
+/* To allow for hash collisions, filter search continues at these
+ * increments from the first possible entry selected by the hash. */
+static			uint16_t
+efx_filter_tbl_increment(
+	__in		uint32_t key)
+{
+	return ((uint16_t)(key * 2 - 1));
+}
+
+static	__checkReturn	boolean_t
+efx_filter_test_used(
+	__in		efx_filter_tbl_t *eftp,
+	__in		unsigned int index)
+{
+	EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
+	return ((eftp->eft_bitmap[index / 32] & (1 << (index % 32))) != 0);
+}
+
+static			void
+efx_filter_set_used(
+	__in		efx_filter_tbl_t *eftp,
+	__in		unsigned int index)
+{
+	EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
+	eftp->eft_bitmap[index / 32] |= (1 << (index % 32));
+	++eftp->eft_used;
+}
+
+static			void
+efx_filter_clear_used(
+	__in		efx_filter_tbl_t *eftp,
+	__in		unsigned int index)
+{
+	EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
+	eftp->eft_bitmap[index / 32] &= ~(1 << (index % 32));
+
+	--eftp->eft_used;
+	EFSYS_ASSERT3U(eftp->eft_used, >=, 0);
+}
+
+
+static 			efx_filter_tbl_id_t
+efx_filter_tbl_id(
+	__in		efx_filter_type_t type)
+{
+	efx_filter_tbl_id_t tbl_id;
+
+	switch (type)
+	{
+	case EFX_FILTER_RX_TCP_FULL:
+	case EFX_FILTER_RX_TCP_WILD:
+	case EFX_FILTER_RX_UDP_FULL:
+	case EFX_FILTER_RX_UDP_WILD:
+		tbl_id = EFX_FILTER_TBL_RX_IP;
+		break;
+
+#if EFSYS_OPT_SIENA
+	case EFX_FILTER_RX_MAC_FULL:
+	case EFX_FILTER_RX_MAC_WILD:
+		tbl_id = EFX_FILTER_TBL_RX_MAC;
+		break;
+
+	case EFX_FILTER_TX_TCP_FULL:
+	case EFX_FILTER_TX_TCP_WILD:
+	case EFX_FILTER_TX_UDP_FULL:
+	case EFX_FILTER_TX_UDP_WILD:
+		tbl_id = EFX_FILTER_TBL_TX_IP;
+		break;
+
+	case EFX_FILTER_TX_MAC_FULL:
+	case EFX_FILTER_TX_MAC_WILD:
+		tbl_id = EFX_FILTER_TBL_RX_MAC;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		EFSYS_ASSERT(B_FALSE);
+		break;
+	}
+	return (tbl_id);
+}
+
+static			void
+efx_filter_reset_search_depth(
+	__inout		efx_filter_t *efp,
+	__in		efx_filter_tbl_id_t tbl_id)
+{
+	switch (tbl_id)
+	{
+	case EFX_FILTER_TBL_RX_IP:
+		efp->ef_depth[EFX_FILTER_RX_TCP_FULL] = 0;
+		efp->ef_depth[EFX_FILTER_RX_TCP_WILD] = 0;
+		efp->ef_depth[EFX_FILTER_RX_UDP_FULL] = 0;
+		efp->ef_depth[EFX_FILTER_RX_UDP_WILD] = 0;
+		break;
+
+#if EFSYS_OPT_SIENA
+	case EFX_FILTER_TBL_RX_MAC:
+		efp->ef_depth[EFX_FILTER_RX_MAC_FULL] = 0;
+		efp->ef_depth[EFX_FILTER_RX_MAC_WILD] = 0;
+		break;
+
+	case EFX_FILTER_TBL_TX_IP:
+		efp->ef_depth[EFX_FILTER_TX_TCP_FULL] = 0;
+		efp->ef_depth[EFX_FILTER_TX_TCP_WILD] = 0;
+		efp->ef_depth[EFX_FILTER_TX_UDP_FULL] = 0;
+		efp->ef_depth[EFX_FILTER_TX_UDP_WILD] = 0;
+		break;
+
+	case EFX_FILTER_TBL_TX_MAC:
+		efp->ef_depth[EFX_FILTER_TX_MAC_FULL] = 0;
+		efp->ef_depth[EFX_FILTER_TX_MAC_WILD] = 0;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		EFSYS_ASSERT(B_FALSE);
+		break;
+	}
+}
+
+static			void
+efx_filter_push_rx_limits(
+	__in		efx_nic_t *enp)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_oword_t oword;
+
+	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
+	    efp->ef_depth[EFX_FILTER_RX_TCP_FULL] +
+	    FILTER_CTL_SRCH_FUDGE_FULL);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
+	    efp->ef_depth[EFX_FILTER_RX_TCP_WILD] +
+	    FILTER_CTL_SRCH_FUDGE_WILD);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
+	    efp->ef_depth[EFX_FILTER_RX_UDP_FULL] +
+	    FILTER_CTL_SRCH_FUDGE_FULL);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
+	    efp->ef_depth[EFX_FILTER_RX_UDP_WILD] +
+	    FILTER_CTL_SRCH_FUDGE_WILD);
+
+#if EFSYS_OPT_SIENA
+	if (efp->ef_tbl[EFX_FILTER_TBL_RX_MAC].eft_size) {
+		EFX_SET_OWORD_FIELD(oword,
+		    FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
+		    efp->ef_depth[EFX_FILTER_RX_MAC_FULL] +
+		    FILTER_CTL_SRCH_FUDGE_FULL);
+		EFX_SET_OWORD_FIELD(oword,
+		    FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
+		    efp->ef_depth[EFX_FILTER_RX_MAC_WILD] +
+		    FILTER_CTL_SRCH_FUDGE_WILD);
+	}
+#endif /* EFSYS_OPT_SIENA */
+
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
+}
+
+static			void
+efx_filter_push_tx_limits(
+	__in		efx_nic_t *enp)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_oword_t oword;
+
+	if (efp->ef_tbl[EFX_FILTER_TBL_TX_IP].eft_size == 0)
+		return;
+
+	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
+	    efp->ef_depth[EFX_FILTER_TX_TCP_FULL] +
+	    FILTER_CTL_SRCH_FUDGE_FULL);
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
+	    efp->ef_depth[EFX_FILTER_TX_TCP_WILD] +
+	    FILTER_CTL_SRCH_FUDGE_WILD);
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
+	    efp->ef_depth[EFX_FILTER_TX_UDP_FULL] +
+	    FILTER_CTL_SRCH_FUDGE_FULL);
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
+	    efp->ef_depth[EFX_FILTER_TX_UDP_WILD] +
+	    FILTER_CTL_SRCH_FUDGE_WILD);
+
+	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
+}
+
+/* Build a filter entry and return its n-tuple key. */
+static	__checkReturn	uint32_t
+efx_filter_build(
+	__out		efx_oword_t *filter,
+	__in		efx_filter_spec_t *spec)
+{
+	uint32_t dword3;
+	uint32_t key;
+	uint8_t  type  = spec->efs_type;
+	uint8_t  flags = spec->efs_flags;
+
+	switch (efx_filter_tbl_id(type)) {
+	case EFX_FILTER_TBL_RX_IP: {
+		boolean_t is_udp = (type == EFX_FILTER_RX_UDP_FULL ||
+		    type == EFX_FILTER_RX_UDP_WILD);
+		EFX_POPULATE_OWORD_7(*filter,
+		    FRF_BZ_RSS_EN,     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
+		    FRF_BZ_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
+		    FRF_AZ_TCP_UDP, is_udp,
+		    FRF_AZ_RXQ_ID, spec->efs_dmaq_id,
+		    EFX_DWORD_2, spec->efs_dword[2],
+		    EFX_DWORD_1, spec->efs_dword[1],
+		    EFX_DWORD_0, spec->efs_dword[0]);
+		dword3 = is_udp;
+		break;
+	}
+
+#if EFSYS_OPT_SIENA
+	case EFX_FILTER_TBL_RX_MAC: {
+		boolean_t is_wild = (type == EFX_FILTER_RX_MAC_WILD);
+		EFX_POPULATE_OWORD_8(*filter,
+		    FRF_CZ_RMFT_RSS_EN, (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
+		    FRF_CZ_RMFT_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
+		    FRF_CZ_RMFT_IP_OVERRIDE, (flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP) ? 1 : 0,
+		    FRF_CZ_RMFT_RXQ_ID, spec->efs_dmaq_id,
+		    FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
+		    FRF_CZ_RMFT_DEST_MAC_DW1, spec->efs_dword[2],
+		    FRF_CZ_RMFT_DEST_MAC_DW0, spec->efs_dword[1],
+		    FRF_CZ_RMFT_VLAN_ID, spec->efs_dword[0]);
+		dword3 = is_wild;
+		break;
+	}
+#endif /* EFSYS_OPT_SIENA */
+
+	case EFX_FILTER_TBL_TX_IP: {
+		boolean_t is_udp = (type == EFX_FILTER_TX_UDP_FULL ||
+		    type == EFX_FILTER_TX_UDP_WILD);
+		EFX_POPULATE_OWORD_5(*filter,
+		    FRF_CZ_TIFT_TCP_UDP, is_udp,
+		    FRF_CZ_TIFT_TXQ_ID, spec->efs_dmaq_id,
+		    EFX_DWORD_2, spec->efs_dword[2],
+		    EFX_DWORD_1, spec->efs_dword[1],
+		    EFX_DWORD_0, spec->efs_dword[0]);
+		dword3 = is_udp | spec->efs_dmaq_id << 1;
+		break;
+	}
+
+#if EFSYS_OPT_SIENA
+	case EFX_FILTER_TBL_TX_MAC: {
+		boolean_t is_wild = (type == EFX_FILTER_TX_MAC_WILD);
+		EFX_POPULATE_OWORD_5(*filter,
+		    FRF_CZ_TMFT_TXQ_ID, spec->efs_dmaq_id,
+		    FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
+		    FRF_CZ_TMFT_SRC_MAC_DW1, spec->efs_dword[2],
+		    FRF_CZ_TMFT_SRC_MAC_DW0, spec->efs_dword[1],
+		    FRF_CZ_TMFT_VLAN_ID, spec->efs_dword[0]);
+		dword3 = is_wild | spec->efs_dmaq_id << 1;
+		break;
+	}
+#endif /* EFSYS_OPT_SIENA */
+
+	default:
+		EFSYS_ASSERT(B_FALSE);
+	}
+
+	key = spec->efs_dword[0] ^ spec->efs_dword[1] ^ spec->efs_dword[2] ^ dword3;
+	return (key);
+}
+
+static	__checkReturn		int
+efx_filter_push_entry(
+	__inout			efx_nic_t *enp,
+	__in			efx_filter_type_t type,
+	__in			int index,
+	__in			efx_oword_t *eop)
+{
+	int rc;
+
+	switch (type)
+	{
+	case EFX_FILTER_RX_TCP_FULL:
+	case EFX_FILTER_RX_TCP_WILD:
+	case EFX_FILTER_RX_UDP_FULL:
+	case EFX_FILTER_RX_UDP_WILD:
+		EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index, eop);
+		break;
+
+#if EFSYS_OPT_SIENA
+	case EFX_FILTER_RX_MAC_FULL:
+	case EFX_FILTER_RX_MAC_WILD:
+		EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index, eop);
+		break;
+
+	case EFX_FILTER_TX_TCP_FULL:
+	case EFX_FILTER_TX_TCP_WILD:
+	case EFX_FILTER_TX_UDP_FULL:
+	case EFX_FILTER_TX_UDP_WILD:
+		EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index, eop);
+		break;
+
+	case EFX_FILTER_TX_MAC_FULL:
+	case EFX_FILTER_TX_MAC_WILD:
+		EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index, eop);
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+	return (0);
+
+fail1:
+	return (rc);
+}
+
+
+static	__checkReturn	boolean_t
+efx_filter_equal(
+	__in		const efx_filter_spec_t *left,
+	__in		const efx_filter_spec_t *right)
+{
+	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(left->efs_type);
+
+	if (left->efs_type != right->efs_type)
+		return (B_FALSE);
+
+	if (memcmp(left->efs_dword, right->efs_dword, sizeof(left->efs_dword)))
+		return (B_FALSE);
+
+	if ((tbl_id == EFX_FILTER_TBL_TX_IP ||
+	     tbl_id == EFX_FILTER_TBL_TX_MAC) &&
+	    left->efs_dmaq_id != right->efs_dmaq_id)
+		return (B_FALSE);
+
+	return (B_TRUE);
+}
+
+static	__checkReturn	int
+efx_filter_search(
+	__in		efx_filter_tbl_t *eftp,
+	__in		efx_filter_spec_t *spec,
+	__in		uint32_t key,
+	__in		boolean_t for_insert,
+	__out		int *filter_index,
+	__out		int *depth_required)
+{
+	unsigned hash, incr, filter_idx, depth;
+
+	hash = efx_filter_tbl_hash(key);
+	incr = efx_filter_tbl_increment(key);
+
+	filter_idx = hash & (eftp->eft_size - 1);
+	depth = 1;
+
+	for (;;) {
+		/* Return success if entry is used and matches this spec
+		 * or entry is unused and we are trying to insert.
+		 */
+		if (efx_filter_test_used(eftp, filter_idx) ?
+		    efx_filter_equal(spec, &eftp->eft_spec[filter_idx]) :
+		    for_insert) {
+			*filter_index = filter_idx;
+			*depth_required = depth;
+			return (0);
+		}
+
+		/* Return failure if we reached the maximum search depth */
+		if (depth == FILTER_CTL_SRCH_MAX)
+			return for_insert ? EBUSY : ENOENT;
+
+		filter_idx = (filter_idx + incr) & (eftp->eft_size - 1);
+		++depth;
+	}
+}
+
+	__checkReturn	int
+efx_filter_insert_filter(
+	__in		efx_nic_t *enp,
+	__in		efx_filter_spec_t *spec,
+	__in		boolean_t replace)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
+	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
+	efx_filter_spec_t *saved_spec;
+	efx_oword_t filter;
+	int filter_idx;
+	unsigned int depth;
+	int state;
+	uint32_t key;
+	int rc;
+
+	if (eftp->eft_size == 0)
+		return (EINVAL);
+
+	key = efx_filter_build(&filter, spec);
+
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	rc = efx_filter_search(eftp, spec, key, B_TRUE, &filter_idx, &depth);
+	if (rc != 0)
+		goto done;
+
+	EFSYS_ASSERT3U(filter_idx, <, eftp->eft_size);
+	saved_spec = &eftp->eft_spec[filter_idx];
+
+	if (efx_filter_test_used(eftp, filter_idx)) {
+		if (replace == B_FALSE) {
+			rc = EEXIST;
+			goto done;
+		}
+	}
+	efx_filter_set_used(eftp, filter_idx);
+	*saved_spec = *spec;
+
+	if (efp->ef_depth[spec->efs_type] < depth) {
+		efp->ef_depth[spec->efs_type] = depth;
+		if (tbl_id == EFX_FILTER_TBL_TX_IP ||
+		    tbl_id == EFX_FILTER_TBL_TX_MAC)
+			efx_filter_push_tx_limits(enp);
+		else
+			efx_filter_push_rx_limits(enp);
+	}
+
+	efx_filter_push_entry(enp, spec->efs_type, filter_idx, &filter);
+
+done:
+	EFSYS_UNLOCK(enp->en_eslp, state);
+	return (rc);
+}
+
+static			void
+efx_filter_clear_entry(
+	__in		efx_nic_t *enp,
+	__in		efx_filter_tbl_t *eftp,
+	__in		int index)
+{
+	efx_oword_t filter;
+
+	if (efx_filter_test_used(eftp, index)) {
+		efx_filter_clear_used(eftp, index);
+
+		EFX_ZERO_OWORD(filter);
+		efx_filter_push_entry(enp, eftp->eft_spec[index].efs_type,
+		    index, &filter);
+
+		memset(&eftp->eft_spec[index], 0, sizeof(eftp->eft_spec[0]));
+	}
+}
+
+	__checkReturn	int
+efx_filter_remove_filter(
+	__in		efx_nic_t *enp,
+	__in		efx_filter_spec_t *spec)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
+	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
+	efx_filter_spec_t *saved_spec;
+	efx_oword_t filter;
+	int filter_idx, depth;
+	int state;
+	uint32_t key;
+	int rc;
+
+	key = efx_filter_build(&filter, spec);
+
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	rc = efx_filter_search(eftp, spec, key, B_FALSE, &filter_idx, &depth);
+	if (rc != 0)
+		goto out;
+
+	saved_spec = &eftp->eft_spec[filter_idx];
+
+	efx_filter_clear_entry(enp, eftp, filter_idx);
+	if (eftp->eft_used == 0)
+		efx_filter_reset_search_depth(efp, tbl_id);
+
+	rc = 0;
+
+out:
+	EFSYS_UNLOCK(enp->en_eslp, state);
+	return (rc);
+}
+
+			void
+efx_filter_remove_index(
+	__inout		efx_nic_t *enp,
+	__in		efx_filter_type_t type,
+	__in		int index)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	enum efx_filter_tbl_id tbl_id = efx_filter_tbl_id(type);
+	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
+	int state;
+
+	if (index < 0)
+		return;
+
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	efx_filter_clear_entry(enp, eftp, index);
+	if (eftp->eft_used == 0)
+		efx_filter_reset_search_depth(efp, tbl_id);
+
+	EFSYS_UNLOCK(enp->en_eslp, state);
+}
+
+			void
+efx_filter_tbl_clear(
+	__inout		efx_nic_t *enp,
+	__in		efx_filter_tbl_id_t tbl_id)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
+	int index;
+	int state;
+
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	for (index = 0; index < eftp->eft_size; ++index) {
+		efx_filter_clear_entry(enp, eftp, index);
+	}
+
+	if (eftp->eft_used == 0)
+		efx_filter_reset_search_depth(efp, tbl_id);
+
+	EFSYS_UNLOCK(enp->en_eslp, state);
+}
+
+/* Restore filter state after a reset */
+			void
+efx_filter_restore(
+	__in		efx_nic_t *enp)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_id_t tbl_id;
+	efx_filter_tbl_t *eftp;
+	efx_filter_spec_t *spec;
+	efx_oword_t filter;
+	int filter_idx;
+	int state;
+
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
+		eftp = &efp->ef_tbl[tbl_id];
+		for (filter_idx = 0; filter_idx < eftp->eft_size; filter_idx++) {
+			if (!efx_filter_test_used(eftp, filter_idx))
+				continue;
+
+			spec = &eftp->eft_spec[filter_idx];
+			efx_filter_build(&filter, spec);
+			efx_filter_push_entry(enp, spec->efs_type,
+			    filter_idx, &filter);
+		}
+	}
+
+	efx_filter_push_rx_limits(enp);
+	efx_filter_push_tx_limits(enp);
+
+	EFSYS_UNLOCK(enp->en_eslp, state);
+}
+
+			void
+efx_filter_redirect_index(
+	__inout		efx_nic_t *enp,
+	__in		efx_filter_type_t type,
+	__in		int filter_index,
+	__in		int rxq_index)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_t *eftp =
+		&efp->ef_tbl[efx_filter_tbl_id(type)];
+	efx_filter_spec_t *spec;
+	efx_oword_t filter;
+	int state;
+
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	spec = &eftp->eft_spec[filter_index];
+	spec->efs_dmaq_id = (uint16_t)rxq_index;
+
+	efx_filter_build(&filter, spec);
+	efx_filter_push_entry(enp, spec->efs_type, filter_index, &filter);
+
+	EFSYS_UNLOCK(enp->en_eslp, state);
+}
+
+	__checkReturn	int
+efx_filter_init(
+	__in		efx_nic_t *enp)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_t *eftp;
+	int tbl_id;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
+
+	switch (enp->en_family)
+	{
+#if EFSYS_OPT_FALCON
+	case EFX_FAMILY_FALCON:
+		eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];
+		eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
+		break;
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+	case EFX_FAMILY_SIENA:
+		eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];
+		eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
+
+		eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_MAC];
+		eftp->eft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
+
+		eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_IP];
+		eftp->eft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
+
+		eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_MAC];
+		eftp->eft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
+		unsigned int bitmap_size;
+
+		eftp = &efp->ef_tbl[tbl_id];
+		if (eftp->eft_size == 0)
+			continue;
+
+		EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));
+		bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;
+
+		EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, eftp->eft_bitmap);
+		if (!eftp->eft_bitmap) {
+			rc = ENOMEM;
+			goto fail2;
+		}
+
+		EFSYS_KMEM_ALLOC(enp->en_esip, eftp->eft_size * sizeof(*eftp->eft_spec),
+		    eftp->eft_spec);
+		if (!eftp->eft_spec) {
+			rc = ENOMEM;
+			goto fail2;
+		}
+		memset(eftp->eft_spec, 0, eftp->eft_size * sizeof(*eftp->eft_spec));
+	}
+	enp->en_mod_flags |= EFX_MOD_FILTER;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+	efx_filter_fini(enp);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+			void
+efx_filter_fini(
+	__in		efx_nic_t *enp)
+{
+	efx_filter_t *efp = &enp->en_filter;
+	efx_filter_tbl_id_t tbl_id;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+
+	for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
+		efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
+		unsigned int bitmap_size;
+
+		EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));
+		bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;
+
+		EFSYS_KMEM_FREE(enp->en_esip, bitmap_size, eftp->eft_bitmap);
+		eftp->eft_bitmap = NULL;
+
+		EFSYS_KMEM_FREE(enp->en_esip, eftp->eft_size * sizeof(*eftp->eft_spec),
+		    eftp->eft_spec);
+		eftp->eft_spec = NULL;
+	}
+
+	enp->en_mod_flags &= ~EFX_MOD_FILTER;
+}
+
+extern			void
+efx_filter_spec_rx_ipv4_tcp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_tcp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_tcp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
+		    EFX_FILTER_FLAG_RX_SCATTER)) == 0);
+
+	spec->efs_type = EFX_FILTER_RX_TCP_FULL;
+	spec->efs_flags = (uint8_t)flags;
+	spec->efs_dword[0] = src_tcp | src_ip << 16;
+	spec->efs_dword[1] = dest_tcp << 16 | src_ip >> 16;
+	spec->efs_dword[2] = dest_ip;
+}
+
+extern			void
+efx_filter_spec_rx_ipv4_tcp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_tcp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
+		    EFX_FILTER_FLAG_RX_SCATTER)) == 0);
+
+	spec->efs_type = EFX_FILTER_RX_TCP_WILD;
+	spec->efs_flags = (uint8_t)flags;
+	spec->efs_dword[0] = 0;
+	spec->efs_dword[1] = dest_tcp << 16;
+	spec->efs_dword[2] = dest_ip;
+}
+
+extern			void
+efx_filter_spec_rx_ipv4_udp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_udp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_udp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
+		    EFX_FILTER_FLAG_RX_SCATTER)) == 0);
+
+	spec->efs_type = EFX_FILTER_RX_UDP_FULL;
+	spec->efs_flags = (uint8_t)flags;
+	spec->efs_dword[0] = src_udp | src_ip << 16;
+	spec->efs_dword[1] = dest_udp << 16 | src_ip >> 16;
+	spec->efs_dword[2] = dest_ip;
+}
+
+extern			void
+efx_filter_spec_rx_ipv4_udp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_udp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
+		    EFX_FILTER_FLAG_RX_SCATTER)) == 0);
+
+	spec->efs_type = EFX_FILTER_RX_UDP_WILD;
+	spec->efs_flags = (uint8_t)flags;
+	spec->efs_dword[0] = dest_udp;
+	spec->efs_dword[1] = 0;
+	spec->efs_dword[2] = dest_ip;
+}
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_rx_mac_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint16_t vlan_id,
+	__in		uint8_t *dest_mac)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT3P(dest_mac, !=, NULL);
+	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
+		    EFX_FILTER_FLAG_RX_SCATTER |
+		    EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);
+
+	spec->efs_type = EFX_FILTER_RX_MAC_FULL;
+	spec->efs_flags = (uint8_t)flags;
+	spec->efs_dword[0] = vlan_id;
+	spec->efs_dword[1] =
+	    dest_mac[2] << 24 |
+	    dest_mac[3] << 16 |
+	    dest_mac[4] <<  8 |
+	    dest_mac[5];
+ 	spec->efs_dword[2] =
+	    dest_mac[0] <<  8 |
+	    dest_mac[1];
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_rx_mac_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		unsigned int flags,
+	__in		uint8_t *dest_mac)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT3P(dest_mac, !=, NULL);
+	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
+		    EFX_FILTER_FLAG_RX_SCATTER |
+		    EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);
+
+	spec->efs_type = EFX_FILTER_RX_MAC_WILD;
+	spec->efs_flags = (uint8_t)flags;
+	spec->efs_dword[0] = 0;
+	spec->efs_dword[1] =
+	    dest_mac[2] << 24 |
+	    dest_mac[3] << 16 |
+	    dest_mac[4] <<  8 |
+	    dest_mac[5];
+ 	spec->efs_dword[2] =
+	    dest_mac[0] <<  8 |
+	    dest_mac[1];
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_tx_ipv4_tcp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_tcp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_tcp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_type = EFX_FILTER_TX_TCP_FULL;
+	spec->efs_flags = 0;
+	spec->efs_dword[0] = src_tcp | src_ip << 16;
+	spec->efs_dword[1] = dest_tcp << 16 | src_ip >> 16;
+	spec->efs_dword[2] = dest_ip;
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_tx_ipv4_tcp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_tcp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_type = EFX_FILTER_TX_TCP_WILD;
+	spec->efs_flags = 0;
+	spec->efs_dword[0] = 0;
+	spec->efs_dword[1] = src_tcp << 16;
+	spec->efs_dword[2] = src_ip;
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_tx_ipv4_udp_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_udp,
+	__in		uint32_t dest_ip,
+	__in		uint16_t dest_udp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_type = EFX_FILTER_TX_UDP_FULL;
+	spec->efs_flags = 0;
+	spec->efs_dword[0] = src_udp | src_ip << 16;
+	spec->efs_dword[1] = dest_udp << 16 | src_ip >> 16;
+	spec->efs_dword[2] = dest_ip;
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_tx_ipv4_udp_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint32_t src_ip,
+	__in		uint16_t src_udp)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_type = EFX_FILTER_TX_UDP_WILD;
+	spec->efs_flags = 0;
+	spec->efs_dword[0] = src_udp;
+	spec->efs_dword[1] = 0;
+	spec->efs_dword[2] = src_ip;
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_tx_mac_full(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint16_t vlan_id,
+	__in		uint8_t *src_mac)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT3P(src_mac, !=, NULL);
+
+	spec->efs_type = EFX_FILTER_TX_MAC_FULL;
+	spec->efs_flags = 0;
+	spec->efs_dword[0] = vlan_id;
+	spec->efs_dword[1] =
+	    src_mac[2] << 24 |
+	    src_mac[3] << 16 |
+	    src_mac[4] <<  8 |
+	    src_mac[5];
+ 	spec->efs_dword[2] =
+	    src_mac[0] <<  8 |
+	    src_mac[1];
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_SIENA
+extern			void
+efx_filter_spec_tx_mac_wild(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint8_t *src_mac)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+	EFSYS_ASSERT3P(src_mac, !=, NULL);
+
+	spec->efs_type = EFX_FILTER_TX_MAC_WILD;
+	spec->efs_flags = 0;
+	spec->efs_dword[0] = 0;
+	spec->efs_dword[1] =
+	    src_mac[2] << 24 |
+	    src_mac[3] << 16 |
+	    src_mac[4] <<  8 |
+	    src_mac[5];
+ 	spec->efs_dword[2] =
+	    src_mac[0] <<  8 |
+	    src_mac[1];
+}
+#endif	/* EFSYS_OPT_SIENA */
+
+
+#endif /* EFSYS_OPT_FILTER */
diff --git a/sys/dev/sfxge/common/efx_impl.h b/sys/dev/sfxge/common/efx_impl.h
new file mode 100644
index 0000000..5858e96
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_impl.h
@@ -0,0 +1,734 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_SYS_EFX_IMPL_H
+#define	_SYS_EFX_IMPL_H
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_regs.h"
+
+#if EFSYS_OPT_FALCON
+#include "falcon_impl.h"
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+#include "siena_impl.h"
+#endif	/* EFSYS_OPT_SIENA */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#define	EFX_MOD_MCDI	0x00000001
+#define	EFX_MOD_PROBE	0x00000002
+#define	EFX_MOD_NVRAM	0x00000004
+#define	EFX_MOD_VPD	0x00000008
+#define	EFX_MOD_NIC	0x00000010
+#define	EFX_MOD_INTR	0x00000020
+#define	EFX_MOD_EV	0x00000040
+#define	EFX_MOD_RX	0x00000080
+#define	EFX_MOD_TX	0x00000100
+#define	EFX_MOD_PORT	0x00000200
+#define	EFX_MOD_MON	0x00000400
+#define	EFX_MOD_WOL	0x00000800
+#define	EFX_MOD_FILTER	0x00001000
+
+#define	EFX_RESET_MAC	0x00000001
+#define	EFX_RESET_PHY	0x00000002
+
+typedef enum efx_mac_type_e {
+	EFX_MAC_INVALID = 0,
+	EFX_MAC_FALCON_GMAC,
+	EFX_MAC_FALCON_XMAC,
+	EFX_MAC_SIENA,
+	EFX_MAC_NTYPES
+} efx_mac_type_t;
+
+typedef struct efx_mac_ops_s {
+	int		(*emo_reset)(efx_nic_t *); /* optional */
+	int		(*emo_poll)(efx_nic_t *, efx_link_mode_t *);
+	int		(*emo_up)(efx_nic_t *, boolean_t *);
+	int		(*emo_reconfigure)(efx_nic_t *);
+#if EFSYS_OPT_LOOPBACK
+	int		(*emo_loopback_set)(efx_nic_t *, efx_link_mode_t,
+					    efx_loopback_type_t);
+#endif	/* EFSYS_OPT_LOOPBACK */
+#if EFSYS_OPT_MAC_STATS
+	int		(*emo_stats_upload)(efx_nic_t *, efsys_mem_t *);
+	int		(*emo_stats_periodic)(efx_nic_t *, efsys_mem_t *,
+					      uint16_t, boolean_t);
+	int		(*emo_stats_update)(efx_nic_t *, efsys_mem_t *,
+					    efsys_stat_t *, uint32_t *);
+#endif	/* EFSYS_OPT_MAC_STATS */
+} efx_mac_ops_t;
+
+typedef struct efx_phy_ops_s {
+	int		(*epo_power)(efx_nic_t *, boolean_t); /* optional */
+	int		(*epo_reset)(efx_nic_t *);
+	int		(*epo_reconfigure)(efx_nic_t *);
+	int		(*epo_verify)(efx_nic_t *);
+	int		(*epo_uplink_check)(efx_nic_t *,
+					    boolean_t *); /* optional */
+	int		(*epo_downlink_check)(efx_nic_t *, efx_link_mode_t *,
+					      unsigned int *, uint32_t *);
+	int		(*epo_oui_get)(efx_nic_t *, uint32_t *);
+#if EFSYS_OPT_PHY_STATS
+	int		(*epo_stats_update)(efx_nic_t *, efsys_mem_t *,
+					    uint32_t *);
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	const char	__cs *(*epo_prop_name)(efx_nic_t *, unsigned int);
+#endif	/* EFSYS_OPT_PHY_PROPS */
+	int		(*epo_prop_get)(efx_nic_t *, unsigned int, uint32_t,
+					uint32_t *);
+	int		(*epo_prop_set)(efx_nic_t *, unsigned int, uint32_t);
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	int		(*epo_bist_start)(efx_nic_t *, efx_phy_bist_type_t);
+	int		(*epo_bist_poll)(efx_nic_t *, efx_phy_bist_type_t,
+					 efx_phy_bist_result_t *, uint32_t *,
+					 unsigned long *, size_t);
+	void		(*epo_bist_stop)(efx_nic_t *, efx_phy_bist_type_t);
+#endif	/* EFSYS_OPT_PHY_BIST */
+} efx_phy_ops_t;
+
+typedef struct efx_port_s {
+	efx_mac_type_t		ep_mac_type;
+	uint32_t  		ep_phy_type;
+	uint8_t			ep_port;
+	uint32_t		ep_mac_pdu;
+	uint8_t			ep_mac_addr[6];
+	efx_link_mode_t		ep_link_mode;
+	boolean_t		ep_unicst;
+	boolean_t		ep_brdcst;
+	unsigned int		ep_fcntl;
+	boolean_t		ep_fcntl_autoneg;
+	efx_oword_t		ep_multicst_hash[2];
+#if EFSYS_OPT_LOOPBACK
+	efx_loopback_type_t	ep_loopback_type;
+	efx_link_mode_t		ep_loopback_link_mode;
+#endif	/* EFSYS_OPT_LOOPBACK */
+#if EFSYS_OPT_PHY_FLAGS
+	uint32_t		ep_phy_flags;
+#endif	/* EFSYS_OPT_PHY_FLAGS */
+#if EFSYS_OPT_PHY_LED_CONTROL
+	efx_phy_led_mode_t	ep_phy_led_mode;
+#endif	/* EFSYS_OPT_PHY_LED_CONTROL */
+	efx_phy_media_type_t	ep_fixed_port_type;
+	efx_phy_media_type_t	ep_module_type;
+	uint32_t		ep_adv_cap_mask;
+	uint32_t		ep_lp_cap_mask;
+	uint32_t		ep_default_adv_cap_mask;
+	uint32_t		ep_phy_cap_mask;
+#if EFSYS_OPT_PHY_TXC43128 || EFSYS_OPT_PHY_QT2025C
+	union {
+		struct {
+			unsigned int	bug10934_count;
+		} ep_txc43128;
+		struct {
+			unsigned int	bug17190_count;
+		} ep_qt2025c;
+	};
+#endif
+	boolean_t		ep_mac_poll_needed; /* falcon only */
+	boolean_t		ep_mac_up; /* falcon only */
+	uint32_t		ep_fwver; /* falcon only */
+	boolean_t		ep_mac_drain;
+	boolean_t		ep_mac_stats_pending;
+#if EFSYS_OPT_PHY_BIST
+	efx_phy_bist_type_t	ep_current_bist;
+#endif
+	efx_mac_ops_t		*ep_emop;
+	efx_phy_ops_t		*ep_epop;
+} efx_port_t;
+
+typedef struct efx_mon_ops_s {
+	int	(*emo_reset)(efx_nic_t *);
+	int	(*emo_reconfigure)(efx_nic_t *);
+#if EFSYS_OPT_MON_STATS
+	int	(*emo_stats_update)(efx_nic_t *, efsys_mem_t *,
+				    efx_mon_stat_value_t *);
+#endif	/* EFSYS_OPT_MON_STATS */
+} efx_mon_ops_t;
+
+typedef struct efx_mon_s {
+	efx_mon_type_t	em_type;
+	efx_mon_ops_t	*em_emop;
+} efx_mon_t;
+
+typedef struct efx_intr_s {
+	efx_intr_type_t	ei_type;
+	efsys_mem_t	*ei_esmp;
+	unsigned int	ei_level;
+} efx_intr_t;
+
+typedef struct efx_nic_ops_s {
+	int	(*eno_probe)(efx_nic_t *);
+	int	(*eno_reset)(efx_nic_t *);
+	int	(*eno_init)(efx_nic_t *);
+#if EFSYS_OPT_DIAG
+	int	(*eno_sram_test)(efx_nic_t *, efx_sram_pattern_fn_t);
+	int	(*eno_register_test)(efx_nic_t *);
+#endif	/* EFSYS_OPT_DIAG */
+	void	(*eno_fini)(efx_nic_t *);
+	void	(*eno_unprobe)(efx_nic_t *);
+} efx_nic_ops_t;
+
+#define EFX_TXQ_LIMIT_TARGET 259
+#define EFX_RXQ_LIMIT_TARGET 768
+
+#if EFSYS_OPT_FILTER
+
+typedef enum efx_filter_type_e {
+	EFX_FILTER_RX_TCP_FULL,	/* TCP/IPv4 4-tuple {dIP,dTCP,sIP,sTCP} */
+	EFX_FILTER_RX_TCP_WILD,	/* TCP/IPv4 dest    {dIP,dTCP,  -,   -} */
+	EFX_FILTER_RX_UDP_FULL,	/* UDP/IPv4 4-tuple {dIP,dUDP,sIP,sUDP} */
+	EFX_FILTER_RX_UDP_WILD,	/* UDP/IPv4 dest    {dIP,dUDP,  -,   -} */
+
+#if EFSYS_OPT_SIENA
+	EFX_FILTER_RX_MAC_FULL,	/* Ethernet {dMAC,VLAN} */
+	EFX_FILTER_RX_MAC_WILD,	/* Ethernet {dMAC,   -} */
+
+	EFX_FILTER_TX_TCP_FULL,		/* TCP/IPv4 {dIP,dTCP,sIP,sTCP} */
+	EFX_FILTER_TX_TCP_WILD,		/* TCP/IPv4 {  -,   -,sIP,sTCP} */
+	EFX_FILTER_TX_UDP_FULL,		/* UDP/IPv4 {dIP,dTCP,sIP,sTCP} */
+	EFX_FILTER_TX_UDP_WILD,		/* UDP/IPv4 source (host, port) */
+
+	EFX_FILTER_TX_MAC_FULL,		/* Ethernet source (MAC address, VLAN ID) */
+	EFX_FILTER_TX_MAC_WILD,		/* Ethernet source (MAC address) */
+#endif /* EFSYS_OPT_SIENA */
+
+	EFX_FILTER_NTYPES
+} efx_filter_type_t;
+
+typedef enum efx_filter_tbl_id_e {
+	EFX_FILTER_TBL_RX_IP = 0,
+	EFX_FILTER_TBL_RX_MAC,
+	EFX_FILTER_TBL_TX_IP,
+	EFX_FILTER_TBL_TX_MAC,
+	EFX_FILTER_NTBLS
+} efx_filter_tbl_id_t;
+
+typedef struct efx_filter_tbl_s {
+	int			eft_size;	/* number of entries */
+	int			eft_used;	/* active count */
+	uint32_t		*eft_bitmap;	/* active bitmap */
+	efx_filter_spec_t	*eft_spec;	/* array of saved specs */
+} efx_filter_tbl_t;
+
+typedef struct efx_filter_s {
+	efx_filter_tbl_t	ef_tbl[EFX_FILTER_NTBLS];
+	unsigned int		ef_depth[EFX_FILTER_NTYPES];
+} efx_filter_t;
+
+
+extern	__checkReturn	int
+efx_filter_insert_filter(
+	__in		efx_nic_t *enp,
+	__in		efx_filter_spec_t *spec,
+	__in		boolean_t replace);
+
+extern	__checkReturn	int
+efx_filter_remove_filter(
+	__in		efx_nic_t *enp,
+	__in		efx_filter_spec_t *spec);
+
+extern			void
+efx_filter_remove_index(
+	__inout		efx_nic_t *enp,
+	__in		efx_filter_type_t type,
+	__in		int filter_idx);
+
+extern			void
+efx_filter_redirect_index(
+	__inout		efx_nic_t *enp,
+	__in		efx_filter_type_t type,
+	__in		int filter_index,
+	__in		int rxq_index);
+
+extern	__checkReturn	int
+efx_filter_clear_tbl(
+	__in		efx_nic_t *enp,
+	__in		efx_filter_tbl_id_t tbl);
+
+#endif	/* EFSYS_OPT_FILTER */
+
+#if EFSYS_OPT_NVRAM
+typedef struct efx_nvram_ops_s {
+#if EFSYS_OPT_DIAG
+	int	(*envo_test)(efx_nic_t *);
+#endif	/* EFSYS_OPT_DIAG */
+	int	(*envo_size)(efx_nic_t *, efx_nvram_type_t, size_t *);
+	int	(*envo_get_version)(efx_nic_t *, efx_nvram_type_t,
+				    uint32_t *, uint16_t *);
+	int	(*envo_rw_start)(efx_nic_t *, efx_nvram_type_t, size_t *);
+	int	(*envo_read_chunk)(efx_nic_t *, efx_nvram_type_t,
+				    unsigned int, caddr_t, size_t);
+	int	(*envo_erase)(efx_nic_t *, efx_nvram_type_t);
+	int	(*envo_write_chunk)(efx_nic_t *, efx_nvram_type_t,
+				    unsigned int, caddr_t, size_t);
+	void	(*envo_rw_finish)(efx_nic_t *, efx_nvram_type_t);
+	int	(*envo_set_version)(efx_nic_t *, efx_nvram_type_t, uint16_t *);
+
+} efx_nvram_ops_t;
+#endif /* EFSYS_OPT_NVRAM */
+
+#if EFSYS_OPT_VPD
+typedef struct efx_vpd_ops_s {
+	int	(*evpdo_init)(efx_nic_t *);
+	int	(*evpdo_size)(efx_nic_t *, size_t *);
+	int	(*evpdo_read)(efx_nic_t *, caddr_t, size_t);
+	int	(*evpdo_verify)(efx_nic_t *, caddr_t, size_t);
+	int	(*evpdo_reinit)(efx_nic_t *, caddr_t, size_t);
+	int	(*evpdo_get)(efx_nic_t *, caddr_t, size_t, efx_vpd_value_t *);
+	int	(*evpdo_set)(efx_nic_t *, caddr_t, size_t, efx_vpd_value_t *);
+	int	(*evpdo_next)(efx_nic_t *, caddr_t, size_t, efx_vpd_value_t *,
+			    unsigned int *);
+	int	(*evpdo_write)(efx_nic_t *, caddr_t, size_t);
+	void	(*evpdo_fini)(efx_nic_t *);
+} efx_vpd_ops_t;
+#endif	/* EFSYS_OPT_VPD */
+
+struct efx_nic_s {
+	uint32_t		en_magic;
+	efx_family_t		en_family;
+	uint32_t		en_features;
+	efsys_identifier_t	*en_esip;
+	efsys_lock_t		*en_eslp;
+	efsys_bar_t 		*en_esbp;
+	unsigned int		en_mod_flags;
+	unsigned int		en_reset_flags;
+	efx_nic_cfg_t		en_nic_cfg;
+	efx_port_t		en_port;
+	efx_mon_t		en_mon;
+	efx_intr_t		en_intr;
+	uint32_t		en_ev_qcount;
+	uint32_t		en_rx_qcount;
+	uint32_t		en_tx_qcount;
+	efx_nic_ops_t		*en_enop;
+#if EFSYS_OPT_FILTER
+	efx_filter_t		en_filter;
+#endif	/* EFSYS_OPT_FILTER */
+#if EFSYS_OPT_NVRAM
+	efx_nvram_type_t	en_nvram_locked;
+	efx_nvram_ops_t		*en_envop;
+#endif	/* EFSYS_OPT_NVRAM */
+#if EFSYS_OPT_VPD
+	efx_vpd_ops_t		*en_evpdop;
+#endif	/* EFSYS_OPT_VPD */
+	union {
+#if EFSYS_OPT_FALCON
+		struct {
+			falcon_spi_dev_t	enu_fsd[FALCON_SPI_NTYPES];
+			falcon_i2c_t		enu_fip;
+			boolean_t		enu_i2c_locked;
+#if EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE
+			const uint8_t		*enu_forced_cfg;
+#endif	/* EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE */
+			uint8_t			enu_mon_devid;
+#if EFSYS_OPT_PCIE_TUNE
+			unsigned int 		enu_nlanes;
+#endif	/* EFSYS_OPT_PCIE_TUNE */
+			uint16_t		enu_board_rev;
+			boolean_t		enu_internal_sram;
+			uint8_t			enu_sram_num_bank;
+			uint8_t			enu_sram_bank_size;
+		} falcon;
+#endif	/* EFSYS_OPT_FALCON */
+#if EFSYS_OPT_SIENA
+		struct {
+#if EFSYS_OPT_MCDI
+			efx_mcdi_iface_t	enu_mip;
+#endif	/* EFSYS_OPT_MCDI */
+#if EFSYS_OPT_NVRAM || EFSYS_OPT_VPD
+			unsigned int		enu_partn_mask;
+#endif	/* EFSYS_OPT_NVRAM || EFSYS_OPT_VPD */
+#if EFSYS_OPT_VPD
+			caddr_t			enu_svpd;
+			size_t			enu_svpd_length;
+#endif	/* EFSYS_OPT_VPD */
+		} siena;
+#endif	/* EFSYS_OPT_SIENA */
+	} en_u;
+};
+
+
+#define	EFX_NIC_MAGIC	0x02121996
+
+typedef	boolean_t (*efx_ev_handler_t)(efx_evq_t *, efx_qword_t *,
+    const efx_ev_callbacks_t *, void *);
+
+struct efx_evq_s {
+	uint32_t			ee_magic;
+	efx_nic_t			*ee_enp;
+	unsigned int			ee_index;
+	unsigned int			ee_mask;
+	efsys_mem_t			*ee_esmp;
+#if EFSYS_OPT_QSTATS
+	uint32_t			ee_stat[EV_NQSTATS];
+#endif	/* EFSYS_OPT_QSTATS */
+	efx_ev_handler_t		ee_handler[1 << FSF_AZ_EV_CODE_WIDTH];
+};
+
+#define	EFX_EVQ_MAGIC	0x08081997
+
+#define	EFX_EV_TIMER_QUANTUM	5
+
+struct efx_rxq_s {
+	uint32_t			er_magic;
+	efx_nic_t			*er_enp;
+	unsigned int			er_index;
+	unsigned int			er_mask;
+	efsys_mem_t			*er_esmp;
+};
+
+#define	EFX_RXQ_MAGIC	0x15022005
+
+struct efx_txq_s {
+	uint32_t			et_magic;
+	efx_nic_t			*et_enp;
+	unsigned int			et_index;
+	unsigned int			et_mask;
+	efsys_mem_t			*et_esmp;
+#if EFSYS_OPT_QSTATS
+	uint32_t			et_stat[TX_NQSTATS];
+#endif	/* EFSYS_OPT_QSTATS */
+};
+
+#define	EFX_TXQ_MAGIC	0x05092005
+
+#define	EFX_MAC_ADDR_COPY(_dst, _src)					\
+	do {								\
+		(_dst)[0] = (_src)[0];					\
+		(_dst)[1] = (_src)[1];					\
+		(_dst)[2] = (_src)[2];					\
+		(_dst)[3] = (_src)[3];					\
+		(_dst)[4] = (_src)[4];					\
+		(_dst)[5] = (_src)[5];					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#if EFSYS_OPT_CHECK_REG
+#define	EFX_CHECK_REG(_enp, _reg)					\
+	do {								\
+		const char __cs *name = #_reg;				\
+		char min = name[4];					\
+		char max = name[5];					\
+		char rev;						\
+									\
+		switch ((_enp)->en_family) {				\
+		case EFX_FAMILY_FALCON:					\
+			rev = 'B';					\
+			break;						\
+									\
+		case EFX_FAMILY_SIENA:					\
+			rev = 'C';					\
+			break;						\
+									\
+		default:						\
+			rev = '?';					\
+			break;						\
+		}							\
+									\
+		EFSYS_ASSERT3S(rev, >=, min);				\
+		EFSYS_ASSERT3S(rev, <=, max);				\
+									\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+#else
+#define	EFX_CHECK_REG(_enp, _reg) do {					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while(B_FALSE)
+#endif
+
+#define	EFX_BAR_READD(_enp, _reg, _edp, _lock)				\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_BAR_READD((_enp)->en_esbp, _reg ## _OFST,		\
+		    (_edp), (_lock));					\
+		EFSYS_PROBE3(efx_bar_readd, const char *, #_reg,	\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_WRITED(_enp, _reg, _edp, _lock)				\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE3(efx_bar_writed, const char *, #_reg,	\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+		EFSYS_BAR_WRITED((_enp)->en_esbp, _reg ## _OFST,	\
+		    (_edp), (_lock));					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_READQ(_enp, _reg, _eqp)					\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_BAR_READQ((_enp)->en_esbp, _reg ## _OFST,		\
+		    (_eqp));						\
+		EFSYS_PROBE4(efx_bar_readq, const char *, #_reg,	\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_WRITEQ(_enp, _reg, _eqp)				\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE4(efx_bar_writeq, const char *, #_reg,	\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+		EFSYS_BAR_WRITEQ((_enp)->en_esbp, _reg ## _OFST,	\
+		    (_eqp));						\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_READO(_enp, _reg, _eop)					\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_BAR_READO((_enp)->en_esbp, _reg ## _OFST,		\
+		    (_eop), B_TRUE);					\
+		EFSYS_PROBE6(efx_bar_reado, const char *, #_reg,	\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_WRITEO(_enp, _reg, _eop)				\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE6(efx_bar_writeo, const char *, #_reg,	\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+		EFSYS_BAR_WRITEO((_enp)->en_esbp, _reg ## _OFST,	\
+		    (_eop), B_TRUE);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_READD(_enp, _reg, _index, _edp, _lock)		\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_BAR_READD((_enp)->en_esbp,			\
+		    (_reg ## _OFST + ((_index) * _reg ## _STEP)),	\
+		    (_edp), (_lock));					\
+		EFSYS_PROBE4(efx_bar_tbl_readd, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_WRITED(_enp, _reg, _index, _edp, _lock)		\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE4(efx_bar_tbl_writed, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+		EFSYS_BAR_WRITED((_enp)->en_esbp,			\
+		    (_reg ## _OFST + ((_index) * _reg ## _STEP)),	\
+		    (_edp), (_lock));					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_WRITED3(_enp, _reg, _index, _edp, _lock)		\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE4(efx_bar_tbl_writed, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_edp)->ed_u32[0]);			\
+		EFSYS_BAR_WRITED((_enp)->en_esbp,			\
+		    (_reg ## _OFST +					\
+		    (3 * sizeof (efx_dword_t)) + 			\
+		    ((_index) * _reg ## _STEP)),			\
+		    (_edp), (_lock));					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_READQ(_enp, _reg, _index, _eqp)			\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_BAR_READQ((_enp)->en_esbp,			\
+		    (_reg ## _OFST + ((_index) * _reg ## _STEP)),	\
+		    (_eqp));						\
+		EFSYS_PROBE5(efx_bar_tbl_readq, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_WRITEQ(_enp, _reg, _index, _eqp)			\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE5(efx_bar_tbl_writeq, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eqp)->eq_u32[1],			\
+		    uint32_t, (_eqp)->eq_u32[0]);			\
+		EFSYS_BAR_WRITEQ((_enp)->en_esbp,			\
+		    (_reg ## _OFST + ((_index) * _reg ## _STEP)),	\
+		    (_eqp));						\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_READO(_enp, _reg, _index, _eop)			\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_BAR_READO((_enp)->en_esbp,			\
+		    (_reg ## _OFST + ((_index) * _reg ## _STEP)),	\
+		    (_eop), B_TRUE);					\
+		EFSYS_PROBE7(efx_bar_tbl_reado, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_BAR_TBL_WRITEO(_enp, _reg, _index, _eop)			\
+	do {								\
+		EFX_CHECK_REG((_enp), (_reg));				\
+		EFSYS_PROBE7(efx_bar_tbl_writeo, const char *, #_reg,	\
+		    uint32_t, (_index),					\
+		    uint32_t, _reg ## _OFST,				\
+		    uint32_t, (_eop)->eo_u32[3],			\
+		    uint32_t, (_eop)->eo_u32[2],			\
+		    uint32_t, (_eop)->eo_u32[1],			\
+		    uint32_t, (_eop)->eo_u32[0]);			\
+		EFSYS_BAR_WRITEO((_enp)->en_esbp,			\
+		    (_reg ## _OFST + ((_index) * _reg ## _STEP)),	\
+		    (_eop), B_TRUE);					\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+extern	__checkReturn	int
+efx_mac_select(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+efx_phy_probe(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_phy_unprobe(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_VPD
+
+/* VPD utility functions */
+
+extern	__checkReturn		int
+efx_vpd_hunk_length(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			size_t *lengthp);
+
+extern	__checkReturn		int
+efx_vpd_hunk_verify(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out_opt		boolean_t *cksummedp);
+
+extern	__checkReturn		int
+efx_vpd_hunk_reinit(
+	__in			caddr_t data,
+	__in			size_t size,
+	__in			boolean_t wantpid);
+
+extern	__checkReturn		int
+efx_vpd_hunk_get(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_tag_t tag,
+	__in			efx_vpd_keyword_t keyword,
+	__out			unsigned int *payloadp,
+	__out			uint8_t *paylenp);
+
+extern	__checkReturn			int
+efx_vpd_hunk_next(
+	__in_bcount(size)		caddr_t data,
+	__in				size_t size,
+	__out				efx_vpd_tag_t *tagp,
+	__out				efx_vpd_keyword_t *keyword,
+	__out_bcount_opt(*paylenp)	unsigned int *payloadp,
+	__out_opt			uint8_t *paylenp,
+	__inout				unsigned int *contp);
+
+extern	__checkReturn		int
+efx_vpd_hunk_set(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_value_t *evvp);
+
+#endif	/* EFSYS_OPT_VPD */
+
+#if EFSYS_OPT_DIAG
+
+extern	efx_sram_pattern_fn_t	__cs __efx_sram_pattern_fns[];
+
+typedef struct efx_register_set_s {
+	unsigned int		address;
+	unsigned int		step;
+	unsigned int		rows;
+	efx_oword_t		mask;
+} efx_register_set_t;
+
+extern	__checkReturn	int
+efx_nic_test_registers(
+	__in		efx_nic_t *enp,
+	__in		efx_register_set_t *rsp,
+	__in		size_t count);
+
+extern	__checkReturn	int
+efx_nic_test_tables(
+	__in		efx_nic_t *enp,
+	__in		efx_register_set_t *rsp,
+	__in		efx_pattern_type_t pattern,
+	__in		size_t count);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_EFX_IMPL_H */
diff --git a/sys/dev/sfxge/common/efx_intr.c b/sys/dev/sfxge/common/efx_intr.c
new file mode 100644
index 0000000..77780b1
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_intr.c
@@ -0,0 +1,354 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+	__checkReturn	int
+efx_intr_init(
+	__in		efx_nic_t *enp,
+	__in		efx_intr_type_t type,
+	__in		efsys_mem_t *esmp)
+{
+	efx_intr_t *eip = &(enp->en_intr);
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	if (enp->en_mod_flags & EFX_MOD_INTR) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	enp->en_mod_flags |= EFX_MOD_INTR;
+
+	eip->ei_type = type;
+	eip->ei_esmp = esmp;
+
+	/*
+	 * bug17213 workaround.
+	 *
+	 * Under legacy interrupts, don't share a level between fatal
+	 * interrupts and event queue interrupts. Under MSI-X, they
+	 * must share, or we won't get an interrupt.
+	 */
+	if (enp->en_family == EFX_FAMILY_SIENA &&
+	    eip->ei_type == EFX_INTR_LINE)
+		eip->ei_level = 0x1f;
+	else
+		eip->ei_level = 0;
+
+	/* Enable all the genuinely fatal interrupts */
+	EFX_SET_OWORD(oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_ILL_ADR_INT_KER_EN, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RBUF_OWN_INT_KER_EN, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TBUF_OWN_INT_KER_EN, 0);
+	if (enp->en_family >= EFX_FAMILY_SIENA)
+		EFX_SET_OWORD_FIELD(oword, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_FATAL_INTR_REG_KER, &oword);
+
+	/* Set up the interrupt address register */
+	EFX_POPULATE_OWORD_3(oword,
+	    FRF_AZ_NORM_INT_VEC_DIS_KER, (type == EFX_INTR_MESSAGE) ? 1 : 0,
+	    FRF_AZ_INT_ADR_KER_DW0, EFSYS_MEM_ADDR(esmp) & 0xffffffff,
+	    FRF_AZ_INT_ADR_KER_DW1, EFSYS_MEM_ADDR(esmp) >> 32);
+	EFX_BAR_WRITEO(enp, FR_AZ_INT_ADR_REG_KER, &oword);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			void
+efx_intr_enable(
+	__in		efx_nic_t *enp)
+{
+	efx_intr_t *eip = &(enp->en_intr);
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, eip->ei_level);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+}
+
+			void
+efx_intr_disable(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+
+	EFSYS_SPIN(10);
+}
+
+			void
+efx_intr_disable_unlocked(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	EFSYS_BAR_READO(enp->en_esbp, FR_AZ_INT_EN_REG_KER_OFST,
+			&oword, B_FALSE);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 0);
+	EFSYS_BAR_WRITEO(enp->en_esbp, FR_AZ_INT_EN_REG_KER_OFST,
+	    &oword, B_FALSE);
+}
+
+	__checkReturn	int
+efx_intr_trigger(
+	__in		efx_nic_t *enp,
+	__in		unsigned int level)
+{
+	efx_intr_t *eip = &(enp->en_intr);
+	efx_oword_t oword;
+	unsigned int count;
+	uint32_t sel;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	/* bug16757: No event queues can be initialized */
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
+
+	switch (enp->en_family) {
+	case EFX_FAMILY_FALCON:
+		if (level > EFX_NINTR_FALCON) {
+			rc = EINVAL;
+			goto fail1;
+		}
+		break;
+
+	case EFX_FAMILY_SIENA:
+		if (level > EFX_NINTR_SIENA) {
+			rc = EINVAL;
+			goto fail1;
+		}
+		break;
+
+	default:
+		EFSYS_ASSERT(B_FALSE);
+		break;
+	}
+
+	if (level > EFX_MASK32(FRF_AZ_KER_INT_LEVE_SEL))
+		return (ENOTSUP); /* avoid EFSYS_PROBE() */
+
+	sel = level;
+
+	/* Trigger a test interrupt */
+	EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, sel);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_KER, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+
+	/*
+	 * Wait up to 100ms for the interrupt to be raised before restoring
+	 * KER_INT_LEVE_SEL. Ignore a failure to raise (the caller will
+	 * observe this soon enough anyway), but always reset KER_INT_LEVE_SEL
+	 */
+	count = 0;
+	do {
+		EFSYS_SPIN(100);	/* 100us */
+
+		EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+	} while (EFX_OWORD_FIELD(oword, FRF_AZ_KER_INT_KER) && ++count < 1000);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, eip->ei_level);
+	EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn	boolean_t
+efx_intr_check_fatal(
+	__in		efx_nic_t *enp)
+{
+	efx_intr_t *eip = &(enp->en_intr);
+	efsys_mem_t *esmp = eip->ei_esmp;
+	efx_oword_t oword;
+
+	/* Read the syndrome */
+	EFSYS_MEM_READO(esmp, 0, &oword);
+
+	if (EFX_OWORD_FIELD(oword, FSF_AZ_NET_IVEC_FATAL_INT) != 0) {
+		EFSYS_PROBE(fatal);
+
+		/* Clear the fatal interrupt condition */
+		EFX_SET_OWORD_FIELD(oword, FSF_AZ_NET_IVEC_FATAL_INT, 0);
+		EFSYS_MEM_WRITEO(esmp, 0, &oword);
+
+		return (B_TRUE);
+	}
+
+	return (B_FALSE);
+}
+
+			void
+efx_intr_status_line(
+	__in		efx_nic_t *enp,
+	__out		boolean_t *fatalp,
+	__out		uint32_t *qmaskp)
+{
+	efx_intr_t *eip = &(enp->en_intr);
+	efx_dword_t dword;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	/*
+	 * Read the queue mask and implicitly acknowledge the
+	 * interrupt.
+	 */
+	EFX_BAR_READD(enp, FR_BZ_INT_ISR0_REG, &dword, B_FALSE);
+	*qmaskp = EFX_DWORD_FIELD(dword, EFX_DWORD_0);
+
+	EFSYS_PROBE1(qmask, uint32_t, *qmaskp);
+
+	if (*qmaskp & (1U << eip->ei_level))
+		*fatalp = efx_intr_check_fatal(enp);
+	else
+		*fatalp = B_FALSE;
+}
+
+			void
+efx_intr_status_message(
+	__in		efx_nic_t *enp,
+	__in		unsigned int message,
+	__out		boolean_t *fatalp)
+{
+	efx_intr_t *eip = &(enp->en_intr);
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	if (message == eip->ei_level)
+		*fatalp = efx_intr_check_fatal(enp);
+	else
+		*fatalp = B_FALSE;
+}
+
+		void
+efx_intr_fatal(
+	__in	efx_nic_t *enp)
+{
+#if EFSYS_OPT_DECODE_INTR_FATAL
+	efx_oword_t fatal;
+	efx_oword_t mem_per;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	EFX_BAR_READO(enp, FR_AZ_FATAL_INTR_REG_KER, &fatal);
+	EFX_ZERO_OWORD(mem_per);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRM_PERR_INT_KER) != 0 ||
+	    EFX_OWORD_FIELD(fatal, FRF_AZ_MEM_PERR_INT_KER) != 0)
+		EFX_BAR_READO(enp, FR_AZ_MEM_STAT_REG, &mem_per);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRAM_OOB_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_SRAM_OOB, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_BUFID_DC_OOB_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_BUFID_DC_OOB, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_MEM_PERR_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_MEM_PERR,
+		    EFX_OWORD_FIELD(mem_per, EFX_DWORD_0),
+		    EFX_OWORD_FIELD(mem_per, EFX_DWORD_1));
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_RBUF_OWN_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_RBUF_OWN, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_TBUF_OWN_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_TBUF_OWN, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_RDESCQ_OWN_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_RDESQ_OWN, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_TDESCQ_OWN_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_TDESQ_OWN, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_EVQ_OWN_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_EVQ_OWN, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_EVF_OFLO_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_EVFF_OFLO, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_ILL_ADR_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_ILL_ADDR, 0, 0);
+
+	if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRM_PERR_INT_KER) != 0)
+		EFSYS_ERR(enp->en_esip, EFX_ERR_SRAM_PERR,
+		    EFX_OWORD_FIELD(mem_per, EFX_DWORD_0),
+		    EFX_OWORD_FIELD(mem_per, EFX_DWORD_1));
+#else
+	EFSYS_ASSERT(0);
+#endif
+}
+
+		void
+efx_intr_fini(
+	__in	efx_nic_t *enp)
+{
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
+
+	/* Clear the interrupt address register */
+	EFX_ZERO_OWORD(oword);
+	EFX_BAR_WRITEO(enp, FR_AZ_INT_ADR_REG_KER, &oword);
+
+	enp->en_mod_flags &= ~EFX_MOD_INTR;
+}
diff --git a/sys/dev/sfxge/common/efx_mac.c b/sys/dev/sfxge/common/efx_mac.c
new file mode 100644
index 0000000..6852584
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_mac.c
@@ -0,0 +1,684 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_MAC_FALCON_GMAC
+#include "falcon_gmac.h"
+#endif
+
+#if EFSYS_OPT_MAC_FALCON_XMAC
+#include "falcon_xmac.h"
+#endif
+
+#if EFSYS_OPT_MAC_FALCON_GMAC
+static efx_mac_ops_t	__cs __efx_falcon_gmac_ops = {
+	falcon_gmac_reset,		/* emo_reset */
+	falcon_mac_poll,		/* emo_poll */
+	falcon_mac_up,			/* emo_up */
+	falcon_gmac_reconfigure,	/* emo_reconfigure */
+#if EFSYS_OPT_LOOPBACK
+	falcon_mac_loopback_set,	/* emo_loopback_set */
+#endif	/* EFSYS_OPT_LOOPBACK */
+#if EFSYS_OPT_MAC_STATS
+	falcon_mac_stats_upload,	/* emo_stats_upload */
+	NULL,				/* emo_stats_periodic */
+	falcon_gmac_stats_update	/* emo_stats_update */
+#endif	/* EFSYS_OPT_MAC_STATS */
+};
+#endif	/* EFSYS_OPT_MAC_FALCON_GMAC */
+
+#if EFSYS_OPT_MAC_FALCON_XMAC
+static efx_mac_ops_t	__cs __efx_falcon_xmac_ops = {
+	falcon_xmac_reset,		/* emo_reset */
+	falcon_mac_poll,		/* emo_poll */
+	falcon_mac_up,			/* emo_up */
+	falcon_xmac_reconfigure,	/* emo_reconfigure */
+#if EFSYS_OPT_LOOPBACK
+	falcon_mac_loopback_set,	/* emo_loopback_set */
+#endif	/* EFSYS_OPT_LOOPBACK */
+#if EFSYS_OPT_MAC_STATS
+	falcon_mac_stats_upload,	/* emo_stats_upload */
+	NULL,				/* emo_stats_periodic */
+	falcon_xmac_stats_update	/* emo_stats_update */
+#endif	/* EFSYS_OPT_MAC_STATS */
+};
+#endif	/* EFSYS_OPT_MAC_FALCON_XMAC */
+
+#if EFSYS_OPT_SIENA
+static efx_mac_ops_t	__cs __efx_siena_mac_ops = {
+	NULL,				/* emo_reset */
+	siena_mac_poll,			/* emo_poll */
+	siena_mac_up,			/* emo_up */
+	siena_mac_reconfigure,		/* emo_reconfigure */
+#if EFSYS_OPT_LOOPBACK
+	siena_mac_loopback_set,		/* emo_loopback_set */
+#endif	/* EFSYS_OPT_LOOPBACK */
+#if EFSYS_OPT_MAC_STATS
+	siena_mac_stats_upload,		/* emo_stats_upload */
+	siena_mac_stats_periodic,	/* emo_stats_periodic */
+	siena_mac_stats_update		/* emo_stats_update */
+#endif	/* EFSYS_OPT_MAC_STATS */
+};
+#endif	/* EFSYS_OPT_SIENA */
+
+static efx_mac_ops_t	__cs * __cs __efx_mac_ops[] = {
+	NULL,
+#if EFSYS_OPT_MAC_FALCON_GMAC
+	&__efx_falcon_gmac_ops,
+#else
+	NULL,
+#endif	/* EFSYS_OPT_MAC_FALCON_GMAC */
+#if EFSYS_OPT_MAC_FALCON_XMAC
+	&__efx_falcon_xmac_ops,
+#else
+	NULL,
+#endif	/* EFSYS_OPT_MAC_FALCON_XMAC */
+#if EFSYS_OPT_SIENA
+	&__efx_siena_mac_ops,
+#else
+	NULL,
+#endif	/* EFSYS_OPT_SIENA */
+};
+
+	__checkReturn			int
+efx_mac_pdu_set(
+	__in				efx_nic_t *enp,
+	__in				size_t pdu)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	uint32_t old_pdu;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+	EFSYS_ASSERT(emop != NULL);
+
+	if (pdu < EFX_MAC_PDU_MIN) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (pdu > EFX_MAC_PDU_MAX) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	old_pdu = epp->ep_mac_pdu;
+	epp->ep_mac_pdu = (uint32_t)pdu;
+	if ((rc = emop->emo_reconfigure(enp)) != 0)
+		goto fail3;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+
+	epp->ep_mac_pdu = old_pdu;
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+efx_mac_addr_set(
+	__in				efx_nic_t *enp,
+	__in				uint8_t *addr)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	uint8_t old_addr[6];
+	uint32_t oui;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	if (addr[0] & 0x01) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	oui = addr[0] << 16 | addr[1] << 8 | addr[2];
+	if (oui == 0x000000) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	EFX_MAC_ADDR_COPY(old_addr, epp->ep_mac_addr);
+	EFX_MAC_ADDR_COPY(epp->ep_mac_addr, addr);
+	if ((rc = emop->emo_reconfigure(enp)) != 0)
+		goto fail3;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+
+	EFX_MAC_ADDR_COPY(epp->ep_mac_addr, old_addr);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+efx_mac_filter_set(
+	__in				efx_nic_t *enp,
+	__in				boolean_t unicst,
+	__in				boolean_t brdcst)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	boolean_t old_unicst;
+	boolean_t old_brdcst;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	old_unicst = unicst;
+	old_brdcst = brdcst;
+
+	epp->ep_unicst = unicst;
+	epp->ep_brdcst = brdcst;
+
+	if ((rc = emop->emo_reconfigure(enp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	epp->ep_unicst = old_unicst;
+	epp->ep_brdcst = old_brdcst;
+
+	return (rc);
+}
+
+	__checkReturn			int
+efx_mac_drain(
+	__in				efx_nic_t *enp,
+	__in				boolean_t enabled)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+	EFSYS_ASSERT(emop != NULL);
+
+	if (epp->ep_mac_drain == enabled)
+		return (0);
+
+	epp->ep_mac_drain = enabled;
+
+	if (enabled && emop->emo_reset != NULL) {
+		if ((rc = emop->emo_reset(enp)) != 0)
+			goto fail1;
+
+		EFSYS_ASSERT(enp->en_reset_flags & EFX_RESET_MAC);
+		enp->en_reset_flags &= ~EFX_RESET_PHY;
+	}
+
+	if ((rc = emop->emo_reconfigure(enp)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_mac_up(
+	__in		efx_nic_t *enp,
+	__out		boolean_t *mac_upp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	if ((rc = emop->emo_up(enp, mac_upp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+efx_mac_fcntl_set(
+	__in				efx_nic_t *enp,
+	__in				unsigned int fcntl,
+	__in				boolean_t autoneg)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	efx_phy_ops_t *epop = epp->ep_epop;
+	unsigned int old_fcntl;
+	boolean_t old_autoneg;
+	unsigned int old_adv_cap;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	if ((fcntl & ~(EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE)) != 0) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	/*
+	 * Ignore a request to set flow control autonegotiation
+	 * if the PHY doesn't support it.
+	 */
+	if (~epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN))
+		autoneg = B_FALSE;
+
+	old_fcntl = epp->ep_fcntl;
+	old_autoneg = autoneg;
+	old_adv_cap = epp->ep_adv_cap_mask;
+
+	epp->ep_fcntl = fcntl;
+	epp->ep_fcntl_autoneg = autoneg;
+
+	/*
+	 * If the PHY supports autonegotiation, then encode the flow control
+	 * settings in the advertised capabilities, and restart AN. Otherwise,
+	 * just push the new settings directly to the MAC.
+	 */
+	if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN)) {
+		if (fcntl & EFX_FCNTL_RESPOND)
+			epp->ep_adv_cap_mask |=    (1 << EFX_PHY_CAP_PAUSE |
+						    1 << EFX_PHY_CAP_ASYM);
+		else
+			epp->ep_adv_cap_mask &=   ~(1 << EFX_PHY_CAP_PAUSE |
+						    1 << EFX_PHY_CAP_ASYM);
+
+		if (fcntl & EFX_FCNTL_GENERATE)
+			epp->ep_adv_cap_mask ^= (1 << EFX_PHY_CAP_ASYM);
+
+		if ((rc = epop->epo_reconfigure(enp)) != 0)
+			goto fail2;
+
+	} else {
+		if ((rc = emop->emo_reconfigure(enp)) != 0)
+			goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+
+	epp->ep_fcntl = old_fcntl;
+	epp->ep_fcntl_autoneg = old_autoneg;
+	epp->ep_adv_cap_mask = old_adv_cap;
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			void
+efx_mac_fcntl_get(
+	__in		efx_nic_t *enp,
+	__out		unsigned int *fcntl_wantedp,
+	__out		unsigned int *fcntl_linkp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	unsigned int wanted;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	/*
+	 * If the PHY supports auto negotiation, then the requested flow
+	 * control settings are encoded in the advertised capabilities.
+	 */
+	if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN)) {
+		wanted = 0;
+
+		if (epp->ep_adv_cap_mask & (1 << EFX_PHY_CAP_PAUSE))
+			wanted = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
+		if (epp->ep_adv_cap_mask & (1 << EFX_PHY_CAP_ASYM))
+			wanted ^= EFX_FCNTL_GENERATE;
+	} else
+		wanted = epp->ep_fcntl;
+
+	*fcntl_linkp = epp->ep_fcntl;
+	*fcntl_wantedp = wanted;
+}
+
+	__checkReturn			int
+efx_mac_hash_set(
+	__in				efx_nic_t *enp,
+	__in_ecount(EFX_MAC_HASH_BITS)	unsigned int const *bucket)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	efx_oword_t old_hash[2];
+	unsigned int index;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	memcpy(old_hash, epp->ep_multicst_hash, sizeof (old_hash));
+
+	/* Set the lower 128 bits of the hash */
+	EFX_ZERO_OWORD(epp->ep_multicst_hash[0]);
+	for (index = 0; index < 128; index++) {
+		if (bucket[index] != 0)
+			EFX_SET_OWORD_BIT(epp->ep_multicst_hash[0], index);
+	}
+
+	/* Set the upper 128 bits of the hash */
+	EFX_ZERO_OWORD(epp->ep_multicst_hash[1]);
+	for (index = 0; index < 128; index++) {
+		if (bucket[index + 128] != 0)
+			EFX_SET_OWORD_BIT(epp->ep_multicst_hash[1], index);
+	}
+
+	if ((rc = emop->emo_reconfigure(enp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	memcpy(epp->ep_multicst_hash, old_hash, sizeof (old_hash));
+
+	return (rc);
+}
+
+#if EFSYS_OPT_MAC_STATS
+
+#if EFSYS_OPT_NAMES
+
+/* START MKCONFIG GENERATED EfxMacStatNamesBlock adf707adba80813e */
+static const char 	__cs * __cs __efx_mac_stat_name[] = {
+	"rx_octets",
+	"rx_pkts",
+	"rx_unicst_pkts",
+	"rx_multicst_pkts",
+	"rx_brdcst_pkts",
+	"rx_pause_pkts",
+	"rx_le_64_pkts",
+	"rx_65_to_127_pkts",
+	"rx_128_to_255_pkts",
+	"rx_256_to_511_pkts",
+	"rx_512_to_1023_pkts",
+	"rx_1024_to_15xx_pkts",
+	"rx_ge_15xx_pkts",
+	"rx_errors",
+	"rx_fcs_errors",
+	"rx_drop_events",
+	"rx_false_carrier_errors",
+	"rx_symbol_errors",
+	"rx_align_errors",
+	"rx_internal_errors",
+	"rx_jabber_pkts",
+	"rx_lane0_char_err",
+	"rx_lane1_char_err",
+	"rx_lane2_char_err",
+	"rx_lane3_char_err",
+	"rx_lane0_disp_err",
+	"rx_lane1_disp_err",
+	"rx_lane2_disp_err",
+	"rx_lane3_disp_err",
+	"rx_match_fault",
+	"rx_nodesc_drop_cnt",
+	"tx_octets",
+	"tx_pkts",
+	"tx_unicst_pkts",
+	"tx_multicst_pkts",
+	"tx_brdcst_pkts",
+	"tx_pause_pkts",
+	"tx_le_64_pkts",
+	"tx_65_to_127_pkts",
+	"tx_128_to_255_pkts",
+	"tx_256_to_511_pkts",
+	"tx_512_to_1023_pkts",
+	"tx_1024_to_15xx_pkts",
+	"tx_ge_15xx_pkts",
+	"tx_errors",
+	"tx_sgl_col_pkts",
+	"tx_mult_col_pkts",
+	"tx_ex_col_pkts",
+	"tx_late_col_pkts",
+	"tx_def_pkts",
+	"tx_ex_def_pkts",
+};
+/* END MKCONFIG GENERATED EfxMacStatNamesBlock */
+
+	__checkReturn			const char __cs *
+efx_mac_stat_name(
+	__in				efx_nic_t *enp,
+	__in				unsigned int id)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	EFSYS_ASSERT3U(id, <, EFX_MAC_NSTATS);
+	return (__efx_mac_stat_name[id]);
+}
+
+#endif	/* EFSYS_OPT_STAT_NAME */
+
+	__checkReturn			int
+efx_mac_stats_upload(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+	EFSYS_ASSERT(emop != NULL);
+
+	/*
+	 * Don't assert !ep_mac_stats_pending, because the client might
+	 * have failed to finalise statistics when previously stopping
+	 * the port.
+	 */
+	if ((rc = emop->emo_stats_upload(enp, esmp)) != 0)
+		goto fail1;
+
+	epp->ep_mac_stats_pending = B_TRUE;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+efx_mac_stats_periodic(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__in				uint16_t period_ms,
+	__in				boolean_t events)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	EFSYS_ASSERT(emop != NULL);
+
+	if (emop->emo_stats_periodic == NULL) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if ((rc = emop->emo_stats_periodic(enp, esmp, period_ms, events)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+
+	__checkReturn			int
+efx_mac_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__inout_ecount(EFX_MAC_NSTATS)	efsys_stat_t *essp,
+	__in				uint32_t *generationp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+	EFSYS_ASSERT(emop != NULL);
+
+	rc = emop->emo_stats_update(enp, esmp, essp, generationp);
+	if (rc == 0)
+		epp->ep_mac_stats_pending = B_FALSE;
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_MAC_STATS */
+
+	__checkReturn			int
+efx_mac_select(
+	__in				efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_type_t type = EFX_MAC_INVALID;
+	efx_mac_ops_t *emop;
+	int rc = EINVAL;
+
+#if EFSYS_OPT_SIENA
+	if (enp->en_family == EFX_FAMILY_SIENA) {
+		type = EFX_MAC_SIENA;
+		goto chosen;
+	}
+#endif
+
+#if EFSYS_OPT_FALCON
+	switch (epp->ep_link_mode) {
+#if EFSYS_OPT_MAC_FALCON_GMAC
+	case EFX_LINK_100HDX:
+	case EFX_LINK_100FDX:
+	case EFX_LINK_1000HDX:
+	case EFX_LINK_1000FDX:
+		type = EFX_MAC_FALCON_GMAC;
+		goto chosen;
+#endif	/* EFSYS_OPT_FALCON_GMAC */
+
+#if EFSYS_OPT_MAC_FALCON_XMAC
+	case EFX_LINK_10000FDX:
+		type = EFX_MAC_FALCON_XMAC;
+		goto chosen;
+#endif	/* EFSYS_OPT_FALCON_XMAC */
+
+	default:
+#if EFSYS_OPT_MAC_FALCON_GMAC && EFSYS_OPT_MAC_FALCON_XMAC
+		/* Only initialise a MAC supported by the PHY */
+		if (epp->ep_phy_cap_mask &
+		    ((1 << EFX_PHY_CAP_1000FDX) |
+		    (1 << EFX_PHY_CAP_1000HDX) |
+		    (1 << EFX_PHY_CAP_100FDX) |
+		    (1 << EFX_PHY_CAP_100HDX) |
+		    (1 << EFX_PHY_CAP_10FDX) |
+		    (1 << EFX_PHY_CAP_10FDX)))
+			type = EFX_MAC_FALCON_GMAC;
+		else
+			type = EFX_MAC_FALCON_XMAC;
+#elif EFSYS_OPT_MAC_FALCON_GMAC
+		type = EFX_MAC_FALCON_GMAC;
+#else
+		type = EFX_MAC_FALCON_XMAC;
+#endif
+		goto chosen;
+	}
+#endif	/* EFSYS_OPT_FALCON */
+
+chosen:
+	EFSYS_ASSERT(type != EFX_MAC_INVALID);
+	EFSYS_ASSERT3U(type, <, EFX_MAC_NTYPES);
+	emop = epp->ep_emop = (efx_mac_ops_t *)__efx_mac_ops[type];
+	EFSYS_ASSERT(emop != NULL);
+
+	epp->ep_mac_type = type;
+	
+	if (emop->emo_reset != NULL) {
+		if ((rc = emop->emo_reset(enp)) != 0)
+			goto fail1;
+		
+		EFSYS_ASSERT(enp->en_reset_flags & EFX_RESET_MAC);
+		enp->en_reset_flags &= ~EFX_RESET_MAC;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
diff --git a/sys/dev/sfxge/common/efx_mcdi.c b/sys/dev/sfxge/common/efx_mcdi.c
new file mode 100644
index 0000000..7d01e9b
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_mcdi.c
@@ -0,0 +1,733 @@
+/*-
+ * Copyright 2008-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_regs_mcdi.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_MCDI
+
+/* Shared memory layout */
+
+#define	MCDI_P1_DBL_OFST	0x0
+#define	MCDI_P2_DBL_OFST	0x1
+#define	MCDI_P1_PDU_OFST	0x2
+#define	MCDI_P2_PDU_OFST	0x42
+#define	MCDI_P1_REBOOT_OFST	0x1fe
+#define	MCDI_P2_REBOOT_OFST	0x1ff
+
+/* A reboot/assertion causes the MCDI status word to be set after the
+ * command word is set or a REBOOT event is sent. If we notice a reboot
+ * via these mechanisms then wait 10ms for the status word to be set.
+ */
+#define	MCDI_STATUS_SLEEP_US	10000
+
+			void
+efx_mcdi_request_start(
+	__in		efx_nic_t *enp,
+	__in		efx_mcdi_req_t *emrp,
+	__in		boolean_t ev_cpl)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	efx_dword_t dword;
+	unsigned int seq;
+	unsigned int xflags;
+	unsigned int pdur;
+	unsigned int dbr;
+	unsigned int pos;
+	int state;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
+	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
+
+	switch (emip->emi_port)	{
+	case 1:
+		pdur = MCDI_P1_PDU_OFST;
+		dbr = MCDI_P1_DBL_OFST;
+		break;
+	case 2:
+		pdur = MCDI_P2_PDU_OFST;
+		dbr = MCDI_P2_DBL_OFST;
+		break;
+	default:
+		EFSYS_ASSERT(0);
+		pdur = dbr = 0;
+	};
+
+	/*
+	 * efx_mcdi_request_start() is naturally serialised against both
+	 * efx_mcdi_request_poll() and efx_mcdi_ev_cpl()/efx_mcdi_ev_death(),
+	 * by virtue of there only being one oustanding MCDI request.
+	 * Unfortunately, upper layers may also call efx_mcdi_request_abort()
+	 * at any time, to timeout a pending mcdi request, That request may
+	 * then subsequently complete, meaning efx_mcdi_ev_cpl() or
+	 * efx_mcdi_ev_death() may end up running in parallel with
+	 * efx_mcdi_request_start(). This race is handled by ensuring that
+	 * %emi_pending_req, %emi_ev_cpl and %emi_seq are protected by the
+	 * en_eslp lock.
+	 */
+	EFSYS_LOCK(enp->en_eslp, state);
+	EFSYS_ASSERT(emip->emi_pending_req == NULL);
+	emip->emi_pending_req = emrp;
+	emip->emi_ev_cpl = ev_cpl;
+	emip->emi_poll_cnt = 0;
+	seq = emip->emi_seq++ & 0xf;
+	EFSYS_UNLOCK(enp->en_eslp, state);
+
+	xflags = 0;
+	if (ev_cpl)
+		xflags |= MCDI_HEADER_XFLAGS_EVREQ;
+
+	/* Construct the header in shared memory */
+	EFX_POPULATE_DWORD_6(dword,
+			    MCDI_HEADER_CODE, emrp->emr_cmd,
+			    MCDI_HEADER_RESYNC, 1,
+			    MCDI_HEADER_DATALEN, emrp->emr_in_length,
+			    MCDI_HEADER_SEQ, seq,
+			    MCDI_HEADER_RESPONSE, 0,
+			    MCDI_HEADER_XFLAGS, xflags);
+	EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, pdur, &dword, B_TRUE);
+
+	for (pos = 0; pos < emrp->emr_in_length; pos += sizeof (efx_dword_t)) {
+		memcpy(&dword, MCDI_IN(*emrp, efx_dword_t, pos),
+		    MIN(sizeof (dword), emrp->emr_in_length - pos));
+		EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM,
+		    pdur + 1 + (pos >> 2), &dword, B_FALSE);
+	}
+
+	/* Ring the doorbell */
+	EFX_POPULATE_DWORD_1(dword, EFX_DWORD_0, 0xd004be11);
+	EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, dbr, &dword, B_FALSE);
+}
+
+static			void
+efx_mcdi_request_copyout(
+	__in		efx_nic_t *enp,
+	__in		efx_mcdi_req_t *emrp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	unsigned int pos;
+	unsigned int pdur;
+	efx_dword_t data;
+
+	pdur = (emip->emi_port == 1) ? MCDI_P1_PDU_OFST : MCDI_P2_PDU_OFST;
+
+	/* Copy payload out if caller supplied buffer */
+	if (emrp->emr_out_buf != NULL) {
+		size_t bytes = MIN(emrp->emr_out_length_used,
+				    emrp->emr_out_length);
+		for (pos = 0; pos < bytes; pos += sizeof (efx_dword_t)) {
+			EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM,
+			    pdur + 1 + (pos >> 2), &data, B_FALSE);
+			memcpy(MCDI_OUT(*emrp, efx_dword_t, pos), &data,
+			    MIN(sizeof (data), bytes - pos));
+		}
+	}
+}
+
+static			int
+efx_mcdi_request_errcode(
+	__in		unsigned int err)
+{
+
+	switch (err) {
+	case MC_CMD_ERR_ENOENT:
+		return (ENOENT);
+	case MC_CMD_ERR_EINTR:
+		return (EINTR);
+	case MC_CMD_ERR_EACCES:
+		return (EACCES);
+	case MC_CMD_ERR_EBUSY:
+		return (EBUSY);
+	case MC_CMD_ERR_EINVAL:
+		return (EINVAL);
+	case MC_CMD_ERR_EDEADLK:
+		return (EDEADLK);
+	case MC_CMD_ERR_ENOSYS:
+		return (ENOTSUP);
+	case MC_CMD_ERR_ETIME:
+		return (ETIMEDOUT);
+#ifdef WITH_MCDI_V2
+	case MC_CMD_ERR_EAGAIN:
+		return (EAGAIN);
+	case MC_CMD_ERR_ENOSPC:
+		return (ENOSPC);
+#endif
+	default:
+		EFSYS_PROBE1(mc_pcol_error, int, err);
+		return (EIO);
+	}
+}
+
+static			void
+efx_mcdi_raise_exception(
+	__in		efx_nic_t *enp,
+	__in_opt	efx_mcdi_req_t *emrp,
+	__in		int rc)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	const efx_mcdi_transport_t *emtp = emip->emi_mtp;
+	efx_mcdi_exception_t exception;
+
+	/* Reboot or Assertion failure only */
+	EFSYS_ASSERT(rc == EIO || rc == EINTR);
+
+	/*
+	 * If MC_CMD_REBOOT causes a reboot (dependent on parameters),
+	 * then the EIO is not worthy of an exception.
+	 */
+	if (emrp != NULL && emrp->emr_cmd == MC_CMD_REBOOT && rc == EIO)
+		return;
+
+	exception = (rc == EIO)
+		? EFX_MCDI_EXCEPTION_MC_REBOOT
+		: EFX_MCDI_EXCEPTION_MC_BADASSERT;
+
+	emtp->emt_exception(emtp->emt_context, exception);
+}
+
+static			int
+efx_mcdi_poll_reboot(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	unsigned int rebootr;
+	efx_dword_t dword;
+	uint32_t value;
+
+	EFSYS_ASSERT(emip->emi_port == 1 || emip->emi_port == 2);
+	rebootr = ((emip->emi_port == 1)
+	    ? MCDI_P1_REBOOT_OFST
+	    : MCDI_P2_REBOOT_OFST);
+
+	EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM, rebootr, &dword, B_FALSE);
+	value = EFX_DWORD_FIELD(dword, EFX_DWORD_0);
+
+	if (value == 0)
+		return (0);
+
+	EFX_ZERO_DWORD(dword);
+	EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, rebootr, &dword, B_FALSE);
+
+	if (value == MC_STATUS_DWORD_ASSERT)
+		return (EINTR);
+	else
+		return (EIO);
+}
+
+	__checkReturn	boolean_t
+efx_mcdi_request_poll(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	efx_mcdi_req_t *emrp;
+	efx_dword_t dword;
+	unsigned int pdur;
+	unsigned int seq;
+	unsigned int length;
+	int state;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
+
+	/* Serialise against post-watchdog efx_mcdi_ev* */
+	EFSYS_LOCK(enp->en_eslp, state);
+
+	EFSYS_ASSERT(emip->emi_pending_req != NULL);
+	EFSYS_ASSERT(!emip->emi_ev_cpl);
+	emrp = emip->emi_pending_req;
+
+	/* Check for reboot atomically w.r.t efx_mcdi_request_start */
+	if (emip->emi_poll_cnt++ == 0) {
+		if ((rc = efx_mcdi_poll_reboot(enp)) != 0) {
+			emip->emi_pending_req = NULL;
+			EFSYS_UNLOCK(enp->en_eslp, state);
+
+			goto fail1;
+		}
+	}
+
+	EFSYS_ASSERT(emip->emi_port == 1 || emip->emi_port == 2);
+	pdur = (emip->emi_port == 1) ? MCDI_P1_PDU_OFST : MCDI_P2_PDU_OFST;
+
+	/* Read the command header */
+	EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM, pdur, &dword, B_FALSE);
+	if (EFX_DWORD_FIELD(dword, MCDI_HEADER_RESPONSE) == 0) {
+		EFSYS_UNLOCK(enp->en_eslp, state);
+		return (B_FALSE);
+	}
+
+	/* Request complete */
+	emip->emi_pending_req = NULL;
+	seq = (emip->emi_seq - 1) & 0xf;
+
+	/* Check for synchronous reboot */
+	if (EFX_DWORD_FIELD(dword, MCDI_HEADER_ERROR) != 0 &&
+	    EFX_DWORD_FIELD(dword, MCDI_HEADER_DATALEN) == 0) {
+		/* Consume status word */
+		EFSYS_SPIN(MCDI_STATUS_SLEEP_US);
+		efx_mcdi_poll_reboot(enp);
+		EFSYS_UNLOCK(enp->en_eslp, state);
+		rc = EIO;
+		goto fail2;
+	}
+
+	EFSYS_UNLOCK(enp->en_eslp, state);
+
+	/* Check that the returned data is consistent */
+	if (EFX_DWORD_FIELD(dword, MCDI_HEADER_CODE) != emrp->emr_cmd ||
+	    EFX_DWORD_FIELD(dword, MCDI_HEADER_SEQ) != seq) {
+		/* Response is for a different request */
+		rc = EIO;
+		goto fail3;
+	}
+
+	length = EFX_DWORD_FIELD(dword, MCDI_HEADER_DATALEN);
+	if (EFX_DWORD_FIELD(dword, MCDI_HEADER_ERROR)) {
+		efx_dword_t errdword;
+		int errcode;
+
+		EFSYS_ASSERT3U(length, ==, 4);
+		EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM,
+		    pdur + 1 + (MC_CMD_ERR_CODE_OFST >> 2),
+		    &errdword, B_FALSE);
+		errcode = EFX_DWORD_FIELD(errdword, EFX_DWORD_0);
+		rc = efx_mcdi_request_errcode(errcode);
+		EFSYS_PROBE2(mcdi_err, int, emrp->emr_cmd, int, errcode);
+		goto fail4;
+
+	} else {
+		emrp->emr_out_length_used = length;
+		emrp->emr_rc = 0;
+		efx_mcdi_request_copyout(enp, emrp);
+	}
+
+	goto out;
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	/* Fill out error state */
+	emrp->emr_rc = rc;
+	emrp->emr_out_length_used = 0;
+
+	/* Reboot/Assertion */
+	if (rc == EIO || rc == EINTR)
+		efx_mcdi_raise_exception(enp, emrp, rc);
+
+out:
+	return (B_TRUE);
+}
+
+			void
+efx_mcdi_execute(
+	__in		efx_nic_t *enp,
+	__in		efx_mcdi_req_t *emrp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	const efx_mcdi_transport_t *emtp = emip->emi_mtp;
+
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
+
+	emtp->emt_execute(emtp->emt_context, emrp);
+}
+
+			void
+efx_mcdi_ev_cpl(
+	__in		efx_nic_t *enp,
+	__in		unsigned int seq,
+	__in		unsigned int outlen,
+	__in		int errcode)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	const efx_mcdi_transport_t *emtp = emip->emi_mtp;
+	efx_mcdi_req_t *emrp;
+	int state;
+
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
+
+	/*
+	 * Serialise against efx_mcdi_request_poll()/efx_mcdi_request_start()
+	 * when we're completing an aborted request.
+	 */
+	EFSYS_LOCK(enp->en_eslp, state);
+	if (emip->emi_pending_req == NULL || !emip->emi_ev_cpl ||
+	    (seq != ((emip->emi_seq - 1) & 0xf))) {
+		EFSYS_ASSERT(emip->emi_aborted > 0);
+		if (emip->emi_aborted > 0)
+			--emip->emi_aborted;
+		EFSYS_UNLOCK(enp->en_eslp, state);
+		return;
+	}
+
+	emrp = emip->emi_pending_req;
+	emip->emi_pending_req = NULL;
+	EFSYS_UNLOCK(enp->en_eslp, state);
+
+	/*
+	 * Fill out the remaining hdr fields, and copyout the payload
+	 * if the user supplied an output buffer.
+	 */
+	if (errcode != 0) {
+		EFSYS_PROBE2(mcdi_err, int, emrp->emr_cmd,
+		    int, errcode);
+		emrp->emr_out_length_used = 0;
+		emrp->emr_rc = efx_mcdi_request_errcode(errcode);
+	} else {
+		emrp->emr_out_length_used = outlen;
+		emrp->emr_rc = 0;
+		efx_mcdi_request_copyout(enp, emrp);
+	}
+
+	emtp->emt_ev_cpl(emtp->emt_context);
+}
+
+			void
+efx_mcdi_ev_death(
+	__in		efx_nic_t *enp,
+	__in		int rc)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	const efx_mcdi_transport_t *emtp = emip->emi_mtp;
+	efx_mcdi_req_t *emrp = NULL;
+	boolean_t ev_cpl;
+	int state;
+
+	/*
+	 * The MCDI request (if there is one) has been terminated, either
+	 * by a BADASSERT or REBOOT event.
+	 *
+	 * If there is an oustanding event-completed MCDI operation, then we
+	 * will never receive the completion event (because both MCDI
+	 * completions and BADASSERT events are sent to the same evq). So
+	 * complete this MCDI op.
+	 *
+	 * This function might run in parallel with efx_mcdi_request_poll()
+	 * for poll completed mcdi requests, and also with
+	 * efx_mcdi_request_start() for post-watchdog completions.
+	 */
+	EFSYS_LOCK(enp->en_eslp, state);
+	emrp = emip->emi_pending_req;
+	ev_cpl = emip->emi_ev_cpl;
+	if (emrp != NULL && emip->emi_ev_cpl) {
+		emip->emi_pending_req = NULL;
+
+		emrp->emr_out_length_used = 0;
+		emrp->emr_rc = rc;
+		++emip->emi_aborted;
+	}
+
+	/* Since we're running in parallel with a request, consume the
+	 * status word before dropping the lock.
+	 */
+	if (rc == EIO || rc == EINTR) {
+		EFSYS_SPIN(MCDI_STATUS_SLEEP_US);
+		(void) efx_mcdi_poll_reboot(enp);
+	}
+
+	EFSYS_UNLOCK(enp->en_eslp, state);
+
+	efx_mcdi_raise_exception(enp, emrp, rc);
+
+	if (emrp != NULL && ev_cpl)
+		emtp->emt_ev_cpl(emtp->emt_context);
+}
+
+	__checkReturn		int
+efx_mcdi_version(
+	__in			efx_nic_t *enp,
+	__out_ecount_opt(4)	uint16_t versionp[4],
+	__out_opt		uint32_t *buildp,
+	__out_opt		efx_mcdi_boot_t *statusp)
+{
+	uint8_t outbuf[MAX(MC_CMD_GET_VERSION_OUT_LEN,
+		    MC_CMD_GET_BOOT_STATUS_OUT_LEN)];
+	efx_mcdi_req_t req;
+	efx_word_t *ver_words;
+	uint16_t version[4];
+	uint32_t build;
+	efx_mcdi_boot_t status;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
+
+	EFX_STATIC_ASSERT(MC_CMD_GET_VERSION_IN_LEN == 0);
+	req.emr_cmd = MC_CMD_GET_VERSION;
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = MC_CMD_GET_VERSION_OUT_LEN;
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	/* bootrom support */
+	if (req.emr_out_length_used == MC_CMD_GET_VERSION_V0_OUT_LEN) {
+		version[0] = version[1] = version[2] = version[3] = 0;
+		build = MCDI_OUT_DWORD(req, GET_VERSION_OUT_FIRMWARE);
+
+		goto version;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_VERSION_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	ver_words = MCDI_OUT2(req, efx_word_t, GET_VERSION_OUT_VERSION);
+	version[0] = EFX_WORD_FIELD(ver_words[0], EFX_WORD_0);
+	version[1] = EFX_WORD_FIELD(ver_words[1], EFX_WORD_0);
+	version[2] = EFX_WORD_FIELD(ver_words[2], EFX_WORD_0);
+	version[3] = EFX_WORD_FIELD(ver_words[3], EFX_WORD_0);
+	build = MCDI_OUT_DWORD(req, GET_VERSION_OUT_FIRMWARE);
+
+version:
+	/* The bootrom doesn't understand BOOT_STATUS */
+	if (build == MC_CMD_GET_VERSION_OUT_FIRMWARE_BOOTROM) {
+		status = EFX_MCDI_BOOT_ROM;
+		goto out;
+	}
+
+	req.emr_cmd = MC_CMD_GET_BOOT_STATUS;
+	EFX_STATIC_ASSERT(MC_CMD_GET_BOOT_STATUS_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = MC_CMD_GET_BOOT_STATUS_OUT_LEN;
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail3;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_BOOT_STATUS_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail4;
+	}
+
+	if (MCDI_OUT_DWORD_FIELD(req, GET_BOOT_STATUS_OUT_FLAGS,
+	    GET_BOOT_STATUS_OUT_FLAGS_PRIMARY))
+		status = EFX_MCDI_BOOT_PRIMARY;
+	else
+		status = EFX_MCDI_BOOT_SECONDARY;
+
+out:
+	if (versionp != NULL)
+		memcpy(versionp, version, sizeof (version));
+	if (buildp != NULL)
+		*buildp = build;
+	if (statusp != NULL)
+		*statusp = status;
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_mcdi_init(
+	__in		efx_nic_t *enp,
+	__in		const efx_mcdi_transport_t *mtp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	efx_oword_t oword;
+	unsigned int portnum;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0);
+	enp->en_mod_flags |= EFX_MOD_MCDI;
+
+	if (enp->en_family == EFX_FAMILY_FALCON)
+		return (0);
+
+	emip->emi_mtp = mtp;
+
+	/* Determine the port number to use for MCDI */
+	EFX_BAR_READO(enp, FR_AZ_CS_DEBUG_REG, &oword);
+	portnum = EFX_OWORD_FIELD(oword, FRF_CZ_CS_PORT_NUM);
+
+	if (portnum == 0) {
+		/* Presumably booted from ROM; only MCDI port 1 will work */
+		emip->emi_port = 1;
+	} else if (portnum <= 2) {
+		emip->emi_port = portnum;
+	} else {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	/*
+	 * Wipe the atomic reboot status so subsequent MCDI requests succeed.
+	 * BOOT_STATUS is preserved so eno_nic_probe() can boot out of the
+	 * assertion handler.
+	 */
+	(void) efx_mcdi_poll_reboot(enp);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	enp->en_mod_flags &= ~EFX_MOD_MCDI;
+
+	return (rc);
+}
+
+
+	__checkReturn	int
+efx_mcdi_reboot(
+	__in		efx_nic_t *enp)
+{
+	uint8_t payload[MC_CMD_REBOOT_IN_LEN];
+	efx_mcdi_req_t req;
+	int rc;
+
+	/*
+	 * We could require the caller to have caused en_mod_flags=0 to
+	 * call this function. This doesn't help the other port though,
+	 * who's about to get the MC ripped out from underneath them.
+	 * Since they have to cope with the subsequent fallout of MCDI
+	 * failures, we should as well.
+	 */
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	req.emr_cmd = MC_CMD_REBOOT;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_REBOOT_IN_LEN;
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, REBOOT_IN_FLAGS, 0);
+
+	efx_mcdi_execute(enp, &req);
+
+	/* Invert EIO */
+	if (req.emr_rc != EIO) {
+		rc = EIO;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	boolean_t
+efx_mcdi_request_abort(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	efx_mcdi_req_t *emrp;
+	boolean_t aborted;
+	int state;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
+
+	/*
+	 * efx_mcdi_ev_* may have already completed this event, and be
+	 * spinning/blocked on the upper layer lock. So it *is* legitimate
+	 * to for emi_pending_req to be NULL. If there is a pending event
+	 * completed request, then provide a "credit" to allow
+	 * efx_mcdi_ev_cpl() to accept a single spurious completion.
+	 */
+	EFSYS_LOCK(enp->en_eslp, state);
+	emrp = emip->emi_pending_req;
+	aborted = (emrp != NULL);
+	if (aborted) {
+		emip->emi_pending_req = NULL;
+
+		/* Error the request */
+		emrp->emr_out_length_used = 0;
+		emrp->emr_rc = ETIMEDOUT;
+
+		/* Provide a credit for seqno/emr_pending_req mismatches */
+		if (emip->emi_ev_cpl)
+			++emip->emi_aborted;
+
+		/*
+		 * The upper layer has called us, so we don't
+		 * need to complete the request.
+		 */
+	}
+	EFSYS_UNLOCK(enp->en_eslp, state);
+
+	return (aborted);
+}
+
+			void
+efx_mcdi_fini(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+
+	EFSYS_ASSERT3U(enp->en_mod_flags, ==, EFX_MOD_MCDI);
+	enp->en_mod_flags &= ~EFX_MOD_MCDI;
+
+	if (~(enp->en_features) & EFX_FEATURE_MCDI)
+		return;
+
+	emip->emi_mtp = NULL;
+	emip->emi_port = 0;
+	emip->emi_aborted = 0;
+}
+
+#endif	/* EFSYS_OPT_MCDI */
diff --git a/sys/dev/sfxge/common/efx_mcdi.h b/sys/dev/sfxge/common/efx_mcdi.h
new file mode 100644
index 0000000..ef0ba90
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_mcdi.h
@@ -0,0 +1,238 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SYS_EFX_MCDI_H
+#define	_SYS_EFX_MCDI_H
+
+#include "efx.h"
+#include "efx_regs.h"
+#include "efx_regs_mcdi.h"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/* Number of retries attempted for init code */
+#define	EFX_MCDI_REQ_RETRY_INIT 2
+
+struct efx_mcdi_req_s {
+	/* Inputs: Command #, input buffer and length */
+	unsigned int	emr_cmd;
+	uint8_t		*emr_in_buf;
+	size_t		emr_in_length;
+	/* Outputs: retcode, buffer, length, and length used*/
+	int		emr_rc;
+	uint8_t		*emr_out_buf;
+	size_t		emr_out_length;
+	size_t		emr_out_length_used;
+};
+
+typedef struct efx_mcdi_iface_s {
+	const efx_mcdi_transport_t *emi_mtp;
+	unsigned int		emi_port;
+	unsigned int		emi_seq;
+	efx_mcdi_req_t		*emi_pending_req;
+	boolean_t		emi_ev_cpl;
+	int			emi_aborted;
+	uint32_t		emi_poll_cnt;
+} efx_mcdi_iface_t;
+
+extern			void
+efx_mcdi_execute(
+	__in		efx_nic_t *enp,
+	__in		efx_mcdi_req_t *emrp);
+
+extern			void
+efx_mcdi_ev_cpl(
+	__in		efx_nic_t *enp,
+	__in		unsigned int seq,
+	__in		unsigned int outlen,
+	__in		int errcode);
+
+extern			void
+efx_mcdi_ev_death(
+	__in		efx_nic_t *enp,
+	__in		int rc);
+
+typedef enum efx_mcdi_boot_e {
+	EFX_MCDI_BOOT_PRIMARY,
+	EFX_MCDI_BOOT_SECONDARY,
+	EFX_MCDI_BOOT_ROM,
+} efx_mcdi_boot_t;
+
+extern	__checkReturn		int
+efx_mcdi_version(
+	__in			efx_nic_t *enp,
+	__out_ecount_opt(4)	uint16_t versionp[4],
+	__out_opt		uint32_t *buildp,
+	__out_opt		efx_mcdi_boot_t *statusp);
+
+#define	MCDI_IN(_emr, _type, _ofst)					\
+	((_type *)((_emr).emr_in_buf + (_ofst)))
+
+#define	MCDI_IN2(_emr, _type, _ofst)					\
+	MCDI_IN(_emr, _type, MC_CMD_ ## _ofst ## _OFST)
+
+#define	MCDI_IN_SET_BYTE(_emr, _ofst, _value)				\
+	EFX_POPULATE_BYTE_1(*MCDI_IN2(_emr, efx_byte_t, _ofst),		\
+		EFX_BYTE_0, _value)
+
+#define	MCDI_IN_SET_DWORD(_emr, _ofst, _value)				\
+	EFX_POPULATE_DWORD_1(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		EFX_DWORD_0, _value)
+
+#define	MCDI_IN_POPULATE_DWORD_1(_emr, _ofst, _field1, _value1)		\
+	EFX_POPULATE_DWORD_1(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1)
+
+#define	MCDI_IN_POPULATE_DWORD_2(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2)					\
+	EFX_POPULATE_DWORD_2(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2)
+
+#define	MCDI_IN_POPULATE_DWORD_3(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3)			\
+	EFX_POPULATE_DWORD_3(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3)
+
+#define	MCDI_IN_POPULATE_DWORD_4(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3, _field4, _value4)	\
+	EFX_POPULATE_DWORD_4(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4)
+
+#define	MCDI_IN_POPULATE_DWORD_5(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3, _field4, _value4,	\
+		_field5, _value5)					\
+	EFX_POPULATE_DWORD_5(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4,				\
+		MC_CMD_ ## _field5, _value5)
+
+#define	MCDI_IN_POPULATE_DWORD_6(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3, _field4, _value4,	\
+		_field5, _value5, _field6, _value6)			\
+	EFX_POPULATE_DWORD_6(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4,				\
+		MC_CMD_ ## _field5, _value5,				\
+		MC_CMD_ ## _field6, _value6)
+
+#define	MCDI_IN_POPULATE_DWORD_7(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3, _field4, _value4,	\
+		_field5, _value5, _field6, _value6, _field7, _value7)	\
+	EFX_POPULATE_DWORD_7(MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4,				\
+		MC_CMD_ ## _field5, _value5,				\
+		MC_CMD_ ## _field6, _value6,				\
+		MC_CMD_ ## _field7, _value7)
+
+#define	MCDI_IN_POPULATE_DWORD_8(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3, _field4, _value4,	\
+		_field5, _value5, _field6, _value6, _field7, _value7,	\
+		_field8, _value8)					\
+	EFX_POPULATE_DWORD_8(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4,				\
+		MC_CMD_ ## _field5, _value5,				\
+		MC_CMD_ ## _field6, _value6,				\
+		MC_CMD_ ## _field7, _value7,				\
+		MC_CMD_ ## _field8, _value8)
+
+#define	MCDI_IN_POPULATE_DWORD_9(_emr, _ofst, _field1, _value1,		\
+		_field2, _value2, _field3, _value3, _field4, _value4,	\
+		_field5, _value5, _field6, _value6, _field7, _value7,	\
+		_field8, _value8, _field9, _value9)			\
+	EFX_POPULATE_DWORD_9(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4,				\
+		MC_CMD_ ## _field5, _value5,				\
+		MC_CMD_ ## _field6, _value6,				\
+		MC_CMD_ ## _field7, _value7,				\
+		MC_CMD_ ## _field8, _value8,				\
+		MC_CMD_ ## _field9, _value9)
+
+#define	MCDI_IN_POPULATE_DWORD_10(_emr, _ofst, _field1, _value1,	\
+		_field2, _value2, _field3, _value3, _field4, _value4,	\
+		_field5, _value5, _field6, _value6, _field7, _value7,	\
+		_field8, _value8, _field9, _value9, _field10, _value10)	\
+	EFX_POPULATE_DWORD_10(*MCDI_IN2(_emr, efx_dword_t, _ofst),	\
+		MC_CMD_ ## _field1, _value1,				\
+		MC_CMD_ ## _field2, _value2,				\
+		MC_CMD_ ## _field3, _value3,				\
+		MC_CMD_ ## _field4, _value4,				\
+		MC_CMD_ ## _field5, _value5,				\
+		MC_CMD_ ## _field6, _value6,				\
+		MC_CMD_ ## _field7, _value7,				\
+		MC_CMD_ ## _field8, _value8,				\
+		MC_CMD_ ## _field9, _value9,				\
+		MC_CMD_ ## _field10, _value10)
+
+#define	MCDI_OUT(_emr, _type, _ofst)					\
+	((_type *)((_emr).emr_out_buf + (_ofst)))
+
+#define	MCDI_OUT2(_emr, _type, _ofst)					\
+	MCDI_OUT(_emr, _type, MC_CMD_ ## _ofst ## _OFST)
+
+#define	MCDI_OUT_BYTE(_emr, _ofst)					\
+	EFX_BYTE_FIELD(*MCDI_OUT2(_emr, efx_byte_t, _ofst),		\
+		    EFX_BYTE_0)
+
+#define	MCDI_OUT_WORD(_emr, _ofst)					\
+	EFX_WORD_FIELD(*MCDI_OUT2(_emr, efx_word_t, _ofst),		\
+		    EFX_WORD_0)
+
+#define	MCDI_OUT_DWORD(_emr, _ofst)					\
+	EFX_DWORD_FIELD(*MCDI_OUT2(_emr, efx_dword_t, _ofst),		\
+			EFX_DWORD_0)
+
+#define	MCDI_OUT_DWORD_FIELD(_emr, _ofst, _field)			\
+	EFX_DWORD_FIELD(*MCDI_OUT2(_emr, efx_dword_t, _ofst),		\
+			MC_CMD_ ## _field)
+
+#define	MCDI_EV_FIELD(_eqp, _field)					\
+	EFX_QWORD_FIELD(*eqp, MCDI_EVENT_ ## _field)
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_EFX_MCDI_H */
diff --git a/sys/dev/sfxge/common/efx_mon.c b/sys/dev/sfxge/common/efx_mon.c
new file mode 100644
index 0000000..0d3221a
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_mon.c
@@ -0,0 +1,269 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_MON_NULL
+#include "nullmon.h"
+#endif
+
+#if EFSYS_OPT_MON_LM87
+#include "lm87.h"
+#endif
+
+#if EFSYS_OPT_MON_MAX6647
+#include "max6647.h"
+#endif
+
+#if EFSYS_OPT_NAMES
+
+static const char	__cs * __cs __efx_mon_name[] = {
+	"",
+	"nullmon",
+	"lm87",
+	"max6647",
+	"sfx90x0"
+};
+
+		const char __cs *
+efx_mon_name(
+	__in	efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	EFSYS_ASSERT(encp->enc_mon_type != EFX_MON_INVALID);
+	EFSYS_ASSERT3U(encp->enc_mon_type, <, EFX_MON_NTYPES);
+	return (__efx_mon_name[encp->enc_mon_type]);
+}
+
+#endif	/* EFSYS_OPT_NAMES */
+
+#if EFSYS_OPT_MON_NULL
+static efx_mon_ops_t	__cs __efx_mon_null_ops = {
+	nullmon_reset,			/* emo_reset */
+	nullmon_reconfigure,		/* emo_reconfigure */
+#if EFSYS_OPT_MON_STATS
+	nullmon_stats_update		/* emo_stat_update */
+#endif	/* EFSYS_OPT_MON_STATS */
+};
+#endif
+
+#if EFSYS_OPT_MON_LM87
+static efx_mon_ops_t	__cs __efx_mon_lm87_ops = {
+	lm87_reset,			/* emo_reset */
+	lm87_reconfigure,		/* emo_reconfigure */
+#if EFSYS_OPT_MON_STATS
+	lm87_stats_update		/* emo_stat_update */
+#endif	/* EFSYS_OPT_MON_STATS */
+};
+#endif
+
+#if EFSYS_OPT_MON_MAX6647
+static efx_mon_ops_t	__cs __efx_mon_max6647_ops = {
+	max6647_reset,			/* emo_reset */
+	max6647_reconfigure,		/* emo_reconfigure */
+#if EFSYS_OPT_MON_STATS
+	max6647_stats_update		/* emo_stat_update */
+#endif	/* EFSYS_OPT_MON_STATS */
+};
+#endif
+
+#if EFSYS_OPT_MON_SIENA
+static efx_mon_ops_t	__cs __efx_mon_siena_ops = {
+	siena_mon_reset,		/* emo_reset */
+	siena_mon_reconfigure,		/* emo_reconfigure */
+#if EFSYS_OPT_MON_STATS
+	siena_mon_stats_update		/* emo_stat_update */
+#endif	/* EFSYS_OPT_MON_STATS */
+};
+#endif
+
+
+static efx_mon_ops_t	__cs * __cs __efx_mon_ops[] = {
+	NULL,
+#if EFSYS_OPT_MON_NULL
+	&__efx_mon_null_ops,
+#else
+	NULL,
+#endif
+#if EFSYS_OPT_MON_LM87
+	&__efx_mon_lm87_ops,
+#else
+	NULL,
+#endif
+#if EFSYS_OPT_MON_MAX6647
+	&__efx_mon_max6647_ops,
+#else
+	NULL,
+#endif
+#if EFSYS_OPT_MON_SIENA
+	&__efx_mon_siena_ops
+#else
+	NULL
+#endif
+};
+
+	__checkReturn	int
+efx_mon_init(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_mon_t *emp = &(enp->en_mon);
+	efx_mon_ops_t *emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+
+	if (enp->en_mod_flags & EFX_MOD_MON) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	enp->en_mod_flags |= EFX_MOD_MON;
+
+	emp->em_type = encp->enc_mon_type;
+
+	EFSYS_ASSERT(encp->enc_mon_type != EFX_MON_INVALID);
+	EFSYS_ASSERT3U(emp->em_type, <, EFX_MON_NTYPES);
+	if ((emop = (efx_mon_ops_t *)__efx_mon_ops[emp->em_type]) == NULL) {
+		rc = ENOTSUP;
+		goto fail2;
+	}
+
+	if ((rc = emop->emo_reset(enp)) != 0)
+		goto fail3;
+
+	if ((rc = emop->emo_reconfigure(enp)) != 0)
+		goto fail4;
+
+	emp->em_emop = emop;
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail5);
+
+	(void) emop->emo_reset(enp);
+
+fail3:
+	EFSYS_PROBE(fail4);
+fail2:
+	EFSYS_PROBE(fail3);
+
+	emp->em_type = EFX_MON_INVALID;
+
+	enp->en_mod_flags &= ~EFX_MOD_MON;
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_MON_STATS
+
+#if EFSYS_OPT_NAMES
+
+/* START MKCONFIG GENERATED MonitorStatNamesBlock 08518fd1fb4e2612 */
+static const char 	__cs * __cs __mon_stat_name[] = {
+	"value_2_5v",
+	"value_vccp1",
+	"value_vcc",
+	"value_5v",
+	"value_12v",
+	"value_vccp2",
+	"value_ext_temp",
+	"value_int_temp",
+	"value_ain1",
+	"value_ain2",
+	"controller_cooling",
+	"ext_cooling",
+	"1v",
+	"1_2v",
+	"1_8v",
+	"3_3v",
+};
+
+/* END MKCONFIG GENERATED MonitorStatNamesBlock */
+
+extern					const char __cs *
+efx_mon_stat_name(
+	__in				efx_nic_t *enp,
+	__in				efx_mon_stat_t id)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	EFSYS_ASSERT3U(id, <, EFX_MON_NSTATS);
+	return (__mon_stat_name[id]);
+}
+
+#endif	/* EFSYS_OPT_NAMES */
+
+	__checkReturn			int
+efx_mon_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values)
+{
+	efx_mon_t *emp = &(enp->en_mon);
+	efx_mon_ops_t *emop = emp->em_emop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON);
+
+	return (emop->emo_stats_update(enp, esmp, values));
+}
+
+#endif	/* EFSYS_OPT_MON_STATS */
+
+		void
+efx_mon_fini(
+	__in	efx_nic_t *enp)
+{
+	efx_mon_t *emp = &(enp->en_mon);
+	efx_mon_ops_t *emop = emp->em_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON);
+
+	emp->em_emop = NULL;
+
+	rc = emop->emo_reset(enp);
+	if (rc != 0)
+		EFSYS_PROBE1(fail1, int, rc);
+
+	emp->em_type = EFX_MON_INVALID;
+
+	enp->en_mod_flags &= ~EFX_MOD_MON;
+}
diff --git a/sys/dev/sfxge/common/efx_nic.c b/sys/dev/sfxge/common/efx_nic.c
new file mode 100644
index 0000000..07ce009
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_nic.c
@@ -0,0 +1,674 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+	__checkReturn	int
+efx_family(
+	__in		uint16_t venid,
+	__in		uint16_t devid,
+	__out		efx_family_t *efp)
+{
+#if EFSYS_OPT_FALCON
+	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_FALCON) {
+		*efp = EFX_FAMILY_FALCON;
+		return (0);
+	}
+#endif
+#if EFSYS_OPT_SIENA
+	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_BETHPAGE) {
+		*efp = EFX_FAMILY_SIENA;
+		return (0);
+	}
+	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_SIENA) {
+		*efp = EFX_FAMILY_SIENA;
+		return (0);
+	}
+	if (venid == EFX_PCI_VENID_SFC &&
+	    devid == EFX_PCI_DEVID_SIENA_F1_UNINIT) {
+		*efp = EFX_FAMILY_SIENA;
+		return (0);
+	}
+#endif
+	return (ENOTSUP);
+}
+
+/*
+ * To support clients which aren't provided with any PCI context infer
+ * the hardware family by inspecting the hardware. Obviously the caller
+ * must be damn sure they're really talking to a supported device.
+ */
+	__checkReturn	int
+efx_infer_family(
+	__in		efsys_bar_t *esbp,
+	__out		efx_family_t *efp)
+{
+	efx_family_t family;
+	efx_oword_t oword;
+	unsigned int portnum;
+	int rc;
+
+	EFSYS_BAR_READO(esbp, FR_AZ_CS_DEBUG_REG_OFST, &oword, B_TRUE);
+	portnum = EFX_OWORD_FIELD(oword, FRF_CZ_CS_PORT_NUM);
+	switch (portnum) {
+#if EFSYS_OPT_FALCON
+	case 0:
+		family = EFX_FAMILY_FALCON;
+		break;
+#endif
+#if EFSYS_OPT_SIENA
+	case 1:
+	case 2:
+		family = EFX_FAMILY_SIENA;
+		break;
+#endif
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if (efp != NULL)
+		*efp = family;
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+/*
+ * The built-in default value device id for port 1 of Siena is 0x0810.
+ * manftest needs to be able to cope with that.
+ */
+
+#define	EFX_BIU_MAGIC0	0x01234567
+#define	EFX_BIU_MAGIC1	0xfedcba98
+
+static	__checkReturn	int
+efx_nic_biu_test(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+	int rc;
+
+	/*
+	 * Write magic values to scratch registers 0 and 1, then
+	 * verify that the values were written correctly.  Interleave
+	 * the accesses to ensure that the BIU is not just reading
+	 * back the cached value that was last written.
+	 */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword);
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword);
+
+	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword);
+	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
+		rc = EIO;
+		goto fail1;
+	}
+
+	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword);
+	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
+		rc = EIO;
+		goto fail2;
+	}
+
+	/*
+	 * Perform the same test, with the values swapped.  This
+	 * ensures that subsequent tests don't start with the correct
+	 * values already written into the scratch registers.
+	 */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword);
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword);
+
+	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword);
+	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
+		rc = EIO;
+		goto fail3;
+	}
+
+	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword);
+	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
+		rc = EIO;
+		goto fail4;
+	}
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_FALCON
+
+static efx_nic_ops_t	__cs __efx_nic_falcon_ops = {
+	falcon_nic_probe,		/* eno_probe */
+	falcon_nic_reset,		/* eno_reset */
+	falcon_nic_init,		/* eno_init */
+#if EFSYS_OPT_DIAG
+	falcon_sram_test,		/* eno_sram_test */
+	falcon_nic_register_test,	/* eno_register_test */
+#endif	/* EFSYS_OPT_DIAG */
+	falcon_nic_fini,		/* eno_fini */
+	falcon_nic_unprobe,		/* eno_unprobe */
+};
+
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+
+static efx_nic_ops_t	__cs __efx_nic_siena_ops = {
+	siena_nic_probe,		/* eno_probe */
+	siena_nic_reset,		/* eno_reset */
+	siena_nic_init,			/* eno_init */
+#if EFSYS_OPT_DIAG
+	siena_sram_test,		/* eno_sram_test */
+	siena_nic_register_test,	/* eno_register_test */
+#endif	/* EFSYS_OPT_DIAG */
+	siena_nic_fini,			/* eno_fini */
+	siena_nic_unprobe,		/* eno_unprobe */
+};
+
+#endif	/* EFSYS_OPT_SIENA */
+
+	__checkReturn	int
+efx_nic_create(
+	__in		efx_family_t family,
+	__in		efsys_identifier_t *esip,
+	__in		efsys_bar_t *esbp,
+	__in		efsys_lock_t *eslp,
+	__deref_out	efx_nic_t **enpp)
+{
+	efx_nic_t *enp;
+	int rc;
+
+	EFSYS_ASSERT3U(family, >, EFX_FAMILY_INVALID);
+	EFSYS_ASSERT3U(family, <, EFX_FAMILY_NTYPES);
+
+	/* Allocate a NIC object */
+	EFSYS_KMEM_ALLOC(esip, sizeof (efx_nic_t), enp);
+
+	if (enp == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	enp->en_magic = EFX_NIC_MAGIC;
+
+	switch (family) {
+#if EFSYS_OPT_FALCON
+	case EFX_FAMILY_FALCON:
+		enp->en_enop = (efx_nic_ops_t *)&__efx_nic_falcon_ops;
+		enp->en_features = 0;
+		break;
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+	case EFX_FAMILY_SIENA:
+		enp->en_enop = (efx_nic_ops_t *)&__efx_nic_siena_ops;
+		enp->en_features = EFX_FEATURE_IPV6 |
+		    EFX_FEATURE_LFSR_HASH_INSERT |
+		    EFX_FEATURE_LINK_EVENTS | EFX_FEATURE_PERIODIC_MAC_STATS |
+		    EFX_FEATURE_WOL | EFX_FEATURE_MCDI |
+		    EFX_FEATURE_LOOKAHEAD_SPLIT | EFX_FEATURE_MAC_HEADER_FILTERS;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		rc = ENOTSUP;
+		goto fail2;
+	}
+
+	enp->en_family = family;
+	enp->en_esip = esip;
+	enp->en_esbp = esbp;
+	enp->en_eslp = eslp;
+
+	*enpp = enp;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail3);
+
+	enp->en_magic = 0;
+
+	/* Free the NIC object */
+	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_nic_probe(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_ops_t *enop;
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+#if EFSYS_OPT_MCDI
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
+#endif	/* EFSYS_OPT_MCDI */
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_PROBE));
+
+	/* Test BIU */
+	if ((rc = efx_nic_biu_test(enp)) != 0)
+		goto fail1;
+
+	/* Clear the region register */
+	EFX_POPULATE_OWORD_4(oword,
+	    FRF_AZ_ADR_REGION0, 0,
+	    FRF_AZ_ADR_REGION1, (1 << 16),
+	    FRF_AZ_ADR_REGION2, (2 << 16),
+	    FRF_AZ_ADR_REGION3, (3 << 16));
+	EFX_BAR_WRITEO(enp, FR_AZ_ADR_REGION_REG, &oword);
+
+	enop = enp->en_enop;
+	if ((rc = enop->eno_probe(enp)) != 0)
+		goto fail2;
+
+	if ((rc = efx_phy_probe(enp)) != 0)
+		goto fail3;
+
+	enp->en_mod_flags |= EFX_MOD_PROBE;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+
+	enop->eno_unprobe(enp);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_PCIE_TUNE
+
+	__checkReturn	int
+efx_nic_pcie_tune(
+	__in		efx_nic_t *enp,
+	unsigned int	nlanes)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
+
+#if EFSYS_OPT_FALCON
+	if (enp->en_family == EFX_FAMILY_FALCON)
+		return (falcon_nic_pcie_tune(enp, nlanes));
+#endif
+	return (ENOTSUP);
+}
+
+	__checkReturn	int
+efx_nic_pcie_extended_sync(
+	__in		efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
+
+#if EFSYS_OPT_SIENA
+	if (enp->en_family == EFX_FAMILY_SIENA)
+		return (siena_nic_pcie_extended_sync(enp));
+#endif
+
+	return (ENOTSUP);
+}
+
+#endif	/* EFSYS_OPT_PCIE_TUNE */
+
+	__checkReturn	int
+efx_nic_init(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_ops_t *enop = enp->en_enop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+
+	if (enp->en_mod_flags & EFX_MOD_NIC) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if ((rc = enop->eno_init(enp)) != 0)
+		goto fail2;
+
+	enp->en_mod_flags |= EFX_MOD_NIC;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			void
+efx_nic_fini(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_ops_t *enop = enp->en_enop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
+	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_NIC);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
+
+	enop->eno_fini(enp);
+
+	enp->en_mod_flags &= ~EFX_MOD_NIC;
+}
+
+			void
+efx_nic_unprobe(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_ops_t *enop = enp->en_enop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+#if EFSYS_OPT_MCDI
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
+#endif	/* EFSYS_OPT_MCDI */
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
+
+	efx_phy_unprobe(enp);
+
+	enop->eno_unprobe(enp);
+
+	enp->en_mod_flags &= ~EFX_MOD_PROBE;
+}
+
+			void
+efx_nic_destroy(
+	__in	efx_nic_t *enp)
+{
+	efsys_identifier_t *esip = enp->en_esip;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0);
+
+	enp->en_family = 0;
+	enp->en_esip = NULL;
+	enp->en_esbp = NULL;
+	enp->en_eslp = NULL;
+
+	enp->en_enop = NULL;
+
+	enp->en_magic = 0;
+
+	/* Free the NIC object */
+	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
+}
+
+	__checkReturn	int
+efx_nic_reset(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_ops_t *enop = enp->en_enop;
+	unsigned int mod_flags;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
+	/*
+	 * All modules except the MCDI, PROBE, NVRAM, VPD, MON (which we
+	 * do not reset here) must have been shut down or never initialized.
+	 *
+	 * A rule of thumb here is: If the controller or MC reboots, is *any*
+	 * state lost. If it's lost and needs reapplying, then the module
+	 * *must* not be initialised during the reset.
+	 */
+	mod_flags = enp->en_mod_flags;
+	mod_flags &= ~(EFX_MOD_MCDI | EFX_MOD_PROBE | EFX_MOD_NVRAM |
+		    EFX_MOD_VPD | EFX_MOD_MON);
+	EFSYS_ASSERT3U(mod_flags, ==, 0);
+	if (mod_flags != 0) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if ((rc = enop->eno_reset(enp)) != 0)
+		goto fail2;
+
+	enp->en_reset_flags |= EFX_RESET_MAC;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			const efx_nic_cfg_t *
+efx_nic_cfg_get(
+	__in		efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	return (&(enp->en_nic_cfg));
+}
+
+#if EFSYS_OPT_DIAG
+
+	__checkReturn	int
+efx_nic_register_test(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_ops_t *enop = enp->en_enop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
+
+	if ((rc = enop->eno_register_test(enp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_nic_test_registers(
+	__in		efx_nic_t *enp,
+	__in		efx_register_set_t *rsp,
+	__in		size_t count)
+{
+	unsigned int bit;
+	efx_oword_t original;
+	efx_oword_t reg;
+	efx_oword_t buf;
+	int rc;
+
+	while (count > 0) {
+		/* This function is only suitable for registers */
+		EFSYS_ASSERT(rsp->rows == 1);
+
+		/* bit sweep on and off */
+		EFSYS_BAR_READO(enp->en_esbp, rsp->address, &original,
+			    B_TRUE);
+		for (bit = 0; bit < 128; bit++) {
+			/* Is this bit in the mask? */
+			if (~(rsp->mask.eo_u32[bit >> 5]) & (1 << bit))
+				continue;
+
+			/* Test this bit can be set in isolation */
+			reg = original;
+			EFX_AND_OWORD(reg, rsp->mask);
+			EFX_SET_OWORD_BIT(reg, bit);
+
+			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
+				    B_TRUE);
+			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
+				    B_TRUE);
+
+			EFX_AND_OWORD(buf, rsp->mask);
+			if (memcmp(&reg, &buf, sizeof (reg))) {
+				rc = EIO;
+				goto fail1;
+			}
+
+			/* Test this bit can be cleared in isolation */
+			EFX_OR_OWORD(reg, rsp->mask);
+			EFX_CLEAR_OWORD_BIT(reg, bit);
+
+			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
+				    B_TRUE);
+			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
+				    B_TRUE);
+
+			EFX_AND_OWORD(buf, rsp->mask);
+			if (memcmp(&reg, &buf, sizeof (reg))) {
+				rc = EIO;
+				goto fail2;
+			}
+		}
+
+		/* Restore the old value */
+		EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original,
+			    B_TRUE);
+
+		--count;
+		++rsp;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	/* Restore the old value */
+	EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original, B_TRUE);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_nic_test_tables(
+	__in		efx_nic_t *enp,
+	__in		efx_register_set_t *rsp,
+	__in		efx_pattern_type_t pattern,
+	__in		size_t count)
+{
+	efx_sram_pattern_fn_t func;
+	unsigned int index;
+	unsigned int address;
+	efx_oword_t reg;
+	efx_oword_t buf;
+	int rc;
+
+	EFSYS_ASSERT(pattern < EFX_PATTERN_NTYPES);
+	func = __efx_sram_pattern_fns[pattern];
+
+	while (count > 0) {
+		/* Write */
+		address = rsp->address;
+		for (index = 0; index < rsp->rows; ++index) {
+			func(2 * index + 0, B_FALSE, &reg.eo_qword[0]);
+			func(2 * index + 1, B_FALSE, &reg.eo_qword[1]);
+			EFX_AND_OWORD(reg, rsp->mask);
+			EFSYS_BAR_WRITEO(enp->en_esbp, address, &reg, B_TRUE);
+
+			address += rsp->step;
+		}
+
+		/* Read */
+		address = rsp->address;
+		for (index = 0; index < rsp->rows; ++index) {
+			func(2 * index + 0, B_FALSE, &reg.eo_qword[0]);
+			func(2 * index + 1, B_FALSE, &reg.eo_qword[1]);
+			EFX_AND_OWORD(reg, rsp->mask);
+			EFSYS_BAR_READO(enp->en_esbp, address, &buf, B_TRUE);
+			if (memcmp(&reg, &buf, sizeof (reg))) {
+				rc = EIO;
+				goto fail1;
+			}
+
+			address += rsp->step;
+		}
+
+		++rsp;
+		--count;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_DIAG */
diff --git a/sys/dev/sfxge/common/efx_nvram.c b/sys/dev/sfxge/common/efx_nvram.c
new file mode 100644
index 0000000..d4cf741
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_nvram.c
@@ -0,0 +1,372 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_NVRAM
+
+#if EFSYS_OPT_FALCON
+
+static efx_nvram_ops_t	__cs	__efx_nvram_falcon_ops = {
+#if EFSYS_OPT_DIAG
+	falcon_nvram_test,		/* envo_test */
+#endif	/* EFSYS_OPT_DIAG */
+	falcon_nvram_size,		/* envo_size */
+	falcon_nvram_get_version,	/* envo_get_version */
+	falcon_nvram_rw_start,		/* envo_rw_start */
+	falcon_nvram_read_chunk,	/* envo_read_chunk */
+	falcon_nvram_erase,		/* envo_erase */
+	falcon_nvram_write_chunk,	/* envo_write_chunk */
+	falcon_nvram_rw_finish,		/* envo_rw_finish */
+	falcon_nvram_set_version,	/* envo_set_version */
+};
+
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+
+static efx_nvram_ops_t	__cs	__efx_nvram_siena_ops = {
+#if EFSYS_OPT_DIAG
+	siena_nvram_test,		/* envo_test */
+#endif	/* EFSYS_OPT_DIAG */
+	siena_nvram_size,		/* envo_size */
+	siena_nvram_get_version,	/* envo_get_version */
+	siena_nvram_rw_start,		/* envo_rw_start */
+	siena_nvram_read_chunk,		/* envo_read_chunk */
+	siena_nvram_erase,		/* envo_erase */
+	siena_nvram_write_chunk,	/* envo_write_chunk */
+	siena_nvram_rw_finish,		/* envo_rw_finish */
+	siena_nvram_set_version,	/* envo_set_version */
+};
+
+#endif	/* EFSYS_OPT_SIENA */
+
+	__checkReturn	int
+efx_nvram_init(
+	__in		efx_nic_t *enp)
+{
+	efx_nvram_ops_t *envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NVRAM));
+
+	switch (enp->en_family) {
+#if EFSYS_OPT_FALCON
+	case EFX_FAMILY_FALCON:
+		envop = (efx_nvram_ops_t *)&__efx_nvram_falcon_ops;
+		break;
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+	case EFX_FAMILY_SIENA:
+		envop = (efx_nvram_ops_t *)&__efx_nvram_siena_ops;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		EFSYS_ASSERT(0);
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	enp->en_envop = envop;
+	enp->en_mod_flags |= EFX_MOD_NVRAM;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_DIAG
+
+	__checkReturn		int
+efx_nvram_test(
+	__in			efx_nic_t *enp)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	if ((rc = envop->envo_test(enp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_DIAG */
+
+	__checkReturn		int
+efx_nvram_size(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			size_t *sizep)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+
+	if ((rc = envop->envo_size(enp, type, sizep)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_nvram_get_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint32_t *subtypep,
+	__out_ecount(4)		uint16_t version[4])
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+
+	if ((rc = envop->envo_get_version(enp, type, subtypep, version)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_nvram_rw_start(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out_opt		size_t *chunk_sizep)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+	EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID);
+
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, EFX_NVRAM_INVALID);
+
+	if ((rc = envop->envo_rw_start(enp, type, chunk_sizep)) != 0)
+		goto fail1;
+
+	enp->en_nvram_locked = type;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_nvram_read_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+	EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID);
+
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type);
+
+	if ((rc = envop->envo_read_chunk(enp, type, offset, data, size)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_nvram_erase(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+	EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID);
+
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type);
+
+	if ((rc = envop->envo_erase(enp, type)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_nvram_write_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+	EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID);
+
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type);
+
+	if ((rc = envop->envo_write_chunk(enp, type, offset, data, size)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				void
+efx_nvram_rw_finish(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type)
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+	EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID);
+
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type);
+
+	envop->envo_rw_finish(enp, type);
+
+	enp->en_nvram_locked = EFX_NVRAM_INVALID;
+}
+
+	__checkReturn		int
+efx_nvram_set_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint16_t version[4])
+{
+	efx_nvram_ops_t *envop = enp->en_envop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+
+	/*
+	 * The Siena implementation of envo_set_version() will attempt to
+	 * acquire the NVRAM_UPDATE lock for the DYNAMIC_CONFIG sector.
+	 * Therefore, you can't have already acquired the NVRAM_UPDATE lock.
+	 */
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, EFX_NVRAM_INVALID);
+
+	if ((rc = envop->envo_set_version(enp, type, version)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+void
+efx_nvram_fini(
+	__in		efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM);
+
+	EFSYS_ASSERT3U(enp->en_nvram_locked, ==, EFX_NVRAM_INVALID);
+
+	enp->en_envop = NULL;
+	enp->en_mod_flags &= ~EFX_MOD_NVRAM;
+}
+
+#endif	/* EFSYS_OPT_NVRAM */
diff --git a/sys/dev/sfxge/common/efx_phy.c b/sys/dev/sfxge/common/efx_phy.c
new file mode 100644
index 0000000..0b098ec
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_phy.c
@@ -0,0 +1,752 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+#if EFSYS_OPT_FALCON
+#include "falcon_nvram.h"
+#endif
+
+#if EFSYS_OPT_MAC_FALCON_XMAC
+#include "falcon_xmac.h"
+#endif
+
+#if EFSYS_OPT_MAC_FALCON_GMAC
+#include "falcon_gmac.h"
+#endif
+
+#if EFSYS_OPT_PHY_NULL
+#include "nullphy.h"
+#endif
+
+#if EFSYS_OPT_PHY_QT2022C2
+#include "qt2022c2.h"
+#endif
+
+#if EFSYS_OPT_PHY_SFX7101
+#include "sfx7101.h"
+#endif
+
+#if EFSYS_OPT_PHY_TXC43128
+#include "txc43128.h"
+#endif
+
+#if EFSYS_OPT_PHY_SFT9001
+#include "sft9001.h"
+#endif
+
+#if EFSYS_OPT_PHY_QT2025C
+#include "qt2025c.h"
+#endif
+
+#if EFSYS_OPT_PHY_NULL
+static efx_phy_ops_t	__cs __efx_phy_null_ops = {
+	NULL,				/* epo_power */
+	nullphy_reset,			/* epo_reset */
+	nullphy_reconfigure,		/* epo_reconfigure */
+	nullphy_verify,			/* epo_verify */
+	NULL,				/* epo_uplink_check */
+	nullphy_downlink_check,		/* epo_downlink_check */
+	nullphy_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	nullphy_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	nullphy_prop_name,		/* epo_prop_name */
+#endif
+	nullphy_prop_get,		/* epo_prop_get */
+	nullphy_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	NULL,				/* epo_bist_start */
+	NULL,				/* epo_bist_poll */
+	NULL,				/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_PHY_NULL */
+
+#if EFSYS_OPT_PHY_QT2022C2
+static efx_phy_ops_t	__cs __efx_phy_qt2022c2_ops = {
+	NULL,				/* epo_power */
+	qt2022c2_reset,			/* epo_reset */
+	qt2022c2_reconfigure,		/* epo_reconfigure */
+	qt2022c2_verify,		/* epo_verify */
+	qt2022c2_uplink_check,		/* epo_uplink_check */
+	qt2022c2_downlink_check,	/* epo_downlink_check */
+	qt2022c2_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	qt2022c2_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	qt2022c2_prop_name,		/* epo_prop_name */
+#endif
+	qt2022c2_prop_get,		/* epo_prop_get */
+	qt2022c2_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	NULL,				/* epo_bist_start */
+	NULL,				/* epo_bist_poll */
+	NULL,				/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_PHY_QT2022C2 */
+
+#if EFSYS_OPT_PHY_SFX7101
+static efx_phy_ops_t	__cs __efx_phy_sfx7101_ops = {
+	sfx7101_power,			/* epo_power */
+	sfx7101_reset,			/* epo_reset */
+	sfx7101_reconfigure,		/* epo_reconfigure */
+	sfx7101_verify,			/* epo_verify */
+	sfx7101_uplink_check,		/* epo_uplink_check */
+	sfx7101_downlink_check,		/* epo_downlink_check */
+	sfx7101_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	sfx7101_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	sfx7101_prop_name,		/* epo_prop_name */
+#endif
+	sfx7101_prop_get,		/* epo_prop_get */
+	sfx7101_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	NULL,				/* epo_bist_start */
+	NULL,				/* epo_bist_poll */
+	NULL,				/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_PHY_SFX7101 */
+
+#if EFSYS_OPT_PHY_TXC43128
+static efx_phy_ops_t	__cs __efx_phy_txc43128_ops = {
+	NULL,				/* epo_power */
+	txc43128_reset,			/* epo_reset */
+	txc43128_reconfigure,		/* epo_reconfigure */
+	txc43128_verify,		/* epo_verify */
+	txc43128_uplink_check,		/* epo_uplink_check */
+	txc43128_downlink_check,	/* epo_downlink_check */
+	txc43128_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	txc43128_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	txc43128_prop_name,		/* epo_prop_name */
+#endif
+	txc43128_prop_get,		/* epo_prop_get */
+	txc43128_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	NULL,				/* epo_bist_start */
+	NULL,				/* epo_bist_poll */
+	NULL,				/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_PHY_TXC43128 */
+
+#if EFSYS_OPT_PHY_SFT9001
+static efx_phy_ops_t	__cs __efx_phy_sft9001_ops = {
+	NULL,				/* epo_power */
+	sft9001_reset,			/* epo_reset */
+	sft9001_reconfigure,		/* epo_reconfigure */
+	sft9001_verify,			/* epo_verify */
+	sft9001_uplink_check,		/* epo_uplink_check */
+	sft9001_downlink_check,		/* epo_downlink_check */
+	sft9001_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	sft9001_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	sft9001_prop_name,		/* epo_prop_name */
+#endif
+	sft9001_prop_get,		/* epo_prop_get */
+	sft9001_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	sft9001_bist_start,		/* epo_bist_start */
+	sft9001_bist_poll,		/* epo_bist_poll */
+	sft9001_bist_stop,		/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_PHY_SFT9001 */
+
+#if EFSYS_OPT_PHY_QT2025C
+static efx_phy_ops_t	__cs __efx_phy_qt2025c_ops = {
+	NULL,				/* epo_power */
+	qt2025c_reset,			/* epo_reset */
+	qt2025c_reconfigure,		/* epo_reconfigure */
+	qt2025c_verify,			/* epo_verify */
+	qt2025c_uplink_check,		/* epo_uplink_check */
+	qt2025c_downlink_check,		/* epo_downlink_check */
+	qt2025c_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	qt2025c_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	qt2025c_prop_name,		/* epo_prop_name */
+#endif
+	qt2025c_prop_get,		/* epo_prop_get */
+	qt2025c_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	NULL,				/* epo_bist_start */
+	NULL,				/* epo_bist_poll */
+	NULL,				/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_PHY_QT2025C */
+
+#if EFSYS_OPT_SIENA
+static efx_phy_ops_t	__cs __efx_phy_siena_ops = {
+	siena_phy_power,		/* epo_power */
+	NULL,				/* epo_reset */
+	siena_phy_reconfigure,		/* epo_reconfigure */
+	siena_phy_verify,		/* epo_verify */
+	NULL,				/* epo_uplink_check */
+	NULL,				/* epo_downlink_check */
+	siena_phy_oui_get,		/* epo_oui_get */
+#if EFSYS_OPT_PHY_STATS
+	siena_phy_stats_update,		/* epo_stats_update */
+#endif	/* EFSYS_OPT_PHY_STATS */
+#if EFSYS_OPT_PHY_PROPS
+#if EFSYS_OPT_NAMES
+	siena_phy_prop_name,		/* epo_prop_name */
+#endif
+	siena_phy_prop_get,		/* epo_prop_get */
+	siena_phy_prop_set,		/* epo_prop_set */
+#endif	/* EFSYS_OPT_PHY_PROPS */
+#if EFSYS_OPT_PHY_BIST
+	siena_phy_bist_start, 		/* epo_bist_start */
+	siena_phy_bist_poll,		/* epo_bist_poll */
+	siena_phy_bist_stop,		/* epo_bist_stop */
+#endif	/* EFSYS_OPT_PHY_BIST */
+};
+#endif	/* EFSYS_OPT_SIENA */
+
+	__checkReturn	int
+efx_phy_probe(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_phy_ops_t *epop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	epp->ep_port = encp->enc_port;
+	epp->ep_phy_type = encp->enc_phy_type;
+
+	/* Hook in operations structure */
+	switch (enp->en_family) {
+#if EFSYS_OPT_FALCON
+	case EFX_FAMILY_FALCON:
+		switch (epp->ep_phy_type) {
+#if EFSYS_OPT_PHY_NULL
+		case PHY_TYPE_NONE_DECODE:
+			epop = (efx_phy_ops_t *)&__efx_phy_null_ops;
+			break;
+#endif
+#if EFSYS_OPT_PHY_QT2022C2
+		case PHY_TYPE_QT2022C2_DECODE:
+			epop = (efx_phy_ops_t *)&__efx_phy_qt2022c2_ops;
+			break;
+#endif
+#if EFSYS_OPT_PHY_SFX7101
+		case PHY_TYPE_SFX7101_DECODE:
+			epop = (efx_phy_ops_t *)&__efx_phy_sfx7101_ops;
+			break;
+#endif
+#if EFSYS_OPT_PHY_TXC43128
+		case PHY_TYPE_TXC43128_DECODE:
+			epop = (efx_phy_ops_t *)&__efx_phy_txc43128_ops;
+			break;
+#endif
+#if EFSYS_OPT_PHY_SFT9001
+		case PHY_TYPE_SFT9001A_DECODE:
+		case PHY_TYPE_SFT9001B_DECODE:
+			epop = (efx_phy_ops_t *)&__efx_phy_sft9001_ops;
+			break;
+#endif
+#if EFSYS_OPT_PHY_QT2025C
+		case EFX_PHY_QT2025C:
+			epop = (efx_phy_ops_t *)&__efx_phy_qt2025c_ops;
+			break;
+#endif
+		default:
+			rc = ENOTSUP;
+			goto fail1;
+		}
+		break;
+#endif	/* EFSYS_OPT_FALCON */
+#if EFSYS_OPT_SIENA
+	case EFX_FAMILY_SIENA:
+		epop = (efx_phy_ops_t *)&__efx_phy_siena_ops;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	epp->ep_epop = epop;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	epp->ep_port = 0;
+	epp->ep_phy_type = 0;
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_phy_verify(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	return (epop->epo_verify(enp));
+}
+
+#if EFSYS_OPT_PHY_LED_CONTROL
+
+	__checkReturn	int
+efx_phy_led_set(
+	__in		efx_nic_t *enp,
+	__in		efx_phy_led_mode_t mode)
+{
+	efx_nic_cfg_t *encp = (&enp->en_nic_cfg);
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+	uint32_t mask;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	if (epp->ep_phy_led_mode == mode)
+		goto done;
+
+	mask = (1 << EFX_PHY_LED_DEFAULT);
+	mask |= encp->enc_led_mask;
+
+	if (!((1 << mode) & mask)) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	EFSYS_ASSERT3U(mode, <, EFX_PHY_LED_NMODES);
+	epp->ep_phy_led_mode = mode;
+
+	if ((rc = epop->epo_reconfigure(enp)) != 0)
+		goto fail2;
+
+done:
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+#endif	/* EFSYS_OPT_PHY_LED_CONTROL */
+
+			void
+efx_phy_adv_cap_get(
+	__in		efx_nic_t *enp,
+	__in		uint32_t flag,
+	__out		uint32_t *maskp)
+{
+	efx_port_t *epp = &(enp->en_port);
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+
+	switch (flag) {
+	case EFX_PHY_CAP_CURRENT:
+		*maskp = epp->ep_adv_cap_mask;
+		break;
+	case EFX_PHY_CAP_DEFAULT:
+		*maskp = epp->ep_default_adv_cap_mask;
+		break;
+	case EFX_PHY_CAP_PERM:
+		*maskp = epp->ep_phy_cap_mask;
+		break;
+	default:
+		EFSYS_ASSERT(B_FALSE);
+		break;
+	}
+}
+
+	__checkReturn	int
+efx_phy_adv_cap_set(
+	__in		efx_nic_t *enp,
+	__in		uint32_t mask)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	if ((mask & ~epp->ep_phy_cap_mask) != 0) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if (epp->ep_adv_cap_mask == mask)
+		goto done;
+
+	epp->ep_adv_cap_mask = mask;
+
+	if ((rc = epop->epo_reconfigure(enp)) != 0)
+		goto fail2;
+
+done:
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	void
+efx_phy_lp_cap_get(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *maskp)
+{
+	efx_port_t *epp = &(enp->en_port);
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	*maskp = epp->ep_lp_cap_mask;
+}
+
+	__checkReturn	int
+efx_phy_oui_get(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *ouip)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	return (epop->epo_oui_get(enp, ouip));
+}
+
+			void
+efx_phy_media_type_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_media_type_t *typep)
+{
+	efx_port_t *epp = &(enp->en_port);
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	if (epp->ep_module_type != EFX_PHY_MEDIA_INVALID)
+		*typep = epp->ep_module_type;
+	else
+		*typep = epp->ep_fixed_port_type;
+}
+
+#if EFSYS_OPT_PHY_STATS
+
+#if EFSYS_OPT_NAMES
+
+/* START MKCONFIG GENERATED PhyStatNamesBlock 271268f3da0e804f */
+static const char 	__cs * __cs __efx_phy_stat_name[] = {
+	"oui",
+	"pma_pmd_link_up",
+	"pma_pmd_rx_fault",
+	"pma_pmd_tx_fault",
+	"pma_pmd_rev_a",
+	"pma_pmd_rev_b",
+	"pma_pmd_rev_c",
+	"pma_pmd_rev_d",
+	"pcs_link_up",
+	"pcs_rx_fault",
+	"pcs_tx_fault",
+	"pcs_ber",
+	"pcs_block_errors",
+	"phy_xs_link_up",
+	"phy_xs_rx_fault",
+	"phy_xs_tx_fault",
+	"phy_xs_align",
+	"phy_xs_sync_a",
+	"phy_xs_sync_b",
+	"phy_xs_sync_c",
+	"phy_xs_sync_d",
+	"an_link_up",
+	"an_master",
+	"an_local_rx_ok",
+	"an_remote_rx_ok",
+	"cl22ext_link_up",
+	"snr_a",
+	"snr_b",
+	"snr_c",
+	"snr_d",
+	"pma_pmd_signal_a",
+	"pma_pmd_signal_b",
+	"pma_pmd_signal_c",
+	"pma_pmd_signal_d",
+	"an_complete",
+	"pma_pmd_rev_major",
+	"pma_pmd_rev_minor",
+	"pma_pmd_rev_micro",
+	"pcs_fw_version_0",
+	"pcs_fw_version_1",
+	"pcs_fw_version_2",
+	"pcs_fw_version_3",
+	"pcs_fw_build_yy",
+	"pcs_fw_build_mm",
+	"pcs_fw_build_dd",
+	"pcs_op_mode",
+};
+
+/* END MKCONFIG GENERATED PhyStatNamesBlock */
+
+					const char __cs *
+efx_phy_stat_name(
+	__in				efx_nic_t *enp,
+	__in				efx_phy_stat_t type)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(type, <, EFX_PHY_NSTATS);
+
+	return (__efx_phy_stat_name[type]);
+}
+
+#endif	/* EFSYS_OPT_NAMES */
+
+	__checkReturn			int
+efx_phy_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_PHY_NSTATS)	uint32_t *stat)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	return (epop->epo_stats_update(enp, esmp, stat));
+}
+
+#endif	/* EFSYS_OPT_PHY_STATS */
+
+#if EFSYS_OPT_PHY_PROPS
+
+#if EFSYS_OPT_NAMES
+		const char __cs *
+efx_phy_prop_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+
+	return (epop->epo_prop_name(enp, id));
+}
+#endif	/* EFSYS_OPT_NAMES */
+
+	__checkReturn	int
+efx_phy_prop_get(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t flags,
+	__out		uint32_t *valp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	return (epop->epo_prop_get(enp, id, flags, valp));
+}
+
+	__checkReturn	int
+efx_phy_prop_set(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t val)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	return (epop->epo_prop_set(enp, id, val));
+}
+#endif	/* EFSYS_OPT_PHY_STATS */
+
+#if EFSYS_OPT_PHY_BIST
+
+	__checkReturn		int
+efx_phy_bist_start(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	EFSYS_ASSERT3U(type, !=, EFX_PHY_BIST_TYPE_UNKNOWN);
+	EFSYS_ASSERT3U(type, <, EFX_PHY_BIST_TYPE_NTYPES);
+	EFSYS_ASSERT3U(epp->ep_current_bist, ==, EFX_PHY_BIST_TYPE_UNKNOWN);
+
+	if (epop->epo_bist_start == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = epop->epo_bist_start(enp, type)) != 0)
+		goto fail2;
+
+	epp->ep_current_bist = type;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_phy_bist_poll(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type,
+	__out			efx_phy_bist_result_t *resultp,
+	__out_opt		uint32_t *value_maskp,
+	__out_ecount_opt(count)	unsigned long *valuesp,
+	__in			size_t count)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	EFSYS_ASSERT3U(type, !=, EFX_PHY_BIST_TYPE_UNKNOWN);
+	EFSYS_ASSERT3U(type, <, EFX_PHY_BIST_TYPE_NTYPES);
+	EFSYS_ASSERT3U(epp->ep_current_bist, ==, type);
+
+	EFSYS_ASSERT(epop->epo_bist_poll != NULL);
+	if (epop->epo_bist_poll == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = epop->epo_bist_poll(enp, type, resultp, value_maskp,
+	    valuesp, count)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			void
+efx_phy_bist_stop(
+	__in		efx_nic_t *enp,
+	__in		efx_phy_bist_type_t type)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	EFSYS_ASSERT3U(type, !=, EFX_PHY_BIST_TYPE_UNKNOWN);
+	EFSYS_ASSERT3U(type, <, EFX_PHY_BIST_TYPE_NTYPES);
+	EFSYS_ASSERT3U(epp->ep_current_bist, ==, type);
+
+	EFSYS_ASSERT(epop->epo_bist_stop != NULL);
+
+	if (epop->epo_bist_stop != NULL)
+		epop->epo_bist_stop(enp, type);
+
+	epp->ep_current_bist = EFX_PHY_BIST_TYPE_UNKNOWN;
+}
+
+#endif	/* EFSYS_OPT_PHY_BIST */
+			void
+efx_phy_unprobe(
+	__in	efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	epp->ep_epop = NULL;
+
+	epp->ep_adv_cap_mask = 0;
+
+	epp->ep_port = 0;
+	epp->ep_phy_type = 0;
+}
diff --git a/sys/dev/sfxge/common/efx_port.c b/sys/dev/sfxge/common/efx_port.c
new file mode 100644
index 0000000..cbb0b3b
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_port.c
@@ -0,0 +1,226 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_impl.h"
+
+	__checkReturn	int
+efx_port_init(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	if (enp->en_mod_flags & EFX_MOD_PORT) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	enp->en_mod_flags |= EFX_MOD_PORT;
+
+	epp->ep_mac_type = EFX_MAC_INVALID;
+	epp->ep_link_mode = EFX_LINK_UNKNOWN;
+	epp->ep_mac_poll_needed = B_TRUE;
+	epp->ep_mac_drain = B_TRUE;
+
+	/* Configure the MAC */
+	if ((rc = efx_mac_select(enp)) != 0)
+		goto fail1;
+
+	epp->ep_emop->emo_reconfigure(enp);
+
+	/*
+	 * Turn on the PHY if available, otherwise reset it, and
+	 * reconfigure it with the current configuration.
+	 */
+	if (epop->epo_power != NULL) {
+		if ((rc = epop->epo_power(enp, B_TRUE)) != 0)
+			goto fail2;
+	} else {
+		if ((rc = epop->epo_reset(enp)) != 0)
+			goto fail2;
+	}
+
+	EFSYS_ASSERT(enp->en_reset_flags & EFX_RESET_PHY);
+	enp->en_reset_flags &= ~EFX_RESET_PHY;
+
+	if ((rc = epop->epo_reconfigure(enp)) != 0)
+		goto fail3;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	enp->en_mod_flags &= ~EFX_MOD_PORT;
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_port_poll(
+	__in		efx_nic_t *enp,
+	__out		efx_link_mode_t	*link_modep)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	efx_link_mode_t ignore_link_mode;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	EFSYS_ASSERT(emop != NULL);
+	EFSYS_ASSERT(!epp->ep_mac_stats_pending);
+
+	if (link_modep == NULL)
+		link_modep = &ignore_link_mode;
+
+	if ((rc = emop->emo_poll(enp, link_modep)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_LOOPBACK
+
+	__checkReturn	int
+efx_port_loopback_set(
+	__in		efx_nic_t *enp,
+	__in		efx_link_mode_t link_mode,
+	__in		efx_loopback_type_t loopback_type)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_mac_ops_t *emop = epp->ep_emop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+	EFSYS_ASSERT(emop != NULL);
+
+	EFSYS_ASSERT(link_mode < EFX_LINK_NMODES);
+	if ((1 << loopback_type) & ~encp->enc_loopback_types[link_mode]) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if (epp->ep_loopback_type == loopback_type &&
+	    epp->ep_loopback_link_mode == link_mode)
+		return (0);
+
+	if ((rc = emop->emo_loopback_set(enp, link_mode, loopback_type)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_NAMES
+
+static const char 	__cs * __cs __efx_loopback_type_name[] = {
+	"OFF",
+	"DATA",
+	"GMAC",
+	"XGMII",
+	"XGXS",
+	"XAUI",
+	"GMII",
+	"SGMII",
+	"XGBR",
+	"XFI",
+	"XAUI_FAR",
+	"GMII_FAR",
+	"SGMII_FAR",
+	"XFI_FAR",
+	"GPHY",
+	"PHY_XS",
+	"PCS",
+	"PMA_PMD",
+};
+
+	__checkReturn	const char __cs *
+efx_loopback_type_name(
+	__in		efx_nic_t *enp,
+	__in		efx_loopback_type_t type)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(type, <, EFX_LOOPBACK_NTYPES);
+
+	return (__efx_loopback_type_name[type]);
+}
+
+#endif	/* EFSYS_OPT_NAMES */
+
+#endif	/* EFSYS_OPT_LOOPBACK */
+
+			void
+efx_port_fini(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT);
+
+	EFSYS_ASSERT(epp->ep_mac_drain);
+
+	epp->ep_emop = NULL;
+	epp->ep_mac_type = EFX_MAC_INVALID;
+	epp->ep_mac_drain = B_FALSE;
+	epp->ep_mac_poll_needed = B_FALSE;
+
+	/* Turn off the PHY */
+	if (epop->epo_power != NULL)
+		(void) epop->epo_power(enp, B_FALSE);
+
+	enp->en_mod_flags &= ~EFX_MOD_PORT;
+}
diff --git a/sys/dev/sfxge/common/efx_regs.h b/sys/dev/sfxge/common/efx_regs.h
new file mode 100644
index 0000000..c31c33e
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_regs.h
@@ -0,0 +1,3846 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_SYS_EFX_REGS_H
+#define	_SYS_EFX_REGS_H
+
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+
+/*
+ * FR_AB_EE_VPD_CFG0_REG_SF(128bit):
+ * SPI/VPD configuration register 0
+ */
+#define	FR_AB_EE_VPD_CFG0_REG_SF_OFST 0x00000300
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AB_EE_VPD_CFG0_REG(128bit):
+ * SPI/VPD configuration register 0
+ */
+#define	FR_AB_EE_VPD_CFG0_REG_OFST 0x00000140
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_SF_FASTRD_EN_LBN 127
+#define	FRF_AB_EE_SF_FASTRD_EN_WIDTH 1
+#define	FRF_AB_EE_SF_CLOCK_DIV_LBN 120
+#define	FRF_AB_EE_SF_CLOCK_DIV_WIDTH 7
+#define	FRF_AB_EE_VPD_WIP_POLL_LBN 119
+#define	FRF_AB_EE_VPD_WIP_POLL_WIDTH 1
+#define	FRF_AB_EE_EE_CLOCK_DIV_LBN 112
+#define	FRF_AB_EE_EE_CLOCK_DIV_WIDTH 7
+#define	FRF_AB_EE_EE_WR_TMR_VALUE_LBN 96
+#define	FRF_AB_EE_EE_WR_TMR_VALUE_WIDTH 16
+#define	FRF_AB_EE_VPDW_LENGTH_LBN 80
+#define	FRF_AB_EE_VPDW_LENGTH_WIDTH 15
+#define	FRF_AB_EE_VPDW_BASE_LBN 64
+#define	FRF_AB_EE_VPDW_BASE_WIDTH 15
+#define	FRF_AB_EE_VPD_WR_CMD_EN_LBN 56
+#define	FRF_AB_EE_VPD_WR_CMD_EN_WIDTH 8
+#define	FRF_AB_EE_VPD_BASE_LBN 32
+#define	FRF_AB_EE_VPD_BASE_WIDTH 24
+#define	FRF_AB_EE_VPD_LENGTH_LBN 16
+#define	FRF_AB_EE_VPD_LENGTH_WIDTH 15
+#define	FRF_AB_EE_VPD_AD_SIZE_LBN 8
+#define	FRF_AB_EE_VPD_AD_SIZE_WIDTH 5
+#define	FRF_AB_EE_VPD_ACCESS_ON_LBN 5
+#define	FRF_AB_EE_VPD_ACCESS_ON_WIDTH 1
+#define	FRF_AB_EE_VPD_ACCESS_BLOCK_LBN 4
+#define	FRF_AB_EE_VPD_ACCESS_BLOCK_WIDTH 1
+#define	FRF_AB_EE_VPD_DEV_SF_SEL_LBN 2
+#define	FRF_AB_EE_VPD_DEV_SF_SEL_WIDTH 1
+#define	FRF_AB_EE_VPD_EN_AD9_MODE_LBN 1
+#define	FRF_AB_EE_VPD_EN_AD9_MODE_WIDTH 1
+#define	FRF_AB_EE_VPD_EN_LBN 0
+#define	FRF_AB_EE_VPD_EN_WIDTH 1
+
+
+/*
+ * FR_AB_PCIE_SD_CTL0123_REG_SF(128bit):
+ * PCIE SerDes control register 0 to 3
+ */
+#define	FR_AB_PCIE_SD_CTL0123_REG_SF_OFST 0x00000320
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AB_PCIE_SD_CTL0123_REG(128bit):
+ * PCIE SerDes control register 0 to 3
+ */
+#define	FR_AB_PCIE_SD_CTL0123_REG_OFST 0x00000320
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_PCIE_TESTSIG_H_LBN 96
+#define	FRF_AB_PCIE_TESTSIG_H_WIDTH 19
+#define	FRF_AB_PCIE_TESTSIG_L_LBN 64
+#define	FRF_AB_PCIE_TESTSIG_L_WIDTH 19
+#define	FRF_AB_PCIE_OFFSET_LBN 56
+#define	FRF_AB_PCIE_OFFSET_WIDTH 8
+#define	FRF_AB_PCIE_OFFSETEN_H_LBN 55
+#define	FRF_AB_PCIE_OFFSETEN_H_WIDTH 1
+#define	FRF_AB_PCIE_OFFSETEN_L_LBN 54
+#define	FRF_AB_PCIE_OFFSETEN_L_WIDTH 1
+#define	FRF_AB_PCIE_HIVMODE_H_LBN 53
+#define	FRF_AB_PCIE_HIVMODE_H_WIDTH 1
+#define	FRF_AB_PCIE_HIVMODE_L_LBN 52
+#define	FRF_AB_PCIE_HIVMODE_L_WIDTH 1
+#define	FRF_AB_PCIE_PARRESET_H_LBN 51
+#define	FRF_AB_PCIE_PARRESET_H_WIDTH 1
+#define	FRF_AB_PCIE_PARRESET_L_LBN 50
+#define	FRF_AB_PCIE_PARRESET_L_WIDTH 1
+#define	FRF_AB_PCIE_LPBKWDRV_H_LBN 49
+#define	FRF_AB_PCIE_LPBKWDRV_H_WIDTH 1
+#define	FRF_AB_PCIE_LPBKWDRV_L_LBN 48
+#define	FRF_AB_PCIE_LPBKWDRV_L_WIDTH 1
+#define	FRF_AB_PCIE_LPBK_LBN 40
+#define	FRF_AB_PCIE_LPBK_WIDTH 8
+#define	FRF_AB_PCIE_PARLPBK_LBN 32
+#define	FRF_AB_PCIE_PARLPBK_WIDTH 8
+#define	FRF_AB_PCIE_RXTERMADJ_H_LBN 30
+#define	FRF_AB_PCIE_RXTERMADJ_H_WIDTH 2
+#define	FRF_AB_PCIE_RXTERMADJ_L_LBN 28
+#define	FRF_AB_PCIE_RXTERMADJ_L_WIDTH 2
+#define	FFE_AB_PCIE_RXTERMADJ_MIN15PCNT 3
+#define	FFE_AB_PCIE_RXTERMADJ_PL10PCNT 2
+#define	FFE_AB_PCIE_RXTERMADJ_MIN17PCNT 1
+#define	FFE_AB_PCIE_RXTERMADJ_NOMNL 0
+#define	FRF_AB_PCIE_TXTERMADJ_H_LBN 26
+#define	FRF_AB_PCIE_TXTERMADJ_H_WIDTH 2
+#define	FRF_AB_PCIE_TXTERMADJ_L_LBN 24
+#define	FRF_AB_PCIE_TXTERMADJ_L_WIDTH 2
+#define	FFE_AB_PCIE_TXTERMADJ_MIN15PCNT 3
+#define	FFE_AB_PCIE_TXTERMADJ_PL10PCNT 2
+#define	FFE_AB_PCIE_TXTERMADJ_MIN17PCNT 1
+#define	FFE_AB_PCIE_TXTERMADJ_NOMNL 0
+#define	FRF_AB_PCIE_RXEQCTL_H_LBN 18
+#define	FRF_AB_PCIE_RXEQCTL_H_WIDTH 2
+#define	FRF_AB_PCIE_RXEQCTL_L_LBN 16
+#define	FRF_AB_PCIE_RXEQCTL_L_WIDTH 2
+#define	FFE_AB_PCIE_RXEQCTL_OFF_ALT 3
+#define	FFE_AB_PCIE_RXEQCTL_OFF 2
+#define	FFE_AB_PCIE_RXEQCTL_MIN 1
+#define	FFE_AB_PCIE_RXEQCTL_MAX 0
+#define	FRF_AB_PCIE_HIDRV_LBN 8
+#define	FRF_AB_PCIE_HIDRV_WIDTH 8
+#define	FRF_AB_PCIE_LODRV_LBN 0
+#define	FRF_AB_PCIE_LODRV_WIDTH 8
+
+
+/*
+ * FR_AB_PCIE_SD_CTL45_REG_SF(128bit):
+ * PCIE SerDes control register 4 and 5
+ */
+#define	FR_AB_PCIE_SD_CTL45_REG_SF_OFST 0x00000330
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AB_PCIE_SD_CTL45_REG(128bit):
+ * PCIE SerDes control register 4 and 5
+ */
+#define	FR_AB_PCIE_SD_CTL45_REG_OFST 0x00000330
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_PCIE_DTX7_LBN 60
+#define	FRF_AB_PCIE_DTX7_WIDTH 4
+#define	FRF_AB_PCIE_DTX6_LBN 56
+#define	FRF_AB_PCIE_DTX6_WIDTH 4
+#define	FRF_AB_PCIE_DTX5_LBN 52
+#define	FRF_AB_PCIE_DTX5_WIDTH 4
+#define	FRF_AB_PCIE_DTX4_LBN 48
+#define	FRF_AB_PCIE_DTX4_WIDTH 4
+#define	FRF_AB_PCIE_DTX3_LBN 44
+#define	FRF_AB_PCIE_DTX3_WIDTH 4
+#define	FRF_AB_PCIE_DTX2_LBN 40
+#define	FRF_AB_PCIE_DTX2_WIDTH 4
+#define	FRF_AB_PCIE_DTX1_LBN 36
+#define	FRF_AB_PCIE_DTX1_WIDTH 4
+#define	FRF_AB_PCIE_DTX0_LBN 32
+#define	FRF_AB_PCIE_DTX0_WIDTH 4
+#define	FRF_AB_PCIE_DEQ7_LBN 28
+#define	FRF_AB_PCIE_DEQ7_WIDTH 4
+#define	FRF_AB_PCIE_DEQ6_LBN 24
+#define	FRF_AB_PCIE_DEQ6_WIDTH 4
+#define	FRF_AB_PCIE_DEQ5_LBN 20
+#define	FRF_AB_PCIE_DEQ5_WIDTH 4
+#define	FRF_AB_PCIE_DEQ4_LBN 16
+#define	FRF_AB_PCIE_DEQ4_WIDTH 4
+#define	FRF_AB_PCIE_DEQ3_LBN 12
+#define	FRF_AB_PCIE_DEQ3_WIDTH 4
+#define	FRF_AB_PCIE_DEQ2_LBN 8
+#define	FRF_AB_PCIE_DEQ2_WIDTH 4
+#define	FRF_AB_PCIE_DEQ1_LBN 4
+#define	FRF_AB_PCIE_DEQ1_WIDTH 4
+#define	FRF_AB_PCIE_DEQ0_LBN 0
+#define	FRF_AB_PCIE_DEQ0_WIDTH 4
+
+
+/*
+ * FR_AB_PCIE_PCS_CTL_STAT_REG_SF(128bit):
+ * PCIE PCS control and status register
+ */
+#define	FR_AB_PCIE_PCS_CTL_STAT_REG_SF_OFST 0x00000340
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AB_PCIE_PCS_CTL_STAT_REG(128bit):
+ * PCIE PCS control and status register
+ */
+#define	FR_AB_PCIE_PCS_CTL_STAT_REG_OFST 0x00000340
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_PCIE_PRBSERRCOUNT0_H_LBN 52
+#define	FRF_AB_PCIE_PRBSERRCOUNT0_H_WIDTH 4
+#define	FRF_AB_PCIE_PRBSERRCOUNT0_L_LBN 48
+#define	FRF_AB_PCIE_PRBSERRCOUNT0_L_WIDTH 4
+#define	FRF_AB_PCIE_PRBSERR_LBN 40
+#define	FRF_AB_PCIE_PRBSERR_WIDTH 8
+#define	FRF_AB_PCIE_PRBSERRH0_LBN 32
+#define	FRF_AB_PCIE_PRBSERRH0_WIDTH 8
+#define	FRF_AB_PCIE_FASTINIT_H_LBN 15
+#define	FRF_AB_PCIE_FASTINIT_H_WIDTH 1
+#define	FRF_AB_PCIE_FASTINIT_L_LBN 14
+#define	FRF_AB_PCIE_FASTINIT_L_WIDTH 1
+#define	FRF_AB_PCIE_CTCDISABLE_H_LBN 13
+#define	FRF_AB_PCIE_CTCDISABLE_H_WIDTH 1
+#define	FRF_AB_PCIE_CTCDISABLE_L_LBN 12
+#define	FRF_AB_PCIE_CTCDISABLE_L_WIDTH 1
+#define	FRF_AB_PCIE_PRBSSYNC_H_LBN 11
+#define	FRF_AB_PCIE_PRBSSYNC_H_WIDTH 1
+#define	FRF_AB_PCIE_PRBSSYNC_L_LBN 10
+#define	FRF_AB_PCIE_PRBSSYNC_L_WIDTH 1
+#define	FRF_AB_PCIE_PRBSERRACK_H_LBN 9
+#define	FRF_AB_PCIE_PRBSERRACK_H_WIDTH 1
+#define	FRF_AB_PCIE_PRBSERRACK_L_LBN 8
+#define	FRF_AB_PCIE_PRBSERRACK_L_WIDTH 1
+#define	FRF_AB_PCIE_PRBSSEL_LBN 0
+#define	FRF_AB_PCIE_PRBSSEL_WIDTH 8
+
+
+/*
+ * FR_AB_HW_INIT_REG_SF(128bit):
+ * Hardware initialization register
+ */
+#define	FR_AB_HW_INIT_REG_SF_OFST 0x00000350
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AZ_HW_INIT_REG(128bit):
+ * Hardware initialization register
+ */
+#define	FR_AZ_HW_INIT_REG_OFST 0x000000c0
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_BB_BDMRD_CPLF_FULL_LBN 124
+#define	FRF_BB_BDMRD_CPLF_FULL_WIDTH 1
+#define	FRF_BB_PCIE_CPL_TIMEOUT_CTRL_LBN 121
+#define	FRF_BB_PCIE_CPL_TIMEOUT_CTRL_WIDTH 3
+#define	FRF_CZ_TX_MRG_TAGS_LBN 120
+#define	FRF_CZ_TX_MRG_TAGS_WIDTH 1
+#define	FRF_AZ_TRGT_MASK_ALL_LBN 100
+#define	FRF_AZ_TRGT_MASK_ALL_WIDTH 1
+#define	FRF_AZ_DOORBELL_DROP_LBN 92
+#define	FRF_AZ_DOORBELL_DROP_WIDTH 8
+#define	FRF_AB_TX_RREQ_MASK_EN_LBN 76
+#define	FRF_AB_TX_RREQ_MASK_EN_WIDTH 1
+#define	FRF_AB_PE_EIDLE_DIS_LBN 75
+#define	FRF_AB_PE_EIDLE_DIS_WIDTH 1
+#define	FRF_AZ_FC_BLOCKING_EN_LBN 45
+#define	FRF_AZ_FC_BLOCKING_EN_WIDTH 1
+#define	FRF_AZ_B2B_REQ_EN_LBN 44
+#define	FRF_AZ_B2B_REQ_EN_WIDTH 1
+#define	FRF_AZ_POST_WR_MASK_LBN 40
+#define	FRF_AZ_POST_WR_MASK_WIDTH 4
+#define	FRF_AZ_TLP_TC_LBN 34
+#define	FRF_AZ_TLP_TC_WIDTH 3
+#define	FRF_AZ_TLP_ATTR_LBN 32
+#define	FRF_AZ_TLP_ATTR_WIDTH 2
+#define	FRF_AB_INTB_VEC_LBN 24
+#define	FRF_AB_INTB_VEC_WIDTH 5
+#define	FRF_AB_INTA_VEC_LBN 16
+#define	FRF_AB_INTA_VEC_WIDTH 5
+#define	FRF_AZ_WD_TIMER_LBN 8
+#define	FRF_AZ_WD_TIMER_WIDTH 8
+#define	FRF_AZ_US_DISABLE_LBN 5
+#define	FRF_AZ_US_DISABLE_WIDTH 1
+#define	FRF_AZ_TLP_EP_LBN 4
+#define	FRF_AZ_TLP_EP_WIDTH 1
+#define	FRF_AZ_ATTR_SEL_LBN 3
+#define	FRF_AZ_ATTR_SEL_WIDTH 1
+#define	FRF_AZ_TD_SEL_LBN 1
+#define	FRF_AZ_TD_SEL_WIDTH 1
+#define	FRF_AZ_TLP_TD_LBN 0
+#define	FRF_AZ_TLP_TD_WIDTH 1
+
+
+/*
+ * FR_AB_NIC_STAT_REG_SF(128bit):
+ * NIC status register
+ */
+#define	FR_AB_NIC_STAT_REG_SF_OFST 0x00000360
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AB_NIC_STAT_REG(128bit):
+ * NIC status register
+ */
+#define	FR_AB_NIC_STAT_REG_OFST 0x00000200
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_BB_AER_DIS_LBN 34
+#define	FRF_BB_AER_DIS_WIDTH 1
+#define	FRF_BB_EE_STRAP_EN_LBN 31
+#define	FRF_BB_EE_STRAP_EN_WIDTH 1
+#define	FRF_BB_EE_STRAP_LBN 24
+#define	FRF_BB_EE_STRAP_WIDTH 4
+#define	FRF_BB_REVISION_ID_LBN 17
+#define	FRF_BB_REVISION_ID_WIDTH 7
+#define	FRF_AB_ONCHIP_SRAM_LBN 16
+#define	FRF_AB_ONCHIP_SRAM_WIDTH 1
+#define	FRF_AB_SF_PRST_LBN 9
+#define	FRF_AB_SF_PRST_WIDTH 1
+#define	FRF_AB_EE_PRST_LBN 8
+#define	FRF_AB_EE_PRST_WIDTH 1
+#define	FRF_AB_ATE_MODE_LBN 3
+#define	FRF_AB_ATE_MODE_WIDTH 1
+#define	FRF_AB_STRAP_PINS_LBN 0
+#define	FRF_AB_STRAP_PINS_WIDTH 3
+
+
+/*
+ * FR_AB_GLB_CTL_REG_SF(128bit):
+ * Global control register
+ */
+#define	FR_AB_GLB_CTL_REG_SF_OFST 0x00000370
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AB_GLB_CTL_REG(128bit):
+ * Global control register
+ */
+#define	FR_AB_GLB_CTL_REG_OFST 0x00000220
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EXT_PHY_RST_CTL_LBN 63
+#define	FRF_AB_EXT_PHY_RST_CTL_WIDTH 1
+#define	FRF_AB_XAUI_SD_RST_CTL_LBN 62
+#define	FRF_AB_XAUI_SD_RST_CTL_WIDTH 1
+#define	FRF_AB_PCIE_SD_RST_CTL_LBN 61
+#define	FRF_AB_PCIE_SD_RST_CTL_WIDTH 1
+#define	FRF_AA_PCIX_RST_CTL_LBN 60
+#define	FRF_AA_PCIX_RST_CTL_WIDTH 1
+#define	FRF_BB_BIU_RST_CTL_LBN 60
+#define	FRF_BB_BIU_RST_CTL_WIDTH 1
+#define	FRF_AB_PCIE_STKY_RST_CTL_LBN 59
+#define	FRF_AB_PCIE_STKY_RST_CTL_WIDTH 1
+#define	FRF_AB_PCIE_NSTKY_RST_CTL_LBN 58
+#define	FRF_AB_PCIE_NSTKY_RST_CTL_WIDTH 1
+#define	FRF_AB_PCIE_CORE_RST_CTL_LBN 57
+#define	FRF_AB_PCIE_CORE_RST_CTL_WIDTH 1
+#define	FRF_AB_XGRX_RST_CTL_LBN 56
+#define	FRF_AB_XGRX_RST_CTL_WIDTH 1
+#define	FRF_AB_XGTX_RST_CTL_LBN 55
+#define	FRF_AB_XGTX_RST_CTL_WIDTH 1
+#define	FRF_AB_EM_RST_CTL_LBN 54
+#define	FRF_AB_EM_RST_CTL_WIDTH 1
+#define	FRF_AB_EV_RST_CTL_LBN 53
+#define	FRF_AB_EV_RST_CTL_WIDTH 1
+#define	FRF_AB_SR_RST_CTL_LBN 52
+#define	FRF_AB_SR_RST_CTL_WIDTH 1
+#define	FRF_AB_RX_RST_CTL_LBN 51
+#define	FRF_AB_RX_RST_CTL_WIDTH 1
+#define	FRF_AB_TX_RST_CTL_LBN 50
+#define	FRF_AB_TX_RST_CTL_WIDTH 1
+#define	FRF_AB_EE_RST_CTL_LBN 49
+#define	FRF_AB_EE_RST_CTL_WIDTH 1
+#define	FRF_AB_CS_RST_CTL_LBN 48
+#define	FRF_AB_CS_RST_CTL_WIDTH 1
+#define	FRF_AB_HOT_RST_CTL_LBN 40
+#define	FRF_AB_HOT_RST_CTL_WIDTH 2
+#define	FRF_AB_RST_EXT_PHY_LBN 31
+#define	FRF_AB_RST_EXT_PHY_WIDTH 1
+#define	FRF_AB_RST_XAUI_SD_LBN 30
+#define	FRF_AB_RST_XAUI_SD_WIDTH 1
+#define	FRF_AB_RST_PCIE_SD_LBN 29
+#define	FRF_AB_RST_PCIE_SD_WIDTH 1
+#define	FRF_AA_RST_PCIX_LBN 28
+#define	FRF_AA_RST_PCIX_WIDTH 1
+#define	FRF_BB_RST_BIU_LBN 28
+#define	FRF_BB_RST_BIU_WIDTH 1
+#define	FRF_AB_RST_PCIE_STKY_LBN 27
+#define	FRF_AB_RST_PCIE_STKY_WIDTH 1
+#define	FRF_AB_RST_PCIE_NSTKY_LBN 26
+#define	FRF_AB_RST_PCIE_NSTKY_WIDTH 1
+#define	FRF_AB_RST_PCIE_CORE_LBN 25
+#define	FRF_AB_RST_PCIE_CORE_WIDTH 1
+#define	FRF_AB_RST_XGRX_LBN 24
+#define	FRF_AB_RST_XGRX_WIDTH 1
+#define	FRF_AB_RST_XGTX_LBN 23
+#define	FRF_AB_RST_XGTX_WIDTH 1
+#define	FRF_AB_RST_EM_LBN 22
+#define	FRF_AB_RST_EM_WIDTH 1
+#define	FRF_AB_RST_EV_LBN 21
+#define	FRF_AB_RST_EV_WIDTH 1
+#define	FRF_AB_RST_SR_LBN 20
+#define	FRF_AB_RST_SR_WIDTH 1
+#define	FRF_AB_RST_RX_LBN 19
+#define	FRF_AB_RST_RX_WIDTH 1
+#define	FRF_AB_RST_TX_LBN 18
+#define	FRF_AB_RST_TX_WIDTH 1
+#define	FRF_AB_RST_SF_LBN 17
+#define	FRF_AB_RST_SF_WIDTH 1
+#define	FRF_AB_RST_CS_LBN 16
+#define	FRF_AB_RST_CS_WIDTH 1
+#define	FRF_AB_INT_RST_DUR_LBN 4
+#define	FRF_AB_INT_RST_DUR_WIDTH 3
+#define	FRF_AB_EXT_PHY_RST_DUR_LBN 1
+#define	FRF_AB_EXT_PHY_RST_DUR_WIDTH 3
+#define	FFE_AB_EXT_PHY_RST_DUR_10240US 7
+#define	FFE_AB_EXT_PHY_RST_DUR_5120US 6
+#define	FFE_AB_EXT_PHY_RST_DUR_2560US 5
+#define	FFE_AB_EXT_PHY_RST_DUR_1280US 4
+#define	FFE_AB_EXT_PHY_RST_DUR_640US 3
+#define	FFE_AB_EXT_PHY_RST_DUR_320US 2
+#define	FFE_AB_EXT_PHY_RST_DUR_160US 1
+#define	FFE_AB_EXT_PHY_RST_DUR_80US 0
+#define	FRF_AB_SWRST_LBN 0
+#define	FRF_AB_SWRST_WIDTH 1
+
+
+/*
+ * FR_AZ_IOM_IND_ADR_REG(32bit):
+ * IO-mapped indirect access address register
+ */
+#define	FR_AZ_IOM_IND_ADR_REG_OFST 0x00000000
+/* falcona0,falconb0,sienaa0=net_func_bar0 */
+
+#define	FRF_AZ_IOM_AUTO_ADR_INC_EN_LBN 24
+#define	FRF_AZ_IOM_AUTO_ADR_INC_EN_WIDTH 1
+#define	FRF_AZ_IOM_IND_ADR_LBN 0
+#define	FRF_AZ_IOM_IND_ADR_WIDTH 24
+
+
+/*
+ * FR_AZ_IOM_IND_DAT_REG(32bit):
+ * IO-mapped indirect access data register
+ */
+#define	FR_AZ_IOM_IND_DAT_REG_OFST 0x00000004
+/* falcona0,falconb0,sienaa0=net_func_bar0 */
+
+#define	FRF_AZ_IOM_IND_DAT_LBN 0
+#define	FRF_AZ_IOM_IND_DAT_WIDTH 32
+
+
+/*
+ * FR_AZ_ADR_REGION_REG(128bit):
+ * Address region register
+ */
+#define	FR_AZ_ADR_REGION_REG_OFST 0x00000000
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_ADR_REGION3_LBN 96
+#define	FRF_AZ_ADR_REGION3_WIDTH 18
+#define	FRF_AZ_ADR_REGION2_LBN 64
+#define	FRF_AZ_ADR_REGION2_WIDTH 18
+#define	FRF_AZ_ADR_REGION1_LBN 32
+#define	FRF_AZ_ADR_REGION1_WIDTH 18
+#define	FRF_AZ_ADR_REGION0_LBN 0
+#define	FRF_AZ_ADR_REGION0_WIDTH 18
+
+
+/*
+ * FR_AZ_INT_EN_REG_KER(128bit):
+ * Kernel driver Interrupt enable register
+ */
+#define	FR_AZ_INT_EN_REG_KER_OFST 0x00000010
+/* falcona0,falconb0,sienaa0=net_func_bar2 */
+
+#define	FRF_AZ_KER_INT_LEVE_SEL_LBN 8
+#define	FRF_AZ_KER_INT_LEVE_SEL_WIDTH 6
+#define	FRF_AZ_KER_INT_CHAR_LBN 4
+#define	FRF_AZ_KER_INT_CHAR_WIDTH 1
+#define	FRF_AZ_KER_INT_KER_LBN 3
+#define	FRF_AZ_KER_INT_KER_WIDTH 1
+#define	FRF_AZ_DRV_INT_EN_KER_LBN 0
+#define	FRF_AZ_DRV_INT_EN_KER_WIDTH 1
+
+
+/*
+ * FR_AZ_INT_EN_REG_CHAR(128bit):
+ * Char Driver interrupt enable register
+ */
+#define	FR_AZ_INT_EN_REG_CHAR_OFST 0x00000020
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_CHAR_INT_LEVE_SEL_LBN 8
+#define	FRF_AZ_CHAR_INT_LEVE_SEL_WIDTH 6
+#define	FRF_AZ_CHAR_INT_CHAR_LBN 4
+#define	FRF_AZ_CHAR_INT_CHAR_WIDTH 1
+#define	FRF_AZ_CHAR_INT_KER_LBN 3
+#define	FRF_AZ_CHAR_INT_KER_WIDTH 1
+#define	FRF_AZ_DRV_INT_EN_CHAR_LBN 0
+#define	FRF_AZ_DRV_INT_EN_CHAR_WIDTH 1
+
+
+/*
+ * FR_AZ_INT_ADR_REG_KER(128bit):
+ * Interrupt host address for Kernel driver
+ */
+#define	FR_AZ_INT_ADR_REG_KER_OFST 0x00000030
+/* falcona0,falconb0,sienaa0=net_func_bar2 */
+
+#define	FRF_AZ_NORM_INT_VEC_DIS_KER_LBN 64
+#define	FRF_AZ_NORM_INT_VEC_DIS_KER_WIDTH 1
+#define	FRF_AZ_INT_ADR_KER_LBN 0
+#define	FRF_AZ_INT_ADR_KER_WIDTH 64
+#define	FRF_AZ_INT_ADR_KER_DW0_LBN 0
+#define	FRF_AZ_INT_ADR_KER_DW0_WIDTH 32
+#define	FRF_AZ_INT_ADR_KER_DW1_LBN 32
+#define	FRF_AZ_INT_ADR_KER_DW1_WIDTH 32
+
+
+/*
+ * FR_AZ_INT_ADR_REG_CHAR(128bit):
+ * Interrupt host address for Char driver
+ */
+#define	FR_AZ_INT_ADR_REG_CHAR_OFST 0x00000040
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_NORM_INT_VEC_DIS_CHAR_LBN 64
+#define	FRF_AZ_NORM_INT_VEC_DIS_CHAR_WIDTH 1
+#define	FRF_AZ_INT_ADR_CHAR_LBN 0
+#define	FRF_AZ_INT_ADR_CHAR_WIDTH 64
+#define	FRF_AZ_INT_ADR_CHAR_DW0_LBN 0
+#define	FRF_AZ_INT_ADR_CHAR_DW0_WIDTH 32
+#define	FRF_AZ_INT_ADR_CHAR_DW1_LBN 32
+#define	FRF_AZ_INT_ADR_CHAR_DW1_WIDTH 32
+
+
+/*
+ * FR_AA_INT_ACK_KER(32bit):
+ * Kernel interrupt acknowledge register
+ */
+#define	FR_AA_INT_ACK_KER_OFST 0x00000050
+/* falcona0=net_func_bar2 */
+
+#define	FRF_AA_INT_ACK_KER_FIELD_LBN 0
+#define	FRF_AA_INT_ACK_KER_FIELD_WIDTH 32
+
+
+/*
+ * FR_BZ_INT_ISR0_REG(128bit):
+ * Function 0 Interrupt Acknowlege Status register
+ */
+#define	FR_BZ_INT_ISR0_REG_OFST 0x00000090
+/* falconb0,sienaa0=net_func_bar2 */
+
+#define	FRF_BZ_INT_ISR_REG_LBN 0
+#define	FRF_BZ_INT_ISR_REG_WIDTH 64
+#define	FRF_BZ_INT_ISR_REG_DW0_LBN 0
+#define	FRF_BZ_INT_ISR_REG_DW0_WIDTH 32
+#define	FRF_BZ_INT_ISR_REG_DW1_LBN 32
+#define	FRF_BZ_INT_ISR_REG_DW1_WIDTH 32
+
+
+/*
+ * FR_AB_EE_SPI_HCMD_REG(128bit):
+ * SPI host command register
+ */
+#define	FR_AB_EE_SPI_HCMD_REG_OFST 0x00000100
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_SPI_HCMD_CMD_EN_LBN 31
+#define	FRF_AB_EE_SPI_HCMD_CMD_EN_WIDTH 1
+#define	FRF_AB_EE_WR_TIMER_ACTIVE_LBN 28
+#define	FRF_AB_EE_WR_TIMER_ACTIVE_WIDTH 1
+#define	FRF_AB_EE_SPI_HCMD_SF_SEL_LBN 24
+#define	FRF_AB_EE_SPI_HCMD_SF_SEL_WIDTH 1
+#define	FRF_AB_EE_SPI_HCMD_DABCNT_LBN 16
+#define	FRF_AB_EE_SPI_HCMD_DABCNT_WIDTH 5
+#define	FRF_AB_EE_SPI_HCMD_READ_LBN 15
+#define	FRF_AB_EE_SPI_HCMD_READ_WIDTH 1
+#define	FRF_AB_EE_SPI_HCMD_DUBCNT_LBN 12
+#define	FRF_AB_EE_SPI_HCMD_DUBCNT_WIDTH 2
+#define	FRF_AB_EE_SPI_HCMD_ADBCNT_LBN 8
+#define	FRF_AB_EE_SPI_HCMD_ADBCNT_WIDTH 2
+#define	FRF_AB_EE_SPI_HCMD_ENC_LBN 0
+#define	FRF_AB_EE_SPI_HCMD_ENC_WIDTH 8
+
+
+/*
+ * FR_CZ_USR_EV_CFG(32bit):
+ * User Level Event Configuration register
+ */
+#define	FR_CZ_USR_EV_CFG_OFST 0x00000100
+/* sienaa0=net_func_bar2 */
+
+#define	FRF_CZ_USREV_DIS_LBN 16
+#define	FRF_CZ_USREV_DIS_WIDTH 1
+#define	FRF_CZ_DFLT_EVQ_LBN 0
+#define	FRF_CZ_DFLT_EVQ_WIDTH 10
+
+
+/*
+ * FR_AB_EE_SPI_HADR_REG(128bit):
+ * SPI host address register
+ */
+#define	FR_AB_EE_SPI_HADR_REG_OFST 0x00000110
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_SPI_HADR_DUBYTE_LBN 24
+#define	FRF_AB_EE_SPI_HADR_DUBYTE_WIDTH 8
+#define	FRF_AB_EE_SPI_HADR_ADR_LBN 0
+#define	FRF_AB_EE_SPI_HADR_ADR_WIDTH 24
+
+
+/*
+ * FR_AB_EE_SPI_HDATA_REG(128bit):
+ * SPI host data register
+ */
+#define	FR_AB_EE_SPI_HDATA_REG_OFST 0x00000120
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_SPI_HDATA3_LBN 96
+#define	FRF_AB_EE_SPI_HDATA3_WIDTH 32
+#define	FRF_AB_EE_SPI_HDATA2_LBN 64
+#define	FRF_AB_EE_SPI_HDATA2_WIDTH 32
+#define	FRF_AB_EE_SPI_HDATA1_LBN 32
+#define	FRF_AB_EE_SPI_HDATA1_WIDTH 32
+#define	FRF_AB_EE_SPI_HDATA0_LBN 0
+#define	FRF_AB_EE_SPI_HDATA0_WIDTH 32
+
+
+/*
+ * FR_AB_EE_BASE_PAGE_REG(128bit):
+ * Expansion ROM base mirror register
+ */
+#define	FR_AB_EE_BASE_PAGE_REG_OFST 0x00000130
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_EXPROM_MASK_LBN 16
+#define	FRF_AB_EE_EXPROM_MASK_WIDTH 13
+#define	FRF_AB_EE_EXP_ROM_WINDOW_BASE_LBN 0
+#define	FRF_AB_EE_EXP_ROM_WINDOW_BASE_WIDTH 13
+
+
+/*
+ * FR_AB_EE_VPD_SW_CNTL_REG(128bit):
+ * VPD access SW control register
+ */
+#define	FR_AB_EE_VPD_SW_CNTL_REG_OFST 0x00000150
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_VPD_CYCLE_PENDING_LBN 31
+#define	FRF_AB_EE_VPD_CYCLE_PENDING_WIDTH 1
+#define	FRF_AB_EE_VPD_CYC_WRITE_LBN 28
+#define	FRF_AB_EE_VPD_CYC_WRITE_WIDTH 1
+#define	FRF_AB_EE_VPD_CYC_ADR_LBN 0
+#define	FRF_AB_EE_VPD_CYC_ADR_WIDTH 15
+
+
+/*
+ * FR_AB_EE_VPD_SW_DATA_REG(128bit):
+ * VPD access SW data register
+ */
+#define	FR_AB_EE_VPD_SW_DATA_REG_OFST 0x00000160
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_EE_VPD_CYC_DAT_LBN 0
+#define	FRF_AB_EE_VPD_CYC_DAT_WIDTH 32
+
+
+/*
+ * FR_BB_PCIE_CORE_INDIRECT_REG(64bit):
+ * Indirect Access to PCIE Core registers
+ */
+#define	FR_BB_PCIE_CORE_INDIRECT_REG_OFST 0x000001f0
+/* falconb0=net_func_bar2 */
+
+#define	FRF_BB_PCIE_CORE_TARGET_DATA_LBN 32
+#define	FRF_BB_PCIE_CORE_TARGET_DATA_WIDTH 32
+#define	FRF_BB_PCIE_CORE_INDIRECT_ACCESS_DIR_LBN 15
+#define	FRF_BB_PCIE_CORE_INDIRECT_ACCESS_DIR_WIDTH 1
+#define	FRF_BB_PCIE_CORE_TARGET_REG_ADRS_LBN 0
+#define	FRF_BB_PCIE_CORE_TARGET_REG_ADRS_WIDTH 12
+
+
+/*
+ * FR_AB_GPIO_CTL_REG(128bit):
+ * GPIO control register
+ */
+#define	FR_AB_GPIO_CTL_REG_OFST 0x00000210
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GPIO15_OEN_LBN 63
+#define	FRF_AB_GPIO15_OEN_WIDTH 1
+#define	FRF_AB_GPIO14_OEN_LBN 62
+#define	FRF_AB_GPIO14_OEN_WIDTH 1
+#define	FRF_AB_GPIO13_OEN_LBN 61
+#define	FRF_AB_GPIO13_OEN_WIDTH 1
+#define	FRF_AB_GPIO12_OEN_LBN 60
+#define	FRF_AB_GPIO12_OEN_WIDTH 1
+#define	FRF_AB_GPIO11_OEN_LBN 59
+#define	FRF_AB_GPIO11_OEN_WIDTH 1
+#define	FRF_AB_GPIO10_OEN_LBN 58
+#define	FRF_AB_GPIO10_OEN_WIDTH 1
+#define	FRF_AB_GPIO9_OEN_LBN 57
+#define	FRF_AB_GPIO9_OEN_WIDTH 1
+#define	FRF_AB_GPIO8_OEN_LBN 56
+#define	FRF_AB_GPIO8_OEN_WIDTH 1
+#define	FRF_AB_GPIO15_OUT_LBN 55
+#define	FRF_AB_GPIO15_OUT_WIDTH 1
+#define	FRF_AB_GPIO14_OUT_LBN 54
+#define	FRF_AB_GPIO14_OUT_WIDTH 1
+#define	FRF_AB_GPIO13_OUT_LBN 53
+#define	FRF_AB_GPIO13_OUT_WIDTH 1
+#define	FRF_AB_GPIO12_OUT_LBN 52
+#define	FRF_AB_GPIO12_OUT_WIDTH 1
+#define	FRF_AB_GPIO11_OUT_LBN 51
+#define	FRF_AB_GPIO11_OUT_WIDTH 1
+#define	FRF_AB_GPIO10_OUT_LBN 50
+#define	FRF_AB_GPIO10_OUT_WIDTH 1
+#define	FRF_AB_GPIO9_OUT_LBN 49
+#define	FRF_AB_GPIO9_OUT_WIDTH 1
+#define	FRF_AB_GPIO8_OUT_LBN 48
+#define	FRF_AB_GPIO8_OUT_WIDTH 1
+#define	FRF_AB_GPIO15_IN_LBN 47
+#define	FRF_AB_GPIO15_IN_WIDTH 1
+#define	FRF_AB_GPIO14_IN_LBN 46
+#define	FRF_AB_GPIO14_IN_WIDTH 1
+#define	FRF_AB_GPIO13_IN_LBN 45
+#define	FRF_AB_GPIO13_IN_WIDTH 1
+#define	FRF_AB_GPIO12_IN_LBN 44
+#define	FRF_AB_GPIO12_IN_WIDTH 1
+#define	FRF_AB_GPIO11_IN_LBN 43
+#define	FRF_AB_GPIO11_IN_WIDTH 1
+#define	FRF_AB_GPIO10_IN_LBN 42
+#define	FRF_AB_GPIO10_IN_WIDTH 1
+#define	FRF_AB_GPIO9_IN_LBN 41
+#define	FRF_AB_GPIO9_IN_WIDTH 1
+#define	FRF_AB_GPIO8_IN_LBN 40
+#define	FRF_AB_GPIO8_IN_WIDTH 1
+#define	FRF_AB_GPIO15_PWRUP_VALUE_LBN 39
+#define	FRF_AB_GPIO15_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO14_PWRUP_VALUE_LBN 38
+#define	FRF_AB_GPIO14_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO13_PWRUP_VALUE_LBN 37
+#define	FRF_AB_GPIO13_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO12_PWRUP_VALUE_LBN 36
+#define	FRF_AB_GPIO12_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO11_PWRUP_VALUE_LBN 35
+#define	FRF_AB_GPIO11_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO10_PWRUP_VALUE_LBN 34
+#define	FRF_AB_GPIO10_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO9_PWRUP_VALUE_LBN 33
+#define	FRF_AB_GPIO9_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO8_PWRUP_VALUE_LBN 32
+#define	FRF_AB_GPIO8_PWRUP_VALUE_WIDTH 1
+#define	FRF_BB_CLK156_OUT_EN_LBN 31
+#define	FRF_BB_CLK156_OUT_EN_WIDTH 1
+#define	FRF_BB_USE_NIC_CLK_LBN 30
+#define	FRF_BB_USE_NIC_CLK_WIDTH 1
+#define	FRF_AB_GPIO5_OEN_LBN 29
+#define	FRF_AB_GPIO5_OEN_WIDTH 1
+#define	FRF_AB_GPIO4_OEN_LBN 28
+#define	FRF_AB_GPIO4_OEN_WIDTH 1
+#define	FRF_AB_GPIO3_OEN_LBN 27
+#define	FRF_AB_GPIO3_OEN_WIDTH 1
+#define	FRF_AB_GPIO2_OEN_LBN 26
+#define	FRF_AB_GPIO2_OEN_WIDTH 1
+#define	FRF_AB_GPIO1_OEN_LBN 25
+#define	FRF_AB_GPIO1_OEN_WIDTH 1
+#define	FRF_AB_GPIO0_OEN_LBN 24
+#define	FRF_AB_GPIO0_OEN_WIDTH 1
+#define	FRF_AB_GPIO5_OUT_LBN 21
+#define	FRF_AB_GPIO5_OUT_WIDTH 1
+#define	FRF_AB_GPIO4_OUT_LBN 20
+#define	FRF_AB_GPIO4_OUT_WIDTH 1
+#define	FRF_AB_GPIO3_OUT_LBN 19
+#define	FRF_AB_GPIO3_OUT_WIDTH 1
+#define	FRF_AB_GPIO2_OUT_LBN 18
+#define	FRF_AB_GPIO2_OUT_WIDTH 1
+#define	FRF_AB_GPIO1_OUT_LBN 17
+#define	FRF_AB_GPIO1_OUT_WIDTH 1
+#define	FRF_AB_GPIO0_OUT_LBN 16
+#define	FRF_AB_GPIO0_OUT_WIDTH 1
+#define	FRF_AB_GPIO5_IN_LBN 13
+#define	FRF_AB_GPIO5_IN_WIDTH 1
+#define	FRF_AB_GPIO4_IN_LBN 12
+#define	FRF_AB_GPIO4_IN_WIDTH 1
+#define	FRF_AB_GPIO3_IN_LBN 11
+#define	FRF_AB_GPIO3_IN_WIDTH 1
+#define	FRF_AB_GPIO2_IN_LBN 10
+#define	FRF_AB_GPIO2_IN_WIDTH 1
+#define	FRF_AB_GPIO1_IN_LBN 9
+#define	FRF_AB_GPIO1_IN_WIDTH 1
+#define	FRF_AB_GPIO0_IN_LBN 8
+#define	FRF_AB_GPIO0_IN_WIDTH 1
+#define	FRF_AB_GPIO5_PWRUP_VALUE_LBN 5
+#define	FRF_AB_GPIO5_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO4_PWRUP_VALUE_LBN 4
+#define	FRF_AB_GPIO4_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO3_PWRUP_VALUE_LBN 3
+#define	FRF_AB_GPIO3_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO2_PWRUP_VALUE_LBN 2
+#define	FRF_AB_GPIO2_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO1_PWRUP_VALUE_LBN 1
+#define	FRF_AB_GPIO1_PWRUP_VALUE_WIDTH 1
+#define	FRF_AB_GPIO0_PWRUP_VALUE_LBN 0
+#define	FRF_AB_GPIO0_PWRUP_VALUE_WIDTH 1
+
+
+/*
+ * FR_AZ_FATAL_INTR_REG_KER(128bit):
+ * Fatal interrupt register for Kernel
+ */
+#define	FR_AZ_FATAL_INTR_REG_KER_OFST 0x00000230
+/* falcona0,falconb0,sienaa0=net_func_bar2 */
+
+#define	FRF_CZ_SRAM_PERR_INT_P_KER_EN_LBN 44
+#define	FRF_CZ_SRAM_PERR_INT_P_KER_EN_WIDTH 1
+#define	FRF_AB_PCI_BUSERR_INT_KER_EN_LBN 43
+#define	FRF_AB_PCI_BUSERR_INT_KER_EN_WIDTH 1
+#define	FRF_CZ_MBU_PERR_INT_KER_EN_LBN 43
+#define	FRF_CZ_MBU_PERR_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_SRAM_OOB_INT_KER_EN_LBN 42
+#define	FRF_AZ_SRAM_OOB_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_BUFID_OOB_INT_KER_EN_LBN 41
+#define	FRF_AZ_BUFID_OOB_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_MEM_PERR_INT_KER_EN_LBN 40
+#define	FRF_AZ_MEM_PERR_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_RBUF_OWN_INT_KER_EN_LBN 39
+#define	FRF_AZ_RBUF_OWN_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_TBUF_OWN_INT_KER_EN_LBN 38
+#define	FRF_AZ_TBUF_OWN_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_RDESCQ_OWN_INT_KER_EN_LBN 37
+#define	FRF_AZ_RDESCQ_OWN_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_TDESCQ_OWN_INT_KER_EN_LBN 36
+#define	FRF_AZ_TDESCQ_OWN_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_EVQ_OWN_INT_KER_EN_LBN 35
+#define	FRF_AZ_EVQ_OWN_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_EVF_OFLO_INT_KER_EN_LBN 34
+#define	FRF_AZ_EVF_OFLO_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_ILL_ADR_INT_KER_EN_LBN 33
+#define	FRF_AZ_ILL_ADR_INT_KER_EN_WIDTH 1
+#define	FRF_AZ_SRM_PERR_INT_KER_EN_LBN 32
+#define	FRF_AZ_SRM_PERR_INT_KER_EN_WIDTH 1
+#define	FRF_CZ_SRAM_PERR_INT_P_KER_LBN 12
+#define	FRF_CZ_SRAM_PERR_INT_P_KER_WIDTH 1
+#define	FRF_AB_PCI_BUSERR_INT_KER_LBN 11
+#define	FRF_AB_PCI_BUSERR_INT_KER_WIDTH 1
+#define	FRF_CZ_MBU_PERR_INT_KER_LBN 11
+#define	FRF_CZ_MBU_PERR_INT_KER_WIDTH 1
+#define	FRF_AZ_SRAM_OOB_INT_KER_LBN 10
+#define	FRF_AZ_SRAM_OOB_INT_KER_WIDTH 1
+#define	FRF_AZ_BUFID_DC_OOB_INT_KER_LBN 9
+#define	FRF_AZ_BUFID_DC_OOB_INT_KER_WIDTH 1
+#define	FRF_AZ_MEM_PERR_INT_KER_LBN 8
+#define	FRF_AZ_MEM_PERR_INT_KER_WIDTH 1
+#define	FRF_AZ_RBUF_OWN_INT_KER_LBN 7
+#define	FRF_AZ_RBUF_OWN_INT_KER_WIDTH 1
+#define	FRF_AZ_TBUF_OWN_INT_KER_LBN 6
+#define	FRF_AZ_TBUF_OWN_INT_KER_WIDTH 1
+#define	FRF_AZ_RDESCQ_OWN_INT_KER_LBN 5
+#define	FRF_AZ_RDESCQ_OWN_INT_KER_WIDTH 1
+#define	FRF_AZ_TDESCQ_OWN_INT_KER_LBN 4
+#define	FRF_AZ_TDESCQ_OWN_INT_KER_WIDTH 1
+#define	FRF_AZ_EVQ_OWN_INT_KER_LBN 3
+#define	FRF_AZ_EVQ_OWN_INT_KER_WIDTH 1
+#define	FRF_AZ_EVF_OFLO_INT_KER_LBN 2
+#define	FRF_AZ_EVF_OFLO_INT_KER_WIDTH 1
+#define	FRF_AZ_ILL_ADR_INT_KER_LBN 1
+#define	FRF_AZ_ILL_ADR_INT_KER_WIDTH 1
+#define	FRF_AZ_SRM_PERR_INT_KER_LBN 0
+#define	FRF_AZ_SRM_PERR_INT_KER_WIDTH 1
+
+
+/*
+ * FR_AZ_FATAL_INTR_REG_CHAR(128bit):
+ * Fatal interrupt register for Char
+ */
+#define	FR_AZ_FATAL_INTR_REG_CHAR_OFST 0x00000240
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_CZ_SRAM_PERR_INT_P_CHAR_EN_LBN 44
+#define	FRF_CZ_SRAM_PERR_INT_P_CHAR_EN_WIDTH 1
+#define	FRF_AB_PCI_BUSERR_INT_CHAR_EN_LBN 43
+#define	FRF_AB_PCI_BUSERR_INT_CHAR_EN_WIDTH 1
+#define	FRF_CZ_MBU_PERR_INT_CHAR_EN_LBN 43
+#define	FRF_CZ_MBU_PERR_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_SRAM_OOB_INT_CHAR_EN_LBN 42
+#define	FRF_AZ_SRAM_OOB_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_BUFID_OOB_INT_CHAR_EN_LBN 41
+#define	FRF_AZ_BUFID_OOB_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_MEM_PERR_INT_CHAR_EN_LBN 40
+#define	FRF_AZ_MEM_PERR_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_RBUF_OWN_INT_CHAR_EN_LBN 39
+#define	FRF_AZ_RBUF_OWN_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_TBUF_OWN_INT_CHAR_EN_LBN 38
+#define	FRF_AZ_TBUF_OWN_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_RDESCQ_OWN_INT_CHAR_EN_LBN 37
+#define	FRF_AZ_RDESCQ_OWN_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_TDESCQ_OWN_INT_CHAR_EN_LBN 36
+#define	FRF_AZ_TDESCQ_OWN_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_EVQ_OWN_INT_CHAR_EN_LBN 35
+#define	FRF_AZ_EVQ_OWN_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_EVF_OFLO_INT_CHAR_EN_LBN 34
+#define	FRF_AZ_EVF_OFLO_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_ILL_ADR_INT_CHAR_EN_LBN 33
+#define	FRF_AZ_ILL_ADR_INT_CHAR_EN_WIDTH 1
+#define	FRF_AZ_SRM_PERR_INT_CHAR_EN_LBN 32
+#define	FRF_AZ_SRM_PERR_INT_CHAR_EN_WIDTH 1
+#define	FRF_CZ_SRAM_PERR_INT_P_CHAR_LBN 12
+#define	FRF_CZ_SRAM_PERR_INT_P_CHAR_WIDTH 1
+#define	FRF_AB_PCI_BUSERR_INT_CHAR_LBN 11
+#define	FRF_AB_PCI_BUSERR_INT_CHAR_WIDTH 1
+#define	FRF_CZ_MBU_PERR_INT_CHAR_LBN 11
+#define	FRF_CZ_MBU_PERR_INT_CHAR_WIDTH 1
+#define	FRF_AZ_SRAM_OOB_INT_CHAR_LBN 10
+#define	FRF_AZ_SRAM_OOB_INT_CHAR_WIDTH 1
+#define	FRF_AZ_BUFID_DC_OOB_INT_CHAR_LBN 9
+#define	FRF_AZ_BUFID_DC_OOB_INT_CHAR_WIDTH 1
+#define	FRF_AZ_MEM_PERR_INT_CHAR_LBN 8
+#define	FRF_AZ_MEM_PERR_INT_CHAR_WIDTH 1
+#define	FRF_AZ_RBUF_OWN_INT_CHAR_LBN 7
+#define	FRF_AZ_RBUF_OWN_INT_CHAR_WIDTH 1
+#define	FRF_AZ_TBUF_OWN_INT_CHAR_LBN 6
+#define	FRF_AZ_TBUF_OWN_INT_CHAR_WIDTH 1
+#define	FRF_AZ_RDESCQ_OWN_INT_CHAR_LBN 5
+#define	FRF_AZ_RDESCQ_OWN_INT_CHAR_WIDTH 1
+#define	FRF_AZ_TDESCQ_OWN_INT_CHAR_LBN 4
+#define	FRF_AZ_TDESCQ_OWN_INT_CHAR_WIDTH 1
+#define	FRF_AZ_EVQ_OWN_INT_CHAR_LBN 3
+#define	FRF_AZ_EVQ_OWN_INT_CHAR_WIDTH 1
+#define	FRF_AZ_EVF_OFLO_INT_CHAR_LBN 2
+#define	FRF_AZ_EVF_OFLO_INT_CHAR_WIDTH 1
+#define	FRF_AZ_ILL_ADR_INT_CHAR_LBN 1
+#define	FRF_AZ_ILL_ADR_INT_CHAR_WIDTH 1
+#define	FRF_AZ_SRM_PERR_INT_CHAR_LBN 0
+#define	FRF_AZ_SRM_PERR_INT_CHAR_WIDTH 1
+
+
+/*
+ * FR_AZ_DP_CTRL_REG(128bit):
+ * Datapath control register
+ */
+#define	FR_AZ_DP_CTRL_REG_OFST 0x00000250
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_FLS_EVQ_ID_LBN 0
+#define	FRF_AZ_FLS_EVQ_ID_WIDTH 12
+
+
+/*
+ * FR_AZ_MEM_STAT_REG(128bit):
+ * Memory status register
+ */
+#define	FR_AZ_MEM_STAT_REG_OFST 0x00000260
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MEM_PERR_VEC_LBN 53
+#define	FRF_AB_MEM_PERR_VEC_WIDTH 40
+#define	FRF_AB_MEM_PERR_VEC_DW0_LBN 53
+#define	FRF_AB_MEM_PERR_VEC_DW0_WIDTH 32
+#define	FRF_AB_MEM_PERR_VEC_DW1_LBN 85
+#define	FRF_AB_MEM_PERR_VEC_DW1_WIDTH 6
+#define	FRF_AB_MBIST_CORR_LBN 38
+#define	FRF_AB_MBIST_CORR_WIDTH 15
+#define	FRF_AB_MBIST_ERR_LBN 0
+#define	FRF_AB_MBIST_ERR_WIDTH 40
+#define	FRF_AB_MBIST_ERR_DW0_LBN 0
+#define	FRF_AB_MBIST_ERR_DW0_WIDTH 32
+#define	FRF_AB_MBIST_ERR_DW1_LBN 32
+#define	FRF_AB_MBIST_ERR_DW1_WIDTH 6
+#define	FRF_CZ_MEM_PERR_VEC_LBN 0
+#define	FRF_CZ_MEM_PERR_VEC_WIDTH 35
+#define	FRF_CZ_MEM_PERR_VEC_DW0_LBN 0
+#define	FRF_CZ_MEM_PERR_VEC_DW0_WIDTH 32
+#define	FRF_CZ_MEM_PERR_VEC_DW1_LBN 32
+#define	FRF_CZ_MEM_PERR_VEC_DW1_WIDTH 3
+
+
+/*
+ * FR_PORT0_CS_DEBUG_REG(128bit):
+ * Debug register
+ */
+
+#define	FR_AZ_CS_DEBUG_REG_OFST 0x00000270
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GLB_DEBUG2_SEL_LBN 50
+#define	FRF_AB_GLB_DEBUG2_SEL_WIDTH 3
+#define	FRF_AB_DEBUG_BLK_SEL2_LBN 47
+#define	FRF_AB_DEBUG_BLK_SEL2_WIDTH 3
+#define	FRF_AB_DEBUG_BLK_SEL1_LBN 44
+#define	FRF_AB_DEBUG_BLK_SEL1_WIDTH 3
+#define	FRF_AB_DEBUG_BLK_SEL0_LBN 41
+#define	FRF_AB_DEBUG_BLK_SEL0_WIDTH 3
+#define	FRF_CZ_CS_PORT_NUM_LBN 40
+#define	FRF_CZ_CS_PORT_NUM_WIDTH 2
+#define	FRF_AB_MISC_DEBUG_ADDR_LBN 36
+#define	FRF_AB_MISC_DEBUG_ADDR_WIDTH 5
+#define	FRF_CZ_CS_RESERVED_LBN 36
+#define	FRF_CZ_CS_RESERVED_WIDTH 4
+#define	FRF_AB_SERDES_DEBUG_ADDR_LBN 31
+#define	FRF_AB_SERDES_DEBUG_ADDR_WIDTH 5
+#define	FRF_CZ_CS_PORT_FPE_DW0_LBN 1
+#define	FRF_CZ_CS_PORT_FPE_DW0_WIDTH 32
+#define	FRF_CZ_CS_PORT_FPE_DW1_LBN 33
+#define	FRF_CZ_CS_PORT_FPE_DW1_WIDTH 3
+#define	FRF_CZ_CS_PORT_FPE_LBN 1
+#define	FRF_CZ_CS_PORT_FPE_WIDTH 35
+#define	FRF_AB_EM_DEBUG_ADDR_LBN 26
+#define	FRF_AB_EM_DEBUG_ADDR_WIDTH 5
+#define	FRF_AB_SR_DEBUG_ADDR_LBN 21
+#define	FRF_AB_SR_DEBUG_ADDR_WIDTH 5
+#define	FRF_AB_EV_DEBUG_ADDR_LBN 16
+#define	FRF_AB_EV_DEBUG_ADDR_WIDTH 5
+#define	FRF_AB_RX_DEBUG_ADDR_LBN 11
+#define	FRF_AB_RX_DEBUG_ADDR_WIDTH 5
+#define	FRF_AB_TX_DEBUG_ADDR_LBN 6
+#define	FRF_AB_TX_DEBUG_ADDR_WIDTH 5
+#define	FRF_AB_CS_BIU_DEBUG_ADDR_LBN 1
+#define	FRF_AB_CS_BIU_DEBUG_ADDR_WIDTH 5
+#define	FRF_AZ_CS_DEBUG_EN_LBN 0
+#define	FRF_AZ_CS_DEBUG_EN_WIDTH 1
+
+
+/*
+ * FR_AZ_DRIVER_REG(128bit):
+ * Driver scratch register [0-7]
+ */
+#define	FR_AZ_DRIVER_REG_OFST 0x00000280
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_DRIVER_REG_STEP 16
+#define	FR_AZ_DRIVER_REG_ROWS 8
+
+#define	FRF_AZ_DRIVER_DW0_LBN 0
+#define	FRF_AZ_DRIVER_DW0_WIDTH 32
+
+
+/*
+ * FR_AZ_ALTERA_BUILD_REG(128bit):
+ * Altera build register
+ */
+#define	FR_AZ_ALTERA_BUILD_REG_OFST 0x00000300
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_ALTERA_BUILD_VER_LBN 0
+#define	FRF_AZ_ALTERA_BUILD_VER_WIDTH 32
+
+
+/*
+ * FR_AZ_CSR_SPARE_REG(128bit):
+ * Spare register
+ */
+#define	FR_AZ_CSR_SPARE_REG_OFST 0x00000310
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_MEM_PERR_EN_TX_DATA_LBN 72
+#define	FRF_AZ_MEM_PERR_EN_TX_DATA_WIDTH 2
+#define	FRF_AZ_MEM_PERR_EN_LBN 64
+#define	FRF_AZ_MEM_PERR_EN_WIDTH 38
+#define	FRF_AZ_MEM_PERR_EN_DW0_LBN 64
+#define	FRF_AZ_MEM_PERR_EN_DW0_WIDTH 32
+#define	FRF_AZ_MEM_PERR_EN_DW1_LBN 96
+#define	FRF_AZ_MEM_PERR_EN_DW1_WIDTH 6
+#define	FRF_AZ_CSR_SPARE_BITS_LBN 0
+#define	FRF_AZ_CSR_SPARE_BITS_WIDTH 32
+
+
+/*
+ * FR_BZ_DEBUG_DATA_OUT_REG(128bit):
+ * Live Debug and Debug 2 out ports
+ */
+#define	FR_BZ_DEBUG_DATA_OUT_REG_OFST 0x00000350
+/* falconb0,sienaa0=net_func_bar2 */
+
+#define	FRF_BZ_DEBUG2_PORT_LBN 25
+#define	FRF_BZ_DEBUG2_PORT_WIDTH 15
+#define	FRF_BZ_DEBUG1_PORT_LBN 0
+#define	FRF_BZ_DEBUG1_PORT_WIDTH 25
+
+
+/*
+ * FR_BZ_EVQ_RPTR_REGP0(32bit):
+ * Event queue read pointer register
+ */
+#define	FR_BZ_EVQ_RPTR_REGP0_OFST 0x00000400
+/* falconb0,sienaa0=net_func_bar2 */
+#define	FR_BZ_EVQ_RPTR_REGP0_STEP 8192
+#define	FR_BZ_EVQ_RPTR_REGP0_ROWS 1024
+/*
+ * FR_AA_EVQ_RPTR_REG_KER(32bit):
+ * Event queue read pointer register
+ */
+#define	FR_AA_EVQ_RPTR_REG_KER_OFST 0x00011b00
+/* falcona0=net_func_bar2 */
+#define	FR_AA_EVQ_RPTR_REG_KER_STEP 4
+#define	FR_AA_EVQ_RPTR_REG_KER_ROWS 4
+/*
+ * FR_AZ_EVQ_RPTR_REG(32bit):
+ * Event queue read pointer register
+ */
+#define	FR_AZ_EVQ_RPTR_REG_OFST 0x00fa0000
+/* falconb0=net_func_bar2,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_EVQ_RPTR_REG_STEP 16
+#define	FR_AB_EVQ_RPTR_REG_ROWS 4096
+#define	FR_CZ_EVQ_RPTR_REG_ROWS 1024
+/*
+ * FR_BB_EVQ_RPTR_REGP123(32bit):
+ * Event queue read pointer register
+ */
+#define	FR_BB_EVQ_RPTR_REGP123_OFST 0x01000400
+/* falconb0=net_func_bar2 */
+#define	FR_BB_EVQ_RPTR_REGP123_STEP 8192
+#define	FR_BB_EVQ_RPTR_REGP123_ROWS 3072
+
+#define	FRF_AZ_EVQ_RPTR_VLD_LBN 15
+#define	FRF_AZ_EVQ_RPTR_VLD_WIDTH 1
+#define	FRF_AZ_EVQ_RPTR_LBN 0
+#define	FRF_AZ_EVQ_RPTR_WIDTH 15
+
+
+/*
+ * FR_BZ_TIMER_COMMAND_REGP0(128bit):
+ * Timer Command Registers
+ */
+#define	FR_BZ_TIMER_COMMAND_REGP0_OFST 0x00000420
+/* falconb0,sienaa0=net_func_bar2 */
+#define	FR_BZ_TIMER_COMMAND_REGP0_STEP 8192
+#define	FR_BZ_TIMER_COMMAND_REGP0_ROWS 1024
+/*
+ * FR_AA_TIMER_COMMAND_REG_KER(128bit):
+ * Timer Command Registers
+ */
+#define	FR_AA_TIMER_COMMAND_REG_KER_OFST 0x00000420
+/* falcona0=net_func_bar2 */
+#define	FR_AA_TIMER_COMMAND_REG_KER_STEP 8192
+#define	FR_AA_TIMER_COMMAND_REG_KER_ROWS 4
+/*
+ * FR_AB_TIMER_COMMAND_REGP123(128bit):
+ * Timer Command Registers
+ */
+#define	FR_AB_TIMER_COMMAND_REGP123_OFST 0x01000420
+/* falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AB_TIMER_COMMAND_REGP123_STEP 8192
+#define	FR_AB_TIMER_COMMAND_REGP123_ROWS 3072
+/*
+ * FR_AA_TIMER_COMMAND_REGP0(128bit):
+ * Timer Command Registers
+ */
+#define	FR_AA_TIMER_COMMAND_REGP0_OFST 0x00008420
+/* falcona0=char_func_bar0 */
+#define	FR_AA_TIMER_COMMAND_REGP0_STEP 8192
+#define	FR_AA_TIMER_COMMAND_REGP0_ROWS 1020
+
+#define	FRF_CZ_TC_TIMER_MODE_LBN 14
+#define	FRF_CZ_TC_TIMER_MODE_WIDTH 2
+#define	FRF_AB_TC_TIMER_MODE_LBN 12
+#define	FRF_AB_TC_TIMER_MODE_WIDTH 2
+#define	FRF_CZ_TC_TIMER_VAL_LBN 0
+#define	FRF_CZ_TC_TIMER_VAL_WIDTH 14
+#define	FRF_AB_TC_TIMER_VAL_LBN 0
+#define	FRF_AB_TC_TIMER_VAL_WIDTH 12
+
+
+/*
+ * FR_AZ_DRV_EV_REG(128bit):
+ * Driver generated event register
+ */
+#define	FR_AZ_DRV_EV_REG_OFST 0x00000440
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_DRV_EV_QID_LBN 64
+#define	FRF_AZ_DRV_EV_QID_WIDTH 12
+#define	FRF_AZ_DRV_EV_DATA_LBN 0
+#define	FRF_AZ_DRV_EV_DATA_WIDTH 64
+#define	FRF_AZ_DRV_EV_DATA_DW0_LBN 0
+#define	FRF_AZ_DRV_EV_DATA_DW0_WIDTH 32
+#define	FRF_AZ_DRV_EV_DATA_DW1_LBN 32
+#define	FRF_AZ_DRV_EV_DATA_DW1_WIDTH 32
+
+
+/*
+ * FR_AZ_EVQ_CTL_REG(128bit):
+ * Event queue control register
+ */
+#define	FR_AZ_EVQ_CTL_REG_OFST 0x00000450
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_CZ_RX_EVQ_WAKEUP_MASK_LBN 15
+#define	FRF_CZ_RX_EVQ_WAKEUP_MASK_WIDTH 10
+#define	FRF_BB_RX_EVQ_WAKEUP_MASK_LBN 15
+#define	FRF_BB_RX_EVQ_WAKEUP_MASK_WIDTH 6
+#define	FRF_AZ_EVQ_OWNERR_CTL_LBN 14
+#define	FRF_AZ_EVQ_OWNERR_CTL_WIDTH 1
+#define	FRF_AZ_EVQ_FIFO_AF_TH_LBN 7
+#define	FRF_AZ_EVQ_FIFO_AF_TH_WIDTH 7
+#define	FRF_AZ_EVQ_FIFO_NOTAF_TH_LBN 0
+#define	FRF_AZ_EVQ_FIFO_NOTAF_TH_WIDTH 7
+
+
+/*
+ * FR_AZ_EVQ_CNT1_REG(128bit):
+ * Event counter 1 register
+ */
+#define	FR_AZ_EVQ_CNT1_REG_OFST 0x00000460
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_EVQ_CNT_PRE_FIFO_LBN 120
+#define	FRF_AZ_EVQ_CNT_PRE_FIFO_WIDTH 7
+#define	FRF_AZ_EVQ_CNT_TOBIU_LBN 100
+#define	FRF_AZ_EVQ_CNT_TOBIU_WIDTH 20
+#define	FRF_AZ_EVQ_TX_REQ_CNT_LBN 80
+#define	FRF_AZ_EVQ_TX_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_RX_REQ_CNT_LBN 60
+#define	FRF_AZ_EVQ_RX_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_EM_REQ_CNT_LBN 40
+#define	FRF_AZ_EVQ_EM_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_CSR_REQ_CNT_LBN 20
+#define	FRF_AZ_EVQ_CSR_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_ERR_REQ_CNT_LBN 0
+#define	FRF_AZ_EVQ_ERR_REQ_CNT_WIDTH 20
+
+
+/*
+ * FR_AZ_EVQ_CNT2_REG(128bit):
+ * Event counter 2 register
+ */
+#define	FR_AZ_EVQ_CNT2_REG_OFST 0x00000470
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_EVQ_UPD_REQ_CNT_LBN 104
+#define	FRF_AZ_EVQ_UPD_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_CLR_REQ_CNT_LBN 84
+#define	FRF_AZ_EVQ_CLR_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_RDY_CNT_LBN 80
+#define	FRF_AZ_EVQ_RDY_CNT_WIDTH 4
+#define	FRF_AZ_EVQ_WU_REQ_CNT_LBN 60
+#define	FRF_AZ_EVQ_WU_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_WET_REQ_CNT_LBN 40
+#define	FRF_AZ_EVQ_WET_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_INIT_REQ_CNT_LBN 20
+#define	FRF_AZ_EVQ_INIT_REQ_CNT_WIDTH 20
+#define	FRF_AZ_EVQ_TM_REQ_CNT_LBN 0
+#define	FRF_AZ_EVQ_TM_REQ_CNT_WIDTH 20
+
+
+/*
+ * FR_CZ_USR_EV_REG(32bit):
+ * Event mailbox register
+ */
+#define	FR_CZ_USR_EV_REG_OFST 0x00000540
+/* sienaa0=net_func_bar2 */
+#define	FR_CZ_USR_EV_REG_STEP 8192
+#define	FR_CZ_USR_EV_REG_ROWS 1024
+
+#define	FRF_CZ_USR_EV_DATA_LBN 0
+#define	FRF_CZ_USR_EV_DATA_WIDTH 32
+
+
+/*
+ * FR_AZ_BUF_TBL_CFG_REG(128bit):
+ * Buffer table configuration register
+ */
+#define	FR_AZ_BUF_TBL_CFG_REG_OFST 0x00000600
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_BUF_TBL_MODE_LBN 3
+#define	FRF_AZ_BUF_TBL_MODE_WIDTH 1
+
+
+/*
+ * FR_AZ_SRM_RX_DC_CFG_REG(128bit):
+ * SRAM receive descriptor cache configuration register
+ */
+#define	FR_AZ_SRM_RX_DC_CFG_REG_OFST 0x00000610
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_SRM_CLK_TMP_EN_LBN 21
+#define	FRF_AZ_SRM_CLK_TMP_EN_WIDTH 1
+#define	FRF_AZ_SRM_RX_DC_BASE_ADR_LBN 0
+#define	FRF_AZ_SRM_RX_DC_BASE_ADR_WIDTH 21
+
+
+/*
+ * FR_AZ_SRM_TX_DC_CFG_REG(128bit):
+ * SRAM transmit descriptor cache configuration register
+ */
+#define	FR_AZ_SRM_TX_DC_CFG_REG_OFST 0x00000620
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_SRM_TX_DC_BASE_ADR_LBN 0
+#define	FRF_AZ_SRM_TX_DC_BASE_ADR_WIDTH 21
+
+
+/*
+ * FR_AZ_SRM_CFG_REG(128bit):
+ * SRAM configuration register
+ */
+#define	FR_AZ_SRM_CFG_REG_SF_OFST 0x00000380
+/* falcona0,falconb0=eeprom_flash */
+/*
+ * FR_AZ_SRM_CFG_REG(128bit):
+ * SRAM configuration register
+ */
+#define	FR_AZ_SRM_CFG_REG_OFST 0x00000630
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_SRM_OOB_ADR_INTEN_LBN 5
+#define	FRF_AZ_SRM_OOB_ADR_INTEN_WIDTH 1
+#define	FRF_AZ_SRM_OOB_BUF_INTEN_LBN 4
+#define	FRF_AZ_SRM_OOB_BUF_INTEN_WIDTH 1
+#define	FRF_AZ_SRM_INIT_EN_LBN 3
+#define	FRF_AZ_SRM_INIT_EN_WIDTH 1
+#define	FRF_AZ_SRM_NUM_BANK_LBN 2
+#define	FRF_AZ_SRM_NUM_BANK_WIDTH 1
+#define	FRF_AZ_SRM_BANK_SIZE_LBN 0
+#define	FRF_AZ_SRM_BANK_SIZE_WIDTH 2
+
+
+/*
+ * FR_AZ_BUF_TBL_UPD_REG(128bit):
+ * Buffer table update register
+ */
+#define	FR_AZ_BUF_TBL_UPD_REG_OFST 0x00000650
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_BUF_UPD_CMD_LBN 63
+#define	FRF_AZ_BUF_UPD_CMD_WIDTH 1
+#define	FRF_AZ_BUF_CLR_CMD_LBN 62
+#define	FRF_AZ_BUF_CLR_CMD_WIDTH 1
+#define	FRF_AZ_BUF_CLR_END_ID_LBN 32
+#define	FRF_AZ_BUF_CLR_END_ID_WIDTH 20
+#define	FRF_AZ_BUF_CLR_START_ID_LBN 0
+#define	FRF_AZ_BUF_CLR_START_ID_WIDTH 20
+
+
+/*
+ * FR_AZ_SRM_UPD_EVQ_REG(128bit):
+ * Buffer table update register
+ */
+#define	FR_AZ_SRM_UPD_EVQ_REG_OFST 0x00000660
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_SRM_UPD_EVQ_ID_LBN 0
+#define	FRF_AZ_SRM_UPD_EVQ_ID_WIDTH 12
+
+
+/*
+ * FR_AZ_SRAM_PARITY_REG(128bit):
+ * SRAM parity register.
+ */
+#define	FR_AZ_SRAM_PARITY_REG_OFST 0x00000670
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_CZ_BYPASS_ECC_LBN 3
+#define	FRF_CZ_BYPASS_ECC_WIDTH 1
+#define	FRF_CZ_SEC_INT_LBN 2
+#define	FRF_CZ_SEC_INT_WIDTH 1
+#define	FRF_CZ_FORCE_SRAM_DOUBLE_ERR_LBN 1
+#define	FRF_CZ_FORCE_SRAM_DOUBLE_ERR_WIDTH 1
+#define	FRF_CZ_FORCE_SRAM_SINGLE_ERR_LBN 0
+#define	FRF_CZ_FORCE_SRAM_SINGLE_ERR_WIDTH 1
+#define	FRF_AB_FORCE_SRAM_PERR_LBN 0
+#define	FRF_AB_FORCE_SRAM_PERR_WIDTH 1
+
+
+/*
+ * FR_AZ_RX_CFG_REG(128bit):
+ * Receive configuration register
+ */
+#define	FR_AZ_RX_CFG_REG_OFST 0x00000800
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_CZ_RX_HDR_SPLIT_EN_LBN 71
+#define	FRF_CZ_RX_HDR_SPLIT_EN_WIDTH 1
+#define	FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_LBN 62
+#define	FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH 9
+#define	FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_LBN 53
+#define	FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH 9
+#define	FRF_CZ_RX_PRE_RFF_IPG_LBN 49
+#define	FRF_CZ_RX_PRE_RFF_IPG_WIDTH 4
+#define	FRF_BZ_RX_TCP_SUP_LBN 48
+#define	FRF_BZ_RX_TCP_SUP_WIDTH 1
+#define	FRF_BZ_RX_INGR_EN_LBN 47
+#define	FRF_BZ_RX_INGR_EN_WIDTH 1
+#define	FRF_BZ_RX_IP_HASH_LBN 46
+#define	FRF_BZ_RX_IP_HASH_WIDTH 1
+#define	FRF_BZ_RX_HASH_ALG_LBN 45
+#define	FRF_BZ_RX_HASH_ALG_WIDTH 1
+#define	FRF_BZ_RX_HASH_INSRT_HDR_LBN 44
+#define	FRF_BZ_RX_HASH_INSRT_HDR_WIDTH 1
+#define	FRF_BZ_RX_DESC_PUSH_EN_LBN 43
+#define	FRF_BZ_RX_DESC_PUSH_EN_WIDTH 1
+#define	FRF_BZ_RX_RDW_PATCH_EN_LBN 42
+#define	FRF_BZ_RX_RDW_PATCH_EN_WIDTH 1
+#define	FRF_BB_RX_PCI_BURST_SIZE_LBN 39
+#define	FRF_BB_RX_PCI_BURST_SIZE_WIDTH 3
+#define	FRF_BZ_RX_OWNERR_CTL_LBN 38
+#define	FRF_BZ_RX_OWNERR_CTL_WIDTH 1
+#define	FRF_BZ_RX_XON_TX_TH_LBN 33
+#define	FRF_BZ_RX_XON_TX_TH_WIDTH 5
+#define	FRF_AA_RX_DESC_PUSH_EN_LBN 35
+#define	FRF_AA_RX_DESC_PUSH_EN_WIDTH 1
+#define	FRF_AA_RX_RDW_PATCH_EN_LBN 34
+#define	FRF_AA_RX_RDW_PATCH_EN_WIDTH 1
+#define	FRF_AA_RX_PCI_BURST_SIZE_LBN 31
+#define	FRF_AA_RX_PCI_BURST_SIZE_WIDTH 3
+#define	FRF_BZ_RX_XOFF_TX_TH_LBN 28
+#define	FRF_BZ_RX_XOFF_TX_TH_WIDTH 5
+#define	FRF_AA_RX_OWNERR_CTL_LBN 30
+#define	FRF_AA_RX_OWNERR_CTL_WIDTH 1
+#define	FRF_AA_RX_XON_TX_TH_LBN 25
+#define	FRF_AA_RX_XON_TX_TH_WIDTH 5
+#define	FRF_BZ_RX_USR_BUF_SIZE_LBN 19
+#define	FRF_BZ_RX_USR_BUF_SIZE_WIDTH 9
+#define	FRF_AA_RX_XOFF_TX_TH_LBN 20
+#define	FRF_AA_RX_XOFF_TX_TH_WIDTH 5
+#define	FRF_AA_RX_USR_BUF_SIZE_LBN 11
+#define	FRF_AA_RX_USR_BUF_SIZE_WIDTH 9
+#define	FRF_BZ_RX_XON_MAC_TH_LBN 10
+#define	FRF_BZ_RX_XON_MAC_TH_WIDTH 9
+#define	FRF_AA_RX_XON_MAC_TH_LBN 6
+#define	FRF_AA_RX_XON_MAC_TH_WIDTH 5
+#define	FRF_BZ_RX_XOFF_MAC_TH_LBN 1
+#define	FRF_BZ_RX_XOFF_MAC_TH_WIDTH 9
+#define	FRF_AA_RX_XOFF_MAC_TH_LBN 1
+#define	FRF_AA_RX_XOFF_MAC_TH_WIDTH 5
+#define	FRF_AZ_RX_XOFF_MAC_EN_LBN 0
+#define	FRF_AZ_RX_XOFF_MAC_EN_WIDTH 1
+
+
+/*
+ * FR_AZ_RX_FILTER_CTL_REG(128bit):
+ * Receive filter control registers
+ */
+#define	FR_AZ_RX_FILTER_CTL_REG_OFST 0x00000810
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT_LBN 94
+#define	FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT_WIDTH 8
+#define	FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT_LBN 86
+#define	FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT_WIDTH 8
+#define	FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES_LBN 85
+#define	FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES_WIDTH 1
+#define	FRF_CZ_RX_VLAN_MATCH_ETHERTYPE_LBN 69
+#define	FRF_CZ_RX_VLAN_MATCH_ETHERTYPE_WIDTH 16
+#define	FRF_CZ_MULTICAST_NOMATCH_Q_ID_LBN 57
+#define	FRF_CZ_MULTICAST_NOMATCH_Q_ID_WIDTH 12
+#define	FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED_LBN 56
+#define	FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED_WIDTH 1
+#define	FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE_LBN 55
+#define	FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE_WIDTH 1
+#define	FRF_CZ_UNICAST_NOMATCH_Q_ID_LBN 43
+#define	FRF_CZ_UNICAST_NOMATCH_Q_ID_WIDTH 12
+#define	FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED_LBN 42
+#define	FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED_WIDTH 1
+#define	FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE_LBN 41
+#define	FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE_WIDTH 1
+#define	FRF_BZ_SCATTER_ENBL_NO_MATCH_Q_LBN 40
+#define	FRF_BZ_SCATTER_ENBL_NO_MATCH_Q_WIDTH 1
+#define	FRF_AZ_UDP_FULL_SRCH_LIMIT_LBN 32
+#define	FRF_AZ_UDP_FULL_SRCH_LIMIT_WIDTH 8
+#define	FRF_AZ_NUM_KER_LBN 24
+#define	FRF_AZ_NUM_KER_WIDTH 2
+#define	FRF_AZ_UDP_WILD_SRCH_LIMIT_LBN 16
+#define	FRF_AZ_UDP_WILD_SRCH_LIMIT_WIDTH 8
+#define	FRF_AZ_TCP_WILD_SRCH_LIMIT_LBN 8
+#define	FRF_AZ_TCP_WILD_SRCH_LIMIT_WIDTH 8
+#define	FRF_AZ_TCP_FULL_SRCH_LIMIT_LBN 0
+#define	FRF_AZ_TCP_FULL_SRCH_LIMIT_WIDTH 8
+
+
+/*
+ * FR_AZ_RX_FLUSH_DESCQ_REG(128bit):
+ * Receive flush descriptor queue register
+ */
+#define	FR_AZ_RX_FLUSH_DESCQ_REG_OFST 0x00000820
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_FLUSH_DESCQ_CMD_LBN 24
+#define	FRF_AZ_RX_FLUSH_DESCQ_CMD_WIDTH 1
+#define	FRF_AZ_RX_FLUSH_DESCQ_LBN 0
+#define	FRF_AZ_RX_FLUSH_DESCQ_WIDTH 12
+
+
+/*
+ * FR_BZ_RX_DESC_UPD_REGP0(128bit):
+ * Receive descriptor update register.
+ */
+#define	FR_BZ_RX_DESC_UPD_REGP0_OFST 0x00000830
+/* falconb0,sienaa0=net_func_bar2 */
+#define	FR_BZ_RX_DESC_UPD_REGP0_STEP 8192
+#define	FR_BZ_RX_DESC_UPD_REGP0_ROWS 1024
+/*
+ * FR_AA_RX_DESC_UPD_REG_KER(128bit):
+ * Receive descriptor update register.
+ */
+#define	FR_AA_RX_DESC_UPD_REG_KER_OFST 0x00000830
+/* falcona0=net_func_bar2 */
+#define	FR_AA_RX_DESC_UPD_REG_KER_STEP 8192
+#define	FR_AA_RX_DESC_UPD_REG_KER_ROWS 4
+/*
+ * FR_AB_RX_DESC_UPD_REGP123(128bit):
+ * Receive descriptor update register.
+ */
+#define	FR_AB_RX_DESC_UPD_REGP123_OFST 0x01000830
+/* falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AB_RX_DESC_UPD_REGP123_STEP 8192
+#define	FR_AB_RX_DESC_UPD_REGP123_ROWS 3072
+/*
+ * FR_AA_RX_DESC_UPD_REGP0(128bit):
+ * Receive descriptor update register.
+ */
+#define	FR_AA_RX_DESC_UPD_REGP0_OFST 0x00008830
+/* falcona0=char_func_bar0 */
+#define	FR_AA_RX_DESC_UPD_REGP0_STEP 8192
+#define	FR_AA_RX_DESC_UPD_REGP0_ROWS 1020
+
+#define	FRF_AZ_RX_DESC_WPTR_LBN 96
+#define	FRF_AZ_RX_DESC_WPTR_WIDTH 12
+#define	FRF_AZ_RX_DESC_PUSH_CMD_LBN 95
+#define	FRF_AZ_RX_DESC_PUSH_CMD_WIDTH 1
+#define	FRF_AZ_RX_DESC_LBN 0
+#define	FRF_AZ_RX_DESC_WIDTH 64
+#define	FRF_AZ_RX_DESC_DW0_LBN 0
+#define	FRF_AZ_RX_DESC_DW0_WIDTH 32
+#define	FRF_AZ_RX_DESC_DW1_LBN 32
+#define	FRF_AZ_RX_DESC_DW1_WIDTH 32
+
+
+/*
+ * FR_AZ_RX_DC_CFG_REG(128bit):
+ * Receive descriptor cache configuration register
+ */
+#define	FR_AZ_RX_DC_CFG_REG_OFST 0x00000840
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_MAX_PF_LBN 2
+#define	FRF_AZ_RX_MAX_PF_WIDTH 2
+#define	FRF_AZ_RX_DC_SIZE_LBN 0
+#define	FRF_AZ_RX_DC_SIZE_WIDTH 2
+#define	FFE_AZ_RX_DC_SIZE_64 3
+#define	FFE_AZ_RX_DC_SIZE_32 2
+#define	FFE_AZ_RX_DC_SIZE_16 1
+#define	FFE_AZ_RX_DC_SIZE_8 0
+
+
+/*
+ * FR_AZ_RX_DC_PF_WM_REG(128bit):
+ * Receive descriptor cache pre-fetch watermark register
+ */
+#define	FR_AZ_RX_DC_PF_WM_REG_OFST 0x00000850
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_DC_PF_HWM_LBN 6
+#define	FRF_AZ_RX_DC_PF_HWM_WIDTH 6
+#define	FRF_AZ_RX_DC_PF_LWM_LBN 0
+#define	FRF_AZ_RX_DC_PF_LWM_WIDTH 6
+
+
+/*
+ * FR_BZ_RX_RSS_TKEY_REG(128bit):
+ * RSS Toeplitz hash key
+ */
+#define	FR_BZ_RX_RSS_TKEY_REG_OFST 0x00000860
+/* falconb0,sienaa0=net_func_bar2 */
+
+#define	FRF_BZ_RX_RSS_TKEY_LBN 96
+#define	FRF_BZ_RX_RSS_TKEY_WIDTH 32
+#define	FRF_BZ_RX_RSS_TKEY_DW3_LBN 96
+#define	FRF_BZ_RX_RSS_TKEY_DW3_WIDTH 32
+#define	FRF_BZ_RX_RSS_TKEY_DW2_LBN 64
+#define	FRF_BZ_RX_RSS_TKEY_DW2_WIDTH 32
+#define	FRF_BZ_RX_RSS_TKEY_DW1_LBN 32
+#define	FRF_BZ_RX_RSS_TKEY_DW1_WIDTH 32
+#define	FRF_BZ_RX_RSS_TKEY_DW0_LBN 0
+#define	FRF_BZ_RX_RSS_TKEY_DW0_WIDTH 32
+
+
+/*
+ * FR_AZ_RX_NODESC_DROP_REG(128bit):
+ * Receive dropped packet counter register
+ */
+#define	FR_AZ_RX_NODESC_DROP_REG_OFST 0x00000880
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_NODESC_DROP_CNT_LBN 0
+#define	FRF_AZ_RX_NODESC_DROP_CNT_WIDTH 16
+
+
+/*
+ * FR_AZ_RX_SELF_RST_REG(128bit):
+ * Receive self reset register
+ */
+#define	FR_AZ_RX_SELF_RST_REG_OFST 0x00000890
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_ISCSI_DIS_LBN 17
+#define	FRF_AZ_RX_ISCSI_DIS_WIDTH 1
+#define	FRF_AB_RX_SW_RST_REG_LBN 16
+#define	FRF_AB_RX_SW_RST_REG_WIDTH 1
+#define	FRF_AB_RX_SELF_RST_EN_LBN 8
+#define	FRF_AB_RX_SELF_RST_EN_WIDTH 1
+#define	FRF_AZ_RX_MAX_PF_LAT_LBN 4
+#define	FRF_AZ_RX_MAX_PF_LAT_WIDTH 4
+#define	FRF_AZ_RX_MAX_LU_LAT_LBN 0
+#define	FRF_AZ_RX_MAX_LU_LAT_WIDTH 4
+
+
+/*
+ * FR_AZ_RX_DEBUG_REG(128bit):
+ * undocumented register
+ */
+#define	FR_AZ_RX_DEBUG_REG_OFST 0x000008a0
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_DEBUG_LBN 0
+#define	FRF_AZ_RX_DEBUG_WIDTH 64
+#define	FRF_AZ_RX_DEBUG_DW0_LBN 0
+#define	FRF_AZ_RX_DEBUG_DW0_WIDTH 32
+#define	FRF_AZ_RX_DEBUG_DW1_LBN 32
+#define	FRF_AZ_RX_DEBUG_DW1_WIDTH 32
+
+
+/*
+ * FR_AZ_RX_PUSH_DROP_REG(128bit):
+ * Receive descriptor push dropped counter register
+ */
+#define	FR_AZ_RX_PUSH_DROP_REG_OFST 0x000008b0
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_RX_PUSH_DROP_CNT_LBN 0
+#define	FRF_AZ_RX_PUSH_DROP_CNT_WIDTH 32
+
+
+/*
+ * FR_CZ_RX_RSS_IPV6_REG1(128bit):
+ * IPv6 RSS Toeplitz hash key low bytes
+ */
+#define	FR_CZ_RX_RSS_IPV6_REG1_OFST 0x000008d0
+/* sienaa0=net_func_bar2 */
+
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN 0
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH 128
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW0_LBN 0
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW0_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW1_LBN 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW1_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW2_LBN 64
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW2_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW3_LBN 96
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW3_WIDTH 32
+
+
+/*
+ * FR_CZ_RX_RSS_IPV6_REG2(128bit):
+ * IPv6 RSS Toeplitz hash key middle bytes
+ */
+#define	FR_CZ_RX_RSS_IPV6_REG2_OFST 0x000008e0
+/* sienaa0=net_func_bar2 */
+
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN 0
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH 128
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW0_LBN 0
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW0_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW1_LBN 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW1_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW2_LBN 64
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW2_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW3_LBN 96
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW3_WIDTH 32
+
+
+/*
+ * FR_CZ_RX_RSS_IPV6_REG3(128bit):
+ * IPv6 RSS Toeplitz hash key upper bytes and IPv6 RSS settings
+ */
+#define	FR_CZ_RX_RSS_IPV6_REG3_OFST 0x000008f0
+/* sienaa0=net_func_bar2 */
+
+#define	FRF_CZ_RX_RSS_IPV6_THASH_ENABLE_LBN 66
+#define	FRF_CZ_RX_RSS_IPV6_THASH_ENABLE_WIDTH 1
+#define	FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE_LBN 65
+#define	FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE_WIDTH 1
+#define	FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS_LBN 64
+#define	FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS_WIDTH 1
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN 0
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH 64
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW0_LBN 0
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW0_WIDTH 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW1_LBN 32
+#define	FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW1_WIDTH 32
+
+
+/*
+ * FR_AZ_TX_FLUSH_DESCQ_REG(128bit):
+ * Transmit flush descriptor queue register
+ */
+#define	FR_AZ_TX_FLUSH_DESCQ_REG_OFST 0x00000a00
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_FLUSH_DESCQ_CMD_LBN 12
+#define	FRF_AZ_TX_FLUSH_DESCQ_CMD_WIDTH 1
+#define	FRF_AZ_TX_FLUSH_DESCQ_LBN 0
+#define	FRF_AZ_TX_FLUSH_DESCQ_WIDTH 12
+
+
+/*
+ * FR_BZ_TX_DESC_UPD_REGP0(128bit):
+ * Transmit descriptor update register.
+ */
+#define	FR_BZ_TX_DESC_UPD_REGP0_OFST 0x00000a10
+/* falconb0,sienaa0=net_func_bar2 */
+#define	FR_BZ_TX_DESC_UPD_REGP0_STEP 8192
+#define	FR_BZ_TX_DESC_UPD_REGP0_ROWS 1024
+/*
+ * FR_AA_TX_DESC_UPD_REG_KER(128bit):
+ * Transmit descriptor update register.
+ */
+#define	FR_AA_TX_DESC_UPD_REG_KER_OFST 0x00000a10
+/* falcona0=net_func_bar2 */
+#define	FR_AA_TX_DESC_UPD_REG_KER_STEP 8192
+#define	FR_AA_TX_DESC_UPD_REG_KER_ROWS 8
+/*
+ * FR_AB_TX_DESC_UPD_REGP123(128bit):
+ * Transmit descriptor update register.
+ */
+#define	FR_AB_TX_DESC_UPD_REGP123_OFST 0x01000a10
+/* falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AB_TX_DESC_UPD_REGP123_STEP 8192
+#define	FR_AB_TX_DESC_UPD_REGP123_ROWS 3072
+/*
+ * FR_AA_TX_DESC_UPD_REGP0(128bit):
+ * Transmit descriptor update register.
+ */
+#define	FR_AA_TX_DESC_UPD_REGP0_OFST 0x00008a10
+/* falcona0=char_func_bar0 */
+#define	FR_AA_TX_DESC_UPD_REGP0_STEP 8192
+#define	FR_AA_TX_DESC_UPD_REGP0_ROWS 1020
+
+#define	FRF_AZ_TX_DESC_WPTR_LBN 96
+#define	FRF_AZ_TX_DESC_WPTR_WIDTH 12
+#define	FRF_AZ_TX_DESC_PUSH_CMD_LBN 95
+#define	FRF_AZ_TX_DESC_PUSH_CMD_WIDTH 1
+#define	FRF_AZ_TX_DESC_LBN 0
+#define	FRF_AZ_TX_DESC_WIDTH 95
+#define	FRF_AZ_TX_DESC_DW0_LBN 0
+#define	FRF_AZ_TX_DESC_DW0_WIDTH 32
+#define	FRF_AZ_TX_DESC_DW1_LBN 32
+#define	FRF_AZ_TX_DESC_DW1_WIDTH 32
+#define	FRF_AZ_TX_DESC_DW2_LBN 64
+#define	FRF_AZ_TX_DESC_DW2_WIDTH 31
+
+
+/*
+ * FR_AZ_TX_DC_CFG_REG(128bit):
+ * Transmit descriptor cache configuration register
+ */
+#define	FR_AZ_TX_DC_CFG_REG_OFST 0x00000a20
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_DC_SIZE_LBN 0
+#define	FRF_AZ_TX_DC_SIZE_WIDTH 2
+#define	FFE_AZ_TX_DC_SIZE_32 2
+#define	FFE_AZ_TX_DC_SIZE_16 1
+#define	FFE_AZ_TX_DC_SIZE_8 0
+
+
+/*
+ * FR_AA_TX_CHKSM_CFG_REG(128bit):
+ * Transmit checksum configuration register
+ */
+#define	FR_AA_TX_CHKSM_CFG_REG_OFST 0x00000a30
+/* falcona0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AA_TX_Q_CHKSM_DIS_96_127_LBN 96
+#define	FRF_AA_TX_Q_CHKSM_DIS_96_127_WIDTH 32
+#define	FRF_AA_TX_Q_CHKSM_DIS_64_95_LBN 64
+#define	FRF_AA_TX_Q_CHKSM_DIS_64_95_WIDTH 32
+#define	FRF_AA_TX_Q_CHKSM_DIS_32_63_LBN 32
+#define	FRF_AA_TX_Q_CHKSM_DIS_32_63_WIDTH 32
+#define	FRF_AA_TX_Q_CHKSM_DIS_0_31_LBN 0
+#define	FRF_AA_TX_Q_CHKSM_DIS_0_31_WIDTH 32
+
+
+/*
+ * FR_AZ_TX_CFG_REG(128bit):
+ * Transmit configuration register
+ */
+#define	FR_AZ_TX_CFG_REG_OFST 0x00000a50
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_CZ_TX_CONT_LOOKUP_THRESH_RANGE_LBN 114
+#define	FRF_CZ_TX_CONT_LOOKUP_THRESH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_FILTER_TEST_MODE_BIT_LBN 113
+#define	FRF_CZ_TX_FILTER_TEST_MODE_BIT_WIDTH 1
+#define	FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE_LBN 105
+#define	FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE_LBN 97
+#define	FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE_LBN 89
+#define	FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE_LBN 81
+#define	FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE_LBN 73
+#define	FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE_LBN 65
+#define	FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE_WIDTH 8
+#define	FRF_CZ_TX_FILTER_ALL_VLAN_ETHERTYPES_BIT_LBN 64
+#define	FRF_CZ_TX_FILTER_ALL_VLAN_ETHERTYPES_BIT_WIDTH 1
+#define	FRF_CZ_TX_VLAN_MATCH_ETHERTYPE_RANGE_LBN 48
+#define	FRF_CZ_TX_VLAN_MATCH_ETHERTYPE_RANGE_WIDTH 16
+#define	FRF_CZ_TX_FILTER_EN_BIT_LBN 47
+#define	FRF_CZ_TX_FILTER_EN_BIT_WIDTH 1
+#define	FRF_AZ_TX_IP_ID_P0_OFS_LBN 16
+#define	FRF_AZ_TX_IP_ID_P0_OFS_WIDTH 15
+#define	FRF_AZ_TX_NO_EOP_DISC_EN_LBN 5
+#define	FRF_AZ_TX_NO_EOP_DISC_EN_WIDTH 1
+#define	FRF_AZ_TX_P1_PRI_EN_LBN 4
+#define	FRF_AZ_TX_P1_PRI_EN_WIDTH 1
+#define	FRF_AZ_TX_OWNERR_CTL_LBN 2
+#define	FRF_AZ_TX_OWNERR_CTL_WIDTH 1
+#define	FRF_AA_TX_NON_IP_DROP_DIS_LBN 1
+#define	FRF_AA_TX_NON_IP_DROP_DIS_WIDTH 1
+#define	FRF_AZ_TX_IP_ID_REP_EN_LBN 0
+#define	FRF_AZ_TX_IP_ID_REP_EN_WIDTH 1
+
+
+/*
+ * FR_AZ_TX_PUSH_DROP_REG(128bit):
+ * Transmit push dropped register
+ */
+#define	FR_AZ_TX_PUSH_DROP_REG_OFST 0x00000a60
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_PUSH_DROP_CNT_LBN 0
+#define	FRF_AZ_TX_PUSH_DROP_CNT_WIDTH 32
+
+
+/*
+ * FR_AZ_TX_RESERVED_REG(128bit):
+ * Transmit configuration register
+ */
+#define	FR_AZ_TX_RESERVED_REG_OFST 0x00000a80
+/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_EVT_CNT_LBN 121
+#define	FRF_AZ_TX_EVT_CNT_WIDTH 7
+#define	FRF_AZ_TX_PREF_AGE_CNT_LBN 119
+#define	FRF_AZ_TX_PREF_AGE_CNT_WIDTH 2
+#define	FRF_AZ_TX_RD_COMP_TMR_LBN 96
+#define	FRF_AZ_TX_RD_COMP_TMR_WIDTH 23
+#define	FRF_AZ_TX_PUSH_EN_LBN 89
+#define	FRF_AZ_TX_PUSH_EN_WIDTH 1
+#define	FRF_AZ_TX_PUSH_CHK_DIS_LBN 88
+#define	FRF_AZ_TX_PUSH_CHK_DIS_WIDTH 1
+#define	FRF_AZ_TX_D_FF_FULL_P0_LBN 85
+#define	FRF_AZ_TX_D_FF_FULL_P0_WIDTH 1
+#define	FRF_AZ_TX_DMAR_ST_P0_LBN 81
+#define	FRF_AZ_TX_DMAR_ST_P0_WIDTH 1
+#define	FRF_AZ_TX_DMAQ_ST_LBN 78
+#define	FRF_AZ_TX_DMAQ_ST_WIDTH 1
+#define	FRF_AZ_TX_RX_SPACER_LBN 64
+#define	FRF_AZ_TX_RX_SPACER_WIDTH 8
+#define	FRF_AZ_TX_DROP_ABORT_EN_LBN 60
+#define	FRF_AZ_TX_DROP_ABORT_EN_WIDTH 1
+#define	FRF_AZ_TX_SOFT_EVT_EN_LBN 59
+#define	FRF_AZ_TX_SOFT_EVT_EN_WIDTH 1
+#define	FRF_AZ_TX_PS_EVT_DIS_LBN 58
+#define	FRF_AZ_TX_PS_EVT_DIS_WIDTH 1
+#define	FRF_AZ_TX_RX_SPACER_EN_LBN 57
+#define	FRF_AZ_TX_RX_SPACER_EN_WIDTH 1
+#define	FRF_AZ_TX_XP_TIMER_LBN 52
+#define	FRF_AZ_TX_XP_TIMER_WIDTH 5
+#define	FRF_AZ_TX_PREF_SPACER_LBN 44
+#define	FRF_AZ_TX_PREF_SPACER_WIDTH 8
+#define	FRF_AZ_TX_PREF_WD_TMR_LBN 22
+#define	FRF_AZ_TX_PREF_WD_TMR_WIDTH 22
+#define	FRF_AZ_TX_ONLY1TAG_LBN 21
+#define	FRF_AZ_TX_ONLY1TAG_WIDTH 1
+#define	FRF_AZ_TX_PREF_THRESHOLD_LBN 19
+#define	FRF_AZ_TX_PREF_THRESHOLD_WIDTH 2
+#define	FRF_AZ_TX_ONE_PKT_PER_Q_LBN 18
+#define	FRF_AZ_TX_ONE_PKT_PER_Q_WIDTH 1
+#define	FRF_AZ_TX_DIS_NON_IP_EV_LBN 17
+#define	FRF_AZ_TX_DIS_NON_IP_EV_WIDTH 1
+#define	FRF_AA_TX_DMA_FF_THR_LBN 16
+#define	FRF_AA_TX_DMA_FF_THR_WIDTH 1
+#define	FRF_AZ_TX_DMA_SPACER_LBN 8
+#define	FRF_AZ_TX_DMA_SPACER_WIDTH 8
+#define	FRF_AA_TX_TCP_DIS_LBN 7
+#define	FRF_AA_TX_TCP_DIS_WIDTH 1
+#define	FRF_BZ_TX_FLUSH_MIN_LEN_EN_LBN 7
+#define	FRF_BZ_TX_FLUSH_MIN_LEN_EN_WIDTH 1
+#define	FRF_AA_TX_IP_DIS_LBN 6
+#define	FRF_AA_TX_IP_DIS_WIDTH 1
+#define	FRF_AZ_TX_MAX_CPL_LBN 2
+#define	FRF_AZ_TX_MAX_CPL_WIDTH 2
+#define	FFE_AZ_TX_MAX_CPL_16 3
+#define	FFE_AZ_TX_MAX_CPL_8 2
+#define	FFE_AZ_TX_MAX_CPL_4 1
+#define	FFE_AZ_TX_MAX_CPL_NOLIMIT 0
+#define	FRF_AZ_TX_MAX_PREF_LBN 0
+#define	FRF_AZ_TX_MAX_PREF_WIDTH 2
+#define	FFE_AZ_TX_MAX_PREF_32 3
+#define	FFE_AZ_TX_MAX_PREF_16 2
+#define	FFE_AZ_TX_MAX_PREF_8 1
+#define	FFE_AZ_TX_MAX_PREF_OFF 0
+
+
+/*
+ * FR_BZ_TX_PACE_REG(128bit):
+ * Transmit pace control register
+ */
+#define	FR_BZ_TX_PACE_REG_OFST 0x00000a90
+/* falconb0,sienaa0=net_func_bar2 */
+/*
+ * FR_AA_TX_PACE_REG(128bit):
+ * Transmit pace control register
+ */
+#define	FR_AA_TX_PACE_REG_OFST 0x00f80000
+/* falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_PACE_SB_NOT_AF_LBN 19
+#define	FRF_AZ_TX_PACE_SB_NOT_AF_WIDTH 10
+#define	FRF_AZ_TX_PACE_SB_AF_LBN 9
+#define	FRF_AZ_TX_PACE_SB_AF_WIDTH 10
+#define	FRF_AZ_TX_PACE_FB_BASE_LBN 5
+#define	FRF_AZ_TX_PACE_FB_BASE_WIDTH 4
+#define	FRF_AZ_TX_PACE_BIN_TH_LBN 0
+#define	FRF_AZ_TX_PACE_BIN_TH_WIDTH 5
+
+
+/*
+ * FR_AZ_TX_PACE_DROP_QID_REG(128bit):
+ * PACE Drop QID Counter
+ */
+#define	FR_AZ_TX_PACE_DROP_QID_REG_OFST 0x00000aa0
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_PACE_QID_DRP_CNT_LBN 0
+#define	FRF_AZ_TX_PACE_QID_DRP_CNT_WIDTH 16
+
+
+/*
+ * FR_AB_TX_VLAN_REG(128bit):
+ * Transmit VLAN tag register
+ */
+#define	FR_AB_TX_VLAN_REG_OFST 0x00000ae0
+/* falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_TX_VLAN_EN_LBN 127
+#define	FRF_AB_TX_VLAN_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN7_PORT1_EN_LBN 125
+#define	FRF_AB_TX_VLAN7_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN7_PORT0_EN_LBN 124
+#define	FRF_AB_TX_VLAN7_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN7_LBN 112
+#define	FRF_AB_TX_VLAN7_WIDTH 12
+#define	FRF_AB_TX_VLAN6_PORT1_EN_LBN 109
+#define	FRF_AB_TX_VLAN6_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN6_PORT0_EN_LBN 108
+#define	FRF_AB_TX_VLAN6_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN6_LBN 96
+#define	FRF_AB_TX_VLAN6_WIDTH 12
+#define	FRF_AB_TX_VLAN5_PORT1_EN_LBN 93
+#define	FRF_AB_TX_VLAN5_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN5_PORT0_EN_LBN 92
+#define	FRF_AB_TX_VLAN5_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN5_LBN 80
+#define	FRF_AB_TX_VLAN5_WIDTH 12
+#define	FRF_AB_TX_VLAN4_PORT1_EN_LBN 77
+#define	FRF_AB_TX_VLAN4_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN4_PORT0_EN_LBN 76
+#define	FRF_AB_TX_VLAN4_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN4_LBN 64
+#define	FRF_AB_TX_VLAN4_WIDTH 12
+#define	FRF_AB_TX_VLAN3_PORT1_EN_LBN 61
+#define	FRF_AB_TX_VLAN3_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN3_PORT0_EN_LBN 60
+#define	FRF_AB_TX_VLAN3_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN3_LBN 48
+#define	FRF_AB_TX_VLAN3_WIDTH 12
+#define	FRF_AB_TX_VLAN2_PORT1_EN_LBN 45
+#define	FRF_AB_TX_VLAN2_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN2_PORT0_EN_LBN 44
+#define	FRF_AB_TX_VLAN2_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN2_LBN 32
+#define	FRF_AB_TX_VLAN2_WIDTH 12
+#define	FRF_AB_TX_VLAN1_PORT1_EN_LBN 29
+#define	FRF_AB_TX_VLAN1_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN1_PORT0_EN_LBN 28
+#define	FRF_AB_TX_VLAN1_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN1_LBN 16
+#define	FRF_AB_TX_VLAN1_WIDTH 12
+#define	FRF_AB_TX_VLAN0_PORT1_EN_LBN 13
+#define	FRF_AB_TX_VLAN0_PORT1_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN0_PORT0_EN_LBN 12
+#define	FRF_AB_TX_VLAN0_PORT0_EN_WIDTH 1
+#define	FRF_AB_TX_VLAN0_LBN 0
+#define	FRF_AB_TX_VLAN0_WIDTH 12
+
+
+/*
+ * FR_AZ_TX_IPFIL_PORTEN_REG(128bit):
+ * Transmit filter control register
+ */
+#define	FR_AZ_TX_IPFIL_PORTEN_REG_OFST 0x00000af0
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AZ_TX_MADR0_FIL_EN_LBN 64
+#define	FRF_AZ_TX_MADR0_FIL_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL31_PORT_EN_LBN 62
+#define	FRF_AB_TX_IPFIL31_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL30_PORT_EN_LBN 60
+#define	FRF_AB_TX_IPFIL30_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL29_PORT_EN_LBN 58
+#define	FRF_AB_TX_IPFIL29_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL28_PORT_EN_LBN 56
+#define	FRF_AB_TX_IPFIL28_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL27_PORT_EN_LBN 54
+#define	FRF_AB_TX_IPFIL27_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL26_PORT_EN_LBN 52
+#define	FRF_AB_TX_IPFIL26_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL25_PORT_EN_LBN 50
+#define	FRF_AB_TX_IPFIL25_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL24_PORT_EN_LBN 48
+#define	FRF_AB_TX_IPFIL24_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL23_PORT_EN_LBN 46
+#define	FRF_AB_TX_IPFIL23_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL22_PORT_EN_LBN 44
+#define	FRF_AB_TX_IPFIL22_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL21_PORT_EN_LBN 42
+#define	FRF_AB_TX_IPFIL21_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL20_PORT_EN_LBN 40
+#define	FRF_AB_TX_IPFIL20_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL19_PORT_EN_LBN 38
+#define	FRF_AB_TX_IPFIL19_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL18_PORT_EN_LBN 36
+#define	FRF_AB_TX_IPFIL18_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL17_PORT_EN_LBN 34
+#define	FRF_AB_TX_IPFIL17_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL16_PORT_EN_LBN 32
+#define	FRF_AB_TX_IPFIL16_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL15_PORT_EN_LBN 30
+#define	FRF_AB_TX_IPFIL15_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL14_PORT_EN_LBN 28
+#define	FRF_AB_TX_IPFIL14_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL13_PORT_EN_LBN 26
+#define	FRF_AB_TX_IPFIL13_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL12_PORT_EN_LBN 24
+#define	FRF_AB_TX_IPFIL12_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL11_PORT_EN_LBN 22
+#define	FRF_AB_TX_IPFIL11_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL10_PORT_EN_LBN 20
+#define	FRF_AB_TX_IPFIL10_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL9_PORT_EN_LBN 18
+#define	FRF_AB_TX_IPFIL9_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL8_PORT_EN_LBN 16
+#define	FRF_AB_TX_IPFIL8_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL7_PORT_EN_LBN 14
+#define	FRF_AB_TX_IPFIL7_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL6_PORT_EN_LBN 12
+#define	FRF_AB_TX_IPFIL6_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL5_PORT_EN_LBN 10
+#define	FRF_AB_TX_IPFIL5_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL4_PORT_EN_LBN 8
+#define	FRF_AB_TX_IPFIL4_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL3_PORT_EN_LBN 6
+#define	FRF_AB_TX_IPFIL3_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL2_PORT_EN_LBN 4
+#define	FRF_AB_TX_IPFIL2_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL1_PORT_EN_LBN 2
+#define	FRF_AB_TX_IPFIL1_PORT_EN_WIDTH 1
+#define	FRF_AB_TX_IPFIL0_PORT_EN_LBN 0
+#define	FRF_AB_TX_IPFIL0_PORT_EN_WIDTH 1
+
+
+/*
+ * FR_AB_TX_IPFIL_TBL(128bit):
+ * Transmit IP source address filter table
+ */
+#define	FR_AB_TX_IPFIL_TBL_OFST 0x00000b00
+/* falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AB_TX_IPFIL_TBL_STEP 16
+#define	FR_AB_TX_IPFIL_TBL_ROWS 16
+
+#define	FRF_AB_TX_IPFIL_MASK_1_LBN 96
+#define	FRF_AB_TX_IPFIL_MASK_1_WIDTH 32
+#define	FRF_AB_TX_IP_SRC_ADR_1_LBN 64
+#define	FRF_AB_TX_IP_SRC_ADR_1_WIDTH 32
+#define	FRF_AB_TX_IPFIL_MASK_0_LBN 32
+#define	FRF_AB_TX_IPFIL_MASK_0_WIDTH 32
+#define	FRF_AB_TX_IP_SRC_ADR_0_LBN 0
+#define	FRF_AB_TX_IP_SRC_ADR_0_WIDTH 32
+
+
+/*
+ * FR_AB_MD_TXD_REG(128bit):
+ * PHY management transmit data register
+ */
+#define	FR_AB_MD_TXD_REG_OFST 0x00000c00
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MD_TXD_LBN 0
+#define	FRF_AB_MD_TXD_WIDTH 16
+
+
+/*
+ * FR_AB_MD_RXD_REG(128bit):
+ * PHY management receive data register
+ */
+#define	FR_AB_MD_RXD_REG_OFST 0x00000c10
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MD_RXD_LBN 0
+#define	FRF_AB_MD_RXD_WIDTH 16
+
+
+/*
+ * FR_AB_MD_CS_REG(128bit):
+ * PHY management configuration & status register
+ */
+#define	FR_AB_MD_CS_REG_OFST 0x00000c20
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MD_RD_EN_LBN 15
+#define	FRF_AB_MD_RD_EN_WIDTH 1
+#define	FRF_AB_MD_WR_EN_LBN 14
+#define	FRF_AB_MD_WR_EN_WIDTH 1
+#define	FRF_AB_MD_ADDR_CMD_LBN 13
+#define	FRF_AB_MD_ADDR_CMD_WIDTH 1
+#define	FRF_AB_MD_PT_LBN 7
+#define	FRF_AB_MD_PT_WIDTH 3
+#define	FRF_AB_MD_PL_LBN 6
+#define	FRF_AB_MD_PL_WIDTH 1
+#define	FRF_AB_MD_INT_CLR_LBN 5
+#define	FRF_AB_MD_INT_CLR_WIDTH 1
+#define	FRF_AB_MD_GC_LBN 4
+#define	FRF_AB_MD_GC_WIDTH 1
+#define	FRF_AB_MD_PRSP_LBN 3
+#define	FRF_AB_MD_PRSP_WIDTH 1
+#define	FRF_AB_MD_RIC_LBN 2
+#define	FRF_AB_MD_RIC_WIDTH 1
+#define	FRF_AB_MD_RDC_LBN 1
+#define	FRF_AB_MD_RDC_WIDTH 1
+#define	FRF_AB_MD_WRC_LBN 0
+#define	FRF_AB_MD_WRC_WIDTH 1
+
+
+/*
+ * FR_AB_MD_PHY_ADR_REG(128bit):
+ * PHY management PHY address register
+ */
+#define	FR_AB_MD_PHY_ADR_REG_OFST 0x00000c30
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MD_PHY_ADR_LBN 0
+#define	FRF_AB_MD_PHY_ADR_WIDTH 16
+
+
+/*
+ * FR_AB_MD_ID_REG(128bit):
+ * PHY management ID register
+ */
+#define	FR_AB_MD_ID_REG_OFST 0x00000c40
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MD_PRT_ADR_LBN 11
+#define	FRF_AB_MD_PRT_ADR_WIDTH 5
+#define	FRF_AB_MD_DEV_ADR_LBN 6
+#define	FRF_AB_MD_DEV_ADR_WIDTH 5
+
+
+/*
+ * FR_AB_MD_STAT_REG(128bit):
+ * PHY management status & mask register
+ */
+#define	FR_AB_MD_STAT_REG_OFST 0x00000c50
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MD_PINT_LBN 4
+#define	FRF_AB_MD_PINT_WIDTH 1
+#define	FRF_AB_MD_DONE_LBN 3
+#define	FRF_AB_MD_DONE_WIDTH 1
+#define	FRF_AB_MD_BSERR_LBN 2
+#define	FRF_AB_MD_BSERR_WIDTH 1
+#define	FRF_AB_MD_LNFL_LBN 1
+#define	FRF_AB_MD_LNFL_WIDTH 1
+#define	FRF_AB_MD_BSY_LBN 0
+#define	FRF_AB_MD_BSY_WIDTH 1
+
+
+/*
+ * FR_AB_MAC_STAT_DMA_REG(128bit):
+ * Port MAC statistical counter DMA register
+ */
+#define	FR_AB_MAC_STAT_DMA_REG_OFST 0x00000c60
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MAC_STAT_DMA_CMD_LBN 48
+#define	FRF_AB_MAC_STAT_DMA_CMD_WIDTH 1
+#define	FRF_AB_MAC_STAT_DMA_ADR_LBN 0
+#define	FRF_AB_MAC_STAT_DMA_ADR_WIDTH 48
+#define	FRF_AB_MAC_STAT_DMA_ADR_DW0_LBN 0
+#define	FRF_AB_MAC_STAT_DMA_ADR_DW0_WIDTH 32
+#define	FRF_AB_MAC_STAT_DMA_ADR_DW1_LBN 32
+#define	FRF_AB_MAC_STAT_DMA_ADR_DW1_WIDTH 16
+
+
+/*
+ * FR_AB_MAC_CTRL_REG(128bit):
+ * Port MAC control register
+ */
+#define	FR_AB_MAC_CTRL_REG_OFST 0x00000c80
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MAC_XOFF_VAL_LBN 16
+#define	FRF_AB_MAC_XOFF_VAL_WIDTH 16
+#define	FRF_BB_TXFIFO_DRAIN_EN_LBN 7
+#define	FRF_BB_TXFIFO_DRAIN_EN_WIDTH 1
+#define	FRF_AB_MAC_XG_DISTXCRC_LBN 5
+#define	FRF_AB_MAC_XG_DISTXCRC_WIDTH 1
+#define	FRF_AB_MAC_BCAD_ACPT_LBN 4
+#define	FRF_AB_MAC_BCAD_ACPT_WIDTH 1
+#define	FRF_AB_MAC_UC_PROM_LBN 3
+#define	FRF_AB_MAC_UC_PROM_WIDTH 1
+#define	FRF_AB_MAC_LINK_STATUS_LBN 2
+#define	FRF_AB_MAC_LINK_STATUS_WIDTH 1
+#define	FRF_AB_MAC_SPEED_LBN 0
+#define	FRF_AB_MAC_SPEED_WIDTH 2
+#define	FRF_AB_MAC_SPEED_10M 0
+#define	FRF_AB_MAC_SPEED_100M 1
+#define	FRF_AB_MAC_SPEED_1G 2
+#define	FRF_AB_MAC_SPEED_10G 3
+
+/*
+ * FR_BB_GEN_MODE_REG(128bit):
+ * General Purpose mode register (external interrupt mask)
+ */
+#define	FR_BB_GEN_MODE_REG_OFST 0x00000c90
+/* falconb0=net_func_bar2 */
+
+#define	FRF_BB_XFP_PHY_INT_POL_SEL_LBN 3
+#define	FRF_BB_XFP_PHY_INT_POL_SEL_WIDTH 1
+#define	FRF_BB_XG_PHY_INT_POL_SEL_LBN 2
+#define	FRF_BB_XG_PHY_INT_POL_SEL_WIDTH 1
+#define	FRF_BB_XFP_PHY_INT_MASK_LBN 1
+#define	FRF_BB_XFP_PHY_INT_MASK_WIDTH 1
+#define	FRF_BB_XG_PHY_INT_MASK_LBN 0
+#define	FRF_BB_XG_PHY_INT_MASK_WIDTH 1
+
+
+/*
+ * FR_AB_MAC_MC_HASH_REG0(128bit):
+ * Multicast address hash table
+ */
+#define	FR_AB_MAC_MC_HASH0_REG_OFST 0x00000ca0
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MAC_MCAST_HASH0_LBN 0
+#define	FRF_AB_MAC_MCAST_HASH0_WIDTH 128
+#define	FRF_AB_MAC_MCAST_HASH0_DW0_LBN 0
+#define	FRF_AB_MAC_MCAST_HASH0_DW0_WIDTH 32
+#define	FRF_AB_MAC_MCAST_HASH0_DW1_LBN 32
+#define	FRF_AB_MAC_MCAST_HASH0_DW1_WIDTH 32
+#define	FRF_AB_MAC_MCAST_HASH0_DW2_LBN 64
+#define	FRF_AB_MAC_MCAST_HASH0_DW2_WIDTH 32
+#define	FRF_AB_MAC_MCAST_HASH0_DW3_LBN 96
+#define	FRF_AB_MAC_MCAST_HASH0_DW3_WIDTH 32
+
+
+/*
+ * FR_AB_MAC_MC_HASH_REG1(128bit):
+ * Multicast address hash table
+ */
+#define	FR_AB_MAC_MC_HASH1_REG_OFST 0x00000cb0
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_MAC_MCAST_HASH1_LBN 0
+#define	FRF_AB_MAC_MCAST_HASH1_WIDTH 128
+#define	FRF_AB_MAC_MCAST_HASH1_DW0_LBN 0
+#define	FRF_AB_MAC_MCAST_HASH1_DW0_WIDTH 32
+#define	FRF_AB_MAC_MCAST_HASH1_DW1_LBN 32
+#define	FRF_AB_MAC_MCAST_HASH1_DW1_WIDTH 32
+#define	FRF_AB_MAC_MCAST_HASH1_DW2_LBN 64
+#define	FRF_AB_MAC_MCAST_HASH1_DW2_WIDTH 32
+#define	FRF_AB_MAC_MCAST_HASH1_DW3_LBN 96
+#define	FRF_AB_MAC_MCAST_HASH1_DW3_WIDTH 32
+
+
+/*
+ * FR_AB_GM_CFG1_REG(32bit):
+ * GMAC configuration register 1
+ */
+#define	FR_AB_GM_CFG1_REG_OFST 0x00000e00
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_SW_RST_LBN 31
+#define	FRF_AB_GM_SW_RST_WIDTH 1
+#define	FRF_AB_GM_SIM_RST_LBN 30
+#define	FRF_AB_GM_SIM_RST_WIDTH 1
+#define	FRF_AB_GM_RST_RX_MAC_CTL_LBN 19
+#define	FRF_AB_GM_RST_RX_MAC_CTL_WIDTH 1
+#define	FRF_AB_GM_RST_TX_MAC_CTL_LBN 18
+#define	FRF_AB_GM_RST_TX_MAC_CTL_WIDTH 1
+#define	FRF_AB_GM_RST_RX_FUNC_LBN 17
+#define	FRF_AB_GM_RST_RX_FUNC_WIDTH 1
+#define	FRF_AB_GM_RST_TX_FUNC_LBN 16
+#define	FRF_AB_GM_RST_TX_FUNC_WIDTH 1
+#define	FRF_AB_GM_LOOP_LBN 8
+#define	FRF_AB_GM_LOOP_WIDTH 1
+#define	FRF_AB_GM_RX_FC_EN_LBN 5
+#define	FRF_AB_GM_RX_FC_EN_WIDTH 1
+#define	FRF_AB_GM_TX_FC_EN_LBN 4
+#define	FRF_AB_GM_TX_FC_EN_WIDTH 1
+#define	FRF_AB_GM_SYNC_RXEN_LBN 3
+#define	FRF_AB_GM_SYNC_RXEN_WIDTH 1
+#define	FRF_AB_GM_RX_EN_LBN 2
+#define	FRF_AB_GM_RX_EN_WIDTH 1
+#define	FRF_AB_GM_SYNC_TXEN_LBN 1
+#define	FRF_AB_GM_SYNC_TXEN_WIDTH 1
+#define	FRF_AB_GM_TX_EN_LBN 0
+#define	FRF_AB_GM_TX_EN_WIDTH 1
+
+
+/*
+ * FR_AB_GM_CFG2_REG(32bit):
+ * GMAC configuration register 2
+ */
+#define	FR_AB_GM_CFG2_REG_OFST 0x00000e10
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_PAMBL_LEN_LBN 12
+#define	FRF_AB_GM_PAMBL_LEN_WIDTH 4
+#define	FRF_AB_GM_IF_MODE_LBN 8
+#define	FRF_AB_GM_IF_MODE_WIDTH 2
+#define	FRF_AB_GM_IF_MODE_BYTE_MODE 2
+#define	FRF_AB_GM_IF_MODE_NIBBLE_MODE 1
+#define	FRF_AB_GM_HUGE_FRM_EN_LBN 5
+#define	FRF_AB_GM_HUGE_FRM_EN_WIDTH 1
+#define	FRF_AB_GM_LEN_CHK_LBN 4
+#define	FRF_AB_GM_LEN_CHK_WIDTH 1
+#define	FRF_AB_GM_PAD_CRC_EN_LBN 2
+#define	FRF_AB_GM_PAD_CRC_EN_WIDTH 1
+#define	FRF_AB_GM_CRC_EN_LBN 1
+#define	FRF_AB_GM_CRC_EN_WIDTH 1
+#define	FRF_AB_GM_FD_LBN 0
+#define	FRF_AB_GM_FD_WIDTH 1
+
+
+/*
+ * FR_AB_GM_IPG_REG(32bit):
+ * GMAC IPG register
+ */
+#define	FR_AB_GM_IPG_REG_OFST 0x00000e20
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_NONB2B_IPG1_LBN 24
+#define	FRF_AB_GM_NONB2B_IPG1_WIDTH 7
+#define	FRF_AB_GM_NONB2B_IPG2_LBN 16
+#define	FRF_AB_GM_NONB2B_IPG2_WIDTH 7
+#define	FRF_AB_GM_MIN_IPG_ENF_LBN 8
+#define	FRF_AB_GM_MIN_IPG_ENF_WIDTH 8
+#define	FRF_AB_GM_B2B_IPG_LBN 0
+#define	FRF_AB_GM_B2B_IPG_WIDTH 7
+
+
+/*
+ * FR_AB_GM_HD_REG(32bit):
+ * GMAC half duplex register
+ */
+#define	FR_AB_GM_HD_REG_OFST 0x00000e30
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_ALT_BOFF_VAL_LBN 20
+#define	FRF_AB_GM_ALT_BOFF_VAL_WIDTH 4
+#define	FRF_AB_GM_ALT_BOFF_EN_LBN 19
+#define	FRF_AB_GM_ALT_BOFF_EN_WIDTH 1
+#define	FRF_AB_GM_BP_NO_BOFF_LBN 18
+#define	FRF_AB_GM_BP_NO_BOFF_WIDTH 1
+#define	FRF_AB_GM_DIS_BOFF_LBN 17
+#define	FRF_AB_GM_DIS_BOFF_WIDTH 1
+#define	FRF_AB_GM_EXDEF_TX_EN_LBN 16
+#define	FRF_AB_GM_EXDEF_TX_EN_WIDTH 1
+#define	FRF_AB_GM_RTRY_LIMIT_LBN 12
+#define	FRF_AB_GM_RTRY_LIMIT_WIDTH 4
+#define	FRF_AB_GM_COL_WIN_LBN 0
+#define	FRF_AB_GM_COL_WIN_WIDTH 10
+
+
+/*
+ * FR_AB_GM_MAX_FLEN_REG(32bit):
+ * GMAC maximum frame length register
+ */
+#define	FR_AB_GM_MAX_FLEN_REG_OFST 0x00000e40
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_MAX_FLEN_LBN 0
+#define	FRF_AB_GM_MAX_FLEN_WIDTH 16
+
+
+/*
+ * FR_AB_GM_TEST_REG(32bit):
+ * GMAC test register
+ */
+#define	FR_AB_GM_TEST_REG_OFST 0x00000e70
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_MAX_BOFF_LBN 3
+#define	FRF_AB_GM_MAX_BOFF_WIDTH 1
+#define	FRF_AB_GM_REG_TX_FLOW_EN_LBN 2
+#define	FRF_AB_GM_REG_TX_FLOW_EN_WIDTH 1
+#define	FRF_AB_GM_TEST_PAUSE_LBN 1
+#define	FRF_AB_GM_TEST_PAUSE_WIDTH 1
+#define	FRF_AB_GM_SHORT_SLOT_LBN 0
+#define	FRF_AB_GM_SHORT_SLOT_WIDTH 1
+
+
+/*
+ * FR_AB_GM_ADR1_REG(32bit):
+ * GMAC station address register 1
+ */
+#define	FR_AB_GM_ADR1_REG_OFST 0x00000f00
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_ADR_B0_LBN 24
+#define	FRF_AB_GM_ADR_B0_WIDTH 8
+#define	FRF_AB_GM_ADR_B1_LBN 16
+#define	FRF_AB_GM_ADR_B1_WIDTH 8
+#define	FRF_AB_GM_ADR_B2_LBN 8
+#define	FRF_AB_GM_ADR_B2_WIDTH 8
+#define	FRF_AB_GM_ADR_B3_LBN 0
+#define	FRF_AB_GM_ADR_B3_WIDTH 8
+
+
+/*
+ * FR_AB_GM_ADR2_REG(32bit):
+ * GMAC station address register 2
+ */
+#define	FR_AB_GM_ADR2_REG_OFST 0x00000f10
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GM_ADR_B4_LBN 24
+#define	FRF_AB_GM_ADR_B4_WIDTH 8
+#define	FRF_AB_GM_ADR_B5_LBN 16
+#define	FRF_AB_GM_ADR_B5_WIDTH 8
+
+
+/*
+ * FR_AB_GMF_CFG0_REG(32bit):
+ * GMAC FIFO configuration register 0
+ */
+#define	FR_AB_GMF_CFG0_REG_OFST 0x00000f20
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GMF_FTFENRPLY_LBN 20
+#define	FRF_AB_GMF_FTFENRPLY_WIDTH 1
+#define	FRF_AB_GMF_STFENRPLY_LBN 19
+#define	FRF_AB_GMF_STFENRPLY_WIDTH 1
+#define	FRF_AB_GMF_FRFENRPLY_LBN 18
+#define	FRF_AB_GMF_FRFENRPLY_WIDTH 1
+#define	FRF_AB_GMF_SRFENRPLY_LBN 17
+#define	FRF_AB_GMF_SRFENRPLY_WIDTH 1
+#define	FRF_AB_GMF_WTMENRPLY_LBN 16
+#define	FRF_AB_GMF_WTMENRPLY_WIDTH 1
+#define	FRF_AB_GMF_FTFENREQ_LBN 12
+#define	FRF_AB_GMF_FTFENREQ_WIDTH 1
+#define	FRF_AB_GMF_STFENREQ_LBN 11
+#define	FRF_AB_GMF_STFENREQ_WIDTH 1
+#define	FRF_AB_GMF_FRFENREQ_LBN 10
+#define	FRF_AB_GMF_FRFENREQ_WIDTH 1
+#define	FRF_AB_GMF_SRFENREQ_LBN 9
+#define	FRF_AB_GMF_SRFENREQ_WIDTH 1
+#define	FRF_AB_GMF_WTMENREQ_LBN 8
+#define	FRF_AB_GMF_WTMENREQ_WIDTH 1
+#define	FRF_AB_GMF_HSTRSTFT_LBN 4
+#define	FRF_AB_GMF_HSTRSTFT_WIDTH 1
+#define	FRF_AB_GMF_HSTRSTST_LBN 3
+#define	FRF_AB_GMF_HSTRSTST_WIDTH 1
+#define	FRF_AB_GMF_HSTRSTFR_LBN 2
+#define	FRF_AB_GMF_HSTRSTFR_WIDTH 1
+#define	FRF_AB_GMF_HSTRSTSR_LBN 1
+#define	FRF_AB_GMF_HSTRSTSR_WIDTH 1
+#define	FRF_AB_GMF_HSTRSTWT_LBN 0
+#define	FRF_AB_GMF_HSTRSTWT_WIDTH 1
+
+
+/*
+ * FR_AB_GMF_CFG1_REG(32bit):
+ * GMAC FIFO configuration register 1
+ */
+#define	FR_AB_GMF_CFG1_REG_OFST 0x00000f30
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GMF_CFGFRTH_LBN 16
+#define	FRF_AB_GMF_CFGFRTH_WIDTH 5
+#define	FRF_AB_GMF_CFGXOFFRTX_LBN 0
+#define	FRF_AB_GMF_CFGXOFFRTX_WIDTH 16
+
+
+/*
+ * FR_AB_GMF_CFG2_REG(32bit):
+ * GMAC FIFO configuration register 2
+ */
+#define	FR_AB_GMF_CFG2_REG_OFST 0x00000f40
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GMF_CFGHWM_LBN 16
+#define	FRF_AB_GMF_CFGHWM_WIDTH 6
+#define	FRF_AB_GMF_CFGLWM_LBN 0
+#define	FRF_AB_GMF_CFGLWM_WIDTH 6
+
+
+/*
+ * FR_AB_GMF_CFG3_REG(32bit):
+ * GMAC FIFO configuration register 3
+ */
+#define	FR_AB_GMF_CFG3_REG_OFST 0x00000f50
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GMF_CFGHWMFT_LBN 16
+#define	FRF_AB_GMF_CFGHWMFT_WIDTH 6
+#define	FRF_AB_GMF_CFGFTTH_LBN 0
+#define	FRF_AB_GMF_CFGFTTH_WIDTH 6
+
+
+/*
+ * FR_AB_GMF_CFG4_REG(32bit):
+ * GMAC FIFO configuration register 4
+ */
+#define	FR_AB_GMF_CFG4_REG_OFST 0x00000f60
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GMF_HSTFLTRFRM_LBN 0
+#define	FRF_AB_GMF_HSTFLTRFRM_WIDTH 18
+
+
+/*
+ * FR_AB_GMF_CFG5_REG(32bit):
+ * GMAC FIFO configuration register 5
+ */
+#define	FR_AB_GMF_CFG5_REG_OFST 0x00000f70
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_GMF_CFGHDPLX_LBN 22
+#define	FRF_AB_GMF_CFGHDPLX_WIDTH 1
+#define	FRF_AB_GMF_SRFULL_LBN 21
+#define	FRF_AB_GMF_SRFULL_WIDTH 1
+#define	FRF_AB_GMF_HSTSRFULLCLR_LBN 20
+#define	FRF_AB_GMF_HSTSRFULLCLR_WIDTH 1
+#define	FRF_AB_GMF_CFGBYTMODE_LBN 19
+#define	FRF_AB_GMF_CFGBYTMODE_WIDTH 1
+#define	FRF_AB_GMF_HSTDRPLT64_LBN 18
+#define	FRF_AB_GMF_HSTDRPLT64_WIDTH 1
+#define	FRF_AB_GMF_HSTFLTRFRMDC_LBN 0
+#define	FRF_AB_GMF_HSTFLTRFRMDC_WIDTH 18
+
+
+/*
+ * FR_BB_TX_SRC_MAC_TBL(128bit):
+ * Transmit IP source address filter table
+ */
+#define	FR_BB_TX_SRC_MAC_TBL_OFST 0x00001000
+/* falconb0=net_func_bar2 */
+#define	FR_BB_TX_SRC_MAC_TBL_STEP 16
+#define	FR_BB_TX_SRC_MAC_TBL_ROWS 16
+
+#define	FRF_BB_TX_SRC_MAC_ADR_1_LBN 64
+#define	FRF_BB_TX_SRC_MAC_ADR_1_WIDTH 48
+#define	FRF_BB_TX_SRC_MAC_ADR_1_DW0_LBN 64
+#define	FRF_BB_TX_SRC_MAC_ADR_1_DW0_WIDTH 32
+#define	FRF_BB_TX_SRC_MAC_ADR_1_DW1_LBN 96
+#define	FRF_BB_TX_SRC_MAC_ADR_1_DW1_WIDTH 16
+#define	FRF_BB_TX_SRC_MAC_ADR_0_LBN 0
+#define	FRF_BB_TX_SRC_MAC_ADR_0_WIDTH 48
+#define	FRF_BB_TX_SRC_MAC_ADR_0_DW0_LBN 0
+#define	FRF_BB_TX_SRC_MAC_ADR_0_DW0_WIDTH 32
+#define	FRF_BB_TX_SRC_MAC_ADR_0_DW1_LBN 32
+#define	FRF_BB_TX_SRC_MAC_ADR_0_DW1_WIDTH 16
+
+
+/*
+ * FR_BB_TX_SRC_MAC_CTL_REG(128bit):
+ * Transmit MAC source address filter control
+ */
+#define	FR_BB_TX_SRC_MAC_CTL_REG_OFST 0x00001100
+/* falconb0=net_func_bar2 */
+
+#define	FRF_BB_TX_SRC_DROP_CTR_LBN 16
+#define	FRF_BB_TX_SRC_DROP_CTR_WIDTH 16
+#define	FRF_BB_TX_SRC_FLTR_EN_LBN 15
+#define	FRF_BB_TX_SRC_FLTR_EN_WIDTH 1
+#define	FRF_BB_TX_DROP_CTR_CLR_LBN 12
+#define	FRF_BB_TX_DROP_CTR_CLR_WIDTH 1
+#define	FRF_BB_TX_MAC_QID_SEL_LBN 0
+#define	FRF_BB_TX_MAC_QID_SEL_WIDTH 3
+
+
+/*
+ * FR_AB_XM_ADR_LO_REG(128bit):
+ * XGMAC address register low
+ */
+#define	FR_AB_XM_ADR_LO_REG_OFST 0x00001200
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_ADR_LO_LBN 0
+#define	FRF_AB_XM_ADR_LO_WIDTH 32
+
+
+/*
+ * FR_AB_XM_ADR_HI_REG(128bit):
+ * XGMAC address register high
+ */
+#define	FR_AB_XM_ADR_HI_REG_OFST 0x00001210
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_ADR_HI_LBN 0
+#define	FRF_AB_XM_ADR_HI_WIDTH 16
+
+
+/*
+ * FR_AB_XM_GLB_CFG_REG(128bit):
+ * XGMAC global configuration
+ */
+#define	FR_AB_XM_GLB_CFG_REG_OFST 0x00001220
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_RMTFLT_GEN_LBN 17
+#define	FRF_AB_XM_RMTFLT_GEN_WIDTH 1
+#define	FRF_AB_XM_DEBUG_MODE_LBN 16
+#define	FRF_AB_XM_DEBUG_MODE_WIDTH 1
+#define	FRF_AB_XM_RX_STAT_EN_LBN 11
+#define	FRF_AB_XM_RX_STAT_EN_WIDTH 1
+#define	FRF_AB_XM_TX_STAT_EN_LBN 10
+#define	FRF_AB_XM_TX_STAT_EN_WIDTH 1
+#define	FRF_AB_XM_RX_JUMBO_MODE_LBN 6
+#define	FRF_AB_XM_RX_JUMBO_MODE_WIDTH 1
+#define	FRF_AB_XM_WAN_MODE_LBN 5
+#define	FRF_AB_XM_WAN_MODE_WIDTH 1
+#define	FRF_AB_XM_INTCLR_MODE_LBN 3
+#define	FRF_AB_XM_INTCLR_MODE_WIDTH 1
+#define	FRF_AB_XM_CORE_RST_LBN 0
+#define	FRF_AB_XM_CORE_RST_WIDTH 1
+
+
+/*
+ * FR_AB_XM_TX_CFG_REG(128bit):
+ * XGMAC transmit configuration
+ */
+#define	FR_AB_XM_TX_CFG_REG_OFST 0x00001230
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_TX_PROG_LBN 24
+#define	FRF_AB_XM_TX_PROG_WIDTH 1
+#define	FRF_AB_XM_IPG_LBN 16
+#define	FRF_AB_XM_IPG_WIDTH 4
+#define	FRF_AB_XM_FCNTL_LBN 10
+#define	FRF_AB_XM_FCNTL_WIDTH 1
+#define	FRF_AB_XM_TXCRC_LBN 8
+#define	FRF_AB_XM_TXCRC_WIDTH 1
+#define	FRF_AB_XM_EDRC_LBN 6
+#define	FRF_AB_XM_EDRC_WIDTH 1
+#define	FRF_AB_XM_AUTO_PAD_LBN 5
+#define	FRF_AB_XM_AUTO_PAD_WIDTH 1
+#define	FRF_AB_XM_TX_PRMBL_LBN 2
+#define	FRF_AB_XM_TX_PRMBL_WIDTH 1
+#define	FRF_AB_XM_TXEN_LBN 1
+#define	FRF_AB_XM_TXEN_WIDTH 1
+#define	FRF_AB_XM_TX_RST_LBN 0
+#define	FRF_AB_XM_TX_RST_WIDTH 1
+
+
+/*
+ * FR_AB_XM_RX_CFG_REG(128bit):
+ * XGMAC receive configuration
+ */
+#define	FR_AB_XM_RX_CFG_REG_OFST 0x00001240
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_PASS_LENERR_LBN 26
+#define	FRF_AB_XM_PASS_LENERR_WIDTH 1
+#define	FRF_AB_XM_PASS_CRC_ERR_LBN 25
+#define	FRF_AB_XM_PASS_CRC_ERR_WIDTH 1
+#define	FRF_AB_XM_PASS_PRMBLE_ERR_LBN 24
+#define	FRF_AB_XM_PASS_PRMBLE_ERR_WIDTH 1
+#define	FRF_AB_XM_REJ_BCAST_LBN 20
+#define	FRF_AB_XM_REJ_BCAST_WIDTH 1
+#define	FRF_AB_XM_ACPT_ALL_MCAST_LBN 11
+#define	FRF_AB_XM_ACPT_ALL_MCAST_WIDTH 1
+#define	FRF_AB_XM_ACPT_ALL_UCAST_LBN 9
+#define	FRF_AB_XM_ACPT_ALL_UCAST_WIDTH 1
+#define	FRF_AB_XM_AUTO_DEPAD_LBN 8
+#define	FRF_AB_XM_AUTO_DEPAD_WIDTH 1
+#define	FRF_AB_XM_RXCRC_LBN 3
+#define	FRF_AB_XM_RXCRC_WIDTH 1
+#define	FRF_AB_XM_RX_PRMBL_LBN 2
+#define	FRF_AB_XM_RX_PRMBL_WIDTH 1
+#define	FRF_AB_XM_RXEN_LBN 1
+#define	FRF_AB_XM_RXEN_WIDTH 1
+#define	FRF_AB_XM_RX_RST_LBN 0
+#define	FRF_AB_XM_RX_RST_WIDTH 1
+
+
+/*
+ * FR_AB_XM_MGT_INT_MASK(128bit):
+ * documentation to be written for sum_XM_MGT_INT_MASK
+ */
+#define	FR_AB_XM_MGT_INT_MASK_OFST 0x00001250
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_MSK_STA_INTR_LBN 16
+#define	FRF_AB_XM_MSK_STA_INTR_WIDTH 1
+#define	FRF_AB_XM_MSK_STAT_CNTR_HF_LBN 9
+#define	FRF_AB_XM_MSK_STAT_CNTR_HF_WIDTH 1
+#define	FRF_AB_XM_MSK_STAT_CNTR_OF_LBN 8
+#define	FRF_AB_XM_MSK_STAT_CNTR_OF_WIDTH 1
+#define	FRF_AB_XM_MSK_PRMBLE_ERR_LBN 2
+#define	FRF_AB_XM_MSK_PRMBLE_ERR_WIDTH 1
+#define	FRF_AB_XM_MSK_RMTFLT_LBN 1
+#define	FRF_AB_XM_MSK_RMTFLT_WIDTH 1
+#define	FRF_AB_XM_MSK_LCLFLT_LBN 0
+#define	FRF_AB_XM_MSK_LCLFLT_WIDTH 1
+
+
+/*
+ * FR_AB_XM_FC_REG(128bit):
+ * XGMAC flow control register
+ */
+#define	FR_AB_XM_FC_REG_OFST 0x00001270
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_PAUSE_TIME_LBN 16
+#define	FRF_AB_XM_PAUSE_TIME_WIDTH 16
+#define	FRF_AB_XM_RX_MAC_STAT_LBN 11
+#define	FRF_AB_XM_RX_MAC_STAT_WIDTH 1
+#define	FRF_AB_XM_TX_MAC_STAT_LBN 10
+#define	FRF_AB_XM_TX_MAC_STAT_WIDTH 1
+#define	FRF_AB_XM_MCNTL_PASS_LBN 8
+#define	FRF_AB_XM_MCNTL_PASS_WIDTH 2
+#define	FRF_AB_XM_REJ_CNTL_UCAST_LBN 6
+#define	FRF_AB_XM_REJ_CNTL_UCAST_WIDTH 1
+#define	FRF_AB_XM_REJ_CNTL_MCAST_LBN 5
+#define	FRF_AB_XM_REJ_CNTL_MCAST_WIDTH 1
+#define	FRF_AB_XM_ZPAUSE_LBN 2
+#define	FRF_AB_XM_ZPAUSE_WIDTH 1
+#define	FRF_AB_XM_XMIT_PAUSE_LBN 1
+#define	FRF_AB_XM_XMIT_PAUSE_WIDTH 1
+#define	FRF_AB_XM_DIS_FCNTL_LBN 0
+#define	FRF_AB_XM_DIS_FCNTL_WIDTH 1
+
+
+/*
+ * FR_AB_XM_PAUSE_TIME_REG(128bit):
+ * XGMAC pause time register
+ */
+#define	FR_AB_XM_PAUSE_TIME_REG_OFST 0x00001290
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_TX_PAUSE_CNT_LBN 16
+#define	FRF_AB_XM_TX_PAUSE_CNT_WIDTH 16
+#define	FRF_AB_XM_RX_PAUSE_CNT_LBN 0
+#define	FRF_AB_XM_RX_PAUSE_CNT_WIDTH 16
+
+
+/*
+ * FR_AB_XM_TX_PARAM_REG(128bit):
+ * XGMAC transmit parameter register
+ */
+#define	FR_AB_XM_TX_PARAM_REG_OFST 0x000012d0
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_TX_JUMBO_MODE_LBN 31
+#define	FRF_AB_XM_TX_JUMBO_MODE_WIDTH 1
+#define	FRF_AB_XM_MAX_TX_FRM_SIZE_HI_LBN 19
+#define	FRF_AB_XM_MAX_TX_FRM_SIZE_HI_WIDTH 11
+#define	FRF_AB_XM_MAX_TX_FRM_SIZE_LO_LBN 16
+#define	FRF_AB_XM_MAX_TX_FRM_SIZE_LO_WIDTH 3
+#define	FRF_AB_XM_PAD_CHAR_LBN 0
+#define	FRF_AB_XM_PAD_CHAR_WIDTH 8
+
+
+/*
+ * FR_AB_XM_RX_PARAM_REG(128bit):
+ * XGMAC receive parameter register
+ */
+#define	FR_AB_XM_RX_PARAM_REG_OFST 0x000012e0
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_MAX_RX_FRM_SIZE_HI_LBN 3
+#define	FRF_AB_XM_MAX_RX_FRM_SIZE_HI_WIDTH 11
+#define	FRF_AB_XM_MAX_RX_FRM_SIZE_LO_LBN 0
+#define	FRF_AB_XM_MAX_RX_FRM_SIZE_LO_WIDTH 3
+
+
+/*
+ * FR_AB_XM_MGT_INT_MSK_REG(128bit):
+ * XGMAC management interrupt mask register
+ */
+#define	FR_AB_XM_MGT_INT_REG_OFST 0x000012f0
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XM_STAT_CNTR_OF_LBN 9
+#define	FRF_AB_XM_STAT_CNTR_OF_WIDTH 1
+#define	FRF_AB_XM_STAT_CNTR_HF_LBN 8
+#define	FRF_AB_XM_STAT_CNTR_HF_WIDTH 1
+#define	FRF_AB_XM_PRMBLE_ERR_LBN 2
+#define	FRF_AB_XM_PRMBLE_ERR_WIDTH 1
+#define	FRF_AB_XM_RMTFLT_LBN 1
+#define	FRF_AB_XM_RMTFLT_WIDTH 1
+#define	FRF_AB_XM_LCLFLT_LBN 0
+#define	FRF_AB_XM_LCLFLT_WIDTH 1
+
+
+/*
+ * FR_AB_XX_PWR_RST_REG(128bit):
+ * XGXS/XAUI powerdown/reset register
+ */
+#define	FR_AB_XX_PWR_RST_REG_OFST 0x00001300
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_PWRDND_SIG_LBN 31
+#define	FRF_AB_XX_PWRDND_SIG_WIDTH 1
+#define	FRF_AB_XX_PWRDNC_SIG_LBN 30
+#define	FRF_AB_XX_PWRDNC_SIG_WIDTH 1
+#define	FRF_AB_XX_PWRDNB_SIG_LBN 29
+#define	FRF_AB_XX_PWRDNB_SIG_WIDTH 1
+#define	FRF_AB_XX_PWRDNA_SIG_LBN 28
+#define	FRF_AB_XX_PWRDNA_SIG_WIDTH 1
+#define	FRF_AB_XX_SIM_MODE_LBN 27
+#define	FRF_AB_XX_SIM_MODE_WIDTH 1
+#define	FRF_AB_XX_RSTPLLCD_SIG_LBN 25
+#define	FRF_AB_XX_RSTPLLCD_SIG_WIDTH 1
+#define	FRF_AB_XX_RSTPLLAB_SIG_LBN 24
+#define	FRF_AB_XX_RSTPLLAB_SIG_WIDTH 1
+#define	FRF_AB_XX_RESETD_SIG_LBN 23
+#define	FRF_AB_XX_RESETD_SIG_WIDTH 1
+#define	FRF_AB_XX_RESETC_SIG_LBN 22
+#define	FRF_AB_XX_RESETC_SIG_WIDTH 1
+#define	FRF_AB_XX_RESETB_SIG_LBN 21
+#define	FRF_AB_XX_RESETB_SIG_WIDTH 1
+#define	FRF_AB_XX_RESETA_SIG_LBN 20
+#define	FRF_AB_XX_RESETA_SIG_WIDTH 1
+#define	FRF_AB_XX_RSTXGXSRX_SIG_LBN 18
+#define	FRF_AB_XX_RSTXGXSRX_SIG_WIDTH 1
+#define	FRF_AB_XX_RSTXGXSTX_SIG_LBN 17
+#define	FRF_AB_XX_RSTXGXSTX_SIG_WIDTH 1
+#define	FRF_AB_XX_SD_RST_ACT_LBN 16
+#define	FRF_AB_XX_SD_RST_ACT_WIDTH 1
+#define	FRF_AB_XX_PWRDND_EN_LBN 15
+#define	FRF_AB_XX_PWRDND_EN_WIDTH 1
+#define	FRF_AB_XX_PWRDNC_EN_LBN 14
+#define	FRF_AB_XX_PWRDNC_EN_WIDTH 1
+#define	FRF_AB_XX_PWRDNB_EN_LBN 13
+#define	FRF_AB_XX_PWRDNB_EN_WIDTH 1
+#define	FRF_AB_XX_PWRDNA_EN_LBN 12
+#define	FRF_AB_XX_PWRDNA_EN_WIDTH 1
+#define	FRF_AB_XX_RSTPLLCD_EN_LBN 9
+#define	FRF_AB_XX_RSTPLLCD_EN_WIDTH 1
+#define	FRF_AB_XX_RSTPLLAB_EN_LBN 8
+#define	FRF_AB_XX_RSTPLLAB_EN_WIDTH 1
+#define	FRF_AB_XX_RESETD_EN_LBN 7
+#define	FRF_AB_XX_RESETD_EN_WIDTH 1
+#define	FRF_AB_XX_RESETC_EN_LBN 6
+#define	FRF_AB_XX_RESETC_EN_WIDTH 1
+#define	FRF_AB_XX_RESETB_EN_LBN 5
+#define	FRF_AB_XX_RESETB_EN_WIDTH 1
+#define	FRF_AB_XX_RESETA_EN_LBN 4
+#define	FRF_AB_XX_RESETA_EN_WIDTH 1
+#define	FRF_AB_XX_RSTXGXSRX_EN_LBN 2
+#define	FRF_AB_XX_RSTXGXSRX_EN_WIDTH 1
+#define	FRF_AB_XX_RSTXGXSTX_EN_LBN 1
+#define	FRF_AB_XX_RSTXGXSTX_EN_WIDTH 1
+#define	FRF_AB_XX_RST_XX_EN_LBN 0
+#define	FRF_AB_XX_RST_XX_EN_WIDTH 1
+
+
+/*
+ * FR_AB_XX_SD_CTL_REG(128bit):
+ * XGXS/XAUI powerdown/reset control register
+ */
+#define	FR_AB_XX_SD_CTL_REG_OFST 0x00001310
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_TERMADJ1_LBN 17
+#define	FRF_AB_XX_TERMADJ1_WIDTH 1
+#define	FRF_AB_XX_TERMADJ0_LBN 16
+#define	FRF_AB_XX_TERMADJ0_WIDTH 1
+#define	FRF_AB_XX_HIDRVD_LBN 15
+#define	FRF_AB_XX_HIDRVD_WIDTH 1
+#define	FRF_AB_XX_LODRVD_LBN 14
+#define	FRF_AB_XX_LODRVD_WIDTH 1
+#define	FRF_AB_XX_HIDRVC_LBN 13
+#define	FRF_AB_XX_HIDRVC_WIDTH 1
+#define	FRF_AB_XX_LODRVC_LBN 12
+#define	FRF_AB_XX_LODRVC_WIDTH 1
+#define	FRF_AB_XX_HIDRVB_LBN 11
+#define	FRF_AB_XX_HIDRVB_WIDTH 1
+#define	FRF_AB_XX_LODRVB_LBN 10
+#define	FRF_AB_XX_LODRVB_WIDTH 1
+#define	FRF_AB_XX_HIDRVA_LBN 9
+#define	FRF_AB_XX_HIDRVA_WIDTH 1
+#define	FRF_AB_XX_LODRVA_LBN 8
+#define	FRF_AB_XX_LODRVA_WIDTH 1
+#define	FRF_AB_XX_LPBKD_LBN 3
+#define	FRF_AB_XX_LPBKD_WIDTH 1
+#define	FRF_AB_XX_LPBKC_LBN 2
+#define	FRF_AB_XX_LPBKC_WIDTH 1
+#define	FRF_AB_XX_LPBKB_LBN 1
+#define	FRF_AB_XX_LPBKB_WIDTH 1
+#define	FRF_AB_XX_LPBKA_LBN 0
+#define	FRF_AB_XX_LPBKA_WIDTH 1
+
+
+/*
+ * FR_AB_XX_TXDRV_CTL_REG(128bit):
+ * XAUI SerDes transmit drive control register
+ */
+#define	FR_AB_XX_TXDRV_CTL_REG_OFST 0x00001320
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_DEQD_LBN 28
+#define	FRF_AB_XX_DEQD_WIDTH 4
+#define	FRF_AB_XX_DEQC_LBN 24
+#define	FRF_AB_XX_DEQC_WIDTH 4
+#define	FRF_AB_XX_DEQB_LBN 20
+#define	FRF_AB_XX_DEQB_WIDTH 4
+#define	FRF_AB_XX_DEQA_LBN 16
+#define	FRF_AB_XX_DEQA_WIDTH 4
+#define	FRF_AB_XX_DTXD_LBN 12
+#define	FRF_AB_XX_DTXD_WIDTH 4
+#define	FRF_AB_XX_DTXC_LBN 8
+#define	FRF_AB_XX_DTXC_WIDTH 4
+#define	FRF_AB_XX_DTXB_LBN 4
+#define	FRF_AB_XX_DTXB_WIDTH 4
+#define	FRF_AB_XX_DTXA_LBN 0
+#define	FRF_AB_XX_DTXA_WIDTH 4
+
+
+/*
+ * FR_AB_XX_PRBS_CTL_REG(128bit):
+ * documentation to be written for sum_XX_PRBS_CTL_REG
+ */
+#define	FR_AB_XX_PRBS_CTL_REG_OFST 0x00001330
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_CH3_RX_PRBS_SEL_LBN 30
+#define	FRF_AB_XX_CH3_RX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH3_RX_PRBS_INV_LBN 29
+#define	FRF_AB_XX_CH3_RX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH3_RX_PRBS_CHKEN_LBN 28
+#define	FRF_AB_XX_CH3_RX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH2_RX_PRBS_SEL_LBN 26
+#define	FRF_AB_XX_CH2_RX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH2_RX_PRBS_INV_LBN 25
+#define	FRF_AB_XX_CH2_RX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH2_RX_PRBS_CHKEN_LBN 24
+#define	FRF_AB_XX_CH2_RX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH1_RX_PRBS_SEL_LBN 22
+#define	FRF_AB_XX_CH1_RX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH1_RX_PRBS_INV_LBN 21
+#define	FRF_AB_XX_CH1_RX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH1_RX_PRBS_CHKEN_LBN 20
+#define	FRF_AB_XX_CH1_RX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH0_RX_PRBS_SEL_LBN 18
+#define	FRF_AB_XX_CH0_RX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH0_RX_PRBS_INV_LBN 17
+#define	FRF_AB_XX_CH0_RX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH0_RX_PRBS_CHKEN_LBN 16
+#define	FRF_AB_XX_CH0_RX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH3_TX_PRBS_SEL_LBN 14
+#define	FRF_AB_XX_CH3_TX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH3_TX_PRBS_INV_LBN 13
+#define	FRF_AB_XX_CH3_TX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH3_TX_PRBS_CHKEN_LBN 12
+#define	FRF_AB_XX_CH3_TX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH2_TX_PRBS_SEL_LBN 10
+#define	FRF_AB_XX_CH2_TX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH2_TX_PRBS_INV_LBN 9
+#define	FRF_AB_XX_CH2_TX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH2_TX_PRBS_CHKEN_LBN 8
+#define	FRF_AB_XX_CH2_TX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH1_TX_PRBS_SEL_LBN 6
+#define	FRF_AB_XX_CH1_TX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH1_TX_PRBS_INV_LBN 5
+#define	FRF_AB_XX_CH1_TX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH1_TX_PRBS_CHKEN_LBN 4
+#define	FRF_AB_XX_CH1_TX_PRBS_CHKEN_WIDTH 1
+#define	FRF_AB_XX_CH0_TX_PRBS_SEL_LBN 2
+#define	FRF_AB_XX_CH0_TX_PRBS_SEL_WIDTH 2
+#define	FRF_AB_XX_CH0_TX_PRBS_INV_LBN 1
+#define	FRF_AB_XX_CH0_TX_PRBS_INV_WIDTH 1
+#define	FRF_AB_XX_CH0_TX_PRBS_CHKEN_LBN 0
+#define	FRF_AB_XX_CH0_TX_PRBS_CHKEN_WIDTH 1
+
+
+/*
+ * FR_AB_XX_PRBS_CHK_REG(128bit):
+ * documentation to be written for sum_XX_PRBS_CHK_REG
+ */
+#define	FR_AB_XX_PRBS_CHK_REG_OFST 0x00001340
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_REV_LB_EN_LBN 16
+#define	FRF_AB_XX_REV_LB_EN_WIDTH 1
+#define	FRF_AB_XX_CH3_DEG_DET_LBN 15
+#define	FRF_AB_XX_CH3_DEG_DET_WIDTH 1
+#define	FRF_AB_XX_CH3_LFSR_LOCK_IND_LBN 14
+#define	FRF_AB_XX_CH3_LFSR_LOCK_IND_WIDTH 1
+#define	FRF_AB_XX_CH3_PRBS_FRUN_LBN 13
+#define	FRF_AB_XX_CH3_PRBS_FRUN_WIDTH 1
+#define	FRF_AB_XX_CH3_ERR_CHK_LBN 12
+#define	FRF_AB_XX_CH3_ERR_CHK_WIDTH 1
+#define	FRF_AB_XX_CH2_DEG_DET_LBN 11
+#define	FRF_AB_XX_CH2_DEG_DET_WIDTH 1
+#define	FRF_AB_XX_CH2_LFSR_LOCK_IND_LBN 10
+#define	FRF_AB_XX_CH2_LFSR_LOCK_IND_WIDTH 1
+#define	FRF_AB_XX_CH2_PRBS_FRUN_LBN 9
+#define	FRF_AB_XX_CH2_PRBS_FRUN_WIDTH 1
+#define	FRF_AB_XX_CH2_ERR_CHK_LBN 8
+#define	FRF_AB_XX_CH2_ERR_CHK_WIDTH 1
+#define	FRF_AB_XX_CH1_DEG_DET_LBN 7
+#define	FRF_AB_XX_CH1_DEG_DET_WIDTH 1
+#define	FRF_AB_XX_CH1_LFSR_LOCK_IND_LBN 6
+#define	FRF_AB_XX_CH1_LFSR_LOCK_IND_WIDTH 1
+#define	FRF_AB_XX_CH1_PRBS_FRUN_LBN 5
+#define	FRF_AB_XX_CH1_PRBS_FRUN_WIDTH 1
+#define	FRF_AB_XX_CH1_ERR_CHK_LBN 4
+#define	FRF_AB_XX_CH1_ERR_CHK_WIDTH 1
+#define	FRF_AB_XX_CH0_DEG_DET_LBN 3
+#define	FRF_AB_XX_CH0_DEG_DET_WIDTH 1
+#define	FRF_AB_XX_CH0_LFSR_LOCK_IND_LBN 2
+#define	FRF_AB_XX_CH0_LFSR_LOCK_IND_WIDTH 1
+#define	FRF_AB_XX_CH0_PRBS_FRUN_LBN 1
+#define	FRF_AB_XX_CH0_PRBS_FRUN_WIDTH 1
+#define	FRF_AB_XX_CH0_ERR_CHK_LBN 0
+#define	FRF_AB_XX_CH0_ERR_CHK_WIDTH 1
+
+
+/*
+ * FR_AB_XX_PRBS_ERR_REG(128bit):
+ * documentation to be written for sum_XX_PRBS_ERR_REG
+ */
+#define	FR_AB_XX_PRBS_ERR_REG_OFST 0x00001350
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_CH3_PRBS_ERR_CNT_LBN 24
+#define	FRF_AB_XX_CH3_PRBS_ERR_CNT_WIDTH 8
+#define	FRF_AB_XX_CH2_PRBS_ERR_CNT_LBN 16
+#define	FRF_AB_XX_CH2_PRBS_ERR_CNT_WIDTH 8
+#define	FRF_AB_XX_CH1_PRBS_ERR_CNT_LBN 8
+#define	FRF_AB_XX_CH1_PRBS_ERR_CNT_WIDTH 8
+#define	FRF_AB_XX_CH0_PRBS_ERR_CNT_LBN 0
+#define	FRF_AB_XX_CH0_PRBS_ERR_CNT_WIDTH 8
+
+
+/*
+ * FR_AB_XX_CORE_STAT_REG(128bit):
+ * XAUI XGXS core status register
+ */
+#define	FR_AB_XX_CORE_STAT_REG_OFST 0x00001360
+/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+
+#define	FRF_AB_XX_FORCE_SIG3_LBN 31
+#define	FRF_AB_XX_FORCE_SIG3_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG3_VAL_LBN 30
+#define	FRF_AB_XX_FORCE_SIG3_VAL_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG2_LBN 29
+#define	FRF_AB_XX_FORCE_SIG2_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG2_VAL_LBN 28
+#define	FRF_AB_XX_FORCE_SIG2_VAL_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG1_LBN 27
+#define	FRF_AB_XX_FORCE_SIG1_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG1_VAL_LBN 26
+#define	FRF_AB_XX_FORCE_SIG1_VAL_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG0_LBN 25
+#define	FRF_AB_XX_FORCE_SIG0_WIDTH 1
+#define	FRF_AB_XX_FORCE_SIG0_VAL_LBN 24
+#define	FRF_AB_XX_FORCE_SIG0_VAL_WIDTH 1
+#define	FRF_AB_XX_XGXS_LB_EN_LBN 23
+#define	FRF_AB_XX_XGXS_LB_EN_WIDTH 1
+#define	FRF_AB_XX_XGMII_LB_EN_LBN 22
+#define	FRF_AB_XX_XGMII_LB_EN_WIDTH 1
+#define	FRF_AB_XX_MATCH_FAULT_LBN 21
+#define	FRF_AB_XX_MATCH_FAULT_WIDTH 1
+#define	FRF_AB_XX_ALIGN_DONE_LBN 20
+#define	FRF_AB_XX_ALIGN_DONE_WIDTH 1
+#define	FRF_AB_XX_SYNC_STAT3_LBN 19
+#define	FRF_AB_XX_SYNC_STAT3_WIDTH 1
+#define	FRF_AB_XX_SYNC_STAT2_LBN 18
+#define	FRF_AB_XX_SYNC_STAT2_WIDTH 1
+#define	FRF_AB_XX_SYNC_STAT1_LBN 17
+#define	FRF_AB_XX_SYNC_STAT1_WIDTH 1
+#define	FRF_AB_XX_SYNC_STAT0_LBN 16
+#define	FRF_AB_XX_SYNC_STAT0_WIDTH 1
+#define	FRF_AB_XX_COMMA_DET_CH3_LBN 15
+#define	FRF_AB_XX_COMMA_DET_CH3_WIDTH 1
+#define	FRF_AB_XX_COMMA_DET_CH2_LBN 14
+#define	FRF_AB_XX_COMMA_DET_CH2_WIDTH 1
+#define	FRF_AB_XX_COMMA_DET_CH1_LBN 13
+#define	FRF_AB_XX_COMMA_DET_CH1_WIDTH 1
+#define	FRF_AB_XX_COMMA_DET_CH0_LBN 12
+#define	FRF_AB_XX_COMMA_DET_CH0_WIDTH 1
+#define	FRF_AB_XX_CGRP_ALIGN_CH3_LBN 11
+#define	FRF_AB_XX_CGRP_ALIGN_CH3_WIDTH 1
+#define	FRF_AB_XX_CGRP_ALIGN_CH2_LBN 10
+#define	FRF_AB_XX_CGRP_ALIGN_CH2_WIDTH 1
+#define	FRF_AB_XX_CGRP_ALIGN_CH1_LBN 9
+#define	FRF_AB_XX_CGRP_ALIGN_CH1_WIDTH 1
+#define	FRF_AB_XX_CGRP_ALIGN_CH0_LBN 8
+#define	FRF_AB_XX_CGRP_ALIGN_CH0_WIDTH 1
+#define	FRF_AB_XX_CHAR_ERR_CH3_LBN 7
+#define	FRF_AB_XX_CHAR_ERR_CH3_WIDTH 1
+#define	FRF_AB_XX_CHAR_ERR_CH2_LBN 6
+#define	FRF_AB_XX_CHAR_ERR_CH2_WIDTH 1
+#define	FRF_AB_XX_CHAR_ERR_CH1_LBN 5
+#define	FRF_AB_XX_CHAR_ERR_CH1_WIDTH 1
+#define	FRF_AB_XX_CHAR_ERR_CH0_LBN 4
+#define	FRF_AB_XX_CHAR_ERR_CH0_WIDTH 1
+#define	FRF_AB_XX_DISPERR_CH3_LBN 3
+#define	FRF_AB_XX_DISPERR_CH3_WIDTH 1
+#define	FRF_AB_XX_DISPERR_CH2_LBN 2
+#define	FRF_AB_XX_DISPERR_CH2_WIDTH 1
+#define	FRF_AB_XX_DISPERR_CH1_LBN 1
+#define	FRF_AB_XX_DISPERR_CH1_WIDTH 1
+#define	FRF_AB_XX_DISPERR_CH0_LBN 0
+#define	FRF_AB_XX_DISPERR_CH0_WIDTH 1
+
+
+/*
+ * FR_AA_RX_DESC_PTR_TBL_KER(128bit):
+ * Receive descriptor pointer table
+ */
+#define	FR_AA_RX_DESC_PTR_TBL_KER_OFST 0x00011800
+/* falcona0=net_func_bar2 */
+#define	FR_AA_RX_DESC_PTR_TBL_KER_STEP 16
+#define	FR_AA_RX_DESC_PTR_TBL_KER_ROWS 4
+/*
+ * FR_AZ_RX_DESC_PTR_TBL(128bit):
+ * Receive descriptor pointer table
+ */
+#define	FR_AZ_RX_DESC_PTR_TBL_OFST 0x00f40000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_RX_DESC_PTR_TBL_STEP 16
+#define	FR_CZ_RX_DESC_PTR_TBL_ROWS 1024
+#define	FR_AB_RX_DESC_PTR_TBL_ROWS 4096
+
+#define	FRF_CZ_RX_HDR_SPLIT_LBN 90
+#define	FRF_CZ_RX_HDR_SPLIT_WIDTH 1
+#define	FRF_AZ_RX_RESET_LBN 89
+#define	FRF_AZ_RX_RESET_WIDTH 1
+#define	FRF_AZ_RX_ISCSI_DDIG_EN_LBN 88
+#define	FRF_AZ_RX_ISCSI_DDIG_EN_WIDTH 1
+#define	FRF_AZ_RX_ISCSI_HDIG_EN_LBN 87
+#define	FRF_AZ_RX_ISCSI_HDIG_EN_WIDTH 1
+#define	FRF_AZ_RX_DESC_PREF_ACT_LBN 86
+#define	FRF_AZ_RX_DESC_PREF_ACT_WIDTH 1
+#define	FRF_AZ_RX_DC_HW_RPTR_LBN 80
+#define	FRF_AZ_RX_DC_HW_RPTR_WIDTH 6
+#define	FRF_AZ_RX_DESCQ_HW_RPTR_LBN 68
+#define	FRF_AZ_RX_DESCQ_HW_RPTR_WIDTH 12
+#define	FRF_AZ_RX_DESCQ_SW_WPTR_LBN 56
+#define	FRF_AZ_RX_DESCQ_SW_WPTR_WIDTH 12
+#define	FRF_AZ_RX_DESCQ_BUF_BASE_ID_LBN 36
+#define	FRF_AZ_RX_DESCQ_BUF_BASE_ID_WIDTH 20
+#define	FRF_AZ_RX_DESCQ_EVQ_ID_LBN 24
+#define	FRF_AZ_RX_DESCQ_EVQ_ID_WIDTH 12
+#define	FRF_AZ_RX_DESCQ_OWNER_ID_LBN 10
+#define	FRF_AZ_RX_DESCQ_OWNER_ID_WIDTH 14
+#define	FRF_AZ_RX_DESCQ_LABEL_LBN 5
+#define	FRF_AZ_RX_DESCQ_LABEL_WIDTH 5
+#define	FRF_AZ_RX_DESCQ_SIZE_LBN 3
+#define	FRF_AZ_RX_DESCQ_SIZE_WIDTH 2
+#define	FFE_AZ_RX_DESCQ_SIZE_4K 3
+#define	FFE_AZ_RX_DESCQ_SIZE_2K 2
+#define	FFE_AZ_RX_DESCQ_SIZE_1K 1
+#define	FFE_AZ_RX_DESCQ_SIZE_512 0
+#define	FRF_AZ_RX_DESCQ_TYPE_LBN 2
+#define	FRF_AZ_RX_DESCQ_TYPE_WIDTH 1
+#define	FRF_AZ_RX_DESCQ_JUMBO_LBN 1
+#define	FRF_AZ_RX_DESCQ_JUMBO_WIDTH 1
+#define	FRF_AZ_RX_DESCQ_EN_LBN 0
+#define	FRF_AZ_RX_DESCQ_EN_WIDTH 1
+
+
+/*
+ * FR_AA_TX_DESC_PTR_TBL_KER(128bit):
+ * Transmit descriptor pointer
+ */
+#define	FR_AA_TX_DESC_PTR_TBL_KER_OFST 0x00011900
+/* falcona0=net_func_bar2 */
+#define	FR_AA_TX_DESC_PTR_TBL_KER_STEP 16
+#define	FR_AA_TX_DESC_PTR_TBL_KER_ROWS 8
+/*
+ * FR_AZ_TX_DESC_PTR_TBL(128bit):
+ * Transmit descriptor pointer
+ */
+#define	FR_AZ_TX_DESC_PTR_TBL_OFST 0x00f50000
+/* falconb0=net_func_bar2,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_TX_DESC_PTR_TBL_STEP 16
+#define	FR_AB_TX_DESC_PTR_TBL_ROWS 4096
+#define	FR_CZ_TX_DESC_PTR_TBL_ROWS 1024
+
+#define	FRF_CZ_TX_DPT_Q_MASK_WIDTH_LBN 94
+#define	FRF_CZ_TX_DPT_Q_MASK_WIDTH_WIDTH 2
+#define	FRF_CZ_TX_DPT_ETH_FILT_EN_LBN 93
+#define	FRF_CZ_TX_DPT_ETH_FILT_EN_WIDTH 1
+#define	FRF_CZ_TX_DPT_IP_FILT_EN_LBN 92
+#define	FRF_CZ_TX_DPT_IP_FILT_EN_WIDTH 1
+#define	FRF_BZ_TX_NON_IP_DROP_DIS_LBN 91
+#define	FRF_BZ_TX_NON_IP_DROP_DIS_WIDTH 1
+#define	FRF_BZ_TX_IP_CHKSM_DIS_LBN 90
+#define	FRF_BZ_TX_IP_CHKSM_DIS_WIDTH 1
+#define	FRF_BZ_TX_TCP_CHKSM_DIS_LBN 89
+#define	FRF_BZ_TX_TCP_CHKSM_DIS_WIDTH 1
+#define	FRF_AZ_TX_DESCQ_EN_LBN 88
+#define	FRF_AZ_TX_DESCQ_EN_WIDTH 1
+#define	FRF_AZ_TX_ISCSI_DDIG_EN_LBN 87
+#define	FRF_AZ_TX_ISCSI_DDIG_EN_WIDTH 1
+#define	FRF_AZ_TX_ISCSI_HDIG_EN_LBN 86
+#define	FRF_AZ_TX_ISCSI_HDIG_EN_WIDTH 1
+#define	FRF_AZ_TX_DC_HW_RPTR_LBN 80
+#define	FRF_AZ_TX_DC_HW_RPTR_WIDTH 6
+#define	FRF_AZ_TX_DESCQ_HW_RPTR_LBN 68
+#define	FRF_AZ_TX_DESCQ_HW_RPTR_WIDTH 12
+#define	FRF_AZ_TX_DESCQ_SW_WPTR_LBN 56
+#define	FRF_AZ_TX_DESCQ_SW_WPTR_WIDTH 12
+#define	FRF_AZ_TX_DESCQ_BUF_BASE_ID_LBN 36
+#define	FRF_AZ_TX_DESCQ_BUF_BASE_ID_WIDTH 20
+#define	FRF_AZ_TX_DESCQ_EVQ_ID_LBN 24
+#define	FRF_AZ_TX_DESCQ_EVQ_ID_WIDTH 12
+#define	FRF_AZ_TX_DESCQ_OWNER_ID_LBN 10
+#define	FRF_AZ_TX_DESCQ_OWNER_ID_WIDTH 14
+#define	FRF_AZ_TX_DESCQ_LABEL_LBN 5
+#define	FRF_AZ_TX_DESCQ_LABEL_WIDTH 5
+#define	FRF_AZ_TX_DESCQ_SIZE_LBN 3
+#define	FRF_AZ_TX_DESCQ_SIZE_WIDTH 2
+#define	FFE_AZ_TX_DESCQ_SIZE_4K 3
+#define	FFE_AZ_TX_DESCQ_SIZE_2K 2
+#define	FFE_AZ_TX_DESCQ_SIZE_1K 1
+#define	FFE_AZ_TX_DESCQ_SIZE_512 0
+#define	FRF_AZ_TX_DESCQ_TYPE_LBN 1
+#define	FRF_AZ_TX_DESCQ_TYPE_WIDTH 2
+#define	FRF_AZ_TX_DESCQ_FLUSH_LBN 0
+#define	FRF_AZ_TX_DESCQ_FLUSH_WIDTH 1
+
+
+/*
+ * FR_AA_EVQ_PTR_TBL_KER(128bit):
+ * Event queue pointer table
+ */
+#define	FR_AA_EVQ_PTR_TBL_KER_OFST 0x00011a00
+/* falcona0=net_func_bar2 */
+#define	FR_AA_EVQ_PTR_TBL_KER_STEP 16
+#define	FR_AA_EVQ_PTR_TBL_KER_ROWS 4
+/*
+ * FR_AZ_EVQ_PTR_TBL(128bit):
+ * Event queue pointer table
+ */
+#define	FR_AZ_EVQ_PTR_TBL_OFST 0x00f60000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_EVQ_PTR_TBL_STEP 16
+#define	FR_CZ_EVQ_PTR_TBL_ROWS 1024
+#define	FR_AB_EVQ_PTR_TBL_ROWS 4096
+
+#define	FRF_BZ_EVQ_RPTR_IGN_LBN 40
+#define	FRF_BZ_EVQ_RPTR_IGN_WIDTH 1
+#define	FRF_AZ_EVQ_WKUP_OR_INT_EN_LBN 39
+#define	FRF_AZ_EVQ_WKUP_OR_INT_EN_WIDTH 1
+#define	FRF_AZ_EVQ_NXT_WPTR_LBN 24
+#define	FRF_AZ_EVQ_NXT_WPTR_WIDTH 15
+#define	FRF_AZ_EVQ_EN_LBN 23
+#define	FRF_AZ_EVQ_EN_WIDTH 1
+#define	FRF_AZ_EVQ_SIZE_LBN 20
+#define	FRF_AZ_EVQ_SIZE_WIDTH 3
+#define	FFE_AZ_EVQ_SIZE_32K 6
+#define	FFE_AZ_EVQ_SIZE_16K 5
+#define	FFE_AZ_EVQ_SIZE_8K 4
+#define	FFE_AZ_EVQ_SIZE_4K 3
+#define	FFE_AZ_EVQ_SIZE_2K 2
+#define	FFE_AZ_EVQ_SIZE_1K 1
+#define	FFE_AZ_EVQ_SIZE_512 0
+#define	FRF_AZ_EVQ_BUF_BASE_ID_LBN 0
+#define	FRF_AZ_EVQ_BUF_BASE_ID_WIDTH 20
+
+
+/*
+ * FR_AA_BUF_HALF_TBL_KER(64bit):
+ * Buffer table in half buffer table mode direct access by driver
+ */
+#define	FR_AA_BUF_HALF_TBL_KER_OFST 0x00018000
+/* falcona0=net_func_bar2 */
+#define	FR_AA_BUF_HALF_TBL_KER_STEP 8
+#define	FR_AA_BUF_HALF_TBL_KER_ROWS 4096
+/*
+ * FR_AZ_BUF_HALF_TBL(64bit):
+ * Buffer table in half buffer table mode direct access by driver
+ */
+#define	FR_AZ_BUF_HALF_TBL_OFST 0x00800000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_BUF_HALF_TBL_STEP 8
+#define	FR_CZ_BUF_HALF_TBL_ROWS 147456
+#define	FR_AB_BUF_HALF_TBL_ROWS 524288
+
+#define	FRF_AZ_BUF_ADR_HBUF_ODD_LBN 44
+#define	FRF_AZ_BUF_ADR_HBUF_ODD_WIDTH 20
+#define	FRF_AZ_BUF_OWNER_ID_HBUF_ODD_LBN 32
+#define	FRF_AZ_BUF_OWNER_ID_HBUF_ODD_WIDTH 12
+#define	FRF_AZ_BUF_ADR_HBUF_EVEN_LBN 12
+#define	FRF_AZ_BUF_ADR_HBUF_EVEN_WIDTH 20
+#define	FRF_AZ_BUF_OWNER_ID_HBUF_EVEN_LBN 0
+#define	FRF_AZ_BUF_OWNER_ID_HBUF_EVEN_WIDTH 12
+
+
+/*
+ * FR_AA_BUF_FULL_TBL_KER(64bit):
+ * Buffer table in full buffer table mode direct access by driver
+ */
+#define	FR_AA_BUF_FULL_TBL_KER_OFST 0x00018000
+/* falcona0=net_func_bar2 */
+#define	FR_AA_BUF_FULL_TBL_KER_STEP 8
+#define	FR_AA_BUF_FULL_TBL_KER_ROWS 4096
+/*
+ * FR_AZ_BUF_FULL_TBL(64bit):
+ * Buffer table in full buffer table mode direct access by driver
+ */
+#define	FR_AZ_BUF_FULL_TBL_OFST 0x00800000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_BUF_FULL_TBL_STEP 8
+
+#define	FR_CZ_BUF_FULL_TBL_ROWS 147456
+#define	FR_AB_BUF_FULL_TBL_ROWS 917504
+
+#define	FRF_AZ_BUF_FULL_UNUSED_LBN 51
+#define	FRF_AZ_BUF_FULL_UNUSED_WIDTH 13
+#define	FRF_AZ_IP_DAT_BUF_SIZE_LBN 50
+#define	FRF_AZ_IP_DAT_BUF_SIZE_WIDTH 1
+#define	FRF_AZ_BUF_ADR_REGION_LBN 48
+#define	FRF_AZ_BUF_ADR_REGION_WIDTH 2
+#define	FFE_AZ_BUF_ADR_REGN3 3
+#define	FFE_AZ_BUF_ADR_REGN2 2
+#define	FFE_AZ_BUF_ADR_REGN1 1
+#define	FFE_AZ_BUF_ADR_REGN0 0
+#define	FRF_AZ_BUF_ADR_FBUF_LBN 14
+#define	FRF_AZ_BUF_ADR_FBUF_WIDTH 34
+#define	FRF_AZ_BUF_ADR_FBUF_DW0_LBN 14
+#define	FRF_AZ_BUF_ADR_FBUF_DW0_WIDTH 32
+#define	FRF_AZ_BUF_ADR_FBUF_DW1_LBN 46
+#define	FRF_AZ_BUF_ADR_FBUF_DW1_WIDTH 2
+#define	FRF_AZ_BUF_OWNER_ID_FBUF_LBN 0
+#define	FRF_AZ_BUF_OWNER_ID_FBUF_WIDTH 14
+
+
+/*
+ * FR_AZ_RX_FILTER_TBL0(128bit):
+ * TCP/IPv4 Receive filter table
+ */
+#define	FR_AZ_RX_FILTER_TBL0_OFST 0x00f00000
+/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_RX_FILTER_TBL0_STEP 32
+#define	FR_AZ_RX_FILTER_TBL0_ROWS 8192
+/*
+ * FR_AB_RX_FILTER_TBL1(128bit):
+ * TCP/IPv4 Receive filter table
+ */
+#define	FR_AB_RX_FILTER_TBL1_OFST 0x00f00010
+/* falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AB_RX_FILTER_TBL1_STEP 32
+#define	FR_AB_RX_FILTER_TBL1_ROWS 8192
+
+#define	FRF_BZ_RSS_EN_LBN 110
+#define	FRF_BZ_RSS_EN_WIDTH 1
+#define	FRF_BZ_SCATTER_EN_LBN 109
+#define	FRF_BZ_SCATTER_EN_WIDTH 1
+#define	FRF_AZ_TCP_UDP_LBN 108
+#define	FRF_AZ_TCP_UDP_WIDTH 1
+#define	FRF_AZ_RXQ_ID_LBN 96
+#define	FRF_AZ_RXQ_ID_WIDTH 12
+#define	FRF_AZ_DEST_IP_LBN 64
+#define	FRF_AZ_DEST_IP_WIDTH 32
+#define	FRF_AZ_DEST_PORT_TCP_LBN 48
+#define	FRF_AZ_DEST_PORT_TCP_WIDTH 16
+#define	FRF_AZ_SRC_IP_LBN 16
+#define	FRF_AZ_SRC_IP_WIDTH 32
+#define	FRF_AZ_SRC_TCP_DEST_UDP_LBN 0
+#define	FRF_AZ_SRC_TCP_DEST_UDP_WIDTH 16
+
+
+/*
+ * FR_CZ_RX_MAC_FILTER_TBL0(128bit):
+ * Receive Ethernet filter table
+ */
+#define	FR_CZ_RX_MAC_FILTER_TBL0_OFST 0x00f00010
+/* sienaa0=net_func_bar2 */
+#define	FR_CZ_RX_MAC_FILTER_TBL0_STEP 32
+#define	FR_CZ_RX_MAC_FILTER_TBL0_ROWS 512
+
+#define	FRF_CZ_RMFT_RSS_EN_LBN 75
+#define	FRF_CZ_RMFT_RSS_EN_WIDTH 1
+#define	FRF_CZ_RMFT_SCATTER_EN_LBN 74
+#define	FRF_CZ_RMFT_SCATTER_EN_WIDTH 1
+#define	FRF_CZ_RMFT_IP_OVERRIDE_LBN 73
+#define	FRF_CZ_RMFT_IP_OVERRIDE_WIDTH 1
+#define	FRF_CZ_RMFT_RXQ_ID_LBN 61
+#define	FRF_CZ_RMFT_RXQ_ID_WIDTH 12
+#define	FRF_CZ_RMFT_WILDCARD_MATCH_LBN 60
+#define	FRF_CZ_RMFT_WILDCARD_MATCH_WIDTH 1
+#define	FRF_CZ_RMFT_DEST_MAC_LBN 12
+#define	FRF_CZ_RMFT_DEST_MAC_WIDTH 48
+#define	FRF_CZ_RMFT_DEST_MAC_DW0_LBN 12
+#define	FRF_CZ_RMFT_DEST_MAC_DW0_WIDTH 32
+#define	FRF_CZ_RMFT_DEST_MAC_DW1_LBN 44
+#define	FRF_CZ_RMFT_DEST_MAC_DW1_WIDTH 16
+#define	FRF_CZ_RMFT_VLAN_ID_LBN 0
+#define	FRF_CZ_RMFT_VLAN_ID_WIDTH 12
+
+
+/*
+ * FR_AZ_TIMER_TBL(128bit):
+ * Timer table
+ */
+#define	FR_AZ_TIMER_TBL_OFST 0x00f70000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_TIMER_TBL_STEP 16
+#define	FR_CZ_TIMER_TBL_ROWS 1024
+#define	FR_AB_TIMER_TBL_ROWS 4096
+
+#define	FRF_CZ_TIMER_Q_EN_LBN 33
+#define	FRF_CZ_TIMER_Q_EN_WIDTH 1
+#define	FRF_CZ_INT_ARMD_LBN 32
+#define	FRF_CZ_INT_ARMD_WIDTH 1
+#define	FRF_CZ_INT_PEND_LBN 31
+#define	FRF_CZ_INT_PEND_WIDTH 1
+#define	FRF_CZ_HOST_NOTIFY_MODE_LBN 30
+#define	FRF_CZ_HOST_NOTIFY_MODE_WIDTH 1
+#define	FRF_CZ_RELOAD_TIMER_VAL_LBN 16
+#define	FRF_CZ_RELOAD_TIMER_VAL_WIDTH 14
+#define	FRF_CZ_TIMER_MODE_LBN 14
+#define	FRF_CZ_TIMER_MODE_WIDTH 2
+#define	FFE_CZ_TIMER_MODE_INT_HLDOFF 3
+#define	FFE_CZ_TIMER_MODE_TRIG_START 2
+#define	FFE_CZ_TIMER_MODE_IMMED_START 1
+#define	FFE_CZ_TIMER_MODE_DIS 0
+#define	FRF_AB_TIMER_MODE_LBN 12
+#define	FRF_AB_TIMER_MODE_WIDTH 2
+#define	FFE_AB_TIMER_MODE_INT_HLDOFF 2
+#define	FFE_AB_TIMER_MODE_TRIG_START 2
+#define	FFE_AB_TIMER_MODE_IMMED_START 1
+#define	FFE_AB_TIMER_MODE_DIS 0
+#define	FRF_CZ_TIMER_VAL_LBN 0
+#define	FRF_CZ_TIMER_VAL_WIDTH 14
+#define	FRF_AB_TIMER_VAL_LBN 0
+#define	FRF_AB_TIMER_VAL_WIDTH 12
+
+
+/*
+ * FR_BZ_TX_PACE_TBL(128bit):
+ * Transmit pacing table
+ */
+#define	FR_BZ_TX_PACE_TBL_OFST 0x00f80000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2 */
+#define	FR_AZ_TX_PACE_TBL_STEP 16
+#define	FR_CZ_TX_PACE_TBL_ROWS 1024
+#define	FR_BB_TX_PACE_TBL_ROWS 4096
+/*
+ * FR_AA_TX_PACE_TBL(128bit):
+ * Transmit pacing table
+ */
+#define	FR_AA_TX_PACE_TBL_OFST 0x00f80040
+/* falcona0=char_func_bar0 */
+/* FR_AZ_TX_PACE_TBL_STEP 16 */
+#define	FR_AA_TX_PACE_TBL_ROWS 4092
+
+#define	FRF_AZ_TX_PACE_LBN 0
+#define	FRF_AZ_TX_PACE_WIDTH 5
+
+
+/*
+ * FR_BZ_RX_INDIRECTION_TBL(7bit):
+ * RX Indirection Table
+ */
+#define	FR_BZ_RX_INDIRECTION_TBL_OFST 0x00fb0000
+/* falconb0,sienaa0=net_func_bar2 */
+#define	FR_BZ_RX_INDIRECTION_TBL_STEP 16
+#define	FR_BZ_RX_INDIRECTION_TBL_ROWS 128
+
+#define	FRF_BZ_IT_QUEUE_LBN 0
+#define	FRF_BZ_IT_QUEUE_WIDTH 6
+
+
+/*
+ * FR_CZ_TX_FILTER_TBL0(128bit):
+ * TCP/IPv4 Transmit filter table
+ */
+#define	FR_CZ_TX_FILTER_TBL0_OFST 0x00fc0000
+/* sienaa0=net_func_bar2 */
+#define	FR_CZ_TX_FILTER_TBL0_STEP 16
+#define	FR_CZ_TX_FILTER_TBL0_ROWS 8192
+
+#define	FRF_CZ_TIFT_TCP_UDP_LBN 108
+#define	FRF_CZ_TIFT_TCP_UDP_WIDTH 1
+#define	FRF_CZ_TIFT_TXQ_ID_LBN 96
+#define	FRF_CZ_TIFT_TXQ_ID_WIDTH 12
+#define	FRF_CZ_TIFT_DEST_IP_LBN 64
+#define	FRF_CZ_TIFT_DEST_IP_WIDTH 32
+#define	FRF_CZ_TIFT_DEST_PORT_TCP_LBN 48
+#define	FRF_CZ_TIFT_DEST_PORT_TCP_WIDTH 16
+#define	FRF_CZ_TIFT_SRC_IP_LBN 16
+#define	FRF_CZ_TIFT_SRC_IP_WIDTH 32
+#define	FRF_CZ_TIFT_SRC_TCP_DEST_UDP_LBN 0
+#define	FRF_CZ_TIFT_SRC_TCP_DEST_UDP_WIDTH 16
+
+
+/*
+ * FR_CZ_TX_MAC_FILTER_TBL0(128bit):
+ * Transmit Ethernet filter table
+ */
+#define	FR_CZ_TX_MAC_FILTER_TBL0_OFST 0x00fe0000
+/* sienaa0=net_func_bar2 */
+#define	FR_CZ_TX_MAC_FILTER_TBL0_STEP 16
+#define	FR_CZ_TX_MAC_FILTER_TBL0_ROWS 512
+
+#define	FRF_CZ_TMFT_TXQ_ID_LBN 61
+#define	FRF_CZ_TMFT_TXQ_ID_WIDTH 12
+#define	FRF_CZ_TMFT_WILDCARD_MATCH_LBN 60
+#define	FRF_CZ_TMFT_WILDCARD_MATCH_WIDTH 1
+#define	FRF_CZ_TMFT_SRC_MAC_LBN 12
+#define	FRF_CZ_TMFT_SRC_MAC_WIDTH 48
+#define	FRF_CZ_TMFT_SRC_MAC_DW0_LBN 12
+#define	FRF_CZ_TMFT_SRC_MAC_DW0_WIDTH 32
+#define	FRF_CZ_TMFT_SRC_MAC_DW1_LBN 44
+#define	FRF_CZ_TMFT_SRC_MAC_DW1_WIDTH 16
+#define	FRF_CZ_TMFT_VLAN_ID_LBN 0
+#define	FRF_CZ_TMFT_VLAN_ID_WIDTH 12
+
+
+/*
+ * FR_CZ_MC_TREG_SMEM(32bit):
+ * MC Shared Memory
+ */
+#define	FR_CZ_MC_TREG_SMEM_OFST 0x00ff0000
+/* sienaa0=net_func_bar2 */
+#define	FR_CZ_MC_TREG_SMEM_STEP 4
+#define	FR_CZ_MC_TREG_SMEM_ROWS 512
+
+#define	FRF_CZ_MC_TREG_SMEM_ROW_LBN 0
+#define	FRF_CZ_MC_TREG_SMEM_ROW_WIDTH 32
+
+
+/*
+ * FR_BB_MSIX_VECTOR_TABLE(128bit):
+ * MSIX Vector Table
+ */
+#define	FR_BB_MSIX_VECTOR_TABLE_OFST 0x00ff0000
+/* falconb0=net_func_bar2 */
+#define	FR_BZ_MSIX_VECTOR_TABLE_STEP 16
+#define	FR_BB_MSIX_VECTOR_TABLE_ROWS 64
+/*
+ * FR_CZ_MSIX_VECTOR_TABLE(128bit):
+ * MSIX Vector Table
+ */
+#define	FR_CZ_MSIX_VECTOR_TABLE_OFST 0x00000000
+/* sienaa0=pci_f0_bar4 */
+/* FR_BZ_MSIX_VECTOR_TABLE_STEP 16 */
+#define	FR_CZ_MSIX_VECTOR_TABLE_ROWS 1024
+
+#define	FRF_BZ_MSIX_VECTOR_RESERVED_LBN 97
+#define	FRF_BZ_MSIX_VECTOR_RESERVED_WIDTH 31
+#define	FRF_BZ_MSIX_VECTOR_MASK_LBN 96
+#define	FRF_BZ_MSIX_VECTOR_MASK_WIDTH 1
+#define	FRF_BZ_MSIX_MESSAGE_DATA_LBN 64
+#define	FRF_BZ_MSIX_MESSAGE_DATA_WIDTH 32
+#define	FRF_BZ_MSIX_MESSAGE_ADDRESS_HI_LBN 32
+#define	FRF_BZ_MSIX_MESSAGE_ADDRESS_HI_WIDTH 32
+#define	FRF_BZ_MSIX_MESSAGE_ADDRESS_LO_LBN 0
+#define	FRF_BZ_MSIX_MESSAGE_ADDRESS_LO_WIDTH 32
+
+
+/*
+ * FR_BB_MSIX_PBA_TABLE(32bit):
+ * MSIX Pending Bit Array
+ */
+#define	FR_BB_MSIX_PBA_TABLE_OFST 0x00ff2000
+/* falconb0=net_func_bar2 */
+#define	FR_BZ_MSIX_PBA_TABLE_STEP 4
+#define	FR_BB_MSIX_PBA_TABLE_ROWS 2
+/*
+ * FR_CZ_MSIX_PBA_TABLE(32bit):
+ * MSIX Pending Bit Array
+ */
+#define	FR_CZ_MSIX_PBA_TABLE_OFST 0x00008000
+/* sienaa0=pci_f0_bar4 */
+/* FR_BZ_MSIX_PBA_TABLE_STEP 4 */
+#define	FR_CZ_MSIX_PBA_TABLE_ROWS 32
+
+#define	FRF_BZ_MSIX_PBA_PEND_DWORD_LBN 0
+#define	FRF_BZ_MSIX_PBA_PEND_DWORD_WIDTH 32
+
+
+/*
+ * FR_AZ_SRM_DBG_REG(64bit):
+ * SRAM debug access
+ */
+#define	FR_AZ_SRM_DBG_REG_OFST 0x03000000
+/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */
+#define	FR_AZ_SRM_DBG_REG_STEP 8
+
+#define	FR_CZ_SRM_DBG_REG_ROWS 262144
+#define	FR_AB_SRM_DBG_REG_ROWS 2097152
+
+#define	FRF_AZ_SRM_DBG_LBN 0
+#define	FRF_AZ_SRM_DBG_WIDTH 64
+#define	FRF_AZ_SRM_DBG_DW0_LBN 0
+#define	FRF_AZ_SRM_DBG_DW0_WIDTH 32
+#define	FRF_AZ_SRM_DBG_DW1_LBN 32
+#define	FRF_AZ_SRM_DBG_DW1_WIDTH 32
+
+
+/*
+ * FR_AA_INT_ACK_CHAR(32bit):
+ * CHAR interrupt acknowledge register
+ */
+#define	FR_AA_INT_ACK_CHAR_OFST 0x00000060
+/* falcona0=char_func_bar0 */
+
+#define	FRF_AA_INT_ACK_CHAR_FIELD_LBN 0
+#define	FRF_AA_INT_ACK_CHAR_FIELD_WIDTH 32
+
+
+/* FS_DRIVER_EV */
+#define	FSF_AZ_DRIVER_EV_SUBCODE_LBN 56
+#define	FSF_AZ_DRIVER_EV_SUBCODE_WIDTH 4
+#define	FSE_AZ_TX_DSC_ERROR_EV 15
+#define	FSE_AZ_RX_DSC_ERROR_EV 14
+#define	FSE_AZ_RX_RECOVER_EV 11
+#define	FSE_AZ_TIMER_EV 10
+#define	FSE_AZ_TX_PKT_NON_TCP_UDP 9
+#define	FSE_AZ_WAKE_UP_EV 6
+#define	FSE_AZ_SRM_UPD_DONE_EV 5
+#define	FSE_AZ_EVQ_NOT_EN_EV 3
+#define	FSE_AZ_EVQ_INIT_DONE_EV 2
+#define	FSE_AZ_RX_DESCQ_FLS_DONE_EV 1
+#define	FSE_AZ_TX_DESCQ_FLS_DONE_EV 0
+#define	FSF_AZ_DRIVER_EV_SUBDATA_LBN 0
+#define	FSF_AZ_DRIVER_EV_SUBDATA_WIDTH 14
+
+
+/* FS_EVENT_ENTRY */
+#define	FSF_AZ_EV_CODE_LBN 60
+#define	FSF_AZ_EV_CODE_WIDTH 4
+#define	FSE_AZ_EV_CODE_USER_EV 8
+#define	FSE_AZ_EV_CODE_DRV_GEN_EV 7
+#define	FSE_AZ_EV_CODE_GLOBAL_EV 6
+#define	FSE_AZ_EV_CODE_DRIVER_EV 5
+#define	FSE_AZ_EV_CODE_TX_EV 2
+#define	FSE_AZ_EV_CODE_RX_EV 0
+#define	FSF_AZ_EV_DATA_LBN 0
+#define	FSF_AZ_EV_DATA_WIDTH 60
+#define	FSF_AZ_EV_DATA_DW0_LBN 0
+#define	FSF_AZ_EV_DATA_DW0_WIDTH 32
+#define	FSF_AZ_EV_DATA_DW1_LBN 32
+#define	FSF_AZ_EV_DATA_DW1_WIDTH 28
+
+
+/* FS_GLOBAL_EV */
+#define	FSF_AA_GLB_EV_RX_RECOVERY_LBN 12
+#define	FSF_AA_GLB_EV_RX_RECOVERY_WIDTH 1
+#define	FSF_BZ_GLB_EV_XG_MNT_INTR_LBN 11
+#define	FSF_BZ_GLB_EV_XG_MNT_INTR_WIDTH 1
+#define	FSF_AZ_GLB_EV_XFP_PHY0_INTR_LBN 10
+#define	FSF_AZ_GLB_EV_XFP_PHY0_INTR_WIDTH 1
+#define	FSF_AZ_GLB_EV_XG_PHY0_INTR_LBN 9
+#define	FSF_AZ_GLB_EV_XG_PHY0_INTR_WIDTH 1
+#define	FSF_AZ_GLB_EV_G_PHY0_INTR_LBN 7
+#define	FSF_AZ_GLB_EV_G_PHY0_INTR_WIDTH 1
+
+
+/* FS_RX_EV */
+#define	FSF_CZ_RX_EV_PKT_NOT_PARSED_LBN 58
+#define	FSF_CZ_RX_EV_PKT_NOT_PARSED_WIDTH 1
+#define	FSF_CZ_RX_EV_IPV6_PKT_LBN 57
+#define	FSF_CZ_RX_EV_IPV6_PKT_WIDTH 1
+#define	FSF_AZ_RX_EV_PKT_OK_LBN 56
+#define	FSF_AZ_RX_EV_PKT_OK_WIDTH 1
+#define	FSF_AZ_RX_EV_PAUSE_FRM_ERR_LBN 55
+#define	FSF_AZ_RX_EV_PAUSE_FRM_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_BUF_OWNER_ID_ERR_LBN 54
+#define	FSF_AZ_RX_EV_BUF_OWNER_ID_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_IP_FRAG_ERR_LBN 53
+#define	FSF_AZ_RX_EV_IP_FRAG_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR_LBN 52
+#define	FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR_LBN 51
+#define	FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_ETH_CRC_ERR_LBN 50
+#define	FSF_AZ_RX_EV_ETH_CRC_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_FRM_TRUNC_LBN 49
+#define	FSF_AZ_RX_EV_FRM_TRUNC_WIDTH 1
+#define	FSF_AZ_RX_EV_TOBE_DISC_LBN 47
+#define	FSF_AZ_RX_EV_TOBE_DISC_WIDTH 1
+#define	FSF_AZ_RX_EV_PKT_TYPE_LBN 44
+#define	FSF_AZ_RX_EV_PKT_TYPE_WIDTH 3
+#define	FSE_AZ_RX_EV_PKT_TYPE_VLAN_JUMBO 5
+#define	FSE_AZ_RX_EV_PKT_TYPE_VLAN_LLC 4
+#define	FSE_AZ_RX_EV_PKT_TYPE_VLAN 3
+#define	FSE_AZ_RX_EV_PKT_TYPE_JUMBO 2
+#define	FSE_AZ_RX_EV_PKT_TYPE_LLC 1
+#define	FSE_AZ_RX_EV_PKT_TYPE_ETH 0
+#define	FSF_AZ_RX_EV_HDR_TYPE_LBN 42
+#define	FSF_AZ_RX_EV_HDR_TYPE_WIDTH 2
+#define	FSE_AZ_RX_EV_HDR_TYPE_OTHER 3
+#define	FSE_AZ_RX_EV_HDR_TYPE_IPV4_OTHER 2
+#define	FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER 2
+#define	FSE_AZ_RX_EV_HDR_TYPE_IPV4_UDP 1
+#define	FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP 1
+#define	FSE_AZ_RX_EV_HDR_TYPE_IPV4_TCP 0
+#define	FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP 0
+#define	FSF_AZ_RX_EV_DESC_Q_EMPTY_LBN 41
+#define	FSF_AZ_RX_EV_DESC_Q_EMPTY_WIDTH 1
+#define	FSF_AZ_RX_EV_MCAST_HASH_MATCH_LBN 40
+#define	FSF_AZ_RX_EV_MCAST_HASH_MATCH_WIDTH 1
+#define	FSF_AZ_RX_EV_MCAST_PKT_LBN 39
+#define	FSF_AZ_RX_EV_MCAST_PKT_WIDTH 1
+#define	FSF_AA_RX_EV_RECOVERY_FLAG_LBN 37
+#define	FSF_AA_RX_EV_RECOVERY_FLAG_WIDTH 1
+#define	FSF_AZ_RX_EV_Q_LABEL_LBN 32
+#define	FSF_AZ_RX_EV_Q_LABEL_WIDTH 5
+#define	FSF_AZ_RX_EV_JUMBO_CONT_LBN 31
+#define	FSF_AZ_RX_EV_JUMBO_CONT_WIDTH 1
+#define	FSF_AZ_RX_EV_PORT_LBN 30
+#define	FSF_AZ_RX_EV_PORT_WIDTH 1
+#define	FSF_AZ_RX_EV_BYTE_CNT_LBN 16
+#define	FSF_AZ_RX_EV_BYTE_CNT_WIDTH 14
+#define	FSF_AZ_RX_EV_SOP_LBN 15
+#define	FSF_AZ_RX_EV_SOP_WIDTH 1
+#define	FSF_AZ_RX_EV_ISCSI_PKT_OK_LBN 14
+#define	FSF_AZ_RX_EV_ISCSI_PKT_OK_WIDTH 1
+#define	FSF_AZ_RX_EV_ISCSI_DDIG_ERR_LBN 13
+#define	FSF_AZ_RX_EV_ISCSI_DDIG_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_ISCSI_HDIG_ERR_LBN 12
+#define	FSF_AZ_RX_EV_ISCSI_HDIG_ERR_WIDTH 1
+#define	FSF_AZ_RX_EV_DESC_PTR_LBN 0
+#define	FSF_AZ_RX_EV_DESC_PTR_WIDTH 12
+
+
+/* FS_RX_KER_DESC */
+#define	FSF_AZ_RX_KER_BUF_SIZE_LBN 48
+#define	FSF_AZ_RX_KER_BUF_SIZE_WIDTH 14
+#define	FSF_AZ_RX_KER_BUF_REGION_LBN 46
+#define	FSF_AZ_RX_KER_BUF_REGION_WIDTH 2
+#define	FSF_AZ_RX_KER_BUF_ADDR_LBN 0
+#define	FSF_AZ_RX_KER_BUF_ADDR_WIDTH 46
+#define	FSF_AZ_RX_KER_BUF_ADDR_DW0_LBN 0
+#define	FSF_AZ_RX_KER_BUF_ADDR_DW0_WIDTH 32
+#define	FSF_AZ_RX_KER_BUF_ADDR_DW1_LBN 32
+#define	FSF_AZ_RX_KER_BUF_ADDR_DW1_WIDTH 14
+
+
+/* FS_RX_USER_DESC */
+#define	FSF_AZ_RX_USER_2BYTE_OFFSET_LBN 20
+#define	FSF_AZ_RX_USER_2BYTE_OFFSET_WIDTH 12
+#define	FSF_AZ_RX_USER_BUF_ID_LBN 0
+#define	FSF_AZ_RX_USER_BUF_ID_WIDTH 20
+
+
+/* FS_TX_EV */
+#define	FSF_AZ_TX_EV_PKT_ERR_LBN 38
+#define	FSF_AZ_TX_EV_PKT_ERR_WIDTH 1
+#define	FSF_AZ_TX_EV_PKT_TOO_BIG_LBN 37
+#define	FSF_AZ_TX_EV_PKT_TOO_BIG_WIDTH 1
+#define	FSF_AZ_TX_EV_Q_LABEL_LBN 32
+#define	FSF_AZ_TX_EV_Q_LABEL_WIDTH 5
+#define	FSF_AZ_TX_EV_PORT_LBN 16
+#define	FSF_AZ_TX_EV_PORT_WIDTH 1
+#define	FSF_AZ_TX_EV_WQ_FF_FULL_LBN 15
+#define	FSF_AZ_TX_EV_WQ_FF_FULL_WIDTH 1
+#define	FSF_AZ_TX_EV_BUF_OWNER_ID_ERR_LBN 14
+#define	FSF_AZ_TX_EV_BUF_OWNER_ID_ERR_WIDTH 1
+#define	FSF_AZ_TX_EV_COMP_LBN 12
+#define	FSF_AZ_TX_EV_COMP_WIDTH 1
+#define	FSF_AZ_TX_EV_DESC_PTR_LBN 0
+#define	FSF_AZ_TX_EV_DESC_PTR_WIDTH 12
+
+
+/* FS_TX_KER_DESC */
+#define	FSF_AZ_TX_KER_CONT_LBN 62
+#define	FSF_AZ_TX_KER_CONT_WIDTH 1
+#define	FSF_AZ_TX_KER_BYTE_COUNT_LBN 48
+#define	FSF_AZ_TX_KER_BYTE_COUNT_WIDTH 14
+#define	FSF_AZ_TX_KER_BUF_REGION_LBN 46
+#define	FSF_AZ_TX_KER_BUF_REGION_WIDTH 2
+#define	FSF_AZ_TX_KER_BUF_ADDR_LBN 0
+#define	FSF_AZ_TX_KER_BUF_ADDR_WIDTH 46
+#define	FSF_AZ_TX_KER_BUF_ADDR_DW0_LBN 0
+#define	FSF_AZ_TX_KER_BUF_ADDR_DW0_WIDTH 32
+#define	FSF_AZ_TX_KER_BUF_ADDR_DW1_LBN 32
+#define	FSF_AZ_TX_KER_BUF_ADDR_DW1_WIDTH 14
+
+
+/* FS_TX_USER_DESC */
+#define	FSF_AZ_TX_USER_SW_EV_EN_LBN 48
+#define	FSF_AZ_TX_USER_SW_EV_EN_WIDTH 1
+#define	FSF_AZ_TX_USER_CONT_LBN 46
+#define	FSF_AZ_TX_USER_CONT_WIDTH 1
+#define	FSF_AZ_TX_USER_BYTE_CNT_LBN 33
+#define	FSF_AZ_TX_USER_BYTE_CNT_WIDTH 13
+#define	FSF_AZ_TX_USER_BUF_ID_LBN 13
+#define	FSF_AZ_TX_USER_BUF_ID_WIDTH 20
+#define	FSF_AZ_TX_USER_BYTE_OFS_LBN 0
+#define	FSF_AZ_TX_USER_BYTE_OFS_WIDTH 13
+
+
+/* FS_USER_EV */
+#define	FSF_CZ_USER_QID_LBN 32
+#define	FSF_CZ_USER_QID_WIDTH 10
+#define	FSF_CZ_USER_EV_REG_VALUE_LBN 0
+#define	FSF_CZ_USER_EV_REG_VALUE_WIDTH 32
+
+
+/* FS_NET_IVEC */
+#define	FSF_AZ_NET_IVEC_FATAL_INT_LBN 64
+#define	FSF_AZ_NET_IVEC_FATAL_INT_WIDTH 1
+#define	FSF_AZ_NET_IVEC_INT_Q_LBN 40
+#define	FSF_AZ_NET_IVEC_INT_Q_WIDTH 4
+#define	FSF_AZ_NET_IVEC_INT_FLAG_LBN 32
+#define	FSF_AZ_NET_IVEC_INT_FLAG_WIDTH 1
+#define	FSF_AZ_NET_IVEC_EVQ_FIFO_HF_LBN 1
+#define	FSF_AZ_NET_IVEC_EVQ_FIFO_HF_WIDTH 1
+#define	FSF_AZ_NET_IVEC_EVQ_FIFO_AF_LBN 0
+#define	FSF_AZ_NET_IVEC_EVQ_FIFO_AF_WIDTH 1
+
+
+/* DRIVER_EV */
+/* Sub-fields of an RX flush completion event */
+#define	FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL_LBN 12
+#define	FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL_WIDTH 1
+#define	FSF_AZ_DRIVER_EV_RX_DESCQ_ID_LBN 0
+#define	FSF_AZ_DRIVER_EV_RX_DESCQ_ID_WIDTH 12
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+
+
+
+#endif /* _SYS_EFX_REGS_H */
diff --git a/sys/dev/sfxge/common/efx_regs_ef10.h b/sys/dev/sfxge/common/efx_regs_ef10.h
new file mode 100644
index 0000000..62c0372
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_regs_ef10.h
@@ -0,0 +1,2682 @@
+/*-
+ * Copyright 2007-2010 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_SYS_EFX_EF10_REGS_H
+#define	_SYS_EFX_EF10_REGS_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * BIU_HW_REV_ID_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_BIU_HW_REV_ID_REG 0x00000000
+/* hunta0=pcie_pf_bar2 */
+
+#define	ERF_DZ_HW_REV_ID_LBN 0
+#define	ERF_DZ_HW_REV_ID_WIDTH 32
+
+
+/*
+ * BIU_MC_SFT_STATUS_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_BIU_MC_SFT_STATUS_REG 0x00000010
+/* hunta0=pcie_pf_bar2 */
+#define	ER_DZ_BIU_MC_SFT_STATUS_REG_STEP 4
+#define	ER_DZ_BIU_MC_SFT_STATUS_REG_ROWS 8
+
+#define	ERF_DZ_MC_SFT_STATUS_LBN 0
+#define	ERF_DZ_MC_SFT_STATUS_WIDTH 32
+
+
+/*
+ * BIU_INT_ISR_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_BIU_INT_ISR_REG 0x00000090
+/* hunta0=pcie_pf_bar2 */
+
+#define	ERF_DZ_ISR_REG_LBN 0
+#define	ERF_DZ_ISR_REG_WIDTH 32
+
+
+/*
+ * MC_DB_LWRD_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_MC_DB_LWRD_REG 0x00000200
+/* hunta0=pcie_pf_bar2 */
+
+#define	ERF_DZ_MC_DOORBELL_L_LBN 0
+#define	ERF_DZ_MC_DOORBELL_L_WIDTH 32
+
+
+/*
+ * MC_DB_HWRD_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_MC_DB_HWRD_REG 0x00000204
+/* hunta0=pcie_pf_bar2 */
+
+#define	ERF_DZ_MC_DOORBELL_H_LBN 0
+#define	ERF_DZ_MC_DOORBELL_H_WIDTH 32
+
+
+/*
+ * EVQ_RPTR_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_EVQ_RPTR_REG 0x00000400
+/* hunta0=pcie_pf_bar2 */
+#define	ER_DZ_EVQ_RPTR_REG_STEP 4096
+#define	ER_DZ_EVQ_RPTR_REG_ROWS 2048
+
+#define	ERF_DZ_EVQ_RPTR_VLD_LBN 15
+#define	ERF_DZ_EVQ_RPTR_VLD_WIDTH 1
+#define	ERF_DZ_EVQ_RPTR_LBN 0
+#define	ERF_DZ_EVQ_RPTR_WIDTH 15
+
+
+/*
+ * EVQ_TMR_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_EVQ_TMR_REG 0x00000420
+/* hunta0=pcie_pf_bar2 */
+#define	ER_DZ_EVQ_TMR_REG_STEP 4096
+#define	ER_DZ_EVQ_TMR_REG_ROWS 2048
+
+#define	ERF_DZ_TC_TIMER_MODE_LBN 14
+#define	ERF_DZ_TC_TIMER_MODE_WIDTH 2
+#define	ERF_DZ_TC_TIMER_VAL_LBN 0
+#define	ERF_DZ_TC_TIMER_VAL_WIDTH 14
+
+
+/*
+ * RX_DESC_UPD_REG(32bit):
+ * 
+ */
+
+#define	ER_DZ_RX_DESC_UPD_REG 0x00000830
+/* hunta0=pcie_pf_bar2 */
+#define	ER_DZ_RX_DESC_UPD_REG_STEP 4096
+#define	ER_DZ_RX_DESC_UPD_REG_ROWS 2048
+
+#define	ERF_DZ_RX_DESC_WPTR_LBN 0
+#define	ERF_DZ_RX_DESC_WPTR_WIDTH 12
+
+
+/*
+ * TX_DESC_UPD_REG(76bit):
+ * 
+ */
+
+#define	ER_DZ_TX_DESC_UPD_REG 0x00000a10
+/* hunta0=pcie_pf_bar2 */
+#define	ER_DZ_TX_DESC_UPD_REG_STEP 4096
+#define	ER_DZ_TX_DESC_UPD_REG_ROWS 2048
+
+#define	ERF_DZ_TX_DESC_WPTR_LBN 64
+#define	ERF_DZ_TX_DESC_WPTR_WIDTH 12
+#define	ERF_DZ_TX_DESC_HWORD_LBN 32
+#define	ERF_DZ_TX_DESC_HWORD_WIDTH 32
+#define	ERF_DZ_TX_DESC_LWORD_LBN 0
+#define	ERF_DZ_TX_DESC_LWORD_WIDTH 32
+
+
+/* ES_DRIVER_EV */
+#define	ESF_DZ_DRV_CODE_LBN 60
+#define	ESF_DZ_DRV_CODE_WIDTH 4
+#define	ESF_DZ_DRV_SUB_CODE_LBN 56
+#define	ESF_DZ_DRV_SUB_CODE_WIDTH 4
+#define	ESE_DZ_DRV_TIMER_EV 10
+#define	ESE_DZ_DRV_WAKE_UP_EV 6
+#define	ESF_DZ_DRV_SUB_DATA_DW0_LBN 0
+#define	ESF_DZ_DRV_SUB_DATA_DW0_WIDTH 32
+#define	ESF_DZ_DRV_SUB_DATA_DW1_LBN 32
+#define	ESF_DZ_DRV_SUB_DATA_DW1_WIDTH 24
+#define	ESF_DZ_DRV_SUB_DATA_LBN 0
+#define	ESF_DZ_DRV_SUB_DATA_WIDTH 56
+#define	ESF_DZ_DRV_EVQ_ID_LBN 0
+#define	ESF_DZ_DRV_EVQ_ID_WIDTH 14
+#define	ESF_DZ_DRV_TMR_ID_LBN 0
+#define	ESF_DZ_DRV_TMR_ID_WIDTH 14
+
+
+/* ES_EVENT_ENTRY */
+#define	ESF_DZ_EV_CODE_LBN 60
+#define	ESF_DZ_EV_CODE_WIDTH 4
+#define	ESE_DZ_EV_CODE_MCDI_EV 12
+#define	ESE_DZ_EV_CODE_DRIVER_EV 5
+#define	ESE_DZ_EV_CODE_TX_EV 2
+#define	ESE_DZ_EV_CODE_RX_EV 0
+#define	ESE_DZ_OTHER other
+#define	ESF_DZ_EV_DATA_DW0_LBN 0
+#define	ESF_DZ_EV_DATA_DW0_WIDTH 32
+#define	ESF_DZ_EV_DATA_DW1_LBN 32
+#define	ESF_DZ_EV_DATA_DW1_WIDTH 28
+#define	ESF_DZ_EV_DATA_LBN 0
+#define	ESF_DZ_EV_DATA_WIDTH 60
+
+
+/* ES_FF_UMSG_CPU2DL_DESC_FETCH */
+#define	ESF_DZ_C2DDF_DSCR_CACHE_RPTR_LBN 112
+#define	ESF_DZ_C2DDF_DSCR_CACHE_RPTR_WIDTH 6
+#define	ESF_DZ_C2DDF_QID_LBN 96
+#define	ESF_DZ_C2DDF_QID_WIDTH 11
+#define	ESF_DZ_C2DDF_DSCR_BASE_PAGE_ID_LBN 64
+#define	ESF_DZ_C2DDF_DSCR_BASE_PAGE_ID_WIDTH 18
+#define	ESF_DZ_C2DDF_DSCR_HW_RPTR_LBN 48
+#define	ESF_DZ_C2DDF_DSCR_HW_RPTR_WIDTH 12
+#define	ESF_DZ_C2DDF_DSCR_HW_WPTR_LBN 32
+#define	ESF_DZ_C2DDF_DSCR_HW_WPTR_WIDTH 12
+#define	ESF_DZ_C2DDF_OID_LBN 16
+#define	ESF_DZ_C2DDF_OID_WIDTH 12
+#define	ESF_DZ_C2DDF_DSCR_SIZE_LBN 13
+#define	ESF_DZ_C2DDF_DSCR_SIZE_WIDTH 3
+#define	ESE_DZ_C2DDF_DSCR_SIZE_512 7
+#define	ESE_DZ_C2DDF_DSCR_SIZE_1K 6
+#define	ESE_DZ_C2DDF_DSCR_SIZE_2K 5
+#define	ESE_DZ_C2DDF_DSCR_SIZE_4K 4
+#define	ESF_DZ_C2DDF_BIU_ARGS_LBN 0
+#define	ESF_DZ_C2DDF_BIU_ARGS_WIDTH 13
+
+
+/* ES_FF_UMSG_CPU2DL_DESC_PUSH */
+#define	ESF_DZ_C2DDP_DESC_DW0_LBN 128
+#define	ESF_DZ_C2DDP_DESC_DW0_WIDTH 32
+#define	ESF_DZ_C2DDP_DESC_DW1_LBN 160
+#define	ESF_DZ_C2DDP_DESC_DW1_WIDTH 32
+#define	ESF_DZ_C2DDP_DESC_LBN 128
+#define	ESF_DZ_C2DDP_DESC_WIDTH 64
+#define	ESF_DZ_C2DDP_QID_LBN 96
+#define	ESF_DZ_C2DDP_QID_WIDTH 11
+#define	ESF_DZ_C2DDP_DSCR_HW_RPTR_LBN 48
+#define	ESF_DZ_C2DDP_DSCR_HW_RPTR_WIDTH 12
+#define	ESF_DZ_C2DDP_DSCR_HW_WPTR_LBN 32
+#define	ESF_DZ_C2DDP_DSCR_HW_WPTR_WIDTH 12
+#define	ESF_DZ_C2DDP_OID_LBN 16
+#define	ESF_DZ_C2DDP_OID_WIDTH 12
+#define	ESF_DZ_C2DDP_DSCR_SIZE_LBN 0
+#define	ESF_DZ_C2DDP_DSCR_SIZE_WIDTH 3
+#define	ESE_DZ_C2DDF_DSCR_SIZE_512 7
+#define	ESE_DZ_C2DDF_DSCR_SIZE_1K 6
+#define	ESE_DZ_C2DDF_DSCR_SIZE_2K 5
+#define	ESE_DZ_C2DDF_DSCR_SIZE_4K 4
+
+
+/* ES_FF_UMSG_CPU2DL_GPRD */
+#define	ESF_DZ_C2DG_ENCODED_HOST_ADDR_DW0_LBN 64
+#define	ESF_DZ_C2DG_ENCODED_HOST_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_C2DG_ENCODED_HOST_ADDR_DW1_LBN 96
+#define	ESF_DZ_C2DG_ENCODED_HOST_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_C2DG_ENCODED_HOST_ADDR_LBN 64
+#define	ESF_DZ_C2DG_ENCODED_HOST_ADDR_WIDTH 48
+#define	ESF_DZ_C2DG_SMC_ADDR_LBN 16
+#define	ESF_DZ_C2DG_SMC_ADDR_WIDTH 16
+#define	ESF_DZ_C2DG_BIU_ARGS_LBN 0
+#define	ESF_DZ_C2DG_BIU_ARGS_WIDTH 14
+
+
+/* ES_FF_UMSG_CPU2EV_SOFT */
+#define	ESF_DZ_C2ES_TBD_LBN 0
+#define	ESF_DZ_C2ES_TBD_WIDTH 1
+
+
+/* ES_FF_UMSG_CPU2EV_TXCMPLT */
+#define	ESF_DZ_C2ET_EV_SOFT0_LBN 32
+#define	ESF_DZ_C2ET_EV_SOFT0_WIDTH 16
+#define	ESF_DZ_C2ET_DSCR_IDX_LBN 16
+#define	ESF_DZ_C2ET_DSCR_IDX_WIDTH 16
+#define	ESF_DZ_C2ET_EV_QID_LBN 5
+#define	ESF_DZ_C2ET_EV_QID_WIDTH 11
+#define	ESF_DZ_C2ET_EV_QLABEL_LBN 0
+#define	ESF_DZ_C2ET_EV_QLABEL_WIDTH 5
+
+
+/* ES_FF_UMSG_CPU2RXDP_INGR_BUFOP */
+#define	ESF_DZ_C2RIB_EV_DISABLE_LBN 191
+#define	ESF_DZ_C2RIB_EV_DISABLE_WIDTH 1
+#define	ESF_DZ_C2RIB_EV_SOFT_LBN 188
+#define	ESF_DZ_C2RIB_EV_SOFT_WIDTH 3
+#define	ESF_DZ_C2RIB_EV_DESC_PTR_LBN 176
+#define	ESF_DZ_C2RIB_EV_DESC_PTR_WIDTH 12
+#define	ESF_DZ_C2RIB_EV_ARG1_LBN 160
+#define	ESF_DZ_C2RIB_EV_ARG1_WIDTH 16
+#define	ESF_DZ_C2RIB_ENCODED_HOST_ADDR_DW0_LBN 64
+#define	ESF_DZ_C2RIB_ENCODED_HOST_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_C2RIB_ENCODED_HOST_ADDR_DW1_LBN 96
+#define	ESF_DZ_C2RIB_ENCODED_HOST_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_C2RIB_ENCODED_HOST_ADDR_LBN 64
+#define	ESF_DZ_C2RIB_ENCODED_HOST_ADDR_WIDTH 48
+#define	ESF_DZ_C2RIB_BIU_ARGS_LBN 16
+#define	ESF_DZ_C2RIB_BIU_ARGS_WIDTH 13
+#define	ESF_DZ_C2RIB_EV_QID_LBN 5
+#define	ESF_DZ_C2RIB_EV_QID_WIDTH 11
+#define	ESF_DZ_C2RIB_EV_QLABEL_LBN 0
+#define	ESF_DZ_C2RIB_EV_QLABEL_WIDTH 5
+
+
+/* ES_FF_UMSG_CPU2RXDP_INGR_PDISP */
+#define	ESF_DZ_C2RIP_BUF_LEN_LBN 240
+#define	ESF_DZ_C2RIP_BUF_LEN_WIDTH 16
+#define	ESF_DZ_C2RIP_ENCODED_HOST_ADDR_DW0_LBN 192
+#define	ESF_DZ_C2RIP_ENCODED_HOST_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_C2RIP_ENCODED_HOST_ADDR_DW1_LBN 224
+#define	ESF_DZ_C2RIP_ENCODED_HOST_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_C2RIP_ENCODED_HOST_ADDR_LBN 192
+#define	ESF_DZ_C2RIP_ENCODED_HOST_ADDR_WIDTH 48
+#define	ESF_DZ_C2RIP_EV_DISABLE_LBN 191
+#define	ESF_DZ_C2RIP_EV_DISABLE_WIDTH 1
+#define	ESF_DZ_C2RIP_EV_SOFT_LBN 188
+#define	ESF_DZ_C2RIP_EV_SOFT_WIDTH 3
+#define	ESF_DZ_C2RIP_EV_DESC_PTR_LBN 176
+#define	ESF_DZ_C2RIP_EV_DESC_PTR_WIDTH 12
+#define	ESF_DZ_C2RIP_EV_ARG1_LBN 160
+#define	ESF_DZ_C2RIP_EV_ARG1_WIDTH 16
+#define	ESF_DZ_C2RIP_UPD_CRC_MODE_LBN 157
+#define	ESF_DZ_C2RIP_UPD_CRC_MODE_WIDTH 3
+#define	ESE_DZ_C2RIP_FCOIP_MPA 5
+#define	ESE_DZ_C2RIP_FCOIP_FCOE 4
+#define	ESE_DZ_C2RIP_ISCSI_HDR_AND_PYLD 3
+#define	ESE_DZ_C2RIP_ISCSI_HDR 2
+#define	ESE_DZ_C2RIP_FCOE 1
+#define	ESE_DZ_C2RIP_OFF 0
+#define	ESF_DZ_C2RIP_BIU_ARGS_LBN 144
+#define	ESF_DZ_C2RIP_BIU_ARGS_WIDTH 13
+#define	ESF_DZ_C2RIP_EV_QID_LBN 133
+#define	ESF_DZ_C2RIP_EV_QID_WIDTH 11
+#define	ESF_DZ_C2RIP_EV_QLABEL_LBN 128
+#define	ESF_DZ_C2RIP_EV_QLABEL_WIDTH 5
+#define	ESF_DZ_C2RIP_PEDIT_DELTA_LBN 104
+#define	ESF_DZ_C2RIP_PEDIT_DELTA_WIDTH 8
+#define	ESF_DZ_C2RIP_PYLOAD_OFST_LBN 96
+#define	ESF_DZ_C2RIP_PYLOAD_OFST_WIDTH 8
+#define	ESF_DZ_C2RIP_L4_HDR_OFST_LBN 88
+#define	ESF_DZ_C2RIP_L4_HDR_OFST_WIDTH 8
+#define	ESF_DZ_C2RIP_L3_HDR_OFST_LBN 80
+#define	ESF_DZ_C2RIP_L3_HDR_OFST_WIDTH 8
+#define	ESF_DZ_C2RIP_IS_UDP_LBN 69
+#define	ESF_DZ_C2RIP_IS_UDP_WIDTH 1
+#define	ESF_DZ_C2RIP_IS_TCP_LBN 68
+#define	ESF_DZ_C2RIP_IS_TCP_WIDTH 1
+#define	ESF_DZ_C2RIP_IS_IPV6_LBN 67
+#define	ESF_DZ_C2RIP_IS_IPV6_WIDTH 1
+#define	ESF_DZ_C2RIP_IS_IPV4_LBN 66
+#define	ESF_DZ_C2RIP_IS_IPV4_WIDTH 1
+#define	ESF_DZ_C2RIP_IS_FCOE_LBN 65
+#define	ESF_DZ_C2RIP_IS_FCOE_WIDTH 1
+#define	ESF_DZ_C2RIP_PARSE_INCOMP_LBN 64
+#define	ESF_DZ_C2RIP_PARSE_INCOMP_WIDTH 1
+#define	ESF_DZ_C2RIP_FINFO_WRD3_LBN 48
+#define	ESF_DZ_C2RIP_FINFO_WRD3_WIDTH 16
+#define	ESF_DZ_C2RIP_FINFO_WRD2_LBN 32
+#define	ESF_DZ_C2RIP_FINFO_WRD2_WIDTH 16
+#define	ESF_DZ_C2RIP_FINFO_WRD1_LBN 16
+#define	ESF_DZ_C2RIP_FINFO_WRD1_WIDTH 16
+#define	ESF_DZ_C2RIP_FINFO_SRCDST_LBN 0
+#define	ESF_DZ_C2RIP_FINFO_SRCDST_WIDTH 16
+
+
+/* ES_FF_UMSG_CPU2RXDP_INGR_SOFT */
+#define	ESF_DZ_C2RIS_SOFT3_LBN 48
+#define	ESF_DZ_C2RIS_SOFT3_WIDTH 16
+#define	ESF_DZ_C2RIS_SOFT2_LBN 32
+#define	ESF_DZ_C2RIS_SOFT2_WIDTH 16
+#define	ESF_DZ_C2RIS_SOFT1_LBN 16
+#define	ESF_DZ_C2RIS_SOFT1_WIDTH 16
+#define	ESF_DZ_C2RIS_SOFT0_LBN 0
+#define	ESF_DZ_C2RIS_SOFT0_WIDTH 16
+
+
+/* ES_FF_UMSG_CPU2SMC_BUFLKUP */
+#define	ESF_DZ_C2SB_PAGE_ID_LBN 16
+#define	ESF_DZ_C2SB_PAGE_ID_WIDTH 18
+#define	ESF_DZ_C2SB_EXP_PAGE_ID_LBN 0
+#define	ESF_DZ_C2SB_EXP_PAGE_ID_WIDTH 12
+
+
+/* ES_FF_UMSG_CPU2SMC_DESCOP */
+#define	ESF_DZ_C2SD_LEN_LBN 112
+#define	ESF_DZ_C2SD_LEN_WIDTH 14
+#define	ESF_DZ_C2SD_ENCODED_HOST_ADDR_DW0_LBN 64
+#define	ESF_DZ_C2SD_ENCODED_HOST_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_C2SD_ENCODED_HOST_ADDR_DW1_LBN 96
+#define	ESF_DZ_C2SD_ENCODED_HOST_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_C2SD_ENCODED_HOST_ADDR_LBN 64
+#define	ESF_DZ_C2SD_ENCODED_HOST_ADDR_WIDTH 48
+#define	ESF_DZ_C2SD_OFFSET_LBN 48
+#define	ESF_DZ_C2SD_OFFSET_WIDTH 8
+#define	ESF_DZ_C2SD_QID_LBN 32
+#define	ESF_DZ_C2SD_QID_WIDTH 11
+#define	ESF_DZ_C2SD_CONT_LBN 16
+#define	ESF_DZ_C2SD_CONT_WIDTH 1
+#define	ESF_DZ_C2SD_TYPE_LBN 0
+#define	ESF_DZ_C2SD_TYPE_WIDTH 1
+
+
+/* ES_FF_UMSG_CPU2SMC_GPOP */
+#define	ESF_DZ_C2SG_DATA_DW0_LBN 64
+#define	ESF_DZ_C2SG_DATA_DW0_WIDTH 32
+#define	ESF_DZ_C2SG_DATA_DW1_LBN 96
+#define	ESF_DZ_C2SG_DATA_DW1_WIDTH 32
+#define	ESF_DZ_C2SG_DATA_LBN 64
+#define	ESF_DZ_C2SG_DATA_WIDTH 64
+#define	ESF_DZ_C2SG_SOFT_LBN 48
+#define	ESF_DZ_C2SG_SOFT_WIDTH 4
+#define	ESF_DZ_C2SG_REFLECT_LBN 32
+#define	ESF_DZ_C2SG_REFLECT_WIDTH 1
+#define	ESF_DZ_C2SG_ADDR_LBN 0
+#define	ESF_DZ_C2SG_ADDR_WIDTH 16
+
+
+/* ES_FF_UMSG_CPU2TXDP_DMA_BUFREQ */
+#define	ESF_DZ_C2TDB_BUF_LEN_LBN 176
+#define	ESF_DZ_C2TDB_BUF_LEN_WIDTH 16
+#define	ESF_DZ_C2TDB_ENCODED_HOST_ADDR_DW0_LBN 128
+#define	ESF_DZ_C2TDB_ENCODED_HOST_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_C2TDB_ENCODED_HOST_ADDR_DW1_LBN 160
+#define	ESF_DZ_C2TDB_ENCODED_HOST_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_C2TDB_ENCODED_HOST_ADDR_LBN 128
+#define	ESF_DZ_C2TDB_ENCODED_HOST_ADDR_WIDTH 48
+#define	ESF_DZ_C2TDB_SOFT_LBN 112
+#define	ESF_DZ_C2TDB_SOFT_WIDTH 14
+#define	ESF_DZ_C2TDB_DESC_IDX_LBN 96
+#define	ESF_DZ_C2TDB_DESC_IDX_WIDTH 16
+#define	ESF_DZ_C2TDB_UPD_CRC_MODE_LBN 93
+#define	ESF_DZ_C2TDB_UPD_CRC_MODE_WIDTH 3
+#define	ESE_DZ_C2RIP_FCOIP_MPA 5
+#define	ESE_DZ_C2RIP_FCOIP_FCOE 4
+#define	ESE_DZ_C2RIP_ISCSI_HDR_AND_PYLD 3
+#define	ESE_DZ_C2RIP_ISCSI_HDR 2
+#define	ESE_DZ_C2RIP_FCOE 1
+#define	ESE_DZ_C2RIP_OFF 0
+#define	ESF_DZ_C2TDB_BIU_ARGS_LBN 80
+#define	ESF_DZ_C2TDB_BIU_ARGS_WIDTH 13
+#define	ESF_DZ_C2TDB_CONT_LBN 64
+#define	ESF_DZ_C2TDB_CONT_WIDTH 1
+#define	ESF_DZ_C2TDB_FINFO_WRD3_LBN 48
+#define	ESF_DZ_C2TDB_FINFO_WRD3_WIDTH 16
+#define	ESF_DZ_C2TDB_FINFO_WRD2_LBN 32
+#define	ESF_DZ_C2TDB_FINFO_WRD2_WIDTH 16
+#define	ESF_DZ_C2TDB_FINFO_WRD1_LBN 16
+#define	ESF_DZ_C2TDB_FINFO_WRD1_WIDTH 16
+#define	ESF_DZ_C2TDB_FINFO_SRCDST_LBN 0
+#define	ESF_DZ_C2TDB_FINFO_SRCDST_WIDTH 16
+
+
+/* ES_FF_UMSG_CPU2TXDP_DMA_PKTABORT */
+#define	ESF_DZ_C2TDP_SOFT_LBN 48
+#define	ESF_DZ_C2TDP_SOFT_WIDTH 14
+#define	ESF_DZ_C2TDP_DESC_IDX_LBN 32
+#define	ESF_DZ_C2TDP_DESC_IDX_WIDTH 16
+#define	ESF_DZ_C2TDP_BIU_ARGS_LBN 16
+#define	ESF_DZ_C2TDP_BIU_ARGS_WIDTH 14
+
+
+/* ES_FF_UMSG_CPU2TXDP_DMA_SOFT */
+#define	ESF_DZ_C2TDS_SOFT3_LBN 48
+#define	ESF_DZ_C2TDS_SOFT3_WIDTH 16
+#define	ESF_DZ_C2TDS_SOFT2_LBN 32
+#define	ESF_DZ_C2TDS_SOFT2_WIDTH 16
+#define	ESF_DZ_C2TDS_SOFT1_LBN 16
+#define	ESF_DZ_C2TDS_SOFT1_WIDTH 16
+#define	ESF_DZ_C2TDS_SOFT0_LBN 0
+#define	ESF_DZ_C2TDS_SOFT0_WIDTH 16
+
+
+/* ES_FF_UMSG_CPU2TXDP_EGR */
+#define	ESF_DZ_C2TE_PEDIT_DELTA_LBN 168
+#define	ESF_DZ_C2TE_PEDIT_DELTA_WIDTH 8
+#define	ESF_DZ_C2TE_PYLOAD_OFST_LBN 160
+#define	ESF_DZ_C2TE_PYLOAD_OFST_WIDTH 8
+#define	ESF_DZ_C2TE_L4_HDR_OFST_LBN 152
+#define	ESF_DZ_C2TE_L4_HDR_OFST_WIDTH 8
+#define	ESF_DZ_C2TE_L3_HDR_OFST_LBN 144
+#define	ESF_DZ_C2TE_L3_HDR_OFST_WIDTH 8
+#define	ESF_DZ_C2TE_IS_UDP_LBN 133
+#define	ESF_DZ_C2TE_IS_UDP_WIDTH 1
+#define	ESF_DZ_C2TE_IS_TCP_LBN 132
+#define	ESF_DZ_C2TE_IS_TCP_WIDTH 1
+#define	ESF_DZ_C2TE_IS_IPV6_LBN 131
+#define	ESF_DZ_C2TE_IS_IPV6_WIDTH 1
+#define	ESF_DZ_C2TE_IS_IPV4_LBN 130
+#define	ESF_DZ_C2TE_IS_IPV4_WIDTH 1
+#define	ESF_DZ_C2TE_IS_FCOE_LBN 129
+#define	ESF_DZ_C2TE_IS_FCOE_WIDTH 1
+#define	ESF_DZ_C2TE_PARSE_INCOMP_LBN 128
+#define	ESF_DZ_C2TE_PARSE_INCOMP_WIDTH 1
+#define	ESF_DZ_C2TE_PKT_LEN_LBN 112
+#define	ESF_DZ_C2TE_PKT_LEN_WIDTH 16
+#define	ESF_DZ_C2TE_UPD_TCPUDPCSUM_MODE_LBN 97
+#define	ESF_DZ_C2TE_UPD_TCPUDPCSUM_MODE_WIDTH 1
+#define	ESF_DZ_C2TE_UPD_IPCSUM_MODE_LBN 96
+#define	ESF_DZ_C2TE_UPD_IPCSUM_MODE_WIDTH 1
+#define	ESF_DZ_C2TE_UPD_CRC_MODE_LBN 93
+#define	ESF_DZ_C2TE_UPD_CRC_MODE_WIDTH 3
+#define	ESE_DZ_C2RIP_FCOIP_MPA 5
+#define	ESE_DZ_C2RIP_FCOIP_FCOE 4
+#define	ESE_DZ_C2RIP_ISCSI_HDR_AND_PYLD 3
+#define	ESE_DZ_C2RIP_ISCSI_HDR 2
+#define	ESE_DZ_C2RIP_FCOE 1
+#define	ESE_DZ_C2RIP_OFF 0
+#define	ESF_DZ_C2TE_FINFO_WRD3_LBN 48
+#define	ESF_DZ_C2TE_FINFO_WRD3_WIDTH 16
+#define	ESF_DZ_C2TE_FINFO_WRD2_LBN 32
+#define	ESF_DZ_C2TE_FINFO_WRD2_WIDTH 16
+#define	ESF_DZ_C2TE_FINFO_WRD1_LBN 16
+#define	ESF_DZ_C2TE_FINFO_WRD1_WIDTH 16
+#define	ESF_DZ_C2TE_FINFO_SRCDST_LBN 0
+#define	ESF_DZ_C2TE_FINFO_SRCDST_WIDTH 16
+
+
+/* ES_FF_UMSG_CPU2TXDP_EGR_SOFT */
+#define	ESF_DZ_C2TES_SOFT3_LBN 48
+#define	ESF_DZ_C2TES_SOFT3_WIDTH 16
+#define	ESF_DZ_C2TES_SOFT2_LBN 32
+#define	ESF_DZ_C2TES_SOFT2_WIDTH 16
+#define	ESF_DZ_C2TES_SOFT1_LBN 16
+#define	ESF_DZ_C2TES_SOFT1_WIDTH 16
+#define	ESF_DZ_C2TES_SOFT0_LBN 0
+#define	ESF_DZ_C2TES_SOFT0_WIDTH 16
+
+
+/* ES_FF_UMSG_DL2CPU_DESC_FETCH */
+#define	ESF_DZ_D2CDF_REFL_DSCR_HW_WPTR_LBN 64
+#define	ESF_DZ_D2CDF_REFL_DSCR_HW_WPTR_WIDTH 12
+#define	ESF_DZ_D2CDF_FAIL_LBN 48
+#define	ESF_DZ_D2CDF_FAIL_WIDTH 1
+#define	ESF_DZ_D2CDF_QID_LBN 32
+#define	ESF_DZ_D2CDF_QID_WIDTH 11
+#define	ESF_DZ_D2CDF_NUM_DESC_LBN 16
+#define	ESF_DZ_D2CDF_NUM_DESC_WIDTH 7
+#define	ESF_DZ_D2CDF_NEW_DSCR_HW_RPTR_LBN 0
+#define	ESF_DZ_D2CDF_NEW_DSCR_HW_RPTR_WIDTH 12
+
+
+/* ES_FF_UMSG_DL2CPU_GPRD */
+#define	ESF_DZ_D2CG_BIU_ARGS_LBN 0
+#define	ESF_DZ_D2CG_BIU_ARGS_WIDTH 14
+
+
+/* ES_FF_UMSG_DPCPU_PACER_TXQ_D_R_I_REQ */
+#define	ESF_DZ_FRM_LEN_LBN 16
+#define	ESF_DZ_FRM_LEN_WIDTH 15
+#define	ESF_DZ_TXQ_ID_LBN 0
+#define	ESF_DZ_TXQ_ID_WIDTH 10
+
+
+/* ES_FF_UMSG_PACER_BKT_TBL_RD_REQ */
+#define	ESF_DZ_BKT_ID_LBN 0
+#define	ESF_DZ_BKT_ID_WIDTH 9
+
+
+/* ES_FF_UMSG_PACER_BKT_TBL_RD_RSP */
+#define	ESF_DZ_DUE_TIME_LBN 80
+#define	ESF_DZ_DUE_TIME_WIDTH 16
+#define	ESF_DZ_LAST_FILL_TIME_LBN 64
+#define	ESF_DZ_LAST_FILL_TIME_WIDTH 16
+#define	ESF_DZ_RATE_REC_LBN 48
+#define	ESF_DZ_RATE_REC_WIDTH 16
+#define	ESF_DZ_RATE_LBN 32
+#define	ESF_DZ_RATE_WIDTH 16
+#define	ESF_DZ_FILL_LEVEL_LBN 16
+#define	ESF_DZ_FILL_LEVEL_WIDTH 16
+#define	ESF_DZ_IDLE_LBN 15
+#define	ESF_DZ_IDLE_WIDTH 1
+#define	ESF_DZ_USED_LBN 14
+#define	ESF_DZ_USED_WIDTH 1
+#define	ESF_DZ_MAX_FILL_REG_LBN 12
+#define	ESF_DZ_MAX_FILL_REG_WIDTH 2
+#define	ESF_DZ_BKT_ID_LBN 0
+#define	ESF_DZ_BKT_ID_WIDTH 9
+
+
+/* ES_FF_UMSG_PACER_BKT_TBL_WR_REQ */
+#define	ESF_DZ_RATE_REC_LBN 48
+#define	ESF_DZ_RATE_REC_WIDTH 16
+#define	ESF_DZ_RATE_LBN 32
+#define	ESF_DZ_RATE_WIDTH 16
+#define	ESF_DZ_FILL_LEVEL_LBN 16
+#define	ESF_DZ_FILL_LEVEL_WIDTH 16
+#define	ESF_DZ_IDLE_LBN 15
+#define	ESF_DZ_IDLE_WIDTH 1
+#define	ESF_DZ_USED_LBN 14
+#define	ESF_DZ_USED_WIDTH 1
+#define	ESF_DZ_MAX_FILL_REG_LBN 12
+#define	ESF_DZ_MAX_FILL_REG_WIDTH 2
+#define	ESF_DZ_BKT_ID_LBN 0
+#define	ESF_DZ_BKT_ID_WIDTH 9
+
+
+/* ES_FF_UMSG_PACER_TXQ_TBL_RD_REQ */
+#define	ESF_DZ_TXQ_ID_LBN 0
+#define	ESF_DZ_TXQ_ID_WIDTH 10
+
+
+/* ES_FF_UMSG_PACER_TXQ_TBL_RD_RSP */
+#define	ESF_DZ_MAX_BKT2_LBN 112
+#define	ESF_DZ_MAX_BKT2_WIDTH 9
+#define	ESF_DZ_MAX_BKT1_LBN 96
+#define	ESF_DZ_MAX_BKT1_WIDTH 9
+#define	ESF_DZ_MAX_BKT0_LBN 80
+#define	ESF_DZ_MAX_BKT0_WIDTH 9
+#define	ESF_DZ_MIN_BKT_LBN 64
+#define	ESF_DZ_MIN_BKT_WIDTH 9
+#define	ESF_DZ_LABEL_LBN 48
+#define	ESF_DZ_LABEL_WIDTH 4
+#define	ESF_DZ_PQ_FLAGS_LBN 32
+#define	ESF_DZ_PQ_FLAGS_WIDTH 3
+#define	ESF_DZ_DSBL_LBN 16
+#define	ESF_DZ_DSBL_WIDTH 1
+#define	ESF_DZ_TXQ_ID_LBN 0
+#define	ESF_DZ_TXQ_ID_WIDTH 10
+
+
+/* ES_FF_UMSG_PACER_TXQ_TBL_WR_REQ */
+#define	ESF_DZ_MAX_BKT2_LBN 112
+#define	ESF_DZ_MAX_BKT2_WIDTH 9
+#define	ESF_DZ_MAX_BKT1_LBN 96
+#define	ESF_DZ_MAX_BKT1_WIDTH 9
+#define	ESF_DZ_MAX_BKT0_LBN 80
+#define	ESF_DZ_MAX_BKT0_WIDTH 9
+#define	ESF_DZ_MIN_BKT_LBN 64
+#define	ESF_DZ_MIN_BKT_WIDTH 9
+#define	ESF_DZ_LABEL_LBN 48
+#define	ESF_DZ_LABEL_WIDTH 4
+#define	ESF_DZ_PQ_FLAGS_LBN 32
+#define	ESF_DZ_PQ_FLAGS_WIDTH 3
+#define	ESF_DZ_DSBL_LBN 16
+#define	ESF_DZ_DSBL_WIDTH 1
+#define	ESF_DZ_TXQ_ID_LBN 0
+#define	ESF_DZ_TXQ_ID_WIDTH 10
+
+
+/* ES_FF_UMSG_PE */
+#define	ESF_DZ_PE_PKT_OFST_LBN 47
+#define	ESF_DZ_PE_PKT_OFST_WIDTH 17
+#define	ESF_DZ_PE_PEDIT_DELTA_LBN 40
+#define	ESF_DZ_PE_PEDIT_DELTA_WIDTH 8
+#define	ESF_DZ_PE_PYLOAD_OFST_LBN 32
+#define	ESF_DZ_PE_PYLOAD_OFST_WIDTH 8
+#define	ESF_DZ_PE_L4_HDR_OFST_LBN 24
+#define	ESF_DZ_PE_L4_HDR_OFST_WIDTH 8
+#define	ESF_DZ_PE_L3_HDR_OFST_LBN 16
+#define	ESF_DZ_PE_L3_HDR_OFST_WIDTH 8
+#define	ESF_DZ_PE_HAVE_UDP_HDR_LBN 5
+#define	ESF_DZ_PE_HAVE_UDP_HDR_WIDTH 1
+#define	ESF_DZ_PE_HAVE_TCP_HDR_LBN 4
+#define	ESF_DZ_PE_HAVE_TCP_HDR_WIDTH 1
+#define	ESF_DZ_PE_HAVE_IPV6_HDR_LBN 3
+#define	ESF_DZ_PE_HAVE_IPV6_HDR_WIDTH 1
+#define	ESF_DZ_PE_HAVE_IPV4_HDR_LBN 2
+#define	ESF_DZ_PE_HAVE_IPV4_HDR_WIDTH 1
+#define	ESF_DZ_PE_HAVE_FCOE_LBN 1
+#define	ESF_DZ_PE_HAVE_FCOE_WIDTH 1
+#define	ESF_DZ_PE_PARSE_INCOMP_LBN 0
+#define	ESF_DZ_PE_PARSE_INCOMP_WIDTH 1
+
+
+/* ES_FF_UMSG_RXDP_EGR2CPU_SOFT */
+#define	ESF_DZ_RE2CS_SOFT3_LBN 48
+#define	ESF_DZ_RE2CS_SOFT3_WIDTH 16
+#define	ESF_DZ_RE2CS_SOFT2_LBN 32
+#define	ESF_DZ_RE2CS_SOFT2_WIDTH 16
+#define	ESF_DZ_RE2CS_SOFT1_LBN 16
+#define	ESF_DZ_RE2CS_SOFT1_WIDTH 16
+#define	ESF_DZ_RE2CS_SOFT0_LBN 0
+#define	ESF_DZ_RE2CS_SOFT0_WIDTH 16
+
+
+/* ES_FF_UMSG_RXDP_INGR2CPU */
+#define	ESF_DZ_RI2C_LEN_LBN 208
+#define	ESF_DZ_RI2C_LEN_WIDTH 16
+#define	ESF_DZ_RI2C_L4_CLASS_LBN 202
+#define	ESF_DZ_RI2C_L4_CLASS_WIDTH 3
+#define	ESF_DZ_RI2C_L3_CLASS_LBN 199
+#define	ESF_DZ_RI2C_L3_CLASS_WIDTH 3
+#define	ESF_DZ_RI2C_ETHTAG_CLASS_LBN 196
+#define	ESF_DZ_RI2C_ETHTAG_CLASS_WIDTH 3
+#define	ESF_DZ_RI2C_ETHBASE_CLASS_LBN 193
+#define	ESF_DZ_RI2C_ETHBASE_CLASS_WIDTH 3
+#define	ESF_DZ_RI2C_MAC_CLASS_LBN 192
+#define	ESF_DZ_RI2C_MAC_CLASS_WIDTH 1
+#define	ESF_DZ_RI2C_PKT_OFST_LBN 176
+#define	ESF_DZ_RI2C_PKT_OFST_WIDTH 16
+#define	ESF_DZ_RI2C_PEDIT_DELTA_LBN 168
+#define	ESF_DZ_RI2C_PEDIT_DELTA_WIDTH 8
+#define	ESF_DZ_RI2C_PYLOAD_OFST_LBN 160
+#define	ESF_DZ_RI2C_PYLOAD_OFST_WIDTH 8
+#define	ESF_DZ_RI2C_L4_HDR_OFST_LBN 152
+#define	ESF_DZ_RI2C_L4_HDR_OFST_WIDTH 8
+#define	ESF_DZ_RI2C_L3_HDR_OFST_LBN 144
+#define	ESF_DZ_RI2C_L3_HDR_OFST_WIDTH 8
+#define	ESF_DZ_RI2C_HAVE_UDP_HDR_LBN 133
+#define	ESF_DZ_RI2C_HAVE_UDP_HDR_WIDTH 1
+#define	ESF_DZ_RI2C_HAVE_TCP_HDR_LBN 132
+#define	ESF_DZ_RI2C_HAVE_TCP_HDR_WIDTH 1
+#define	ESF_DZ_RI2C_HAVE_IPV6_HDR_LBN 131
+#define	ESF_DZ_RI2C_HAVE_IPV6_HDR_WIDTH 1
+#define	ESF_DZ_RI2C_HAVE_IPV4_HDR_LBN 130
+#define	ESF_DZ_RI2C_HAVE_IPV4_HDR_WIDTH 1
+#define	ESF_DZ_RI2C_HAVE_FCOE_LBN 129
+#define	ESF_DZ_RI2C_HAVE_FCOE_WIDTH 1
+#define	ESF_DZ_RI2C_PARSE_INCOMP_LBN 128
+#define	ESF_DZ_RI2C_PARSE_INCOMP_WIDTH 1
+#define	ESF_DZ_RI2C_EFINFO_WRD3_LBN 112
+#define	ESF_DZ_RI2C_EFINFO_WRD3_WIDTH 16
+#define	ESF_DZ_RI2C_EFINFO_WRD2_LBN 96
+#define	ESF_DZ_RI2C_EFINFO_WRD2_WIDTH 16
+#define	ESF_DZ_RI2C_EFINFO_WRD1_LBN 80
+#define	ESF_DZ_RI2C_EFINFO_WRD1_WIDTH 16
+#define	ESF_DZ_RI2C_EFINFO_WRD0_LBN 64
+#define	ESF_DZ_RI2C_EFINFO_WRD0_WIDTH 16
+#define	ESF_DZ_RI2C_FINFO_WRD3_LBN 48
+#define	ESF_DZ_RI2C_FINFO_WRD3_WIDTH 16
+#define	ESF_DZ_RI2C_FINFO_WRD2_LBN 32
+#define	ESF_DZ_RI2C_FINFO_WRD2_WIDTH 16
+#define	ESF_DZ_RI2C_FINFO_WRD1_LBN 16
+#define	ESF_DZ_RI2C_FINFO_WRD1_WIDTH 16
+#define	ESF_DZ_RI2C_FINFO_SRCDST_LBN 0
+#define	ESF_DZ_RI2C_FINFO_SRCDST_WIDTH 16
+
+
+/* ES_FF_UMSG_SMC2CPU_BUFLKUP */
+#define	ESF_DZ_S2CB_ENCODED_PAGE_ADDR_DW0_LBN 0
+#define	ESF_DZ_S2CB_ENCODED_PAGE_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_S2CB_ENCODED_PAGE_ADDR_DW1_LBN 32
+#define	ESF_DZ_S2CB_ENCODED_PAGE_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_S2CB_ENCODED_PAGE_ADDR_LBN 0
+#define	ESF_DZ_S2CB_ENCODED_PAGE_ADDR_WIDTH 48
+#define	ESF_DZ_S2CB_FAIL_LBN 32
+#define	ESF_DZ_S2CB_FAIL_WIDTH 1
+
+
+/* ES_FF_UMSG_SMC2CPU_DESCRD */
+#define	ESF_DZ_S2CD_BUF_LEN_LBN 112
+#define	ESF_DZ_S2CD_BUF_LEN_WIDTH 14
+#define	ESF_DZ_S2CD_ENCODED_HOST_ADDR_DW0_LBN 64
+#define	ESF_DZ_S2CD_ENCODED_HOST_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_S2CD_ENCODED_HOST_ADDR_DW1_LBN 96
+#define	ESF_DZ_S2CD_ENCODED_HOST_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_S2CD_ENCODED_HOST_ADDR_LBN 64
+#define	ESF_DZ_S2CD_ENCODED_HOST_ADDR_WIDTH 48
+#define	ESF_DZ_S2CD_CONT_LBN 16
+#define	ESF_DZ_S2CD_CONT_WIDTH 1
+#define	ESF_DZ_S2CD_TYPE_LBN 0
+#define	ESF_DZ_S2CD_TYPE_WIDTH 1
+
+
+/* ES_FF_UMSG_SMC2CPU_GPRD */
+#define	ESF_DZ_S2CG_DATA_DW0_LBN 64
+#define	ESF_DZ_S2CG_DATA_DW0_WIDTH 32
+#define	ESF_DZ_S2CG_DATA_DW1_LBN 96
+#define	ESF_DZ_S2CG_DATA_DW1_WIDTH 32
+#define	ESF_DZ_S2CG_DATA_LBN 64
+#define	ESF_DZ_S2CG_DATA_WIDTH 64
+#define	ESF_DZ_S2CG_SOFT_LBN 48
+#define	ESF_DZ_S2CG_SOFT_WIDTH 4
+#define	ESF_DZ_S2CG_FAIL_LBN 32
+#define	ESF_DZ_S2CG_FAIL_WIDTH 1
+
+
+/* ES_FF_UMSG_TXDP_DMA2CPU_PKTRDY */
+#define	ESF_DZ_TD2CP_L4_CLASS_LBN 250
+#define	ESF_DZ_TD2CP_L4_CLASS_WIDTH 3
+#define	ESF_DZ_TD2CP_L3_CLASS_LBN 247
+#define	ESF_DZ_TD2CP_L3_CLASS_WIDTH 3
+#define	ESF_DZ_TD2CP_ETHTAG_CLASS_LBN 244
+#define	ESF_DZ_TD2CP_ETHTAG_CLASS_WIDTH 3
+#define	ESF_DZ_TD2CP_ETHBASE_CLASS_LBN 241
+#define	ESF_DZ_TD2CP_ETHBASE_CLASS_WIDTH 3
+#define	ESF_DZ_TD2CP_MAC_CLASS_LBN 240
+#define	ESF_DZ_TD2CP_MAC_CLASS_WIDTH 1
+#define	ESF_DZ_TD2CP_SOFT_LBN 226
+#define	ESF_DZ_TD2CP_SOFT_WIDTH 14
+#define	ESF_DZ_TD2CP_PKT_ABORT_LBN 225
+#define	ESF_DZ_TD2CP_PKT_ABORT_WIDTH 1
+#define	ESF_DZ_TD2CP_PCIE_ERR_LBN 224
+#define	ESF_DZ_TD2CP_PCIE_ERR_WIDTH 1
+#define	ESF_DZ_TD2CP_DESC_IDX_LBN 208
+#define	ESF_DZ_TD2CP_DESC_IDX_WIDTH 16
+#define	ESF_DZ_TD2CP_PKT_LEN_LBN 192
+#define	ESF_DZ_TD2CP_PKT_LEN_WIDTH 16
+#define	ESF_DZ_TD2CP_PKT_OFFST_OR_FIRST_DESC_IDX_LBN 176
+#define	ESF_DZ_TD2CP_PKT_OFFST_OR_FIRST_DESC_IDX_WIDTH 7
+#define	ESF_DZ_TD2CP_PEDIT_DELTA_LBN 168
+#define	ESF_DZ_TD2CP_PEDIT_DELTA_WIDTH 8
+#define	ESF_DZ_TD2CP_PYLOAD_OFST_LBN 160
+#define	ESF_DZ_TD2CP_PYLOAD_OFST_WIDTH 8
+#define	ESF_DZ_TD2CP_L4_HDR_OFST_LBN 152
+#define	ESF_DZ_TD2CP_L4_HDR_OFST_WIDTH 8
+#define	ESF_DZ_TD2CP_L3_HDR_OFST_LBN 144
+#define	ESF_DZ_TD2CP_L3_HDR_OFST_WIDTH 8
+#define	ESF_DZ_TD2CP_IS_UDP_LBN 133
+#define	ESF_DZ_TD2CP_IS_UDP_WIDTH 1
+#define	ESF_DZ_TD2CP_IS_TCP_LBN 132
+#define	ESF_DZ_TD2CP_IS_TCP_WIDTH 1
+#define	ESF_DZ_TD2CP_IS_IPV6_LBN 131
+#define	ESF_DZ_TD2CP_IS_IPV6_WIDTH 1
+#define	ESF_DZ_TD2CP_IS_IPV4_LBN 130
+#define	ESF_DZ_TD2CP_IS_IPV4_WIDTH 1
+#define	ESF_DZ_TD2CP_IS_FCOE_LBN 129
+#define	ESF_DZ_TD2CP_IS_FCOE_WIDTH 1
+#define	ESF_DZ_TD2CP_PARSE_INCOMP_LBN 128
+#define	ESF_DZ_TD2CP_PARSE_INCOMP_WIDTH 1
+#define	ESF_DZ_TD2CP_EFINFO_WRD3_LBN 112
+#define	ESF_DZ_TD2CP_EFINFO_WRD3_WIDTH 16
+#define	ESF_DZ_TD2CP_EFINFO_WRD2_LBN 96
+#define	ESF_DZ_TD2CP_EFINFO_WRD2_WIDTH 16
+#define	ESF_DZ_TD2CP_EFINFO_WRD1_LBN 80
+#define	ESF_DZ_TD2CP_EFINFO_WRD1_WIDTH 16
+#define	ESF_DZ_TD2CP_EFINFO_WRD0_LBN 64
+#define	ESF_DZ_TD2CP_EFINFO_WRD0_WIDTH 16
+#define	ESF_DZ_TD2CP_FINFO_WRD3_LBN 48
+#define	ESF_DZ_TD2CP_FINFO_WRD3_WIDTH 16
+#define	ESF_DZ_TD2CP_FINFO_WRD2_LBN 32
+#define	ESF_DZ_TD2CP_FINFO_WRD2_WIDTH 16
+#define	ESF_DZ_TD2CP_FINFO_WRD1_LBN 16
+#define	ESF_DZ_TD2CP_FINFO_WRD1_WIDTH 16
+#define	ESF_DZ_TD2CP_FINFO_SRCDST_LBN 0
+#define	ESF_DZ_TD2CP_FINFO_SRCDST_WIDTH 16
+
+
+/* ES_FF_UMSG_TXDP_DMA2CPU_SOFT */
+#define	ESF_DZ_TD2CS_SOFT3_LBN 48
+#define	ESF_DZ_TD2CS_SOFT3_WIDTH 16
+#define	ESF_DZ_TD2CS_SOFT2_LBN 32
+#define	ESF_DZ_TD2CS_SOFT2_WIDTH 16
+#define	ESF_DZ_TD2CS_SOFT1_LBN 16
+#define	ESF_DZ_TD2CS_SOFT1_WIDTH 16
+#define	ESF_DZ_TD2CS_SOFT0_LBN 0
+#define	ESF_DZ_TD2CS_SOFT0_WIDTH 16
+
+
+/* ES_FF_UMSG_TXDP_EGR2CPU_SOFT */
+#define	ESF_DZ_TE2CS_SOFT3_LBN 48
+#define	ESF_DZ_TE2CS_SOFT3_WIDTH 16
+#define	ESF_DZ_TE2CS_SOFT2_LBN 32
+#define	ESF_DZ_TE2CS_SOFT2_WIDTH 16
+#define	ESF_DZ_TE2CS_SOFT1_LBN 16
+#define	ESF_DZ_TE2CS_SOFT1_WIDTH 16
+#define	ESF_DZ_TE2CS_SOFT0_LBN 0
+#define	ESF_DZ_TE2CS_SOFT0_WIDTH 16
+
+
+/* ES_FF_UMSG_VICTL2CPU */
+#define	ESF_DZ_V2C_DESC_WORD3_LBN 112
+#define	ESF_DZ_V2C_DESC_WORD3_WIDTH 17
+#define	ESF_DZ_V2C_DESC_WORD2_LBN 96
+#define	ESF_DZ_V2C_DESC_WORD2_WIDTH 16
+#define	ESF_DZ_V2C_DESC_WORD1_LBN 80
+#define	ESF_DZ_V2C_DESC_WORD1_WIDTH 16
+#define	ESF_DZ_V2C_DESC_WORD0_LBN 64
+#define	ESF_DZ_V2C_DESC_WORD0_WIDTH 16
+#define	ESF_DZ_V2C_NEW_DSCR_WPTR_LBN 32
+#define	ESF_DZ_V2C_NEW_DSCR_WPTR_WIDTH 12
+#define	ESF_DZ_V2C_DESC_PUSH_LBN 16
+#define	ESF_DZ_V2C_DESC_PUSH_WIDTH 1
+
+
+/* ES_LUE_DB_MATCH_ENTRY */
+#define	ESF_DZ_LUE_DSCRMNTR_LBN 140
+#define	ESF_DZ_LUE_DSCRMNTR_WIDTH 4
+#define	ESF_DZ_LUE_MATCH_VAL_DW0_LBN 44
+#define	ESF_DZ_LUE_MATCH_VAL_DW0_WIDTH 32
+#define	ESF_DZ_LUE_MATCH_VAL_DW1_LBN 76
+#define	ESF_DZ_LUE_MATCH_VAL_DW1_WIDTH 32
+#define	ESF_DZ_LUE_MATCH_VAL_DW2_LBN 108
+#define	ESF_DZ_LUE_MATCH_VAL_DW2_WIDTH 32
+#define	ESF_DZ_LUE_MATCH_VAL_LBN 44
+#define	ESF_DZ_LUE_MATCH_VAL_WIDTH 96
+#define	ESF_DZ_LUE_ME_SOFT_LBN 35
+#define	ESF_DZ_LUE_ME_SOFT_WIDTH 9
+#define	ESF_DZ_LUE_TX_MCAST_LBN 33
+#define	ESF_DZ_LUE_TX_MCAST_WIDTH 2
+#define	ESF_DZ_LUE_TX_DOMAIN_LBN 25
+#define	ESF_DZ_LUE_TX_DOMAIN_WIDTH 8
+#define	ESF_DZ_LUE_RX_MCAST_LBN 24
+#define	ESF_DZ_LUE_RX_MCAST_WIDTH 1
+#define	ESE_DZ_LUE_MULTI 1
+#define	ESE_DZ_LUE_SINGLE 0
+#define	ESF_DZ_LUE_RCPNTR_LBN 0
+#define	ESF_DZ_LUE_RCPNTR_WIDTH 24
+#define	ESF_DZ_LUE_RCPNTR_ME_PTR_LBN 0
+#define	ESF_DZ_LUE_RCPNTR_ME_PTR_WIDTH 14
+
+
+/* ES_LUE_DB_NONMATCH_ENTRY */
+#define	ESF_DZ_LUE_DSCRMNTR_LBN 140
+#define	ESF_DZ_LUE_DSCRMNTR_WIDTH 4
+#define	ESF_DZ_LUE_TERMINAL_LBN 139
+#define	ESF_DZ_LUE_TERMINAL_WIDTH 1
+#define	ESF_DZ_LUE_LAST_LBN 138
+#define	ESF_DZ_LUE_LAST_WIDTH 1
+#define	ESF_DZ_LUE_NE_SOFT_LBN 137
+#define	ESF_DZ_LUE_NE_SOFT_WIDTH 1
+#define	ESF_DZ_LUE_RCPNTR_NUM_LBN 134
+#define	ESF_DZ_LUE_RCPNTR_NUM_WIDTH 3
+#define	ESF_DZ_LUE_RCPNTR0_LBN 110
+#define	ESF_DZ_LUE_RCPNTR0_WIDTH 24
+#define	ESF_DZ_LUE_RCPNTR1_LBN 86
+#define	ESF_DZ_LUE_RCPNTR1_WIDTH 24
+#define	ESF_DZ_LUE_RCPNTR2_LBN 62
+#define	ESF_DZ_LUE_RCPNTR2_WIDTH 24
+#define	ESF_DZ_LUE_RCPNTR3_LBN 38
+#define	ESF_DZ_LUE_RCPNTR3_WIDTH 24
+#define	ESF_DZ_LUE_RCPNTR4_LBN 14
+#define	ESF_DZ_LUE_RCPNTR4_WIDTH 24
+#define	ESF_DZ_LUE_RCPNTR_NE_PTR_LBN 0
+#define	ESF_DZ_LUE_RCPNTR_NE_PTR_WIDTH 14
+
+
+/* ES_LUE_MC_DIRECT_REQUEST_MSG */
+#define	ESF_DZ_MC2L_DR_PAD_DW0_LBN 22
+#define	ESF_DZ_MC2L_DR_PAD_DW0_WIDTH 32
+#define	ESF_DZ_MC2L_DR_PAD_DW1_LBN 54
+#define	ESF_DZ_MC2L_DR_PAD_DW1_WIDTH 32
+#define	ESF_DZ_MC2L_DR_PAD_DW2_LBN 86
+#define	ESF_DZ_MC2L_DR_PAD_DW2_WIDTH 32
+#define	ESF_DZ_MC2L_DR_PAD_DW3_LBN 118
+#define	ESF_DZ_MC2L_DR_PAD_DW3_WIDTH 32
+#define	ESF_DZ_MC2L_DR_PAD_DW4_LBN 150
+#define	ESF_DZ_MC2L_DR_PAD_DW4_WIDTH 16
+#define	ESF_DZ_MC2L_DR_PAD_LBN 22
+#define	ESF_DZ_MC2L_DR_PAD_WIDTH 144
+#define	ESF_DZ_MC2L_DR_ADDR_LBN 8
+#define	ESF_DZ_MC2L_DR_ADDR_WIDTH 14
+#define	ESF_DZ_MC2L_DR_THREAD_ID_LBN 5
+#define	ESF_DZ_MC2L_DR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_MC2L_DR_CLIENT_ID_LBN 2
+#define	ESF_DZ_MC2L_DR_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_MC2L_DR_OP_LBN 0
+#define	ESF_DZ_MC2L_DR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MC_DIRECT_RESPONSE_MSG */
+#define	ESF_DZ_L2MC_DR_PAD_LBN 146
+#define	ESF_DZ_L2MC_DR_PAD_WIDTH 6
+#define	ESF_DZ_L2MC_DR_RCPNT_PTR_LBN 132
+#define	ESF_DZ_L2MC_DR_RCPNT_PTR_WIDTH 14
+#define	ESF_DZ_L2MC_DR_RCPNT4_LBN 108
+#define	ESF_DZ_L2MC_DR_RCPNT4_WIDTH 24
+#define	ESF_DZ_L2MC_DR_RCPNT3_LBN 84
+#define	ESF_DZ_L2MC_DR_RCPNT3_WIDTH 24
+#define	ESF_DZ_L2MC_DR_RCPNT2_LBN 60
+#define	ESF_DZ_L2MC_DR_RCPNT2_WIDTH 24
+#define	ESF_DZ_L2MC_DR_RCPNT1_LBN 36
+#define	ESF_DZ_L2MC_DR_RCPNT1_WIDTH 24
+#define	ESF_DZ_L2MC_DR_RCPNT0_LBN 12
+#define	ESF_DZ_L2MC_DR_RCPNT0_WIDTH 24
+#define	ESF_DZ_L2MC_DR_RCPNT_NUM_LBN 9
+#define	ESF_DZ_L2MC_DR_RCPNT_NUM_WIDTH 3
+#define	ESF_DZ_L2MC_DR_LAST_LBN 8
+#define	ESF_DZ_L2MC_DR_LAST_WIDTH 1
+#define	ESF_DZ_L2MC_DR_THREAD_ID_LBN 5
+#define	ESF_DZ_L2MC_DR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_L2MC_DR_CLIENT_ID_LBN 2
+#define	ESF_DZ_L2MC_DR_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_L2MC_DR_OP_LBN 0
+#define	ESF_DZ_L2MC_DR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MC_GP_RD_REQUEST_MSG */
+#define	ESF_DZ_MC2L_GPR_PAD_DW0_LBN 22
+#define	ESF_DZ_MC2L_GPR_PAD_DW0_WIDTH 32
+#define	ESF_DZ_MC2L_GPR_PAD_DW1_LBN 54
+#define	ESF_DZ_MC2L_GPR_PAD_DW1_WIDTH 32
+#define	ESF_DZ_MC2L_GPR_PAD_DW2_LBN 86
+#define	ESF_DZ_MC2L_GPR_PAD_DW2_WIDTH 32
+#define	ESF_DZ_MC2L_GPR_PAD_DW3_LBN 118
+#define	ESF_DZ_MC2L_GPR_PAD_DW3_WIDTH 32
+#define	ESF_DZ_MC2L_GPR_PAD_DW4_LBN 150
+#define	ESF_DZ_MC2L_GPR_PAD_DW4_WIDTH 16
+#define	ESF_DZ_MC2L_GPR_PAD_LBN 22
+#define	ESF_DZ_MC2L_GPR_PAD_WIDTH 144
+#define	ESF_DZ_MC2L_GPR_ADDR_LBN 8
+#define	ESF_DZ_MC2L_GPR_ADDR_WIDTH 14
+#define	ESF_DZ_MC2L_GPR_THREAD_ID_LBN 5
+#define	ESF_DZ_MC2L_GPR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_MC2L_GPR_CLIENT_ID_LBN 2
+#define	ESF_DZ_MC2L_GPR_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_MC2L_GPR_OP_LBN 0
+#define	ESF_DZ_MC2L_GPR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MC_GP_RD_RESPONSE_MSG */
+#define	ESF_DZ_L2MC_GPR_DATA_DW0_LBN 8
+#define	ESF_DZ_L2MC_GPR_DATA_DW0_WIDTH 32
+#define	ESF_DZ_L2MC_GPR_DATA_DW1_LBN 40
+#define	ESF_DZ_L2MC_GPR_DATA_DW1_WIDTH 32
+#define	ESF_DZ_L2MC_GPR_DATA_DW2_LBN 72
+#define	ESF_DZ_L2MC_GPR_DATA_DW2_WIDTH 32
+#define	ESF_DZ_L2MC_GPR_DATA_DW3_LBN 104
+#define	ESF_DZ_L2MC_GPR_DATA_DW3_WIDTH 32
+#define	ESF_DZ_L2MC_GPR_DATA_DW4_LBN 136
+#define	ESF_DZ_L2MC_GPR_DATA_DW4_WIDTH 16
+#define	ESF_DZ_L2MC_GPR_DATA_LBN 8
+#define	ESF_DZ_L2MC_GPR_DATA_WIDTH 144
+#define	ESF_DZ_L2MC_GPR_THREAD_ID_LBN 5
+#define	ESF_DZ_L2MC_GPR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_L2MC_GPR_CLIENT_ID_LBN 2
+#define	ESF_DZ_L2MC_GPR_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_L2MC_GPR_OP_LBN 0
+#define	ESF_DZ_L2MC_GPR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MC_GP_WR_REQUEST_MSG */
+#define	ESF_DZ_MC2L_GPW_DATA_DW0_LBN 22
+#define	ESF_DZ_MC2L_GPW_DATA_DW0_WIDTH 32
+#define	ESF_DZ_MC2L_GPW_DATA_DW1_LBN 54
+#define	ESF_DZ_MC2L_GPW_DATA_DW1_WIDTH 32
+#define	ESF_DZ_MC2L_GPW_DATA_DW2_LBN 86
+#define	ESF_DZ_MC2L_GPW_DATA_DW2_WIDTH 32
+#define	ESF_DZ_MC2L_GPW_DATA_DW3_LBN 118
+#define	ESF_DZ_MC2L_GPW_DATA_DW3_WIDTH 32
+#define	ESF_DZ_MC2L_GPW_DATA_DW4_LBN 150
+#define	ESF_DZ_MC2L_GPW_DATA_DW4_WIDTH 16
+#define	ESF_DZ_MC2L_GPW_DATA_LBN 22
+#define	ESF_DZ_MC2L_GPW_DATA_WIDTH 144
+#define	ESF_DZ_MC2L_GPW_ADDR_LBN 8
+#define	ESF_DZ_MC2L_GPW_ADDR_WIDTH 14
+#define	ESF_DZ_MC2L_GPW_THREAD_ID_LBN 5
+#define	ESF_DZ_MC2L_GPW_THREAD_ID_WIDTH 3
+#define	ESF_DZ_MC2L_GPW_CLIENT_ID_LBN 2
+#define	ESF_DZ_MC2L_GPW_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_MC2L_GPW_OP_LBN 0
+#define	ESF_DZ_MC2L_GPW_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MC_MATCH_REQUEST_MSG */
+#define	ESF_DZ_MC2L_MR_PAD_LBN 135
+#define	ESF_DZ_MC2L_MR_PAD_WIDTH 31
+#define	ESF_DZ_MC2L_MR_HASH2_LBN 122
+#define	ESF_DZ_MC2L_MR_HASH2_WIDTH 13
+#define	ESF_DZ_MC2L_MR_HASH1_LBN 108
+#define	ESF_DZ_MC2L_MR_HASH1_WIDTH 14
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_DW0_LBN 12
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_DW0_WIDTH 32
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_DW1_LBN 44
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_DW1_WIDTH 32
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_DW2_LBN 76
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_DW2_WIDTH 32
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_LBN 12
+#define	ESF_DZ_MC2L_MR_MATCH_BITS_WIDTH 96
+#define	ESF_DZ_MC2L_MR_DSCRMNTR_LBN 8
+#define	ESF_DZ_MC2L_MR_DSCRMNTR_WIDTH 4
+#define	ESF_DZ_MC2L_MR_THREAD_ID_LBN 5
+#define	ESF_DZ_MC2L_MR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_MC2L_MR_CLIENT_ID_LBN 2
+#define	ESF_DZ_MC2L_MR_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_MC2L_MR_OP_LBN 0
+#define	ESF_DZ_MC2L_MR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MC_MATCH_RESPONSE_MSG */
+#define	ESF_DZ_L2MC_MR_PAD_DW0_LBN 53
+#define	ESF_DZ_L2MC_MR_PAD_DW0_WIDTH 32
+#define	ESF_DZ_L2MC_MR_PAD_DW1_LBN 85
+#define	ESF_DZ_L2MC_MR_PAD_DW1_WIDTH 32
+#define	ESF_DZ_L2MC_MR_PAD_DW2_LBN 117
+#define	ESF_DZ_L2MC_MR_PAD_DW2_WIDTH 32
+#define	ESF_DZ_L2MC_MR_PAD_DW3_LBN 149
+#define	ESF_DZ_L2MC_MR_PAD_DW3_WIDTH 3
+#define	ESF_DZ_L2MC_MR_PAD_LBN 53
+#define	ESF_DZ_L2MC_MR_PAD_WIDTH 99
+#define	ESF_DZ_L2MC_MR_LUE_RCPNT_LBN 29
+#define	ESF_DZ_L2MC_MR_LUE_RCPNT_WIDTH 24
+#define	ESF_DZ_L2MC_MR_RX_MCAST_LBN 28
+#define	ESF_DZ_L2MC_MR_RX_MCAST_WIDTH 1
+#define	ESF_DZ_L2MC_MR_TX_DOMAIN_LBN 20
+#define	ESF_DZ_L2MC_MR_TX_DOMAIN_WIDTH 8
+#define	ESF_DZ_L2MC_MR_TX_MCAST_LBN 18
+#define	ESF_DZ_L2MC_MR_TX_MCAST_WIDTH 2
+#define	ESF_DZ_L2MC_MR_SOFT_LBN 9
+#define	ESF_DZ_L2MC_MR_SOFT_WIDTH 9
+#define	ESF_DZ_L2MC_MR_MATCH_LBN 8
+#define	ESF_DZ_L2MC_MR_MATCH_WIDTH 1
+#define	ESF_DZ_L2MC_MR_THREAD_ID_LBN 5
+#define	ESF_DZ_L2MC_MR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_L2MC_MR_CLIENT_ID_LBN 2
+#define	ESF_DZ_L2MC_MR_CLIENT_ID_WIDTH 3
+#define	ESF_DZ_L2MC_MR_OP_LBN 0
+#define	ESF_DZ_L2MC_MR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_BASE_REQ */
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW0_LBN 8
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW0_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW1_LBN 40
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW1_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW2_LBN 72
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW2_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW3_LBN 104
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW3_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW4_LBN 136
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_DW4_WIDTH 30
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_LBN 8
+#define	ESF_DZ_LUE_HW_REQ_BASE_REQ_MSG_DATA_WIDTH 158
+#define	ESF_DZ_LUE_HW_REQ_BASE_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_REQ_BASE_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_REQ_BASE_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_REQ_BASE_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_REQ_BASE_OP_LBN 0
+#define	ESF_DZ_LUE_HW_REQ_BASE_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_BASE_RESP */
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW0_LBN 8
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW0_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW1_LBN 40
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW1_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW2_LBN 72
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW2_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW3_LBN 104
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW3_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW4_LBN 136
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_DW4_WIDTH 16
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_LBN 8
+#define	ESF_DZ_LUE_HW_RSP_BASE_RSP_DATA_WIDTH 144
+#define	ESF_DZ_LUE_HW_RSP_BASE_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_RSP_BASE_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_RSP_BASE_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_RSP_BASE_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_RSP_BASE_OP_LBN 0
+#define	ESF_DZ_LUE_HW_RSP_BASE_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_DIRECT_REQ */
+#define	ESF_DZ_LUE_HW_REQ_DIR_ADDR_LBN 8
+#define	ESF_DZ_LUE_HW_REQ_DIR_ADDR_WIDTH 14
+#define	ESF_DZ_LUE_HW_REQ_DIR_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_REQ_DIR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_REQ_DIR_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_REQ_DIR_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_REQ_DIR_OP_LBN 0
+#define	ESF_DZ_LUE_HW_REQ_DIR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_DIRECT_RESP */
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT_PTR_LBN 132
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT_PTR_WIDTH 14
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT4_LBN 108
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT4_WIDTH 24
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT3_LBN 84
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT3_WIDTH 24
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT2_LBN 60
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT2_WIDTH 24
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT1_LBN 36
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT1_WIDTH 24
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT0_LBN 12
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT0_WIDTH 24
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT_NUM_LBN 9
+#define	ESF_DZ_LUE_HW_RSP_DIR_RCPNT_NUM_WIDTH 3
+#define	ESF_DZ_LUE_HW_RSP_DIR_LAST_LBN 8
+#define	ESF_DZ_LUE_HW_RSP_DIR_LAST_WIDTH 1
+#define	ESF_DZ_LUE_HW_RSP_DIR_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_RSP_DIR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_RSP_DIR_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_RSP_DIR_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_RSP_DIR_OP_LBN 0
+#define	ESF_DZ_LUE_HW_RSP_DIR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_GP_RD_REQ */
+#define	ESF_DZ_LUE_HW_REQ_GPRD_ADDR_LBN 8
+#define	ESF_DZ_LUE_HW_REQ_GPRD_ADDR_WIDTH 14
+#define	ESF_DZ_LUE_HW_REQ_GPRD_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_REQ_GPRD_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_REQ_GPRD_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_REQ_GPRD_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_REQ_GPRD_OP_LBN 0
+#define	ESF_DZ_LUE_HW_REQ_GPRD_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_GP_RD_RESP */
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW0_LBN 8
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW0_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW1_LBN 40
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW1_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW2_LBN 72
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW2_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW3_LBN 104
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW3_WIDTH 32
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW4_LBN 136
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_DW4_WIDTH 16
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_LBN 8
+#define	ESF_DZ_LUE_HW_RSP_GPRD_LUE_DATA_WIDTH 144
+#define	ESF_DZ_LUE_HW_RSP_GPRD_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_RSP_GPRD_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_RSP_GPRD_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_RSP_GPRD_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_RSP_GPRD_OP_LBN 0
+#define	ESF_DZ_LUE_HW_RSP_GPRD_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_GP_WR_REQ */
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW0_LBN 22
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW0_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW1_LBN 54
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW1_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW2_LBN 86
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW2_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW3_LBN 118
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW3_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW4_LBN 150
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_DW4_WIDTH 16
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_LBN 22
+#define	ESF_DZ_LUE_HW_REQ_GPWR_LUE_DATA_WIDTH 144
+#define	ESF_DZ_LUE_HW_REQ_GPWR_ADDR_LBN 8
+#define	ESF_DZ_LUE_HW_REQ_GPWR_ADDR_WIDTH 14
+#define	ESF_DZ_LUE_HW_REQ_GPWR_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_REQ_GPWR_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_REQ_GPWR_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_REQ_GPWR_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_REQ_GPWR_OP_LBN 0
+#define	ESF_DZ_LUE_HW_REQ_GPWR_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_MATCH_REQ */
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_REQ_COUNT_LBN 135
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_REQ_COUNT_WIDTH 6
+#define	ESF_DZ_LUE_HW_REQ_MATCH_HASH2_LBN 122
+#define	ESF_DZ_LUE_HW_REQ_MATCH_HASH2_WIDTH 13
+#define	ESF_DZ_LUE_HW_REQ_MATCH_HASH1_LBN 108
+#define	ESF_DZ_LUE_HW_REQ_MATCH_HASH1_WIDTH 14
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_DW0_LBN 12
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_DW0_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_DW1_LBN 44
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_DW1_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_DW2_LBN 76
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_DW2_WIDTH 32
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_LBN 12
+#define	ESF_DZ_LUE_HW_REQ_MATCH_MATCH_BITS_WIDTH 96
+#define	ESF_DZ_LUE_HW_REQ_MATCH_DSCRMNTR_LBN 8
+#define	ESF_DZ_LUE_HW_REQ_MATCH_DSCRMNTR_WIDTH 4
+#define	ESF_DZ_LUE_HW_REQ_MATCH_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_REQ_MATCH_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_REQ_MATCH_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_REQ_MATCH_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_REQ_MATCH_OP_LBN 0
+#define	ESF_DZ_LUE_HW_REQ_MATCH_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_MSG_MATCH_RESP */
+#define	ESF_DZ_LUE_HW_RSP_MATCH_LUE_RCPNT_LBN 29
+#define	ESF_DZ_LUE_HW_RSP_MATCH_LUE_RCPNT_WIDTH 24
+#define	ESF_DZ_LUE_HW_RSP_MATCH_RX_MCAST_LBN 28
+#define	ESF_DZ_LUE_HW_RSP_MATCH_RX_MCAST_WIDTH 1
+#define	ESF_DZ_LUE_HW_RSP_MATCH_TX_DOMAIN_LBN 20
+#define	ESF_DZ_LUE_HW_RSP_MATCH_TX_DOMAIN_WIDTH 8
+#define	ESF_DZ_LUE_HW_RSP_MATCH_TX_MCAST_LBN 18
+#define	ESF_DZ_LUE_HW_RSP_MATCH_TX_MCAST_WIDTH 2
+#define	ESF_DZ_LUE_HW_RSP_MATCH_SOFT_LBN 9
+#define	ESF_DZ_LUE_HW_RSP_MATCH_SOFT_WIDTH 9
+#define	ESF_DZ_LUE_HW_RSP_MATCH_MATCH_LBN 8
+#define	ESF_DZ_LUE_HW_RSP_MATCH_MATCH_WIDTH 1
+#define	ESF_DZ_LUE_HW_RSP_MATCH_THREAD_ID_LBN 5
+#define	ESF_DZ_LUE_HW_RSP_MATCH_THREAD_ID_WIDTH 3
+#define	ESF_DZ_LUE_HW_RSP_MATCH_CLIENT_ID_LBN 2
+#define	ESF_DZ_LUE_HW_RSP_MATCH_CLIENT_ID_WIDTH 3
+#define	ESE_DZ_LUE_MC_ID 7
+#define	ESE_DZ_LUE_MATCH_REQ_FIFO_ID 3
+#define	ESE_DZ_LUE_TX_DICPU_ID 1
+#define	ESE_DZ_LUE_RX_DICPU_ID 0
+#define	ESF_DZ_LUE_HW_RSP_MATCH_OP_LBN 0
+#define	ESF_DZ_LUE_HW_RSP_MATCH_OP_WIDTH 2
+#define	ESE_DZ_LUE_GP_WR 3
+#define	ESE_DZ_LUE_GP_RD 2
+#define	ESE_DZ_LUE_DIR_REQ 1
+#define	ESE_DZ_LUE_MATCH_REQ 0
+
+
+/* ES_LUE_RCPNTR_TYPE */
+#define	ESF_DZ_LUE_RXQ_LBN 14
+#define	ESF_DZ_LUE_RXQ_WIDTH 10
+#define	ESF_DZ_LUE_RSS_INFO_LBN 8
+#define	ESF_DZ_LUE_RSS_INFO_WIDTH 6
+#define	ESF_DZ_LUE_DEST_LBN 5
+#define	ESF_DZ_LUE_DEST_WIDTH 3
+#define	ESF_DZ_LUE_SOFT_LBN 0
+#define	ESF_DZ_LUE_SOFT_WIDTH 5
+
+
+/* ES_LUE_UMSG_LU2DI_HASH_RESP */
+#define	ESF_DZ_L2DHR_LASTREC_ENTRY_STATUS_LBN 50
+#define	ESF_DZ_L2DHR_LASTREC_ENTRY_STATUS_WIDTH 1
+#define	ESF_DZ_L2DHR_MULTITYPE_STATUS_LBN 50
+#define	ESF_DZ_L2DHR_MULTITYPE_STATUS_WIDTH 1
+#define	ESF_DZ_L2DHR_LASTREC_STATUS_LBN 49
+#define	ESF_DZ_L2DHR_LASTREC_STATUS_WIDTH 1
+#define	ESF_DZ_L2DHR_MATCH_STATUS_LBN 48
+#define	ESF_DZ_L2DHR_MATCH_STATUS_WIDTH 1
+#define	ESF_DZ_L2DHR_HASH_LBN 0
+#define	ESF_DZ_L2DHR_HASH_WIDTH 32
+
+
+/* ES_LUE_UMSG_LU2DI_RXLU_MULTI_MATCH_RESP */
+#define	ESF_DZ_L2DRMMR_SOFT_LBN 112
+#define	ESF_DZ_L2DRMMR_SOFT_WIDTH 9
+#define	ESF_DZ_L2DRMMR_RCPNTR_PTR_LBN 96
+#define	ESF_DZ_L2DRMMR_RCPNTR_PTR_WIDTH 14
+#define	ESF_DZ_L2DRMMR_TX_MCAST_LBN 80
+#define	ESF_DZ_L2DRMMR_TX_MCAST_WIDTH 2
+#define	ESF_DZ_L2DRMMR_MULTITYPE_STATUS_LBN 67
+#define	ESF_DZ_L2DRMMR_MULTITYPE_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMMR_LASTREC_ENTRY_STATUS_LBN 66
+#define	ESF_DZ_L2DRMMR_LASTREC_ENTRY_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMMR_LASTREC_STATUS_LBN 65
+#define	ESF_DZ_L2DRMMR_LASTREC_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMMR_MATCH_STATUS_LBN 64
+#define	ESF_DZ_L2DRMMR_MATCH_STATUS_WIDTH 1
+
+
+/* ES_LUE_UMSG_LU2DI_RXLU_MULTI_RECORD_RESP */
+#define	ESF_DZ_L2DRMRR_SOFT_LBN 112
+#define	ESF_DZ_L2DRMRR_SOFT_WIDTH 9
+#define	ESF_DZ_L2DRMRR_RCPNTR_PTR_LBN 96
+#define	ESF_DZ_L2DRMRR_RCPNTR_PTR_WIDTH 14
+#define	ESF_DZ_L2DRMRR_RCPNTR_NUM_LBN 80
+#define	ESF_DZ_L2DRMRR_RCPNTR_NUM_WIDTH 3
+#define	ESF_DZ_L2DRMRR_MULTITYPE_STATUS_LBN 67
+#define	ESF_DZ_L2DRMRR_MULTITYPE_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMRR_LASTREC_ENTRY_STATUS_LBN 66
+#define	ESF_DZ_L2DRMRR_LASTREC_ENTRY_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMRR_LASTREC_STATUS_LBN 65
+#define	ESF_DZ_L2DRMRR_LASTREC_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMRR_MATCH_STATUS_LBN 64
+#define	ESF_DZ_L2DRMRR_MATCH_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRMRR_RCPNTR_SOFT_LBN 48
+#define	ESF_DZ_L2DRMRR_RCPNTR_SOFT_WIDTH 6
+#define	ESF_DZ_L2DRMRR_RCPNTR_RSS_INFO_LBN 32
+#define	ESF_DZ_L2DRMRR_RCPNTR_RSS_INFO_WIDTH 5
+#define	ESF_DZ_L2DRMRR_RCPNTR_RXQ_LBN 16
+#define	ESF_DZ_L2DRMRR_RCPNTR_RXQ_WIDTH 10
+#define	ESF_DZ_L2DRMRR_HOST_LBN 7
+#define	ESF_DZ_L2DRMRR_HOST_WIDTH 1
+#define	ESF_DZ_L2DRMRR_MC_LBN 6
+#define	ESF_DZ_L2DRMRR_MC_WIDTH 1
+#define	ESF_DZ_L2DRMRR_PORT0_MAC_LBN 5
+#define	ESF_DZ_L2DRMRR_PORT0_MAC_WIDTH 1
+#define	ESF_DZ_L2DRMRR_PORT1_MAC_LBN 4
+#define	ESF_DZ_L2DRMRR_PORT1_MAC_WIDTH 1
+
+
+/* ES_LUE_UMSG_LU2DI_RXLU_SINGLE_MATCH_RESP */
+#define	ESF_DZ_L2DRSMR_MULTITYPE_STATUS_LBN 67
+#define	ESF_DZ_L2DRSMR_MULTITYPE_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRSMR_LASTREC_ENTRY_STATUS_LBN 66
+#define	ESF_DZ_L2DRSMR_LASTREC_ENTRY_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRSMR_LASTREC_STATUS_LBN 65
+#define	ESF_DZ_L2DRSMR_LASTREC_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRSMR_MATCH_STATUS_LBN 64
+#define	ESF_DZ_L2DRSMR_MATCH_STATUS_WIDTH 1
+#define	ESF_DZ_L2DRSMR_RCPNTR_SOFT_LBN 48
+#define	ESF_DZ_L2DRSMR_RCPNTR_SOFT_WIDTH 6
+#define	ESF_DZ_L2DRSMR_RCPNTR_RSS_INFO_LBN 32
+#define	ESF_DZ_L2DRSMR_RCPNTR_RSS_INFO_WIDTH 5
+#define	ESF_DZ_L2DRSMR_RCPNTR_RXQ_LBN 16
+#define	ESF_DZ_L2DRSMR_RCPNTR_RXQ_WIDTH 10
+#define	ESF_DZ_L2DRSMR_HOST_LBN 7
+#define	ESF_DZ_L2DRSMR_HOST_WIDTH 1
+#define	ESF_DZ_L2DRSMR_MC_LBN 6
+#define	ESF_DZ_L2DRSMR_MC_WIDTH 1
+#define	ESF_DZ_L2DRSMR_PORT0_MAC_LBN 5
+#define	ESF_DZ_L2DRSMR_PORT0_MAC_WIDTH 1
+#define	ESF_DZ_L2DRSMR_PORT1_MAC_LBN 4
+#define	ESF_DZ_L2DRSMR_PORT1_MAC_WIDTH 1
+
+
+/* ES_LUE_UMSG_LU2DI_TXLU_MATCH_RESP */
+#define	ESF_DZ_L2DTMR_RCPNTR_SOFT_LBN 112
+#define	ESF_DZ_L2DTMR_RCPNTR_SOFT_WIDTH 6
+#define	ESF_DZ_L2DTMR_RCPNTR_RSS_INFO_LBN 96
+#define	ESF_DZ_L2DTMR_RCPNTR_RSS_INFO_WIDTH 5
+#define	ESF_DZ_L2DTMR_RCPNTR__RXQ_LBN 80
+#define	ESF_DZ_L2DTMR_RCPNTR__RXQ_WIDTH 10
+#define	ESF_DZ_L2DTMR_MULTITYPE_STATUS_LBN 67
+#define	ESF_DZ_L2DTMR_MULTITYPE_STATUS_WIDTH 1
+#define	ESF_DZ_L2DTMR_LASTREC_ENTRY_STATUS_LBN 66
+#define	ESF_DZ_L2DTMR_LASTREC_ENTRY_STATUS_WIDTH 1
+#define	ESF_DZ_L2DTMR_LASTREC_STATUS_LBN 65
+#define	ESF_DZ_L2DTMR_LASTREC_STATUS_WIDTH 1
+#define	ESF_DZ_L2DTMR_MATCH_STATUS_LBN 64
+#define	ESF_DZ_L2DTMR_MATCH_STATUS_WIDTH 1
+#define	ESF_DZ_L2DTMR_ME_SOFT_LBN 48
+#define	ESF_DZ_L2DTMR_ME_SOFT_WIDTH 9
+#define	ESF_DZ_L2DTMR_TX_MCAST_LBN 32
+#define	ESF_DZ_L2DTMR_TX_MCAST_WIDTH 2
+#define	ESF_DZ_L2DTMR_TX_DOMAIN_LBN 16
+#define	ESF_DZ_L2DTMR_TX_DOMAIN_WIDTH 8
+#define	ESF_DZ_L2DTMR_PORT1_MAC_LBN 6
+#define	ESF_DZ_L2DTMR_PORT1_MAC_WIDTH 1
+#define	ESF_DZ_L2DTMR_PMEM_LBN 6
+#define	ESF_DZ_L2DTMR_PMEM_WIDTH 1
+#define	ESF_DZ_L2DTMR_PORT0_MAC_LBN 5
+#define	ESF_DZ_L2DTMR_PORT0_MAC_WIDTH 1
+
+
+/* ES_MC_EVENT */
+#define	ESF_DZ_MC_CODE_LBN 60
+#define	ESF_DZ_MC_CODE_WIDTH 4
+#define	ESF_DZ_MC_OVERRIDE_HOLDOFF_LBN 59
+#define	ESF_DZ_MC_OVERRIDE_HOLDOFF_WIDTH 1
+#define	ESF_DZ_MC_DROP_EVENT_LBN 58
+#define	ESF_DZ_MC_DROP_EVENT_WIDTH 1
+#define	ESF_DZ_MC_SOFT_DW0_LBN 0
+#define	ESF_DZ_MC_SOFT_DW0_WIDTH 32
+#define	ESF_DZ_MC_SOFT_DW1_LBN 32
+#define	ESF_DZ_MC_SOFT_DW1_WIDTH 26
+#define	ESF_DZ_MC_SOFT_LBN 0
+#define	ESF_DZ_MC_SOFT_WIDTH 58
+
+
+/* ES_MC_XGMAC_FLTR_RULE_DEF */
+#define	ESF_DZ_MC_XFRC_MODE_LBN 416
+#define	ESF_DZ_MC_XFRC_MODE_WIDTH 1
+#define	ESE_DZ_MC_XFRC_MODE_LAYERED 1
+#define	ESE_DZ_MC_XFRC_MODE_SIMPLE 0
+#define	ESF_DZ_MC_XFRC_HASH_LBN 384
+#define	ESF_DZ_MC_XFRC_HASH_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW0_LBN 256
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW0_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW1_LBN 288
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW1_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW2_LBN 320
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW2_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW3_LBN 352
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_DW3_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_LBN 256
+#define	ESF_DZ_MC_XFRC_LAYER4_BYTE_MASK_WIDTH 128
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW0_LBN 128
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW0_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW1_LBN 160
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW1_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW2_LBN 192
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW2_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW3_LBN 224
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_DW3_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_LBN 128
+#define	ESF_DZ_MC_XFRC_LAYER3_BYTE_MASK_WIDTH 128
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW0_LBN 0
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW0_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW1_LBN 32
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW1_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW2_LBN 64
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW2_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW3_LBN 96
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_DW3_WIDTH 32
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_LBN 0
+#define	ESF_DZ_MC_XFRC_LAYER2_OR_SIMPLE_BYTE_MASK_WIDTH 128
+
+
+/* ES_RX_EVENT */
+#define	ESF_DZ_RX_CODE_LBN 60
+#define	ESF_DZ_RX_CODE_WIDTH 4
+#define	ESF_DZ_RX_OVERRIDE_HOLDOFF_LBN 59
+#define	ESF_DZ_RX_OVERRIDE_HOLDOFF_WIDTH 1
+#define	ESF_DZ_RX_DROP_EVENT_LBN 58
+#define	ESF_DZ_RX_DROP_EVENT_WIDTH 1
+#define	ESF_DZ_RX_EV_RSVD2_LBN 55
+#define	ESF_DZ_RX_EV_RSVD2_WIDTH 3
+#define	ESF_DZ_RX_EV_SOFT2_LBN 52
+#define	ESF_DZ_RX_EV_SOFT2_WIDTH 3
+#define	ESF_DZ_RX_DSC_PTR_LBITS_LBN 48
+#define	ESF_DZ_RX_DSC_PTR_LBITS_WIDTH 4
+#define	ESF_DZ_RX_L4_CLASS_LBN 45
+#define	ESF_DZ_RX_L4_CLASS_WIDTH 3
+#define	ESE_DZ_L4_CLASS_RSVD7 7
+#define	ESE_DZ_L4_CLASS_RSVD6 6
+#define	ESE_DZ_L4_CLASS_RSVD5 5
+#define	ESE_DZ_L4_CLASS_RSVD4 4
+#define	ESE_DZ_L4_CLASS_RSVD3 3
+#define	ESE_DZ_L4_CLASS_UDP 2
+#define	ESE_DZ_L4_CLASS_TCP 1
+#define	ESE_DZ_L4_CLASS_UNKNOWN 0
+#define	ESF_DZ_RX_L3_CLASS_LBN 42
+#define	ESF_DZ_RX_L3_CLASS_WIDTH 3
+#define	ESE_DZ_L3_CLASS_RSVD7 7
+#define	ESE_DZ_L3_CLASS_IP6_FRAG 6
+#define	ESE_DZ_L3_CLASS_ARP 5
+#define	ESE_DZ_L3_CLASS_IP4_FRAG 4
+#define	ESE_DZ_L3_CLASS_FCOE 3
+#define	ESE_DZ_L3_CLASS_IP6 2
+#define	ESE_DZ_L3_CLASS_IP4 1
+#define	ESE_DZ_L3_CLASS_UNKNOWN 0
+#define	ESF_DZ_RX_ETH_TAG_CLASS_LBN 39
+#define	ESF_DZ_RX_ETH_TAG_CLASS_WIDTH 3
+#define	ESE_DZ_ETH_TAG_CLASS_RSVD7 7
+#define	ESE_DZ_ETH_TAG_CLASS_RSVD6 6
+#define	ESE_DZ_ETH_TAG_CLASS_RSVD5 5
+#define	ESE_DZ_ETH_TAG_CLASS_RSVD4 4
+#define	ESE_DZ_ETH_TAG_CLASS_RSVD3 3
+#define	ESE_DZ_ETH_TAG_CLASS_VLAN2 2
+#define	ESE_DZ_ETH_TAG_CLASS_VLAN1 1
+#define	ESE_DZ_ETH_TAG_CLASS_NONE 0
+#define	ESF_DZ_RX_ETH_BASE_CLASS_LBN 36
+#define	ESF_DZ_RX_ETH_BASE_CLASS_WIDTH 3
+#define	ESE_DZ_ETH_BASE_CLASS_LLC_SNAP 2
+#define	ESE_DZ_ETH_BASE_CLASS_LLC 1
+#define	ESE_DZ_ETH_BASE_CLASS_ETH2 0
+#define	ESF_DZ_RX_MAC_CLASS_LBN 35
+#define	ESF_DZ_RX_MAC_CLASS_WIDTH 1
+#define	ESE_DZ_MAC_CLASS_MCAST 1
+#define	ESE_DZ_MAC_CLASS_UCAST 0
+#define	ESF_DZ_RX_EV_SOFT1_LBN 32
+#define	ESF_DZ_RX_EV_SOFT1_WIDTH 3
+#define	ESF_DZ_RX_EV_RSVD1_LBN 30
+#define	ESF_DZ_RX_EV_RSVD1_WIDTH 2
+#define	ESF_DZ_RX_ECC_ERR_LBN 29
+#define	ESF_DZ_RX_ECC_ERR_WIDTH 1
+#define	ESF_DZ_RX_CRC1_ERR_LBN 28
+#define	ESF_DZ_RX_CRC1_ERR_WIDTH 1
+#define	ESF_DZ_RX_CRC0_ERR_LBN 27
+#define	ESF_DZ_RX_CRC0_ERR_WIDTH 1
+#define	ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN 26
+#define	ESF_DZ_RX_TCPUDP_CKSUM_ERR_WIDTH 1
+#define	ESF_DZ_RX_IPCKSUM_ERR_LBN 25
+#define	ESF_DZ_RX_IPCKSUM_ERR_WIDTH 1
+#define	ESF_DZ_RX_ECRC_ERR_LBN 24
+#define	ESF_DZ_RX_ECRC_ERR_WIDTH 1
+#define	ESF_DZ_RX_QLABEL_LBN 16
+#define	ESF_DZ_RX_QLABEL_WIDTH 8
+#define	ESF_DZ_RX_PARSE_INCOMPLETE_LBN 15
+#define	ESF_DZ_RX_PARSE_INCOMPLETE_WIDTH 1
+#define	ESF_DZ_RX_CONT_LBN 14
+#define	ESF_DZ_RX_CONT_WIDTH 1
+#define	ESF_DZ_RX_BYTES_LBN 0
+#define	ESF_DZ_RX_BYTES_WIDTH 14
+
+
+/* ES_RX_KER_DESC */
+#define	ESF_DZ_RX_KER_RESERVED_LBN 62
+#define	ESF_DZ_RX_KER_RESERVED_WIDTH 2
+#define	ESF_DZ_RX_KER_BYTE_CNT_LBN 48
+#define	ESF_DZ_RX_KER_BYTE_CNT_WIDTH 14
+#define	ESF_DZ_RX_KER_BUF_ADDR_DW0_LBN 0
+#define	ESF_DZ_RX_KER_BUF_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_RX_KER_BUF_ADDR_DW1_LBN 32
+#define	ESF_DZ_RX_KER_BUF_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_RX_KER_BUF_ADDR_LBN 0
+#define	ESF_DZ_RX_KER_BUF_ADDR_WIDTH 48
+
+
+/* ES_RX_USER_DESC */
+#define	ESF_DZ_RX_USR_RESERVED_LBN 62
+#define	ESF_DZ_RX_USR_RESERVED_WIDTH 2
+#define	ESF_DZ_RX_USR_BYTE_CNT_LBN 48
+#define	ESF_DZ_RX_USR_BYTE_CNT_WIDTH 14
+#define	ESF_DZ_RX_USR_BUF_PAGE_SIZE_LBN 44
+#define	ESF_DZ_RX_USR_BUF_PAGE_SIZE_WIDTH 4
+#define	ESE_DZ_USR_BUF_PAGE_SZ_4MB 10
+#define	ESE_DZ_USR_BUF_PAGE_SZ_1MB 8
+#define	ESE_DZ_USR_BUF_PAGE_SZ_64KB 4
+#define	ESE_DZ_USR_BUF_PAGE_SZ_4KB 0
+#define	ESF_DZ_RX_USR_BUF_ID_OFFSET_DW0_LBN 0
+#define	ESF_DZ_RX_USR_BUF_ID_OFFSET_DW0_WIDTH 32
+#define	ESF_DZ_RX_USR_BUF_ID_OFFSET_DW1_LBN 32
+#define	ESF_DZ_RX_USR_BUF_ID_OFFSET_DW1_WIDTH 12
+#define	ESF_DZ_RX_USR_BUF_ID_OFFSET_LBN 0
+#define	ESF_DZ_RX_USR_BUF_ID_OFFSET_WIDTH 44
+#define	ESF_DZ_RX_USR_4KBPS_BUF_ID_LBN 12
+#define	ESF_DZ_RX_USR_4KBPS_BUF_ID_WIDTH 32
+#define	ESF_DZ_RX_USR_64KBPS_BUF_ID_LBN 16
+#define	ESF_DZ_RX_USR_64KBPS_BUF_ID_WIDTH 28
+#define	ESF_DZ_RX_USR_1MBPS_BUF_ID_LBN 20
+#define	ESF_DZ_RX_USR_1MBPS_BUF_ID_WIDTH 24
+#define	ESF_DZ_RX_USR_4MBPS_BUF_ID_LBN 22
+#define	ESF_DZ_RX_USR_4MBPS_BUF_ID_WIDTH 22
+#define	ESF_DZ_RX_USR_4MBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_RX_USR_4MBPS_BYTE_OFFSET_WIDTH 22
+#define	ESF_DZ_RX_USR_1MBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_RX_USR_1MBPS_BYTE_OFFSET_WIDTH 20
+#define	ESF_DZ_RX_USR_64KBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_RX_USR_64KBPS_BYTE_OFFSET_WIDTH 16
+#define	ESF_DZ_RX_USR_4KBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_RX_USR_4KBPS_BYTE_OFFSET_WIDTH 12
+
+
+/* ES_RX_U_QSTATE_TBL0_ENTRY */
+#define	ESF_DZ_RX_U_DC_FILL_LBN 112
+#define	ESF_DZ_RX_U_DC_FILL_WIDTH 7
+#define	ESF_DZ_RX_U_SOFT7_B1R1_0_LBN 112
+#define	ESF_DZ_RX_U_SOFT7_B1R1_0_WIDTH 7
+#define	ESF_DZ_RX_U_DSCR_HW_RPTR_LBN 96
+#define	ESF_DZ_RX_U_DSCR_HW_RPTR_WIDTH 12
+#define	ESF_DZ_RX_U_SOFT12_B1R2_0_LBN 96
+#define	ESF_DZ_RX_U_SOFT12_B1R2_0_WIDTH 12
+#define	ESF_DZ_RX_U_DC_RPTR_LBN 80
+#define	ESF_DZ_RX_U_DC_RPTR_WIDTH 6
+#define	ESF_DZ_RX_U_SOFT6_B1R1_0_LBN 80
+#define	ESF_DZ_RX_U_SOFT6_B1R1_0_WIDTH 6
+#define	ESF_DZ_RX_U_NOTIFY_PENDING_LBN 70
+#define	ESF_DZ_RX_U_NOTIFY_PENDING_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_6_LBN 70
+#define	ESF_DZ_RX_U_SOFT1_B1R0_6_WIDTH 1
+#define	ESF_DZ_RX_U_DATA_ACTIVE_LBN 69
+#define	ESF_DZ_RX_U_DATA_ACTIVE_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_5_LBN 69
+#define	ESF_DZ_RX_U_SOFT1_B1R0_5_WIDTH 1
+#define	ESF_DZ_RX_U_FAST_PATH_LBN 68
+#define	ESF_DZ_RX_U_FAST_PATH_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_4_LBN 68
+#define	ESF_DZ_RX_U_SOFT1_B1R0_4_WIDTH 1
+#define	ESF_DZ_RX_U_NO_FLUSH_LBN 67
+#define	ESF_DZ_RX_U_NO_FLUSH_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_3_LBN 67
+#define	ESF_DZ_RX_U_SOFT1_B1R0_3_WIDTH 1
+#define	ESF_DZ_RX_U_DESC_ACTIVE_LBN 66
+#define	ESF_DZ_RX_U_DESC_ACTIVE_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_2_LBN 66
+#define	ESF_DZ_RX_U_SOFT1_B1R0_2_WIDTH 1
+#define	ESF_DZ_RX_U_HDR_SPLIT_LBN 65
+#define	ESF_DZ_RX_U_HDR_SPLIT_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_1_LBN 65
+#define	ESF_DZ_RX_U_SOFT1_B1R0_1_WIDTH 1
+#define	ESF_DZ_RX_U_Q_ENABLE_LBN 64
+#define	ESF_DZ_RX_U_Q_ENABLE_WIDTH 1
+#define	ESF_DZ_RX_U_SOFT1_B1R0_0_LBN 64
+#define	ESF_DZ_RX_U_SOFT1_B1R0_0_WIDTH 1
+#define	ESF_DZ_RX_U_UPD_CRC_MODE_LBN 29
+#define	ESF_DZ_RX_U_UPD_CRC_MODE_WIDTH 3
+#define	ESE_DZ_C2RIP_FCOIP_MPA 5
+#define	ESE_DZ_C2RIP_FCOIP_FCOE 4
+#define	ESE_DZ_C2RIP_ISCSI_HDR_AND_PYLD 3
+#define	ESE_DZ_C2RIP_ISCSI_HDR 2
+#define	ESE_DZ_C2RIP_FCOE 1
+#define	ESE_DZ_C2RIP_OFF 0
+#define	ESF_DZ_RX_U_SOFT16_B0R1_LBN 16
+#define	ESF_DZ_RX_U_SOFT16_B0R1_WIDTH 16
+#define	ESF_DZ_RX_U_BIU_ARGS_LBN 16
+#define	ESF_DZ_RX_U_BIU_ARGS_WIDTH 13
+#define	ESF_DZ_RX_U_EV_QID_LBN 5
+#define	ESF_DZ_RX_U_EV_QID_WIDTH 11
+#define	ESF_DZ_RX_U_SOFT16_B0R0_LBN 0
+#define	ESF_DZ_RX_U_SOFT16_B0R0_WIDTH 16
+#define	ESF_DZ_RX_U_EV_QLABEL_LBN 0
+#define	ESF_DZ_RX_U_EV_QLABEL_WIDTH 5
+
+
+/* ES_RX_U_QSTATE_TBL1_ENTRY */
+#define	ESF_DZ_RX_U_DSCR_BASE_PAGE_ID_LBN 64
+#define	ESF_DZ_RX_U_DSCR_BASE_PAGE_ID_WIDTH 18
+#define	ESF_DZ_RX_U_SOFT18_B1R0_0_LBN 64
+#define	ESF_DZ_RX_U_SOFT18_B1R0_0_WIDTH 18
+#define	ESF_DZ_RX_U_QST1_SPARE_LBN 52
+#define	ESF_DZ_RX_U_QST1_SPARE_WIDTH 12
+#define	ESF_DZ_RX_U_SOFT16_B0R3_0_LBN 48
+#define	ESF_DZ_RX_U_SOFT16_B0R3_0_WIDTH 16
+#define	ESF_DZ_RX_U_PKT_EDIT_LBN 51
+#define	ESF_DZ_RX_U_PKT_EDIT_WIDTH 1
+#define	ESF_DZ_RX_U_DOORBELL_ENABLED_LBN 50
+#define	ESF_DZ_RX_U_DOORBELL_ENABLED_WIDTH 1
+#define	ESF_DZ_RX_U_WORK_PENDING_LBN 49
+#define	ESF_DZ_RX_U_WORK_PENDING_WIDTH 1
+#define	ESF_DZ_RX_U_ERROR_LBN 48
+#define	ESF_DZ_RX_U_ERROR_WIDTH 1
+#define	ESF_DZ_RX_U_DSCR_SW_WPTR_LBN 32
+#define	ESF_DZ_RX_U_DSCR_SW_WPTR_WIDTH 12
+#define	ESF_DZ_RX_U_SOFT12_B0R2_0_LBN 32
+#define	ESF_DZ_RX_U_SOFT12_B0R2_0_WIDTH 12
+#define	ESF_DZ_RX_U_OWNER_ID_LBN 16
+#define	ESF_DZ_RX_U_OWNER_ID_WIDTH 12
+#define	ESF_DZ_RX_U_SOFT12_B0R1_0_LBN 16
+#define	ESF_DZ_RX_U_SOFT12_B0R1_0_WIDTH 12
+#define	ESF_DZ_RX_U_DSCR_SIZE_LBN 0
+#define	ESF_DZ_RX_U_DSCR_SIZE_WIDTH 3
+#define	ESE_DZ_RX_DSCR_SIZE_512 7
+#define	ESE_DZ_RX_DSCR_SIZE_1K 6
+#define	ESE_DZ_RX_DSCR_SIZE_2K 5
+#define	ESE_DZ_RX_DSCR_SIZE_4K 4
+#define	ESF_DZ_RX_U_SOFT3_B0R0_0_LBN 0
+#define	ESF_DZ_RX_U_SOFT3_B0R0_0_WIDTH 3
+
+
+/* ES_SMC_BUFTBL_CNTRL_ENTRY */
+#define	ESF_DZ_SMC_SW_CNTXT_DW0_LBN 16
+#define	ESF_DZ_SMC_SW_CNTXT_DW0_WIDTH 32
+#define	ESF_DZ_SMC_SW_CNTXT_DW1_LBN 48
+#define	ESF_DZ_SMC_SW_CNTXT_DW1_WIDTH 24
+#define	ESF_DZ_SMC_SW_CNTXT_LBN 16
+#define	ESF_DZ_SMC_SW_CNTXT_WIDTH 56
+#define	ESF_DZ_SMC_PAGE_SIZE_LBN 12
+#define	ESF_DZ_SMC_PAGE_SIZE_WIDTH 4
+#define	ESF_DZ_SMC_OWNER_ID_LBN 0
+#define	ESF_DZ_SMC_OWNER_ID_WIDTH 12
+
+
+/* ES_SMC_BUFTBL_TRANSL_ENTRY */
+#define	ESF_DZ_SMC_PAGE_INDEX0_DW0_LBN 36
+#define	ESF_DZ_SMC_PAGE_INDEX0_DW0_WIDTH 32
+#define	ESF_DZ_SMC_PAGE_INDEX0_DW1_LBN 68
+#define	ESF_DZ_SMC_PAGE_INDEX0_DW1_WIDTH 4
+#define	ESF_DZ_SMC_PAGE_INDEX0_LBN 36
+#define	ESF_DZ_SMC_PAGE_INDEX0_WIDTH 36
+#define	ESF_DZ_SMC_PAGE_INDEX1_DW0_LBN 0
+#define	ESF_DZ_SMC_PAGE_INDEX1_DW0_WIDTH 32
+#define	ESF_DZ_SMC_PAGE_INDEX1_DW1_LBN 32
+#define	ESF_DZ_SMC_PAGE_INDEX1_DW1_WIDTH 4
+#define	ESF_DZ_SMC_PAGE_INDEX1_LBN 0
+#define	ESF_DZ_SMC_PAGE_INDEX1_WIDTH 36
+
+
+/* ES_SMC_DSCR_CACHE_ENTRY */
+#define	ESF_DZ_SMC_BTE_PAD_LBN 64
+#define	ESF_DZ_SMC_BTE_PAD_WIDTH 8
+#define	ESF_DZ_SMC_DSCR_DW0_LBN 0
+#define	ESF_DZ_SMC_DSCR_DW0_WIDTH 32
+#define	ESF_DZ_SMC_DSCR_DW1_LBN 32
+#define	ESF_DZ_SMC_DSCR_DW1_WIDTH 32
+#define	ESF_DZ_SMC_DSCR_LBN 0
+#define	ESF_DZ_SMC_DSCR_WIDTH 64
+
+
+/* ES_SMC_GEN_STORAGE_ENTRY */
+#define	ESF_DZ_SMC_DATA_DW0_LBN 0
+#define	ESF_DZ_SMC_DATA_DW0_WIDTH 32
+#define	ESF_DZ_SMC_DATA_DW1_LBN 32
+#define	ESF_DZ_SMC_DATA_DW1_WIDTH 32
+#define	ESF_DZ_SMC_DATA_DW2_LBN 64
+#define	ESF_DZ_SMC_DATA_DW2_WIDTH 8
+#define	ESF_DZ_SMC_DATA_LBN 0
+#define	ESF_DZ_SMC_DATA_WIDTH 72
+
+
+/* ES_SMC_MSG_BASE_REQ */
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_DW0_LBN 11
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_DW0_WIDTH 32
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_DW1_LBN 43
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_DW1_WIDTH 32
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_DW2_LBN 75
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_DW2_WIDTH 26
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_LBN 11
+#define	ESF_DZ_MC2S_BASE_REQ_MSG_DATA_WIDTH 90
+#define	ESF_DZ_MC2S_BASE_SOFT_LBN 7
+#define	ESF_DZ_MC2S_BASE_SOFT_WIDTH 4
+#define	ESF_DZ_MC2S_BASE_CLIENT_ID_LBN 3
+#define	ESF_DZ_MC2S_BASE_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_MC2S_BASE_OP_LBN 0
+#define	ESF_DZ_MC2S_BASE_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_RESP_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_RESP_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_RESP_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_RESP_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+#define	ESE_DZ_SMC_RESP_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_BUFTBL_LOOKUP_REQ */
+#define	ESF_DZ_MC2S_BL_BUF_ID_LBN 28
+#define	ESF_DZ_MC2S_BL_BUF_ID_WIDTH 18
+#define	ESF_DZ_MC2S_BL_EXP_PAGE_SIZE_LBN 24
+#define	ESF_DZ_MC2S_BL_EXP_PAGE_SIZE_WIDTH 4
+#define	ESE_DZ_SMC_PAGE_SIZE_4M 10
+#define	ESE_DZ_SMC_PAGE_SIZE_1M 8
+#define	ESE_DZ_SMC_PAGE_SIZE_64K 4
+#define	ESE_DZ_SMC_PAGE_SIZE_4K 0
+#define	ESF_DZ_MC2S_BL_EXP_OWNER_ID_LBN 12
+#define	ESF_DZ_MC2S_BL_EXP_OWNER_ID_WIDTH 12
+#define	ESF_DZ_MC2S_BL_REFLECT_LBN 11
+#define	ESF_DZ_MC2S_BL_REFLECT_WIDTH 1
+#define	ESF_DZ_MC2S_BL_SOFT_LBN 7
+#define	ESF_DZ_MC2S_BL_SOFT_WIDTH 4
+#define	ESF_DZ_MC2S_BL_CLIENT_ID_LBN 3
+#define	ESF_DZ_MC2S_BL_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_MC2S_BL_OP_LBN 0
+#define	ESF_DZ_MC2S_BL_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_BUFTBL_LOOKUP_RESP */
+#define	ESF_DZ_S2MC_BL_BUFTBL_ENTRY_DW0_LBN 12
+#define	ESF_DZ_S2MC_BL_BUFTBL_ENTRY_DW0_WIDTH 32
+#define	ESF_DZ_S2MC_BL_BUFTBL_ENTRY_DW1_LBN 44
+#define	ESF_DZ_S2MC_BL_BUFTBL_ENTRY_DW1_WIDTH 4
+#define	ESF_DZ_S2MC_BL_BUFTBL_ENTRY_LBN 12
+#define	ESF_DZ_S2MC_BL_BUFTBL_ENTRY_WIDTH 36
+#define	ESF_DZ_S2MC_BL_FAIL_LBN 11
+#define	ESF_DZ_S2MC_BL_FAIL_WIDTH 1
+#define	ESF_DZ_S2MC_BL_SOFT_LBN 7
+#define	ESF_DZ_S2MC_BL_SOFT_WIDTH 4
+#define	ESF_DZ_S2MC_BL_CLIENT_ID_LBN 3
+#define	ESF_DZ_S2MC_BL_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_S2MC_BL_OP_LBN 0
+#define	ESF_DZ_S2MC_BL_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_DSCR_RD_REQ */
+#define	ESF_DZ_MC2S_DR_DSCR_OFST_LBN 24
+#define	ESF_DZ_MC2S_DR_DSCR_OFST_WIDTH 6
+#define	ESF_DZ_MC2S_DR_QID_LBN 13
+#define	ESF_DZ_MC2S_DR_QID_WIDTH 11
+#define	ESF_DZ_MC2S_DR_IS_TX_LBN 12
+#define	ESF_DZ_MC2S_DR_IS_TX_WIDTH 1
+#define	ESF_DZ_MC2S_DR_REFLECT_LBN 11
+#define	ESF_DZ_MC2S_DR_REFLECT_WIDTH 1
+#define	ESF_DZ_MC2S_DR_SOFT_LBN 7
+#define	ESF_DZ_MC2S_DR_SOFT_WIDTH 4
+#define	ESF_DZ_MC2S_DR_CLIENT_ID_LBN 3
+#define	ESF_DZ_MC2S_DR_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_MC2S_DR_OP_LBN 0
+#define	ESF_DZ_MC2S_DR_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_DSCR_RD_RESP */
+#define	ESF_DZ_S2MC_DR_DSCR_DW0_LBN 12
+#define	ESF_DZ_S2MC_DR_DSCR_DW0_WIDTH 32
+#define	ESF_DZ_S2MC_DR_DSCR_DW1_LBN 44
+#define	ESF_DZ_S2MC_DR_DSCR_DW1_WIDTH 32
+#define	ESF_DZ_S2MC_DR_DSCR_LBN 12
+#define	ESF_DZ_S2MC_DR_DSCR_WIDTH 64
+#define	ESF_DZ_S2MC_DR_FAIL_LBN 11
+#define	ESF_DZ_S2MC_DR_FAIL_WIDTH 1
+#define	ESF_DZ_S2MC_DR_SOFT_LBN 7
+#define	ESF_DZ_S2MC_DR_SOFT_WIDTH 4
+#define	ESF_DZ_S2MC_DR_CLIENT_ID_LBN 3
+#define	ESF_DZ_S2MC_DR_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_S2MC_DR_OP_LBN 0
+#define	ESF_DZ_S2MC_DR_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_DSCR_WR_REQ */
+#define	ESF_DZ_MC2S_DW_DSCR_DW0_LBN 30
+#define	ESF_DZ_MC2S_DW_DSCR_DW0_WIDTH 32
+#define	ESF_DZ_MC2S_DW_DSCR_DW1_LBN 62
+#define	ESF_DZ_MC2S_DW_DSCR_DW1_WIDTH 32
+#define	ESF_DZ_MC2S_DW_DSCR_LBN 30
+#define	ESF_DZ_MC2S_DW_DSCR_WIDTH 64
+#define	ESF_DZ_MC2S_DW_DSCR_OFST_LBN 24
+#define	ESF_DZ_MC2S_DW_DSCR_OFST_WIDTH 6
+#define	ESF_DZ_MC2S_DW_QID_LBN 13
+#define	ESF_DZ_MC2S_DW_QID_WIDTH 11
+#define	ESF_DZ_MC2S_DW_IS_TX_LBN 12
+#define	ESF_DZ_MC2S_DW_IS_TX_WIDTH 1
+#define	ESF_DZ_MC2S_DW_REFLECT_LBN 11
+#define	ESF_DZ_MC2S_DW_REFLECT_WIDTH 1
+#define	ESF_DZ_MC2S_DW_SOFT_LBN 7
+#define	ESF_DZ_MC2S_DW_SOFT_WIDTH 4
+#define	ESF_DZ_MC2S_DW_CLIENT_ID_LBN 3
+#define	ESF_DZ_MC2S_DW_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_MC2S_DW_OP_LBN 0
+#define	ESF_DZ_MC2S_DW_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_DSCR_WR_RESP */
+#define	ESF_DZ_S2MC_DW_FAIL_LBN 11
+#define	ESF_DZ_S2MC_DW_FAIL_WIDTH 1
+#define	ESF_DZ_S2MC_DW_SOFT_LBN 7
+#define	ESF_DZ_S2MC_DW_SOFT_WIDTH 4
+#define	ESF_DZ_S2MC_DW_CLIENT_ID_LBN 3
+#define	ESF_DZ_S2MC_DW_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_S2MC_DW_OP_LBN 0
+#define	ESF_DZ_S2MC_DW_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_RD_REQ */
+#define	ESF_DZ_MC2S_RD_ADDR_LBN 12
+#define	ESF_DZ_MC2S_RD_ADDR_WIDTH 17
+#define	ESF_DZ_MC2S_RD_REFLECT_LBN 11
+#define	ESF_DZ_MC2S_RD_REFLECT_WIDTH 1
+#define	ESF_DZ_MC2S_RD_SOFT_LBN 7
+#define	ESF_DZ_MC2S_RD_SOFT_WIDTH 4
+#define	ESF_DZ_MC2S_RD_CLIENT_ID_LBN 3
+#define	ESF_DZ_MC2S_RD_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_MC2S_RD_OP_LBN 0
+#define	ESF_DZ_MC2S_RD_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_RD_RESP */
+#define	ESF_DZ_S2MC_RD_DATA_DW0_LBN 12
+#define	ESF_DZ_S2MC_RD_DATA_DW0_WIDTH 32
+#define	ESF_DZ_S2MC_RD_DATA_DW1_LBN 44
+#define	ESF_DZ_S2MC_RD_DATA_DW1_WIDTH 32
+#define	ESF_DZ_S2MC_RD_DATA_DW2_LBN 76
+#define	ESF_DZ_S2MC_RD_DATA_DW2_WIDTH 8
+#define	ESF_DZ_S2MC_RD_DATA_LBN 12
+#define	ESF_DZ_S2MC_RD_DATA_WIDTH 72
+#define	ESF_DZ_S2MC_RD_FAIL_LBN 11
+#define	ESF_DZ_S2MC_RD_FAIL_WIDTH 1
+#define	ESF_DZ_S2MC_RD_SOFT_LBN 7
+#define	ESF_DZ_S2MC_RD_SOFT_WIDTH 4
+#define	ESF_DZ_S2MC_RD_CLIENT_ID_LBN 3
+#define	ESF_DZ_S2MC_RD_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_S2MC_RD_OP_LBN 0
+#define	ESF_DZ_S2MC_RD_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_RESP */
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_DW0_LBN 12
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_DW0_WIDTH 32
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_DW1_LBN 44
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_DW1_WIDTH 32
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_DW2_LBN 76
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_DW2_WIDTH 8
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_LBN 12
+#define	ESF_DZ_S2MC_BASE_RSP_DATA_WIDTH 72
+#define	ESF_DZ_S2MC_BASE_FAIL_LBN 11
+#define	ESF_DZ_S2MC_BASE_FAIL_WIDTH 1
+#define	ESF_DZ_S2MC_BASE_SOFT_LBN 7
+#define	ESF_DZ_S2MC_BASE_SOFT_WIDTH 4
+#define	ESF_DZ_S2MC_BASE_CLIENT_ID_LBN 3
+#define	ESF_DZ_S2MC_BASE_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_S2MC_BASE_OP_LBN 0
+#define	ESF_DZ_S2MC_BASE_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_WR_REQ */
+#define	ESF_DZ_MC2S_WR_DATA_DW0_LBN 29
+#define	ESF_DZ_MC2S_WR_DATA_DW0_WIDTH 32
+#define	ESF_DZ_MC2S_WR_DATA_DW1_LBN 61
+#define	ESF_DZ_MC2S_WR_DATA_DW1_WIDTH 32
+#define	ESF_DZ_MC2S_WR_DATA_DW2_LBN 93
+#define	ESF_DZ_MC2S_WR_DATA_DW2_WIDTH 8
+#define	ESF_DZ_MC2S_WR_DATA_LBN 29
+#define	ESF_DZ_MC2S_WR_DATA_WIDTH 72
+#define	ESF_DZ_MC2S_WR_ADDR_LBN 12
+#define	ESF_DZ_MC2S_WR_ADDR_WIDTH 17
+#define	ESF_DZ_MC2S_WR_REFLECT_LBN 11
+#define	ESF_DZ_MC2S_WR_REFLECT_WIDTH 1
+#define	ESF_DZ_MC2S_WR_SOFT_LBN 7
+#define	ESF_DZ_MC2S_WR_SOFT_WIDTH 4
+#define	ESF_DZ_MC2S_WR_CLIENT_ID_LBN 3
+#define	ESF_DZ_MC2S_WR_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_MC2S_WR_OP_LBN 0
+#define	ESF_DZ_MC2S_WR_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_SMC_MSG_WR_RESP */
+#define	ESF_DZ_S2MC_WR_FAIL_LBN 11
+#define	ESF_DZ_S2MC_WR_FAIL_WIDTH 1
+#define	ESF_DZ_S2MC_WR_SOFT_LBN 7
+#define	ESF_DZ_S2MC_WR_SOFT_WIDTH 4
+#define	ESF_DZ_S2MC_WR_CLIENT_ID_LBN 3
+#define	ESF_DZ_S2MC_WR_CLIENT_ID_WIDTH 4
+#define	ESE_DZ_SMC_MACRO_ENGINE_ID 15
+#define	ESE_DZ_SMC_TX_DICPU_ID 14
+#define	ESE_DZ_SMC_RX_DICPU_ID 13
+#define	ESE_DZ_SMC_MC_ID 12
+#define	ESE_DZ_SMC_DL_ID 10
+#define	ESE_DZ_SMC_EV_ID 8
+#define	ESE_DZ_SMC_TX_DPCPU1_ID 5
+#define	ESE_DZ_SMC_TX_DPCPU0_ID 4
+#define	ESE_DZ_SMC_RX_DPCPU_ID 0
+#define	ESF_DZ_S2MC_WR_OP_LBN 0
+#define	ESF_DZ_S2MC_WR_OP_WIDTH 3
+#define	ESE_DZ_SMC_REQ_WR 4
+#define	ESE_DZ_SMC_REQ_RD 3
+#define	ESE_DZ_SMC_REQ_DSCR_WRITE 2
+#define	ESE_DZ_SMC_REQ_DSCR_READ 1
+#define	ESE_DZ_SMC_REQ_BUFTBL_LOOKUP 0
+
+
+/* ES_TX_EVENT */
+#define	ESF_DZ_TX_CODE_LBN 60
+#define	ESF_DZ_TX_CODE_WIDTH 4
+#define	ESF_DZ_TX_OVERRIDE_HOLDOFF_LBN 59
+#define	ESF_DZ_TX_OVERRIDE_HOLDOFF_WIDTH 1
+#define	ESF_DZ_TX_DROP_EVENT_LBN 58
+#define	ESF_DZ_TX_DROP_EVENT_WIDTH 1
+#define	ESF_DZ_TX_EV_RSVD_LBN 48
+#define	ESF_DZ_TX_EV_RSVD_WIDTH 10
+#define	ESF_DZ_TX_SOFT2_LBN 32
+#define	ESF_DZ_TX_SOFT2_WIDTH 16
+#define	ESF_DZ_TX_SOFT1_LBN 24
+#define	ESF_DZ_TX_SOFT1_WIDTH 8
+#define	ESF_DZ_TX_QLABEL_LBN 16
+#define	ESF_DZ_TX_QLABEL_WIDTH 8
+#define	ESF_DZ_TX_DESCR_INDX_LBN 0
+#define	ESF_DZ_TX_DESCR_INDX_WIDTH 16
+
+
+/* ES_TX_KER_DESC */
+#define	ESF_DZ_TX_KER_TYPE_LBN 63
+#define	ESF_DZ_TX_KER_TYPE_WIDTH 1
+#define	ESF_DZ_TX_KER_CONT_LBN 62
+#define	ESF_DZ_TX_KER_CONT_WIDTH 1
+#define	ESF_DZ_TX_KER_BYTE_CNT_LBN 48
+#define	ESF_DZ_TX_KER_BYTE_CNT_WIDTH 14
+#define	ESF_DZ_TX_KER_BUF_ADDR_DW0_LBN 0
+#define	ESF_DZ_TX_KER_BUF_ADDR_DW0_WIDTH 32
+#define	ESF_DZ_TX_KER_BUF_ADDR_DW1_LBN 32
+#define	ESF_DZ_TX_KER_BUF_ADDR_DW1_WIDTH 16
+#define	ESF_DZ_TX_KER_BUF_ADDR_LBN 0
+#define	ESF_DZ_TX_KER_BUF_ADDR_WIDTH 48
+
+
+/* ES_TX_OPTION_DESC */
+#define	ESF_DZ_TX_DESC_IS_OPT_LBN 63
+#define	ESF_DZ_TX_DESC_IS_OPT_WIDTH 1
+#define	ESF_DZ_TX_OPTION_TYPE_LBN 60
+#define	ESF_DZ_TX_OPTION_TYPE_WIDTH 3
+#define	ESE_DZ_TX_OPTION_DESC_TSO 4
+#define	ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0
+#define	ESF_DZ_TX_TSO_TCP_FLAGS_LBN 48
+#define	ESF_DZ_TX_TSO_TCP_FLAGS_WIDTH 8
+#define	ESF_DZ_TX_TSO_TCP_MSS_LBN 32
+#define	ESF_DZ_TX_TSO_TCP_MSS_WIDTH 16
+#define	ESF_DZ_TX_TSO_TCP_SEQNO_LBN 0
+#define	ESF_DZ_TX_TSO_TCP_SEQNO_WIDTH 32
+#define	ESF_DZ_TX_OPTION_CRC_MODE_LBN 2
+#define	ESF_DZ_TX_OPTION_CRC_MODE_WIDTH 3
+#define	ESE_DZ_TX_OPTION_CRC_FCOIP_MPA 5
+#define	ESE_DZ_TX_OPTION_CRC_FCOIP_FCOE 4
+#define	ESE_DZ_TX_OPTION_CRC_ISCSI_HDR_AND_PYLD 3
+#define	ESE_DZ_TX_OPTION_CRC_ISCSI_HDR 2
+#define	ESE_DZ_TX_OPTION_CRC_FCOE 1
+#define	ESE_DZ_TX_OPTION_CRC_OFF 0
+#define	ESF_DZ_TX_OPTION_UDP_TCP_CSUM_LBN 1
+#define	ESF_DZ_TX_OPTION_UDP_TCP_CSUM_WIDTH 1
+#define	ESF_DZ_TX_OPTION_IP_CSUM_LBN 0
+#define	ESF_DZ_TX_OPTION_IP_CSUM_WIDTH 1
+
+
+/* ES_TX_PACER_BASE_MSG */
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_DW0_LBN 11
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_DW0_WIDTH 32
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_DW1_LBN 43
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_DW1_WIDTH 32
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_DW2_LBN 75
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_DW2_WIDTH 23
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_LBN 11
+#define	ESF_DZ_TXP_BASE_REQ_MSG_DATA_WIDTH 87
+#define	ESF_DZ_TXP_BASE_OP_LBN 2
+#define	ESF_DZ_TXP_BASE_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_BASE_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_BASE_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_BKT_D_R_REQ */
+#define	ESF_DZ_TXP_BKT_D_R_REQ_FRM_LEN_LBN 45
+#define	ESF_DZ_TXP_BKT_D_R_REQ_FRM_LEN_WIDTH 14
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MAX_BKT2_LBN 35
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MAX_BKT2_WIDTH 10
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MAX_BKT1_LBN 25
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MAX_BKT1_WIDTH 10
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MAX_BKT0_LBN 15
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MAX_BKT0_WIDTH 10
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MIN_BKT_LBN 5
+#define	ESF_DZ_TXP_BKT_D_R_REQ_MIN_BKT_WIDTH 10
+#define	ESF_DZ_TXP_BKT_D_R_REQ_OP_LBN 2
+#define	ESF_DZ_TXP_BKT_D_R_REQ_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_BKT_D_R_REQ_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_BKT_D_R_REQ_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_BKT_TBL_D_R_RSP */
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_DUE_TIME_WITH_MIN_BKT_LBN 21
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_DUE_TIME_WITH_MIN_BKT_WIDTH 26
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_DUE_TIME_LBN 5
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_DUE_TIME_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_OP_LBN 2
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_BKT_TBL_D_R_RSP_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_BKT_TBL_RD_REQ */
+#define	ESF_DZ_TXP_BKT_TBL_RD_REQ_BKT_ID_LBN 5
+#define	ESF_DZ_TXP_BKT_TBL_RD_REQ_BKT_ID_WIDTH 10
+#define	ESF_DZ_TXP_BKT_TBL_RD_REQ_OP_LBN 2
+#define	ESF_DZ_TXP_BKT_TBL_RD_REQ_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_BKT_TBL_RD_REQ_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_BKT_TBL_RD_REQ_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_BKT_TBL_RD_RSP */
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_IDLE_LBN 97
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_IDLE_WIDTH 1
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_USED_LBN 96
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_USED_WIDTH 1
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_MAX_FILL_REG_LBN 94
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_MAX_FILL_REG_WIDTH 2
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_RATE_REC_LBN 78
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_RATE_REC_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_RATE_LBN 62
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_RATE_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_FILL_LEVEL_LBN 47
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_FILL_LEVEL_WIDTH 15
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_DUE_TIME_LBN 31
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_DUE_TIME_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_LAST_FILL_TIME_LBN 15
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_LAST_FILL_TIME_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_BKT_ID_LBN 5
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_BKT_ID_WIDTH 10
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_OP_LBN 2
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_BKT_TBL_RD_RSP_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_BKT_TBL_WR_REQ */
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_IDLE_LBN 65
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_IDLE_WIDTH 1
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_USED_LBN 64
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_USED_WIDTH 1
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_MAX_FILL_REG_LBN 62
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_MAX_FILL_REG_WIDTH 2
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_RATE_REC_LBN 46
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_RATE_REC_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_RATE_LBN 30
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_RATE_WIDTH 16
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_FILL_LEVEL_LBN 15
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_FILL_LEVEL_WIDTH 15
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_BKT_ID_LBN 5
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_BKT_ID_WIDTH 10
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_OP_LBN 2
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_BKT_TBL_WR_REQ_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_TXQ_D_R_I_REQ */
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_FRM_LEN_LBN 15
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_FRM_LEN_WIDTH 14
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_TXQ_ID_LBN 5
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_TXQ_ID_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_OP_LBN 2
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_TXQ_D_R_I_REQ_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_TXQ_TBL_RD_REQ */
+#define	ESF_DZ_TXP_TXQ_TBL_RD_REQ_TXQ_ID_LBN 5
+#define	ESF_DZ_TXP_TXQ_TBL_RD_REQ_TXQ_ID_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_RD_REQ_OP_LBN 2
+#define	ESF_DZ_TXP_TXQ_TBL_RD_REQ_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_TXQ_TBL_RD_REQ_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_TXQ_TBL_RD_REQ_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_TXQ_TBL_RD_RSP */
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MAX_BKT2_LBN 53
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MAX_BKT2_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MAX_BKT1_LBN 43
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MAX_BKT1_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MAX_BKT0_LBN 33
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MAX_BKT0_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MIN_BKT_LBN 23
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_MIN_BKT_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_LABEL_LBN 19
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_LABEL_WIDTH 4
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_PQ_FLAGS_LBN 16
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_PQ_FLAGS_WIDTH 3
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_DSBL_LBN 15
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_DSBL_WIDTH 1
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_TXQ_ID_LBN 5
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_TXQ_ID_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_OP_LBN 2
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_TXQ_TBL_RD_RSP_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_PACER_TXQ_TBL_WR_REQ */
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MAX_BKT2_LBN 53
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MAX_BKT2_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MAX_BKT1_LBN 43
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MAX_BKT1_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MAX_BKT0_LBN 33
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MAX_BKT0_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MIN_BKT_LBN 23
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_MIN_BKT_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_LABEL_LBN 19
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_LABEL_WIDTH 4
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_PQ_FLAGS_LBN 16
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_PQ_FLAGS_WIDTH 3
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_DSBL_LBN 15
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_DSBL_WIDTH 1
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_TXQ_ID_LBN 5
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_TXQ_ID_WIDTH 10
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_OP_LBN 2
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_OP_WIDTH 3
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_RD 7
+#define	ESE_DZ_DPCPU_PACER_BKT_TBL_WR 6
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_RD 5
+#define	ESE_DZ_DPCPU_PACER_TXQ_TBL_WR 4
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_NI 3
+#define	ESE_DZ_DPCPU_PACER_TXQ_D_R_I 2
+#define	ESE_DZ_DPCPU_PACER_BKT_D_R_RD 1
+#define	ESE_DZ_DPCPU_PACER_BKT_D_RD 0
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_CLIENT_ID_LBN 0
+#define	ESF_DZ_TXP_TXQ_TBL_WR_REQ_CLIENT_ID_WIDTH 2
+#define	ESE_DZ_DPCPU_PACER_CPU_CLIENT 2
+#define	ESE_DZ_DPCPU_PACER_CMD_CTL_CLIENT 1
+#define	ESE_DZ_DPCPU_PACER_ALRT_CTL_CLIENT 0
+
+
+/* ES_TX_USER_DESC */
+#define	ESF_DZ_TX_USR_TYPE_LBN 63
+#define	ESF_DZ_TX_USR_TYPE_WIDTH 1
+#define	ESF_DZ_TX_USR_CONT_LBN 62
+#define	ESF_DZ_TX_USR_CONT_WIDTH 1
+#define	ESF_DZ_TX_USR_BYTE_CNT_LBN 48
+#define	ESF_DZ_TX_USR_BYTE_CNT_WIDTH 14
+#define	ESF_DZ_TX_USR_BUF_PAGE_SIZE_LBN 44
+#define	ESF_DZ_TX_USR_BUF_PAGE_SIZE_WIDTH 4
+#define	ESE_DZ_USR_BUF_PAGE_SZ_4MB 10
+#define	ESE_DZ_USR_BUF_PAGE_SZ_1MB 8
+#define	ESE_DZ_USR_BUF_PAGE_SZ_64KB 4
+#define	ESE_DZ_USR_BUF_PAGE_SZ_4KB 0
+#define	ESF_DZ_TX_USR_BUF_ID_OFFSET_DW0_LBN 0
+#define	ESF_DZ_TX_USR_BUF_ID_OFFSET_DW0_WIDTH 32
+#define	ESF_DZ_TX_USR_BUF_ID_OFFSET_DW1_LBN 32
+#define	ESF_DZ_TX_USR_BUF_ID_OFFSET_DW1_WIDTH 12
+#define	ESF_DZ_TX_USR_BUF_ID_OFFSET_LBN 0
+#define	ESF_DZ_TX_USR_BUF_ID_OFFSET_WIDTH 44
+#define	ESF_DZ_TX_USR_4KBPS_BUF_ID_LBN 12
+#define	ESF_DZ_TX_USR_4KBPS_BUF_ID_WIDTH 32
+#define	ESF_DZ_TX_USR_64KBPS_BUF_ID_LBN 16
+#define	ESF_DZ_TX_USR_64KBPS_BUF_ID_WIDTH 28
+#define	ESF_DZ_TX_USR_1MBPS_BUF_ID_LBN 20
+#define	ESF_DZ_TX_USR_1MBPS_BUF_ID_WIDTH 24
+#define	ESF_DZ_TX_USR_4MBPS_BUF_ID_LBN 22
+#define	ESF_DZ_TX_USR_4MBPS_BUF_ID_WIDTH 22
+#define	ESF_DZ_TX_USR_4MBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_TX_USR_4MBPS_BYTE_OFFSET_WIDTH 22
+#define	ESF_DZ_TX_USR_1MBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_TX_USR_1MBPS_BYTE_OFFSET_WIDTH 20
+#define	ESF_DZ_TX_USR_64KBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_TX_USR_64KBPS_BYTE_OFFSET_WIDTH 16
+#define	ESF_DZ_TX_USR_4KBPS_BYTE_OFFSET_LBN 0
+#define	ESF_DZ_TX_USR_4KBPS_BYTE_OFFSET_WIDTH 12
+
+
+/* ES_TX_U_QSTATE_TBL0_ENTRY */
+#define	ESF_DZ_TX_U_DC_FILL_LBN 112
+#define	ESF_DZ_TX_U_DC_FILL_WIDTH 7
+#define	ESF_DZ_TX_U_SOFT7_B1R3_LBN 112
+#define	ESF_DZ_TX_U_SOFT7_B1R3_WIDTH 7
+#define	ESF_DZ_TX_U_DSCR_HW_RPTR_LBN 96
+#define	ESF_DZ_TX_U_DSCR_HW_RPTR_WIDTH 12
+#define	ESF_DZ_TX_U_SOFT12_B1R2_LBN 96
+#define	ESF_DZ_TX_U_SOFT12_B1R2_WIDTH 12
+#define	ESF_DZ_TX_U_DC_RPTR_LBN 80
+#define	ESF_DZ_TX_U_DC_RPTR_WIDTH 6
+#define	ESF_DZ_TX_U_SOFT6_B1R1_LBN 80
+#define	ESF_DZ_TX_U_SOFT6_B1R1_WIDTH 6
+#define	ESF_DZ_TX_U_SOFT5_B1R0_LBN 64
+#define	ESF_DZ_TX_U_SOFT5_B1R0_WIDTH 5
+#define	ESF_DZ_TX_U_PREFETCH_ACTIVE_LBN 66
+#define	ESF_DZ_TX_U_PREFETCH_ACTIVE_WIDTH 1
+#define	ESF_DZ_TX_U_PREFETCH_PENDING_LBN 65
+#define	ESF_DZ_TX_U_PREFETCH_PENDING_WIDTH 1
+#define	ESF_DZ_TX_U_DOORBELL_ENABLED_LBN 64
+#define	ESF_DZ_TX_U_DOORBELL_ENABLED_WIDTH 1
+#define	ESF_DZ_TX_U_UPD_UDPTCP_CSUM_MODE_LBN 33
+#define	ESF_DZ_TX_U_UPD_UDPTCP_CSUM_MODE_WIDTH 1
+#define	ESF_DZ_TX_U_SOFT2_B0R2_LBN 32
+#define	ESF_DZ_TX_U_SOFT2_B0R2_WIDTH 2
+#define	ESF_DZ_TX_U_UPD_IP_CSUM_MODE_LBN 32
+#define	ESF_DZ_TX_U_UPD_IP_CSUM_MODE_WIDTH 1
+#define	ESF_DZ_TX_U_UPD_CRC_MODE_LBN 29
+#define	ESF_DZ_TX_U_UPD_CRC_MODE_WIDTH 3
+#define	ESE_DZ_C2RIP_FCOIP_MPA 5
+#define	ESE_DZ_C2RIP_FCOIP_FCOE 4
+#define	ESE_DZ_C2RIP_ISCSI_HDR_AND_PYLD 3
+#define	ESE_DZ_C2RIP_ISCSI_HDR 2
+#define	ESE_DZ_C2RIP_FCOE 1
+#define	ESE_DZ_C2RIP_OFF 0
+#define	ESF_DZ_TX_U_SOFT16_B0R1_LBN 16
+#define	ESF_DZ_TX_U_SOFT16_B0R1_WIDTH 16
+#define	ESF_DZ_TX_U_BIU_ARGS_LBN 16
+#define	ESF_DZ_TX_U_BIU_ARGS_WIDTH 13
+#define	ESF_DZ_TX_U_EV_QID_LBN 5
+#define	ESF_DZ_TX_U_EV_QID_WIDTH 11
+#define	ESF_DZ_TX_U_SOFT16_B0R0_LBN 0
+#define	ESF_DZ_TX_U_SOFT16_B0R0_WIDTH 16
+#define	ESF_DZ_TX_U_EV_QLABEL_LBN 0
+#define	ESF_DZ_TX_U_EV_QLABEL_WIDTH 5
+
+
+/* ES_TX_U_QSTATE_TBL1_ENTRY */
+#define	ESF_DZ_TX_U_DSCR_BASE_PAGE_ID_LBN 64
+#define	ESF_DZ_TX_U_DSCR_BASE_PAGE_ID_WIDTH 18
+#define	ESF_DZ_TX_U_SOFT18_B1R0_LBN 64
+#define	ESF_DZ_TX_U_SOFT18_B1R0_WIDTH 18
+#define	ESF_DZ_TX_U_SOFT16_B0R3_LBN 48
+#define	ESF_DZ_TX_U_SOFT16_B0R3_WIDTH 16
+#define	ESF_DZ_TX_U_QUEUE_ENABLED_LBN 49
+#define	ESF_DZ_TX_U_QUEUE_ENABLED_WIDTH 1
+#define	ESF_DZ_TX_U_FLUSH_PENDING_LBN 48
+#define	ESF_DZ_TX_U_FLUSH_PENDING_WIDTH 1
+#define	ESF_DZ_TX_U_DSCR_HW_WPTR_LBN 32
+#define	ESF_DZ_TX_U_DSCR_HW_WPTR_WIDTH 12
+#define	ESF_DZ_TX_U_SOFT12_B0R2_LBN 32
+#define	ESF_DZ_TX_U_SOFT12_B0R2_WIDTH 12
+#define	ESF_DZ_TX_U_OWNER_ID_LBN 16
+#define	ESF_DZ_TX_U_OWNER_ID_WIDTH 12
+#define	ESF_DZ_TX_U_SOFT12_B0R1_LBN 16
+#define	ESF_DZ_TX_U_SOFT12_B0R1_WIDTH 12
+#define	ESF_DZ_TX_U_DSCR_SIZE_LBN 0
+#define	ESF_DZ_TX_U_DSCR_SIZE_WIDTH 3
+#define	ESF_DZ_TX_U_SOFT3_B0R0_LBN 0
+#define	ESF_DZ_TX_U_SOFT3_B0R0_WIDTH 3
+
+
+/* ES_TX_U_QSTATE_TBL2_ENTRY */
+#define	ESF_DZ_TX_FINFO_WRD3_LBN 48
+#define	ESF_DZ_TX_FINFO_WRD3_WIDTH 16
+#define	ESF_DZ_TX_FINFO_WRD2_LBN 32
+#define	ESF_DZ_TX_FINFO_WRD2_WIDTH 16
+#define	ESF_DZ_TX_FINFO_WRD1_LBN 16
+#define	ESF_DZ_TX_FINFO_WRD1_WIDTH 16
+#define	ESF_DZ_TX_FINFO_SRCDST_LBN 0
+#define	ESF_DZ_TX_FINFO_SRCDST_WIDTH 16
+
+
+/* ES_b2t_cpl_rsp */
+#define	ESF_DZ_B2T_CPL_RSP_CPL_ECC_LBN 268
+#define	ESF_DZ_B2T_CPL_RSP_CPL_ECC_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW0_LBN 27
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW0_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW1_LBN 59
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW1_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW2_LBN 91
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW2_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW3_LBN 123
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW3_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW4_LBN 155
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW4_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW5_LBN 187
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW5_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW6_LBN 219
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW6_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW7_LBN 251
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_DW7_WIDTH 32
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_LBN 27
+#define	ESF_DZ_B2T_CPL_RSP_CPL_DATA_WIDTH 256
+#define	ESF_DZ_B2T_CPL_RSP_CPL_EOT_LBN 283
+#define	ESF_DZ_B2T_CPL_RSP_CPL_EOT_WIDTH -15
+#define	ESF_DZ_B2T_CPL_RSP_CPL_ERROR_LBN 26
+#define	ESF_DZ_B2T_CPL_RSP_CPL_ERROR_WIDTH 1
+#define	ESF_DZ_B2T_CPL_RSP_CPL_LAST_LBN 25
+#define	ESF_DZ_B2T_CPL_RSP_CPL_LAST_WIDTH 1
+#define	ESF_DZ_B2T_CPL_RSP_CPL_TAG_LBN 19
+#define	ESF_DZ_B2T_CPL_RSP_CPL_TAG_WIDTH 6
+#define	ESF_DZ_B2T_CPL_RSP_CPL_LEN_LBN 7
+#define	ESF_DZ_B2T_CPL_RSP_CPL_LEN_WIDTH 12
+#define	ESF_DZ_B2T_CPL_RSP_CPL_ADRS_LBN 0
+#define	ESF_DZ_B2T_CPL_RSP_CPL_ADRS_WIDTH 7
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_EFX_EF10_REGS_H */
diff --git a/sys/dev/sfxge/common/efx_regs_mcdi.h b/sys/dev/sfxge/common/efx_regs_mcdi.h
new file mode 100644
index 0000000..720770c
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_regs_mcdi.h
@@ -0,0 +1,2786 @@
+/*-
+ * Copyright 2008-2011 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*! \cidoxg_firmware_mc_cmd */
+
+#ifndef _SIENA_MC_DRIVER_PCOL_H
+#define	_SIENA_MC_DRIVER_PCOL_H
+
+
+/* Values to be written into FMCR_CZ_RESET_STATE_REG to control boot. */
+/* Power-on reset state */
+#define MC_FW_STATE_POR (1)
+/* If this is set in MC_RESET_STATE_REG then it should be
+ * possible to jump into IMEM without loading code from flash. */
+#define MC_FW_WARM_BOOT_OK (2)
+/* The MC main image has started to boot. */
+#define MC_FW_STATE_BOOTING (4)
+/* The Scheduler has started. */
+#define MC_FW_STATE_SCHED (8)
+
+/* Values to be written to the per-port status dword in shared
+ * memory on reboot and assert */
+#define MC_STATUS_DWORD_REBOOT (0xb007b007)
+#define MC_STATUS_DWORD_ASSERT (0xdeaddead)
+
+/* The current version of the MCDI protocol.
+ *
+ * Note that the ROM burnt into the card only talks V0, so at the very
+ * least every driver must support version 0 and MCDI_PCOL_VERSION
+ */
+#ifdef WITH_MCDI_V2
+#define MCDI_PCOL_VERSION 2
+#else
+#define MCDI_PCOL_VERSION 1
+#endif
+
+/* Unused commands: 0x23, 0x27, 0x30, 0x31 */
+
+/* Unused commands: 0x23, 0x27, 0x30, 0x31 */
+
+/**
+ * MCDI version 1
+ *
+ * Each MCDI request starts with an MCDI_HEADER, which is a 32byte
+ * structure, filled in by the client.
+ *
+ *       0       7  8     16    20     22  23  24    31
+ *      | CODE | R | LEN | SEQ | Rsvd | E | R | XFLAGS |
+ *               |                      |   |
+ *               |                      |   \--- Response
+ *               |                      \------- Error
+ *               \------------------------------ Resync (always set)
+ *
+ * The client writes it's request into MC shared memory, and rings the
+ * doorbell. Each request is completed by either by the MC writting
+ * back into shared memory, or by writting out an event.
+ *
+ * All MCDI commands support completion by shared memory response. Each
+ * request may also contain additional data (accounted for by HEADER.LEN),
+ * and some response's may also contain additional data (again, accounted
+ * for by HEADER.LEN).
+ *
+ * Some MCDI commands support completion by event, in which any associated
+ * response data is included in the event.
+ *
+ * The protocol requires one response to be delivered for every request, a
+ * request should not be sent unless the response for the previous request
+ * has been received (either by polling shared memory, or by receiving
+ * an event).
+ */
+
+/** Request/Response structure */
+#define MCDI_HEADER_OFST 0
+#define MCDI_HEADER_CODE_LBN 0
+#define MCDI_HEADER_CODE_WIDTH 7
+#define MCDI_HEADER_RESYNC_LBN 7
+#define MCDI_HEADER_RESYNC_WIDTH 1
+#define MCDI_HEADER_DATALEN_LBN 8
+#define MCDI_HEADER_DATALEN_WIDTH 8
+#define MCDI_HEADER_SEQ_LBN 16
+#define MCDI_HEADER_RSVD_LBN 20
+#define MCDI_HEADER_RSVD_WIDTH 2
+#define MCDI_HEADER_SEQ_WIDTH 4
+#define MCDI_HEADER_ERROR_LBN 22
+#define MCDI_HEADER_ERROR_WIDTH 1
+#define MCDI_HEADER_RESPONSE_LBN 23
+#define MCDI_HEADER_RESPONSE_WIDTH 1
+#define MCDI_HEADER_XFLAGS_LBN 24
+#define MCDI_HEADER_XFLAGS_WIDTH 8
+/* Request response using event */
+#define MCDI_HEADER_XFLAGS_EVREQ 0x01
+
+/* Maximum number of payload bytes */
+#if MCDI_PCOL_VERSION == 1
+#define MCDI_CTL_SDU_LEN_MAX 0xfc
+#elif  MCDI_PCOL_VERSION == 2
+#define MCDI_CTL_SDU_LEN_MAX 0x400
+#endif
+
+/* The MC can generate events for two reasons:
+ *   - To complete a shared memory request if XFLAGS_EVREQ was set
+ *   - As a notification (link state, i2c event), controlled
+ *     via MC_CMD_LOG_CTRL
+ *
+ * Both events share a common structure:
+ *
+ *  0      32     33      36    44     52     60
+ * | Data | Cont | Level | Src | Code | Rsvd |
+ *           |
+ *           \ There is another event pending in this notification
+ *
+ * If Code==CMDDONE, then the fields are further interpreted as:
+ *
+ *   - LEVEL==INFO    Command succeded
+ *   - LEVEL==ERR     Command failed
+ *
+ *    0     8         16      24     32
+ *   | Seq | Datalen | Errno | Rsvd |
+ *
+ *   These fields are taken directly out of the standard MCDI header, i.e.,
+ *   LEVEL==ERR, Datalen == 0 => Reboot
+ *
+ * Events can be squirted out of the UART (using LOG_CTRL) without a
+ * MCDI header.  An event can be distinguished from a MCDI response by
+ * examining the first byte which is 0xc0.  This corresponds to the
+ * non-existent MCDI command MC_CMD_DEBUG_LOG.
+ *
+ *      0         7        8
+ *     | command | Resync |     = 0xc0
+ *
+ * Since the event is written in big-endian byte order, this works
+ * providing bits 56-63 of the event are 0xc0.
+ *
+ *      56     60  63
+ *     | Rsvd | Code |    = 0xc0
+ *
+ * Which means for convenience the event code is 0xc for all MC
+ * generated events.
+ */
+#define FSE_AZ_EV_CODE_MCDI_EVRESPONSE 0xc
+
+
+/* Non-existent command target */
+#define MC_CMD_ERR_ENOENT 2
+/* assert() has killed the MC */
+#define MC_CMD_ERR_EINTR 4
+/* Caller does not hold required locks */
+#define MC_CMD_ERR_EACCES 13
+/* Resource is currently unavailable (e.g. lock contention) */
+#define MC_CMD_ERR_EBUSY 16
+/* Invalid argument to target */
+#define MC_CMD_ERR_EINVAL 22
+/* Non-recursive resource is already acquired */
+#define MC_CMD_ERR_EDEADLK 35
+/* Operation not implemented */
+#define MC_CMD_ERR_ENOSYS 38
+/* Operation timed out */
+#define MC_CMD_ERR_ETIME 62
+
+#define MC_CMD_ERR_CODE_OFST 0
+
+/* We define 8 "escape" commands to allow
+   for command number space extension */
+
+#define MC_CMD_CMD_SPACE_ESCAPE_0	      0x78
+#define MC_CMD_CMD_SPACE_ESCAPE_1	      0x79
+#define MC_CMD_CMD_SPACE_ESCAPE_2	      0x7A
+#define MC_CMD_CMD_SPACE_ESCAPE_3	      0x7B
+#define MC_CMD_CMD_SPACE_ESCAPE_4	      0x7C
+#define MC_CMD_CMD_SPACE_ESCAPE_5	      0x7D
+#define MC_CMD_CMD_SPACE_ESCAPE_6	      0x7E
+#define MC_CMD_CMD_SPACE_ESCAPE_7	      0x7F
+
+/* Vectors in the boot ROM */
+/* Point to the copycode entry point. */
+#define MC_BOOTROM_COPYCODE_VEC (0x7f4)
+/* Points to the recovery mode entry point. */
+#define MC_BOOTROM_NOFLASH_VEC (0x7f8)
+
+/* The command set exported by the boot ROM (MCDI v0) */
+#define MC_CMD_GET_VERSION_V0_SUPPORTED_FUNCS {		\
+	(1 << MC_CMD_READ32)	|			\
+	(1 << MC_CMD_WRITE32)	|			\
+	(1 << MC_CMD_COPYCODE)	|			\
+	(1 << MC_CMD_GET_VERSION),			\
+	0, 0, 0 }
+
+#define MC_CMD_SENSOR_INFO_OUT_OFFSET_OFST(_x) \
+	(MC_CMD_SENSOR_ENTRY_OFST + (_x))
+
+#define MC_CMD_DBI_WRITE_IN_ADDRESS_OFST(n) (  \
+        (MC_CMD_DBI_WRITE_IN_DBIWROP_OFST+     \
+         MC_CMD_DBIWROP_TYPEDEF_ADDRESS_OFST)+ \
+         ((n)*MC_CMD_DBIWROP_TYPEDEF_LEN))
+
+#define MC_CMD_DBI_WRITE_IN_BYTE_MASK_OFST(n) (  \
+        (MC_CMD_DBI_WRITE_IN_DBIWROP_OFST+     \
+         MC_CMD_DBIWROP_TYPEDEF_BYTE_MASK_OFST)+ \
+         ((n)*MC_CMD_DBIWROP_TYPEDEF_LEN))
+
+#define MC_CMD_DBI_WRITE_IN_VALUE_OFST(n) (  \
+        (MC_CMD_DBI_WRITE_IN_DBIWROP_OFST+     \
+         MC_CMD_DBIWROP_TYPEDEF_VALUE_OFST)+ \
+         ((n)*MC_CMD_DBIWROP_TYPEDEF_LEN))
+
+
+#ifdef WITH_MCDI_V2
+
+/* Version 2 adds an optional argument to error returns: the errno value
+ * may be followed by the (0-based) number of the first argument that
+ * could not be processed.
+ */
+#define MC_CMD_ERR_ARG_OFST 4
+
+/* Try again */
+#define MC_CMD_ERR_EAGAIN 11
+/* No space */
+#define MC_CMD_ERR_ENOSPC 28
+
+#endif
+
+/* MCDI_EVENT structuredef */
+#define	MCDI_EVENT_LEN 8
+#define	MCDI_EVENT_CONT_LBN 32
+#define	MCDI_EVENT_CONT_WIDTH 1
+#define	MCDI_EVENT_LEVEL_LBN 33
+#define	MCDI_EVENT_LEVEL_WIDTH 3
+#define	MCDI_EVENT_LEVEL_INFO  0x0 /* enum */
+#define	MCDI_EVENT_LEVEL_WARN 0x1 /* enum */
+#define	MCDI_EVENT_LEVEL_ERR 0x2 /* enum */
+#define	MCDI_EVENT_LEVEL_FATAL 0x3 /* enum */
+#define	MCDI_EVENT_DATA_OFST 0
+#define	MCDI_EVENT_CMDDONE_SEQ_LBN 0
+#define	MCDI_EVENT_CMDDONE_SEQ_WIDTH 8
+#define	MCDI_EVENT_CMDDONE_DATALEN_LBN 8
+#define	MCDI_EVENT_CMDDONE_DATALEN_WIDTH 8
+#define	MCDI_EVENT_CMDDONE_ERRNO_LBN 16
+#define	MCDI_EVENT_CMDDONE_ERRNO_WIDTH 8
+#define	MCDI_EVENT_LINKCHANGE_LP_CAP_LBN 0
+#define	MCDI_EVENT_LINKCHANGE_LP_CAP_WIDTH 16
+#define	MCDI_EVENT_LINKCHANGE_SPEED_LBN 16
+#define	MCDI_EVENT_LINKCHANGE_SPEED_WIDTH 4
+#define	MCDI_EVENT_LINKCHANGE_SPEED_100M  0x1 /* enum */
+#define	MCDI_EVENT_LINKCHANGE_SPEED_1G  0x2 /* enum */
+#define	MCDI_EVENT_LINKCHANGE_SPEED_10G  0x3 /* enum */
+#define	MCDI_EVENT_LINKCHANGE_FCNTL_LBN 20
+#define	MCDI_EVENT_LINKCHANGE_FCNTL_WIDTH 4
+#define	MCDI_EVENT_LINKCHANGE_LINK_FLAGS_LBN 24
+#define	MCDI_EVENT_LINKCHANGE_LINK_FLAGS_WIDTH 8
+#define	MCDI_EVENT_SENSOREVT_MONITOR_LBN 0
+#define	MCDI_EVENT_SENSOREVT_MONITOR_WIDTH 8
+#define	MCDI_EVENT_SENSOREVT_STATE_LBN 8
+#define	MCDI_EVENT_SENSOREVT_STATE_WIDTH 8
+#define	MCDI_EVENT_SENSOREVT_VALUE_LBN 16
+#define	MCDI_EVENT_SENSOREVT_VALUE_WIDTH 16
+#define	MCDI_EVENT_FWALERT_DATA_LBN 8
+#define	MCDI_EVENT_FWALERT_DATA_WIDTH 24
+#define	MCDI_EVENT_FWALERT_REASON_LBN 0
+#define	MCDI_EVENT_FWALERT_REASON_WIDTH 8
+#define	MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS 0x1 /* enum */
+#define	MCDI_EVENT_FLR_VF_LBN 0
+#define	MCDI_EVENT_FLR_VF_WIDTH 8
+#define	MCDI_EVENT_TX_ERR_TXQ_LBN 0
+#define	MCDI_EVENT_TX_ERR_TXQ_WIDTH 12
+#define	MCDI_EVENT_TX_ERR_TYPE_LBN 12
+#define	MCDI_EVENT_TX_ERR_TYPE_WIDTH 4
+#define	MCDI_EVENT_TX_ERR_DL_FAIL 0x1 /* enum */
+#define	MCDI_EVENT_TX_ERR_NO_EOP 0x2 /* enum */
+#define	MCDI_EVENT_TX_ERR_2BIG 0x3 /* enum */
+#define	MCDI_EVENT_TX_ERR_INFO_LBN 16
+#define	MCDI_EVENT_TX_ERR_INFO_WIDTH 16
+#define	MCDI_EVENT_TX_FLUSH_TXQ_LBN 0
+#define	MCDI_EVENT_TX_FLUSH_TXQ_WIDTH 12
+#define	MCDI_EVENT_DATA_LBN 0
+#define	MCDI_EVENT_DATA_WIDTH 32
+#define	MCDI_EVENT_SRC_LBN 36
+#define	MCDI_EVENT_SRC_WIDTH 8
+#define	MCDI_EVENT_EV_CODE_LBN 60
+#define	MCDI_EVENT_EV_CODE_WIDTH 4
+#define	MCDI_EVENT_CODE_LBN 44
+#define	MCDI_EVENT_CODE_WIDTH 8
+#define	MCDI_EVENT_CODE_BADSSERT 0x1 /* enum */
+#define	MCDI_EVENT_CODE_PMNOTICE 0x2 /* enum */
+#define	MCDI_EVENT_CODE_CMDDONE 0x3 /* enum */
+#define	MCDI_EVENT_CODE_LINKCHANGE 0x4 /* enum */
+#define	MCDI_EVENT_CODE_SENSOREVT 0x5 /* enum */
+#define	MCDI_EVENT_CODE_SCHEDERR 0x6 /* enum */
+#define	MCDI_EVENT_CODE_REBOOT 0x7 /* enum */
+#define	MCDI_EVENT_CODE_MAC_STATS_DMA 0x8 /* enum */
+#define	MCDI_EVENT_CODE_FWALERT 0x9 /* enum */
+#define	MCDI_EVENT_CODE_FLR 0xa /* enum */
+#define	MCDI_EVENT_CODE_TX_ERR 0xb /* enum */
+#define	MCDI_EVENT_CODE_TX_FLUSH  0xc /* enum */
+#define	MCDI_EVENT_CMDDONE_DATA_OFST 0
+#define	MCDI_EVENT_CMDDONE_DATA_LBN 0
+#define	MCDI_EVENT_CMDDONE_DATA_WIDTH 32
+#define	MCDI_EVENT_LINKCHANGE_DATA_OFST 0
+#define	MCDI_EVENT_LINKCHANGE_DATA_LBN 0
+#define	MCDI_EVENT_LINKCHANGE_DATA_WIDTH 32
+#define	MCDI_EVENT_SENSOREVT_DATA_OFST 0
+#define	MCDI_EVENT_SENSOREVT_DATA_LBN 0
+#define	MCDI_EVENT_SENSOREVT_DATA_WIDTH 32
+#define	MCDI_EVENT_MAC_STATS_DMA_GENERATION_OFST 0
+#define	MCDI_EVENT_MAC_STATS_DMA_GENERATION_LBN 0
+#define	MCDI_EVENT_MAC_STATS_DMA_GENERATION_WIDTH 32
+#define	MCDI_EVENT_TX_ERR_DATA_OFST 0
+#define	MCDI_EVENT_TX_ERR_DATA_LBN 0
+#define	MCDI_EVENT_TX_ERR_DATA_WIDTH 32
+
+
+/***********************************/
+/* MC_CMD_READ32
+ * Read multiple 32byte words from MC memory.
+ */
+#define	MC_CMD_READ32 0x1
+
+/* MC_CMD_READ32_IN msgrequest */
+#define	MC_CMD_READ32_IN_LEN 8
+#define	MC_CMD_READ32_IN_ADDR_OFST 0
+#define	MC_CMD_READ32_IN_NUMWORDS_OFST 4
+
+/* MC_CMD_READ32_OUT msgresponse */
+#define	MC_CMD_READ32_OUT_LENMIN 4
+#define	MC_CMD_READ32_OUT_LENMAX 252
+#define	MC_CMD_READ32_OUT_LEN(num) (0+4*(num))
+#define	MC_CMD_READ32_OUT_BUFFER_OFST 0
+#define	MC_CMD_READ32_OUT_BUFFER_LEN 4
+#define	MC_CMD_READ32_OUT_BUFFER_MINNUM 1
+#define	MC_CMD_READ32_OUT_BUFFER_MAXNUM 63
+
+
+/***********************************/
+/* MC_CMD_WRITE32
+ * Write multiple 32byte words to MC memory.
+ */
+#define	MC_CMD_WRITE32 0x2
+
+/* MC_CMD_WRITE32_IN msgrequest */
+#define	MC_CMD_WRITE32_IN_LENMIN 8
+#define	MC_CMD_WRITE32_IN_LENMAX 252
+#define	MC_CMD_WRITE32_IN_LEN(num) (4+4*(num))
+#define	MC_CMD_WRITE32_IN_ADDR_OFST 0
+#define	MC_CMD_WRITE32_IN_BUFFER_OFST 4
+#define	MC_CMD_WRITE32_IN_BUFFER_LEN 4
+#define	MC_CMD_WRITE32_IN_BUFFER_MINNUM 1
+#define	MC_CMD_WRITE32_IN_BUFFER_MAXNUM 62
+
+/* MC_CMD_WRITE32_OUT msgresponse */
+#define	MC_CMD_WRITE32_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_COPYCODE
+ * Copy MC code between two locations and jump.
+ */
+#define	MC_CMD_COPYCODE 0x3
+
+/* MC_CMD_COPYCODE_IN msgrequest */
+#define	MC_CMD_COPYCODE_IN_LEN 16
+#define	MC_CMD_COPYCODE_IN_SRC_ADDR_OFST 0
+#define	MC_CMD_COPYCODE_IN_DEST_ADDR_OFST 4
+#define	MC_CMD_COPYCODE_IN_NUMWORDS_OFST 8
+#define	MC_CMD_COPYCODE_IN_JUMP_OFST 12
+#define	MC_CMD_COPYCODE_JUMP_NONE 0x1 /* enum */
+
+/* MC_CMD_COPYCODE_OUT msgresponse */
+#define	MC_CMD_COPYCODE_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SET_FUNC 
+ */
+#define	MC_CMD_SET_FUNC  0x4
+
+/* MC_CMD_SET_FUNC_IN msgrequest */
+#define	MC_CMD_SET_FUNC_IN_LEN 4
+#define	MC_CMD_SET_FUNC_IN_FUNC_OFST 0
+
+/* MC_CMD_SET_FUNC_OUT msgresponse */
+#define	MC_CMD_SET_FUNC_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_BOOT_STATUS
+ */
+#define	MC_CMD_GET_BOOT_STATUS 0x5
+
+/* MC_CMD_GET_BOOT_STATUS_IN msgrequest */
+#define	MC_CMD_GET_BOOT_STATUS_IN_LEN 0
+
+/* MC_CMD_GET_BOOT_STATUS_OUT msgresponse */
+#define	MC_CMD_GET_BOOT_STATUS_OUT_LEN 8
+#define	MC_CMD_GET_BOOT_STATUS_OUT_BOOT_OFFSET_OFST 0
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_OFST 4
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_WATCHDOG_LBN 0
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_WATCHDOG_WIDTH 1
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_PRIMARY_LBN 1
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_PRIMARY_WIDTH 1
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_BACKUP_LBN 2
+#define	MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_BACKUP_WIDTH 1
+
+
+/***********************************/
+/* MC_CMD_GET_ASSERTS 
+ * Get and clear any assertion status.
+ */
+#define	MC_CMD_GET_ASSERTS  0x6
+
+/* MC_CMD_GET_ASSERTS_IN msgrequest */
+#define	MC_CMD_GET_ASSERTS_IN_LEN 4
+#define	MC_CMD_GET_ASSERTS_IN_CLEAR_OFST 0
+
+/* MC_CMD_GET_ASSERTS_OUT msgresponse */
+#define	MC_CMD_GET_ASSERTS_OUT_LEN 140
+#define	MC_CMD_GET_ASSERTS_OUT_GLOBAL_FLAGS_OFST 0
+#define	MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS 0x1 /* enum */
+#define	MC_CMD_GET_ASSERTS_FLAGS_SYS_FAIL 0x2 /* enum */
+#define	MC_CMD_GET_ASSERTS_FLAGS_THR_FAIL 0x3 /* enum */
+#define	MC_CMD_GET_ASSERTS_FLAGS_WDOG_FIRED 0x4 /* enum */
+#define	MC_CMD_GET_ASSERTS_OUT_SAVED_PC_OFFS_OFST 4
+#define	MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST 8
+#define	MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_LEN 4
+#define	MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_NUM 31
+#define	MC_CMD_GET_ASSERTS_OUT_THREAD_OFFS_OFST 132
+#define	MC_CMD_GET_ASSERTS_OUT_RESERVED_OFST 136
+
+
+/***********************************/
+/* MC_CMD_LOG_CTRL 
+ * Configure the output stream for various events and messages.
+ */
+#define	MC_CMD_LOG_CTRL  0x7
+
+/* MC_CMD_LOG_CTRL_IN msgrequest */
+#define	MC_CMD_LOG_CTRL_IN_LEN 8
+#define	MC_CMD_LOG_CTRL_IN_LOG_DEST_OFST 0
+#define	MC_CMD_LOG_CTRL_IN_LOG_DEST_UART 0x1 /* enum */
+#define	MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ 0x2 /* enum */
+#define	MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ_OFST 4
+
+/* MC_CMD_LOG_CTRL_OUT msgresponse */
+#define	MC_CMD_LOG_CTRL_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_VERSION 
+ * Get version information about the MC firmware.
+ */
+#define	MC_CMD_GET_VERSION  0x8
+
+/* MC_CMD_GET_VERSION_IN msgrequest */
+#define	MC_CMD_GET_VERSION_IN_LEN 0
+
+/* MC_CMD_GET_VERSION_V0_OUT msgresponse */
+#define	MC_CMD_GET_VERSION_V0_OUT_LEN 4
+#define	MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0
+#define	MC_CMD_GET_VERSION_OUT_FIRMWARE_ANY 0xffffffff /* enum */
+#define	MC_CMD_GET_VERSION_OUT_FIRMWARE_BOOTROM 0xb0070000 /* enum */
+
+/* MC_CMD_GET_VERSION_OUT msgresponse */
+#define	MC_CMD_GET_VERSION_OUT_LEN 32
+/*            MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */
+/*            Enum values, see field(s): */
+/*               MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */
+#define	MC_CMD_GET_VERSION_OUT_PCOL_OFST 4
+#define	MC_CMD_GET_VERSION_OUT_SUPPORTED_FUNCS_OFST 8
+#define	MC_CMD_GET_VERSION_OUT_SUPPORTED_FUNCS_LEN 16
+#define	MC_CMD_GET_VERSION_OUT_VERSION_OFST 24
+#define	MC_CMD_GET_VERSION_OUT_VERSION_LEN 8
+#define	MC_CMD_GET_VERSION_OUT_VERSION_LO_OFST 24
+#define	MC_CMD_GET_VERSION_OUT_VERSION_HI_OFST 28
+
+
+/***********************************/
+/* MC_CMD_CSR_READ32 
+ * Read 32bit words from the indirect memory map.
+ */
+#define	MC_CMD_CSR_READ32  0xc
+
+/* MC_CMD_CSR_READ32_IN msgrequest */
+#define	MC_CMD_CSR_READ32_IN_LEN 12
+#define	MC_CMD_CSR_READ32_IN_ADDR_OFST 0
+#define	MC_CMD_CSR_READ32_IN_STEP_OFST 4
+#define	MC_CMD_CSR_READ32_IN_NUMWORDS_OFST 8
+
+/* MC_CMD_CSR_READ32_OUT msgresponse */
+#define	MC_CMD_CSR_READ32_OUT_LENMIN 4
+#define	MC_CMD_CSR_READ32_OUT_LENMAX 252
+#define	MC_CMD_CSR_READ32_OUT_LEN(num) (0+4*(num))
+#define	MC_CMD_CSR_READ32_OUT_BUFFER_OFST 0
+#define	MC_CMD_CSR_READ32_OUT_BUFFER_LEN 4
+#define	MC_CMD_CSR_READ32_OUT_BUFFER_MINNUM 1
+#define	MC_CMD_CSR_READ32_OUT_BUFFER_MAXNUM 63
+
+
+/***********************************/
+/* MC_CMD_CSR_WRITE32 
+ * Write 32bit dwords to the indirect memory map.
+ */
+#define	MC_CMD_CSR_WRITE32  0xd
+
+/* MC_CMD_CSR_WRITE32_IN msgrequest */
+#define	MC_CMD_CSR_WRITE32_IN_LENMIN 12
+#define	MC_CMD_CSR_WRITE32_IN_LENMAX 252
+#define	MC_CMD_CSR_WRITE32_IN_LEN(num) (8+4*(num))
+#define	MC_CMD_CSR_WRITE32_IN_ADDR_OFST 0
+#define	MC_CMD_CSR_WRITE32_IN_STEP_OFST 4
+#define	MC_CMD_CSR_WRITE32_IN_BUFFER_OFST 8
+#define	MC_CMD_CSR_WRITE32_IN_BUFFER_LEN 4
+#define	MC_CMD_CSR_WRITE32_IN_BUFFER_MINNUM 1
+#define	MC_CMD_CSR_WRITE32_IN_BUFFER_MAXNUM 61
+
+/* MC_CMD_CSR_WRITE32_OUT msgresponse */
+#define	MC_CMD_CSR_WRITE32_OUT_LEN 4
+#define	MC_CMD_CSR_WRITE32_OUT_STATUS_OFST 0
+
+
+/***********************************/
+/* MC_CMD_STACKINFO 
+ * Get stack information.
+ */
+#define	MC_CMD_STACKINFO  0xf
+
+/* MC_CMD_STACKINFO_IN msgrequest */
+#define	MC_CMD_STACKINFO_IN_LEN 0
+
+/* MC_CMD_STACKINFO_OUT msgresponse */
+#define	MC_CMD_STACKINFO_OUT_LENMIN 12
+#define	MC_CMD_STACKINFO_OUT_LENMAX 252
+#define	MC_CMD_STACKINFO_OUT_LEN(num) (0+12*(num))
+#define	MC_CMD_STACKINFO_OUT_THREAD_INFO_OFST 0
+#define	MC_CMD_STACKINFO_OUT_THREAD_INFO_LEN 12
+#define	MC_CMD_STACKINFO_OUT_THREAD_INFO_MINNUM 1
+#define	MC_CMD_STACKINFO_OUT_THREAD_INFO_MAXNUM 21
+
+
+/***********************************/
+/* MC_CMD_MDIO_READ 
+ * MDIO register read.
+ */
+#define	MC_CMD_MDIO_READ  0x10
+
+/* MC_CMD_MDIO_READ_IN msgrequest */
+#define	MC_CMD_MDIO_READ_IN_LEN 16
+#define	MC_CMD_MDIO_READ_IN_BUS_OFST 0
+#define	MC_CMD_MDIO_BUS_INTERNAL 0x0 /* enum */
+#define	MC_CMD_MDIO_BUS_EXTERNAL 0x1 /* enum */
+#define	MC_CMD_MDIO_READ_IN_PRTAD_OFST 4
+#define	MC_CMD_MDIO_READ_IN_DEVAD_OFST 8
+#define	MC_CMD_MDIO_CLAUSE22 0x20 /* enum */
+#define	MC_CMD_MDIO_READ_IN_ADDR_OFST 12
+
+/* MC_CMD_MDIO_READ_OUT msgresponse */
+#define	MC_CMD_MDIO_READ_OUT_LEN 8
+#define	MC_CMD_MDIO_READ_OUT_VALUE_OFST 0
+#define	MC_CMD_MDIO_READ_OUT_STATUS_OFST 4
+#define	MC_CMD_MDIO_STATUS_GOOD 0x8 /* enum */
+
+
+/***********************************/
+/* MC_CMD_MDIO_WRITE 
+ * MDIO register write.
+ */
+#define	MC_CMD_MDIO_WRITE  0x11
+
+/* MC_CMD_MDIO_WRITE_IN msgrequest */
+#define	MC_CMD_MDIO_WRITE_IN_LEN 20
+#define	MC_CMD_MDIO_WRITE_IN_BUS_OFST 0
+/*               MC_CMD_MDIO_BUS_INTERNAL 0x0 */
+/*               MC_CMD_MDIO_BUS_EXTERNAL 0x1 */
+#define	MC_CMD_MDIO_WRITE_IN_PRTAD_OFST 4
+#define	MC_CMD_MDIO_WRITE_IN_DEVAD_OFST 8
+/*               MC_CMD_MDIO_CLAUSE22 0x20 */
+#define	MC_CMD_MDIO_WRITE_IN_ADDR_OFST 12
+#define	MC_CMD_MDIO_WRITE_IN_VALUE_OFST 16
+
+/* MC_CMD_MDIO_WRITE_OUT msgresponse */
+#define	MC_CMD_MDIO_WRITE_OUT_LEN 4
+#define	MC_CMD_MDIO_WRITE_OUT_STATUS_OFST 0
+/*               MC_CMD_MDIO_STATUS_GOOD 0x8 */
+
+
+/***********************************/
+/* MC_CMD_DBI_WRITE 
+ * Write DBI register(s).
+ */
+#define	MC_CMD_DBI_WRITE  0x12
+
+/* MC_CMD_DBI_WRITE_IN msgrequest */
+#define	MC_CMD_DBI_WRITE_IN_LENMIN 12
+#define	MC_CMD_DBI_WRITE_IN_LENMAX 252
+#define	MC_CMD_DBI_WRITE_IN_LEN(num) (0+12*(num))
+#define	MC_CMD_DBI_WRITE_IN_DBIWROP_OFST 0
+#define	MC_CMD_DBI_WRITE_IN_DBIWROP_LEN 12
+#define	MC_CMD_DBI_WRITE_IN_DBIWROP_MINNUM 1
+#define	MC_CMD_DBI_WRITE_IN_DBIWROP_MAXNUM 21
+
+/* MC_CMD_DBI_WRITE_OUT msgresponse */
+#define	MC_CMD_DBI_WRITE_OUT_LEN 0
+
+/* MC_CMD_DBIWROP_TYPEDEF structuredef */
+#define	MC_CMD_DBIWROP_TYPEDEF_LEN 12
+#define	MC_CMD_DBIWROP_TYPEDEF_ADDRESS_OFST 0
+#define	MC_CMD_DBIWROP_TYPEDEF_ADDRESS_LBN 0
+#define	MC_CMD_DBIWROP_TYPEDEF_ADDRESS_WIDTH 32
+#define	MC_CMD_DBIWROP_TYPEDEF_BYTE_MASK_OFST 4
+#define	MC_CMD_DBIWROP_TYPEDEF_BYTE_MASK_LBN 32
+#define	MC_CMD_DBIWROP_TYPEDEF_BYTE_MASK_WIDTH 32
+#define	MC_CMD_DBIWROP_TYPEDEF_VALUE_OFST 8
+#define	MC_CMD_DBIWROP_TYPEDEF_VALUE_LBN 64
+#define	MC_CMD_DBIWROP_TYPEDEF_VALUE_WIDTH 32
+
+
+/***********************************/
+/* MC_CMD_PORT_READ32 
+ * Read a 32-bit register from the indirect port register map.
+ */
+#define	MC_CMD_PORT_READ32  0x14
+
+/* MC_CMD_PORT_READ32_IN msgrequest */
+#define	MC_CMD_PORT_READ32_IN_LEN 4
+#define	MC_CMD_PORT_READ32_IN_ADDR_OFST 0
+
+/* MC_CMD_PORT_READ32_OUT msgresponse */
+#define	MC_CMD_PORT_READ32_OUT_LEN 8
+#define	MC_CMD_PORT_READ32_OUT_VALUE_OFST 0
+#define	MC_CMD_PORT_READ32_OUT_STATUS_OFST 4
+
+
+/***********************************/
+/* MC_CMD_PORT_WRITE32 
+ * Write a 32-bit register to the indirect port register map.
+ */
+#define	MC_CMD_PORT_WRITE32  0x15
+
+/* MC_CMD_PORT_WRITE32_IN msgrequest */
+#define	MC_CMD_PORT_WRITE32_IN_LEN 8
+#define	MC_CMD_PORT_WRITE32_IN_ADDR_OFST 0
+#define	MC_CMD_PORT_WRITE32_IN_VALUE_OFST 4
+
+/* MC_CMD_PORT_WRITE32_OUT msgresponse */
+#define	MC_CMD_PORT_WRITE32_OUT_LEN 4
+#define	MC_CMD_PORT_WRITE32_OUT_STATUS_OFST 0
+
+
+/***********************************/
+/* MC_CMD_PORT_READ128 
+ * Read a 128-bit register from the indirect port register map.
+ */
+#define	MC_CMD_PORT_READ128  0x16
+
+/* MC_CMD_PORT_READ128_IN msgrequest */
+#define	MC_CMD_PORT_READ128_IN_LEN 4
+#define	MC_CMD_PORT_READ128_IN_ADDR_OFST 0
+
+/* MC_CMD_PORT_READ128_OUT msgresponse */
+#define	MC_CMD_PORT_READ128_OUT_LEN 20
+#define	MC_CMD_PORT_READ128_OUT_VALUE_OFST 0
+#define	MC_CMD_PORT_READ128_OUT_VALUE_LEN 16
+#define	MC_CMD_PORT_READ128_OUT_STATUS_OFST 16
+
+
+/***********************************/
+/* MC_CMD_PORT_WRITE128 
+ * Write a 128-bit register to the indirect port register map.
+ */
+#define	MC_CMD_PORT_WRITE128  0x17
+
+/* MC_CMD_PORT_WRITE128_IN msgrequest */
+#define	MC_CMD_PORT_WRITE128_IN_LEN 20
+#define	MC_CMD_PORT_WRITE128_IN_ADDR_OFST 0
+#define	MC_CMD_PORT_WRITE128_IN_VALUE_OFST 4
+#define	MC_CMD_PORT_WRITE128_IN_VALUE_LEN 16
+
+/* MC_CMD_PORT_WRITE128_OUT msgresponse */
+#define	MC_CMD_PORT_WRITE128_OUT_LEN 4
+#define	MC_CMD_PORT_WRITE128_OUT_STATUS_OFST 0
+
+
+/***********************************/
+/* MC_CMD_GET_BOARD_CFG 
+ * Returns the MC firmware configuration structure.
+ */
+#define	MC_CMD_GET_BOARD_CFG  0x18
+
+/* MC_CMD_GET_BOARD_CFG_IN msgrequest */
+#define	MC_CMD_GET_BOARD_CFG_IN_LEN 0
+
+/* MC_CMD_GET_BOARD_CFG_OUT msgresponse */
+#define	MC_CMD_GET_BOARD_CFG_OUT_LEN 96
+#define	MC_CMD_GET_BOARD_CFG_OUT_BOARD_TYPE_OFST 0
+#define	MC_CMD_GET_BOARD_CFG_OUT_BOARD_NAME_OFST 4
+#define	MC_CMD_GET_BOARD_CFG_OUT_BOARD_NAME_LEN 32
+#define	MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT0_OFST 36
+#define	MC_CMD_CAPABILITIES_SMALL_BUF_TBL_LBN 0x0 /* enum */
+#define	MC_CMD_CAPABILITIES_SMALL_BUF_TBL_WIDTH 0x1 /* enum */
+#define	MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT1_OFST 40
+/*            Enum values, see field(s): */
+/*               CAPABILITIES_PORT0 */
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_OFST 44
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_LEN 6
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_OFST 50
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_LEN 6
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT0_OFST 56
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT1_OFST 60
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT0_OFST 64
+#define	MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT1_OFST 68
+#define	MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_OFST 72
+#define	MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_LEN 2
+#define	MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_NUM 12
+
+
+/***********************************/
+/* MC_CMD_DBI_READX 
+ * Read DBI register(s).
+ */
+#define	MC_CMD_DBI_READX  0x19
+
+/* MC_CMD_DBI_READX_IN msgrequest */
+#define	MC_CMD_DBI_READX_IN_LENMIN 8
+#define	MC_CMD_DBI_READX_IN_LENMAX 248
+#define	MC_CMD_DBI_READX_IN_LEN(num) (0+8*(num))
+#define	MC_CMD_DBI_READX_IN_DBIRDOP_OFST 0
+#define	MC_CMD_DBI_READX_IN_DBIRDOP_LEN 8
+#define	MC_CMD_DBI_READX_IN_DBIRDOP_LO_OFST 0
+#define	MC_CMD_DBI_READX_IN_DBIRDOP_HI_OFST 4
+#define	MC_CMD_DBI_READX_IN_DBIRDOP_MINNUM 1
+#define	MC_CMD_DBI_READX_IN_DBIRDOP_MAXNUM 31
+
+/* MC_CMD_DBI_READX_OUT msgresponse */
+#define	MC_CMD_DBI_READX_OUT_LENMIN 4
+#define	MC_CMD_DBI_READX_OUT_LENMAX 252
+#define	MC_CMD_DBI_READX_OUT_LEN(num) (0+4*(num))
+#define	MC_CMD_DBI_READX_OUT_VALUE_OFST 0
+#define	MC_CMD_DBI_READX_OUT_VALUE_LEN 4
+#define	MC_CMD_DBI_READX_OUT_VALUE_MINNUM 1
+#define	MC_CMD_DBI_READX_OUT_VALUE_MAXNUM 63
+
+
+/***********************************/
+/* MC_CMD_SET_RAND_SEED 
+ * Set the 16byte seed for the MC psuedo-random generator.
+ */
+#define	MC_CMD_SET_RAND_SEED  0x1a
+
+/* MC_CMD_SET_RAND_SEED_IN msgrequest */
+#define	MC_CMD_SET_RAND_SEED_IN_LEN 16
+#define	MC_CMD_SET_RAND_SEED_IN_SEED_OFST 0
+#define	MC_CMD_SET_RAND_SEED_IN_SEED_LEN 16
+
+/* MC_CMD_SET_RAND_SEED_OUT msgresponse */
+#define	MC_CMD_SET_RAND_SEED_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_LTSSM_HIST 
+ * Retrieve the history of the PCIE LTSSM.
+ */
+#define	MC_CMD_LTSSM_HIST  0x1b
+
+/* MC_CMD_LTSSM_HIST_IN msgrequest */
+#define	MC_CMD_LTSSM_HIST_IN_LEN 0
+
+/* MC_CMD_LTSSM_HIST_OUT msgresponse */
+#define	MC_CMD_LTSSM_HIST_OUT_LENMIN 0
+#define	MC_CMD_LTSSM_HIST_OUT_LENMAX 252
+#define	MC_CMD_LTSSM_HIST_OUT_LEN(num) (0+4*(num))
+#define	MC_CMD_LTSSM_HIST_OUT_DATA_OFST 0
+#define	MC_CMD_LTSSM_HIST_OUT_DATA_LEN 4
+#define	MC_CMD_LTSSM_HIST_OUT_DATA_MINNUM 0
+#define	MC_CMD_LTSSM_HIST_OUT_DATA_MAXNUM 63
+
+
+/***********************************/
+/* MC_CMD_DRV_ATTACH 
+ * Inform MCPU that this port is managed on the host.
+ */
+#define	MC_CMD_DRV_ATTACH  0x1c
+
+/* MC_CMD_DRV_ATTACH_IN msgrequest */
+#define	MC_CMD_DRV_ATTACH_IN_LEN 8
+#define	MC_CMD_DRV_ATTACH_IN_NEW_STATE_OFST 0
+#define	MC_CMD_DRV_ATTACH_IN_UPDATE_OFST 4
+
+/* MC_CMD_DRV_ATTACH_OUT msgresponse */
+#define	MC_CMD_DRV_ATTACH_OUT_LEN 4
+#define	MC_CMD_DRV_ATTACH_OUT_OLD_STATE_OFST 0
+
+
+/***********************************/
+/* MC_CMD_NCSI_PROD 
+ * Trigger an NC-SI event.
+ */
+#define	MC_CMD_NCSI_PROD  0x1d
+
+/* MC_CMD_NCSI_PROD_IN msgrequest */
+#define	MC_CMD_NCSI_PROD_IN_LEN 4
+#define	MC_CMD_NCSI_PROD_IN_EVENTS_OFST 0
+#define	MC_CMD_NCSI_PROD_LINKCHANGE 0x0 /* enum */
+#define	MC_CMD_NCSI_PROD_RESET 0x1 /* enum */
+#define	MC_CMD_NCSI_PROD_DRVATTACH 0x2 /* enum */
+#define	MC_CMD_NCSI_PROD_IN_LINKCHANGE_LBN 0
+#define	MC_CMD_NCSI_PROD_IN_LINKCHANGE_WIDTH 1
+#define	MC_CMD_NCSI_PROD_IN_RESET_LBN 1
+#define	MC_CMD_NCSI_PROD_IN_RESET_WIDTH 1
+#define	MC_CMD_NCSI_PROD_IN_DRVATTACH_LBN 2
+#define	MC_CMD_NCSI_PROD_IN_DRVATTACH_WIDTH 1
+
+/* MC_CMD_NCSI_PROD_OUT msgresponse */
+#define	MC_CMD_NCSI_PROD_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SHMUART 
+ * Route UART output to circular buffer in shared memory instead.
+ */
+#define	MC_CMD_SHMUART  0x1f
+
+/* MC_CMD_SHMUART_IN msgrequest */
+#define	MC_CMD_SHMUART_IN_LEN 4
+#define	MC_CMD_SHMUART_IN_FLAG_OFST 0
+
+/* MC_CMD_SHMUART_OUT msgresponse */
+#define	MC_CMD_SHMUART_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_PORT_RESET 
+ * Generic per-port reset.
+ */
+#define	MC_CMD_PORT_RESET  0x20
+
+/* MC_CMD_PORT_RESET_IN msgrequest */
+#define	MC_CMD_PORT_RESET_IN_LEN 0
+
+/* MC_CMD_PORT_RESET_OUT msgresponse */
+#define	MC_CMD_PORT_RESET_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_PCIE_CREDITS 
+ * Read instantaneous and minimum flow control thresholds.
+ */
+#define	MC_CMD_PCIE_CREDITS  0x21
+
+/* MC_CMD_PCIE_CREDITS_IN msgrequest */
+#define	MC_CMD_PCIE_CREDITS_IN_LEN 8
+#define	MC_CMD_PCIE_CREDITS_IN_POLL_PERIOD_OFST 0
+#define	MC_CMD_PCIE_CREDITS_IN_WIPE_OFST 4
+
+/* MC_CMD_PCIE_CREDITS_OUT msgresponse */
+#define	MC_CMD_PCIE_CREDITS_OUT_LEN 16
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_HDR_OFST 0
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_HDR_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_DATA_OFST 2
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_DATA_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_HDR_OFST 4
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_HDR_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_DATA_OFST 6
+#define	MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_DATA_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_HDR_OFST 8
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_HDR_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_DATA_OFST 10
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_DATA_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_HDR_OFST 12
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_HDR_LEN 2
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_DATA_OFST 14
+#define	MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_DATA_LEN 2
+
+
+/***********************************/
+/* MC_CMD_RXD_MONITOR 
+ * Get histogram of RX queue fill level.
+ */
+#define	MC_CMD_RXD_MONITOR  0x22
+
+/* MC_CMD_RXD_MONITOR_IN msgrequest */
+#define	MC_CMD_RXD_MONITOR_IN_LEN 12
+#define	MC_CMD_RXD_MONITOR_IN_QID_OFST 0
+#define	MC_CMD_RXD_MONITOR_IN_POLL_PERIOD_OFST 4
+#define	MC_CMD_RXD_MONITOR_IN_WIPE_OFST 8
+
+/* MC_CMD_RXD_MONITOR_OUT msgresponse */
+#define	MC_CMD_RXD_MONITOR_OUT_LEN 80
+#define	MC_CMD_RXD_MONITOR_OUT_QID_OFST 0
+#define	MC_CMD_RXD_MONITOR_OUT_RING_FILL_OFST 4
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_FILL_OFST 8
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_1_OFST 12
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_2_OFST 16
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_4_OFST 20
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_8_OFST 24
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_16_OFST 28
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_32_OFST 32
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_64_OFST 36
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_128_OFST 40
+#define	MC_CMD_RXD_MONITOR_OUT_RING_LT_256_OFST 44
+#define	MC_CMD_RXD_MONITOR_OUT_RING_GE_256_OFST 48
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_LT_1_OFST 52
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_LT_2_OFST 56
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_LT_4_OFST 60
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_LT_8_OFST 64
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_LT_16_OFST 68
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_LT_32_OFST 72
+#define	MC_CMD_RXD_MONITOR_OUT_CACHE_GE_32_OFST 76
+
+
+/***********************************/
+/* MC_CMD_PUTS 
+ * puts(3) implementation over MCDI
+ */
+#define	MC_CMD_PUTS  0x23
+
+/* MC_CMD_PUTS_IN msgrequest */
+#define	MC_CMD_PUTS_IN_LENMIN 13
+#define	MC_CMD_PUTS_IN_LENMAX 255
+#define	MC_CMD_PUTS_IN_LEN(num) (12+1*(num))
+#define	MC_CMD_PUTS_IN_DEST_OFST 0
+#define	MC_CMD_PUTS_IN_UART_LBN 0
+#define	MC_CMD_PUTS_IN_UART_WIDTH 1
+#define	MC_CMD_PUTS_IN_PORT_LBN 1
+#define	MC_CMD_PUTS_IN_PORT_WIDTH 1
+#define	MC_CMD_PUTS_IN_DHOST_OFST 4
+#define	MC_CMD_PUTS_IN_DHOST_LEN 6
+#define	MC_CMD_PUTS_IN_STRING_OFST 12
+#define	MC_CMD_PUTS_IN_STRING_LEN 1
+#define	MC_CMD_PUTS_IN_STRING_MINNUM 1
+#define	MC_CMD_PUTS_IN_STRING_MAXNUM 243
+
+/* MC_CMD_PUTS_OUT msgresponse */
+#define	MC_CMD_PUTS_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_PHY_CFG 
+ * Report PHY configuration.
+ */
+#define	MC_CMD_GET_PHY_CFG  0x24
+
+/* MC_CMD_GET_PHY_CFG_IN msgrequest */
+#define	MC_CMD_GET_PHY_CFG_IN_LEN 0
+
+/* MC_CMD_GET_PHY_CFG_OUT msgresponse */
+#define	MC_CMD_GET_PHY_CFG_OUT_LEN 72
+#define	MC_CMD_GET_PHY_CFG_OUT_FLAGS_OFST 0
+#define	MC_CMD_GET_PHY_CFG_OUT_PRESENT_LBN 0
+#define	MC_CMD_GET_PHY_CFG_OUT_PRESENT_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_SHORT_LBN 1
+#define	MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_SHORT_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_LONG_LBN 2
+#define	MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_LONG_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_LOWPOWER_LBN 3
+#define	MC_CMD_GET_PHY_CFG_OUT_LOWPOWER_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_POWEROFF_LBN 4
+#define	MC_CMD_GET_PHY_CFG_OUT_POWEROFF_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_TXDIS_LBN 5
+#define	MC_CMD_GET_PHY_CFG_OUT_TXDIS_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_BIST_LBN 6
+#define	MC_CMD_GET_PHY_CFG_OUT_BIST_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_TYPE_OFST 4
+#define	MC_CMD_GET_PHY_CFG_OUT_SUPPORTED_CAP_OFST 8
+#define	MC_CMD_PHY_CAP_10HDX_LBN 1
+#define	MC_CMD_PHY_CAP_10HDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_10FDX_LBN 2
+#define	MC_CMD_PHY_CAP_10FDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_100HDX_LBN 3
+#define	MC_CMD_PHY_CAP_100HDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_100FDX_LBN 4
+#define	MC_CMD_PHY_CAP_100FDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_1000HDX_LBN 5
+#define	MC_CMD_PHY_CAP_1000HDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_1000FDX_LBN 6
+#define	MC_CMD_PHY_CAP_1000FDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_10000FDX_LBN 7
+#define	MC_CMD_PHY_CAP_10000FDX_WIDTH 1
+#define	MC_CMD_PHY_CAP_PAUSE_LBN 8
+#define	MC_CMD_PHY_CAP_PAUSE_WIDTH 1
+#define	MC_CMD_PHY_CAP_ASYM_LBN 9
+#define	MC_CMD_PHY_CAP_ASYM_WIDTH 1
+#define	MC_CMD_PHY_CAP_AN_LBN 10
+#define	MC_CMD_PHY_CAP_AN_WIDTH 1
+#define	MC_CMD_GET_PHY_CFG_OUT_CHANNEL_OFST 12
+#define	MC_CMD_GET_PHY_CFG_OUT_PRT_OFST 16
+#define	MC_CMD_GET_PHY_CFG_OUT_STATS_MASK_OFST 20
+#define	MC_CMD_GET_PHY_CFG_OUT_NAME_OFST 24
+#define	MC_CMD_GET_PHY_CFG_OUT_NAME_LEN 20
+#define	MC_CMD_GET_PHY_CFG_OUT_MEDIA_TYPE_OFST 44
+#define	MC_CMD_MEDIA_XAUI 0x1 /* enum */
+#define	MC_CMD_MEDIA_CX4 0x2 /* enum */
+#define	MC_CMD_MEDIA_KX4 0x3 /* enum */
+#define	MC_CMD_MEDIA_XFP 0x4 /* enum */
+#define	MC_CMD_MEDIA_SFP_PLUS 0x5 /* enum */
+#define	MC_CMD_MEDIA_BASE_T 0x6 /* enum */
+#define	MC_CMD_GET_PHY_CFG_OUT_MMD_MASK_OFST 48
+#define	MC_CMD_MMD_CLAUSE22 0x0 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_PMAPMD 0x1 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_WIS 0x2 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_PCS 0x3 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_PHYXS 0x4 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_DTEXS 0x5 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_TC 0x6 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_AN 0x7 /* enum */
+#define	MC_CMD_MMD_CLAUSE45_C22EXT 0x1d /* enum */
+#define	MC_CMD_MMD_CLAUSE45_VEND1 0x1e /* enum */
+#define	MC_CMD_MMD_CLAUSE45_VEND2 0x1f /* enum */
+#define	MC_CMD_GET_PHY_CFG_OUT_REVISION_OFST 52
+#define	MC_CMD_GET_PHY_CFG_OUT_REVISION_LEN 20
+
+
+/***********************************/
+/* MC_CMD_START_BIST 
+ * Start a BIST test on the PHY.
+ */
+#define	MC_CMD_START_BIST  0x25
+
+/* MC_CMD_START_BIST_IN msgrequest */
+#define	MC_CMD_START_BIST_IN_LEN 4
+#define	MC_CMD_START_BIST_IN_TYPE_OFST 0
+#define	MC_CMD_PHY_BIST_CABLE_SHORT 0x1 /* enum */
+#define	MC_CMD_PHY_BIST_CABLE_LONG 0x2 /* enum */
+#define	MC_CMD_BPX_SERDES_BIST 0x3 /* enum */
+#define	MC_CMD_MC_LOOPBACK_BIST 0x4 /* enum */
+#define	MC_CMD_PHY_BIST 0x5 /* enum */
+
+/* MC_CMD_START_BIST_OUT msgresponse */
+#define	MC_CMD_START_BIST_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_POLL_BIST 
+ * Poll for BIST completion.
+ */
+#define	MC_CMD_POLL_BIST  0x26
+
+/* MC_CMD_POLL_BIST_IN msgrequest */
+#define	MC_CMD_POLL_BIST_IN_LEN 0
+
+/* MC_CMD_POLL_BIST_OUT msgresponse */
+#define	MC_CMD_POLL_BIST_OUT_LEN 8
+#define	MC_CMD_POLL_BIST_OUT_RESULT_OFST 0
+#define	MC_CMD_POLL_BIST_RUNNING 0x1 /* enum */
+#define	MC_CMD_POLL_BIST_PASSED 0x2 /* enum */
+#define	MC_CMD_POLL_BIST_FAILED 0x3 /* enum */
+#define	MC_CMD_POLL_BIST_TIMEOUT 0x4 /* enum */
+#define	MC_CMD_POLL_BIST_OUT_PRIVATE_OFST 4
+
+/* MC_CMD_POLL_BIST_OUT_SFT9001 msgresponse */
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_LEN 36
+/*            MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */
+/*            Enum values, see field(s): */
+/*               MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A_OFST 4
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B_OFST 8
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C_OFST 12
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D_OFST 16
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_A_OFST 20
+#define	MC_CMD_POLL_BIST_SFT9001_PAIR_OK 0x1 /* enum */
+#define	MC_CMD_POLL_BIST_SFT9001_PAIR_OPEN 0x2 /* enum */
+#define	MC_CMD_POLL_BIST_SFT9001_INTRA_PAIR_SHORT 0x3 /* enum */
+#define	MC_CMD_POLL_BIST_SFT9001_INTER_PAIR_SHORT 0x4 /* enum */
+#define	MC_CMD_POLL_BIST_SFT9001_PAIR_BUSY 0x9 /* enum */
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_B_OFST 24
+/*            Enum values, see field(s): */
+/*               CABLE_STATUS_A */
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_C_OFST 28
+/*            Enum values, see field(s): */
+/*               CABLE_STATUS_A */
+#define	MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_D_OFST 32
+/*            Enum values, see field(s): */
+/*               CABLE_STATUS_A */
+
+/* MC_CMD_POLL_BIST_OUT_MRSFP msgresponse */
+#define	MC_CMD_POLL_BIST_OUT_MRSFP_LEN 8
+/*            MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */
+/*            Enum values, see field(s): */
+/*               MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */
+#define	MC_CMD_POLL_BIST_OUT_MRSFP_TEST_OFST 4
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_COMPLETE 0x0 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_OFF_I2C_WRITE 0x1 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_OFF_I2C_NO_ACCESS_IO_EXP 0x2 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_OFF_I2C_NO_ACCESS_MODULE 0x3 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_IO_EXP_I2C_CONFIGURE 0x4 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_I2C_NO_CROSSTALK 0x5 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_MODULE_PRESENCE 0x6 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_MODULE_ID_I2C_ACCESS 0x7 /* enum */
+#define	MC_CMD_POLL_BIST_MRSFP_TEST_MODULE_ID_SANE_VALUE 0x8 /* enum */
+
+
+/***********************************/
+/* MC_CMD_FLUSH_RX_QUEUES 
+ * Flush receive queue(s).
+ */
+#define	MC_CMD_FLUSH_RX_QUEUES  0x27
+
+/* MC_CMD_FLUSH_RX_QUEUES_IN msgrequest */
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_LENMIN 4
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_LENMAX 252
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_LEN(num) (0+4*(num))
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_OFST 0
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_LEN 4
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MINNUM 1
+#define	MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MAXNUM 63
+
+/* MC_CMD_FLUSH_RX_QUEUES_OUT msgresponse */
+#define	MC_CMD_FLUSH_RX_QUEUES_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_LOOPBACK_MODES 
+ * Get port's loopback modes.
+ */
+#define	MC_CMD_GET_LOOPBACK_MODES  0x28
+
+/* MC_CMD_GET_LOOPBACK_MODES_IN msgrequest */
+#define	MC_CMD_GET_LOOPBACK_MODES_IN_LEN 0
+
+/* MC_CMD_GET_LOOPBACK_MODES_OUT msgresponse */
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_LEN 32
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_100M_OFST 0
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LEN 8
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LO_OFST 0
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_100M_HI_OFST 4
+#define	MC_CMD_LOOPBACK_NONE  0x0 /* enum */
+#define	MC_CMD_LOOPBACK_DATA  0x1 /* enum */
+#define	MC_CMD_LOOPBACK_GMAC  0x2 /* enum */
+#define	MC_CMD_LOOPBACK_XGMII 0x3 /* enum */
+#define	MC_CMD_LOOPBACK_XGXS  0x4 /* enum */
+#define	MC_CMD_LOOPBACK_XAUI  0x5 /* enum */
+#define	MC_CMD_LOOPBACK_GMII  0x6 /* enum */
+#define	MC_CMD_LOOPBACK_SGMII  0x7 /* enum */
+#define	MC_CMD_LOOPBACK_XGBR  0x8 /* enum */
+#define	MC_CMD_LOOPBACK_XFI  0x9 /* enum */
+#define	MC_CMD_LOOPBACK_XAUI_FAR  0xa /* enum */
+#define	MC_CMD_LOOPBACK_GMII_FAR  0xb /* enum */
+#define	MC_CMD_LOOPBACK_SGMII_FAR  0xc /* enum */
+#define	MC_CMD_LOOPBACK_XFI_FAR  0xd /* enum */
+#define	MC_CMD_LOOPBACK_GPHY  0xe /* enum */
+#define	MC_CMD_LOOPBACK_PHYXS  0xf /* enum */
+#define	MC_CMD_LOOPBACK_PCS  0x10 /* enum */
+#define	MC_CMD_LOOPBACK_PMAPMD  0x11 /* enum */
+#define	MC_CMD_LOOPBACK_XPORT  0x12 /* enum */
+#define	MC_CMD_LOOPBACK_XGMII_WS  0x13 /* enum */
+#define	MC_CMD_LOOPBACK_XAUI_WS  0x14 /* enum */
+#define	MC_CMD_LOOPBACK_XAUI_WS_FAR  0x15 /* enum */
+#define	MC_CMD_LOOPBACK_XAUI_WS_NEAR  0x16 /* enum */
+#define	MC_CMD_LOOPBACK_GMII_WS  0x17 /* enum */
+#define	MC_CMD_LOOPBACK_XFI_WS  0x18 /* enum */
+#define	MC_CMD_LOOPBACK_XFI_WS_FAR  0x19 /* enum */
+#define	MC_CMD_LOOPBACK_PHYXS_WS  0x1a /* enum */
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_1G_OFST 8
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LEN 8
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LO_OFST 8
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_1G_HI_OFST 12
+/*            Enum values, see field(s): */
+/*               100M */
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_10G_OFST 16
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LEN 8
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LO_OFST 16
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_10G_HI_OFST 20
+/*            Enum values, see field(s): */
+/*               100M */
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_OFST 24
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LEN 8
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LO_OFST 24
+#define	MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_HI_OFST 28
+/*            Enum values, see field(s): */
+/*               100M */
+
+
+/***********************************/
+/* MC_CMD_GET_LINK 
+ * Read the unified MAC/PHY link state.
+ */
+#define	MC_CMD_GET_LINK  0x29
+
+/* MC_CMD_GET_LINK_IN msgrequest */
+#define	MC_CMD_GET_LINK_IN_LEN 0
+
+/* MC_CMD_GET_LINK_OUT msgresponse */
+#define	MC_CMD_GET_LINK_OUT_LEN 28
+#define	MC_CMD_GET_LINK_OUT_CAP_OFST 0
+#define	MC_CMD_GET_LINK_OUT_LP_CAP_OFST 4
+#define	MC_CMD_GET_LINK_OUT_LINK_SPEED_OFST 8
+#define	MC_CMD_GET_LINK_OUT_LOOPBACK_MODE_OFST 12
+/*            Enum values, see field(s): */
+/*               MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */
+#define	MC_CMD_GET_LINK_OUT_FLAGS_OFST 16
+#define	MC_CMD_GET_LINK_OUT_LINK_UP_LBN 0
+#define	MC_CMD_GET_LINK_OUT_LINK_UP_WIDTH 1
+#define	MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN 1
+#define	MC_CMD_GET_LINK_OUT_FULL_DUPLEX_WIDTH 1
+#define	MC_CMD_GET_LINK_OUT_BPX_LINK_LBN 2
+#define	MC_CMD_GET_LINK_OUT_BPX_LINK_WIDTH 1
+#define	MC_CMD_GET_LINK_OUT_PHY_LINK_LBN 3
+#define	MC_CMD_GET_LINK_OUT_PHY_LINK_WIDTH 1
+#define	MC_CMD_GET_LINK_OUT_FCNTL_OFST 20
+#define	MC_CMD_FCNTL_OFF 0x0 /* enum */
+#define	MC_CMD_FCNTL_RESPOND 0x1 /* enum */
+#define	MC_CMD_FCNTL_BIDIR 0x2 /* enum */
+#define	MC_CMD_GET_LINK_OUT_MAC_FAULT_OFST 24
+#define	MC_CMD_MAC_FAULT_XGMII_LOCAL_LBN 0
+#define	MC_CMD_MAC_FAULT_XGMII_LOCAL_WIDTH 1
+#define	MC_CMD_MAC_FAULT_XGMII_REMOTE_LBN 1
+#define	MC_CMD_MAC_FAULT_XGMII_REMOTE_WIDTH 1
+#define	MC_CMD_MAC_FAULT_SGMII_REMOTE_LBN 2
+#define	MC_CMD_MAC_FAULT_SGMII_REMOTE_WIDTH 1
+#define	MC_CMD_MAC_FAULT_PENDING_RECONFIG_LBN 3
+#define	MC_CMD_MAC_FAULT_PENDING_RECONFIG_WIDTH 1
+
+
+/***********************************/
+/* MC_CMD_SET_LINK 
+ * Write the unified MAC/PHY link configuration.
+ */
+#define	MC_CMD_SET_LINK  0x2a
+
+/* MC_CMD_SET_LINK_IN msgrequest */
+#define	MC_CMD_SET_LINK_IN_LEN 16
+#define	MC_CMD_SET_LINK_IN_CAP_OFST 0
+#define	MC_CMD_SET_LINK_IN_FLAGS_OFST 4
+#define	MC_CMD_SET_LINK_IN_LOWPOWER_LBN 0
+#define	MC_CMD_SET_LINK_IN_LOWPOWER_WIDTH 1
+#define	MC_CMD_SET_LINK_IN_POWEROFF_LBN 1
+#define	MC_CMD_SET_LINK_IN_POWEROFF_WIDTH 1
+#define	MC_CMD_SET_LINK_IN_TXDIS_LBN 2
+#define	MC_CMD_SET_LINK_IN_TXDIS_WIDTH 1
+#define	MC_CMD_SET_LINK_IN_LOOPBACK_MODE_OFST 8
+/*            Enum values, see field(s): */
+/*               MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */
+#define	MC_CMD_SET_LINK_IN_LOOPBACK_SPEED_OFST 12
+
+/* MC_CMD_SET_LINK_OUT msgresponse */
+#define	MC_CMD_SET_LINK_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SET_ID_LED 
+ * Set indentification LED state.
+ */
+#define	MC_CMD_SET_ID_LED  0x2b
+
+/* MC_CMD_SET_ID_LED_IN msgrequest */
+#define	MC_CMD_SET_ID_LED_IN_LEN 4
+#define	MC_CMD_SET_ID_LED_IN_STATE_OFST 0
+#define	MC_CMD_LED_OFF  0x0 /* enum */
+#define	MC_CMD_LED_ON  0x1 /* enum */
+#define	MC_CMD_LED_DEFAULT  0x2 /* enum */
+
+/* MC_CMD_SET_ID_LED_OUT msgresponse */
+#define	MC_CMD_SET_ID_LED_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SET_MAC 
+ * Set MAC configuration.
+ */
+#define	MC_CMD_SET_MAC  0x2c
+
+/* MC_CMD_SET_MAC_IN msgrequest */
+#define	MC_CMD_SET_MAC_IN_LEN 24
+#define	MC_CMD_SET_MAC_IN_MTU_OFST 0
+#define	MC_CMD_SET_MAC_IN_DRAIN_OFST 4
+#define	MC_CMD_SET_MAC_IN_ADDR_OFST 8
+#define	MC_CMD_SET_MAC_IN_ADDR_LEN 8
+#define	MC_CMD_SET_MAC_IN_ADDR_LO_OFST 8
+#define	MC_CMD_SET_MAC_IN_ADDR_HI_OFST 12
+#define	MC_CMD_SET_MAC_IN_REJECT_OFST 16
+#define	MC_CMD_SET_MAC_IN_REJECT_UNCST_LBN 0
+#define	MC_CMD_SET_MAC_IN_REJECT_UNCST_WIDTH 1
+#define	MC_CMD_SET_MAC_IN_REJECT_BRDCST_LBN 1
+#define	MC_CMD_SET_MAC_IN_REJECT_BRDCST_WIDTH 1
+#define	MC_CMD_SET_MAC_IN_FCNTL_OFST 20
+/*               MC_CMD_FCNTL_OFF 0x0 */
+/*               MC_CMD_FCNTL_RESPOND 0x1 */
+/*               MC_CMD_FCNTL_BIDIR 0x2 */
+#define	MC_CMD_FCNTL_AUTO 0x3 /* enum */
+
+/* MC_CMD_SET_MAC_OUT msgresponse */
+#define	MC_CMD_SET_MAC_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_PHY_STATS 
+ * Get generic PHY statistics.
+ */
+#define	MC_CMD_PHY_STATS  0x2d
+
+/* MC_CMD_PHY_STATS_IN msgrequest */
+#define	MC_CMD_PHY_STATS_IN_LEN 8
+#define	MC_CMD_PHY_STATS_IN_DMA_ADDR_OFST 0
+#define	MC_CMD_PHY_STATS_IN_DMA_ADDR_LEN 8
+#define	MC_CMD_PHY_STATS_IN_DMA_ADDR_LO_OFST 0
+#define	MC_CMD_PHY_STATS_IN_DMA_ADDR_HI_OFST 4
+
+/* MC_CMD_PHY_STATS_OUT_DMA msgresponse */
+#define	MC_CMD_PHY_STATS_OUT_DMA_LEN 0
+
+/* MC_CMD_PHY_STATS_OUT_NO_DMA msgresponse */
+#define	MC_CMD_PHY_STATS_OUT_NO_DMA_LEN (((MC_CMD_PHY_NSTATS*32))>>3)
+#define	MC_CMD_PHY_STATS_OUT_NO_DMA_STATISTICS_OFST 0
+#define	MC_CMD_PHY_STATS_OUT_NO_DMA_STATISTICS_LEN 4
+#define	MC_CMD_PHY_STATS_OUT_NO_DMA_STATISTICS_NUM MC_CMD_PHY_NSTATS
+#define	MC_CMD_OUI  0x0 /* enum */
+#define	MC_CMD_PMA_PMD_LINK_UP  0x1 /* enum */
+#define	MC_CMD_PMA_PMD_RX_FAULT  0x2 /* enum */
+#define	MC_CMD_PMA_PMD_TX_FAULT  0x3 /* enum */
+#define	MC_CMD_PMA_PMD_SIGNAL  0x4 /* enum */
+#define	MC_CMD_PMA_PMD_SNR_A  0x5 /* enum */
+#define	MC_CMD_PMA_PMD_SNR_B  0x6 /* enum */
+#define	MC_CMD_PMA_PMD_SNR_C  0x7 /* enum */
+#define	MC_CMD_PMA_PMD_SNR_D  0x8 /* enum */
+#define	MC_CMD_PCS_LINK_UP  0x9 /* enum */
+#define	MC_CMD_PCS_RX_FAULT  0xa /* enum */
+#define	MC_CMD_PCS_TX_FAULT  0xb /* enum */
+#define	MC_CMD_PCS_BER  0xc /* enum */
+#define	MC_CMD_PCS_BLOCK_ERRORS  0xd /* enum */
+#define	MC_CMD_PHYXS_LINK_UP  0xe /* enum */
+#define	MC_CMD_PHYXS_RX_FAULT  0xf /* enum */
+#define	MC_CMD_PHYXS_TX_FAULT  0x10 /* enum */
+#define	MC_CMD_PHYXS_ALIGN  0x11 /* enum */
+#define	MC_CMD_PHYXS_SYNC  0x12 /* enum */
+#define	MC_CMD_AN_LINK_UP  0x13 /* enum */
+#define	MC_CMD_AN_COMPLETE  0x14 /* enum */
+#define	MC_CMD_AN_10GBT_STATUS  0x15 /* enum */
+#define	MC_CMD_CL22_LINK_UP  0x16 /* enum */
+#define	MC_CMD_PHY_NSTATS  0x17 /* enum */
+
+
+/***********************************/
+/* MC_CMD_MAC_STATS 
+ * Get generic MAC statistics.
+ */
+#define	MC_CMD_MAC_STATS  0x2e
+
+/* MC_CMD_MAC_STATS_IN msgrequest */
+#define	MC_CMD_MAC_STATS_IN_LEN 16
+#define	MC_CMD_MAC_STATS_IN_DMA_ADDR_OFST 0
+#define	MC_CMD_MAC_STATS_IN_DMA_ADDR_LEN 8
+#define	MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_OFST 0
+#define	MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_OFST 4
+#define	MC_CMD_MAC_STATS_IN_CMD_OFST 8
+#define	MC_CMD_MAC_STATS_IN_DMA_LBN 0
+#define	MC_CMD_MAC_STATS_IN_DMA_WIDTH 1
+#define	MC_CMD_MAC_STATS_IN_CLEAR_LBN 1
+#define	MC_CMD_MAC_STATS_IN_CLEAR_WIDTH 1
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_CHANGE_LBN 2
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_CHANGE_WIDTH 1
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_ENABLE_LBN 3
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_ENABLE_WIDTH 1
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_CLEAR_LBN 4
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_CLEAR_WIDTH 1
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_NOEVENT_LBN 5
+#define	MC_CMD_MAC_STATS_IN_PERIODIC_NOEVENT_WIDTH 1
+#define	MC_CMD_MAC_STATS_IN_PERIOD_MS_LBN 16
+#define	MC_CMD_MAC_STATS_IN_PERIOD_MS_WIDTH 16
+#define	MC_CMD_MAC_STATS_IN_DMA_LEN_OFST 12
+
+/* MC_CMD_MAC_STATS_OUT_DMA msgresponse */
+#define	MC_CMD_MAC_STATS_OUT_DMA_LEN 0
+
+/* MC_CMD_MAC_STATS_OUT_NO_DMA msgresponse */
+#define	MC_CMD_MAC_STATS_OUT_NO_DMA_LEN (((MC_CMD_MAC_NSTATS*64))>>3)
+#define	MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_OFST 0
+#define	MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LEN 8
+#define	MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LO_OFST 0
+#define	MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_HI_OFST 4
+#define	MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS
+#define	MC_CMD_MAC_GENERATION_START  0x0 /* enum */
+#define	MC_CMD_MAC_TX_PKTS  0x1 /* enum */
+#define	MC_CMD_MAC_TX_PAUSE_PKTS  0x2 /* enum */
+#define	MC_CMD_MAC_TX_CONTROL_PKTS  0x3 /* enum */
+#define	MC_CMD_MAC_TX_UNICAST_PKTS  0x4 /* enum */
+#define	MC_CMD_MAC_TX_MULTICAST_PKTS  0x5 /* enum */
+#define	MC_CMD_MAC_TX_BROADCAST_PKTS  0x6 /* enum */
+#define	MC_CMD_MAC_TX_BYTES  0x7 /* enum */
+#define	MC_CMD_MAC_TX_BAD_BYTES  0x8 /* enum */
+#define	MC_CMD_MAC_TX_LT64_PKTS  0x9 /* enum */
+#define	MC_CMD_MAC_TX_64_PKTS  0xa /* enum */
+#define	MC_CMD_MAC_TX_65_TO_127_PKTS  0xb /* enum */
+#define	MC_CMD_MAC_TX_128_TO_255_PKTS  0xc /* enum */
+#define	MC_CMD_MAC_TX_256_TO_511_PKTS  0xd /* enum */
+#define	MC_CMD_MAC_TX_512_TO_1023_PKTS  0xe /* enum */
+#define	MC_CMD_MAC_TX_1024_TO_15XX_PKTS  0xf /* enum */
+#define	MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS  0x10 /* enum */
+#define	MC_CMD_MAC_TX_GTJUMBO_PKTS  0x11 /* enum */
+#define	MC_CMD_MAC_TX_BAD_FCS_PKTS  0x12 /* enum */
+#define	MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS  0x13 /* enum */
+#define	MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS  0x14 /* enum */
+#define	MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS  0x15 /* enum */
+#define	MC_CMD_MAC_TX_LATE_COLLISION_PKTS  0x16 /* enum */
+#define	MC_CMD_MAC_TX_DEFERRED_PKTS  0x17 /* enum */
+#define	MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS  0x18 /* enum */
+#define	MC_CMD_MAC_TX_NON_TCPUDP_PKTS  0x19 /* enum */
+#define	MC_CMD_MAC_TX_MAC_SRC_ERR_PKTS  0x1a /* enum */
+#define	MC_CMD_MAC_TX_IP_SRC_ERR_PKTS  0x1b /* enum */
+#define	MC_CMD_MAC_RX_PKTS  0x1c /* enum */
+#define	MC_CMD_MAC_RX_PAUSE_PKTS  0x1d /* enum */
+#define	MC_CMD_MAC_RX_GOOD_PKTS  0x1e /* enum */
+#define	MC_CMD_MAC_RX_CONTROL_PKTS  0x1f /* enum */
+#define	MC_CMD_MAC_RX_UNICAST_PKTS  0x20 /* enum */
+#define	MC_CMD_MAC_RX_MULTICAST_PKTS  0x21 /* enum */
+#define	MC_CMD_MAC_RX_BROADCAST_PKTS  0x22 /* enum */
+#define	MC_CMD_MAC_RX_BYTES  0x23 /* enum */
+#define	MC_CMD_MAC_RX_BAD_BYTES  0x24 /* enum */
+#define	MC_CMD_MAC_RX_64_PKTS  0x25 /* enum */
+#define	MC_CMD_MAC_RX_65_TO_127_PKTS  0x26 /* enum */
+#define	MC_CMD_MAC_RX_128_TO_255_PKTS  0x27 /* enum */
+#define	MC_CMD_MAC_RX_256_TO_511_PKTS  0x28 /* enum */
+#define	MC_CMD_MAC_RX_512_TO_1023_PKTS  0x29 /* enum */
+#define	MC_CMD_MAC_RX_1024_TO_15XX_PKTS  0x2a /* enum */
+#define	MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS  0x2b /* enum */
+#define	MC_CMD_MAC_RX_GTJUMBO_PKTS  0x2c /* enum */
+#define	MC_CMD_MAC_RX_UNDERSIZE_PKTS  0x2d /* enum */
+#define	MC_CMD_MAC_RX_BAD_FCS_PKTS  0x2e /* enum */
+#define	MC_CMD_MAC_RX_OVERFLOW_PKTS  0x2f /* enum */
+#define	MC_CMD_MAC_RX_FALSE_CARRIER_PKTS  0x30 /* enum */
+#define	MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS  0x31 /* enum */
+#define	MC_CMD_MAC_RX_ALIGN_ERROR_PKTS  0x32 /* enum */
+#define	MC_CMD_MAC_RX_LENGTH_ERROR_PKTS  0x33 /* enum */
+#define	MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS  0x34 /* enum */
+#define	MC_CMD_MAC_RX_JABBER_PKTS  0x35 /* enum */
+#define	MC_CMD_MAC_RX_NODESC_DROPS  0x36 /* enum */
+#define	MC_CMD_MAC_RX_LANES01_CHAR_ERR  0x37 /* enum */
+#define	MC_CMD_MAC_RX_LANES23_CHAR_ERR  0x38 /* enum */
+#define	MC_CMD_MAC_RX_LANES01_DISP_ERR  0x39 /* enum */
+#define	MC_CMD_MAC_RX_LANES23_DISP_ERR  0x3a /* enum */
+#define	MC_CMD_MAC_RX_MATCH_FAULT  0x3b /* enum */
+#define	MC_CMD_GMAC_DMABUF_START  0x40 /* enum */
+#define	MC_CMD_GMAC_DMABUF_END    0x5f /* enum */
+#define	MC_CMD_MAC_GENERATION_END 0x60 /* enum */
+#define	MC_CMD_MAC_NSTATS  0x61 /* enum */
+
+
+/***********************************/
+/* MC_CMD_SRIOV 
+ * to be documented
+ */
+#define	MC_CMD_SRIOV  0x30
+
+/* MC_CMD_SRIOV_IN msgrequest */
+#define	MC_CMD_SRIOV_IN_LEN 12
+#define	MC_CMD_SRIOV_IN_ENABLE_OFST 0
+#define	MC_CMD_SRIOV_IN_VI_BASE_OFST 4
+#define	MC_CMD_SRIOV_IN_VF_COUNT_OFST 8
+
+/* MC_CMD_SRIOV_OUT msgresponse */
+#define	MC_CMD_SRIOV_OUT_LEN 8
+#define	MC_CMD_SRIOV_OUT_VI_SCALE_OFST 0
+#define	MC_CMD_SRIOV_OUT_VF_TOTAL_OFST 4
+
+/* MC_CMD_MEMCPY_RECORD_TYPEDEF structuredef */
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_LEN 32
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_OFST 0
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_LBN 0
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_WIDTH 32
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_OFST 4
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_LBN 32
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_WIDTH 32
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_OFST 8
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LEN 8
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LO_OFST 8
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_HI_OFST 12
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LBN 64
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_WIDTH 64
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_OFST 16
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE 0x100 /* enum */
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_LBN 128
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_WIDTH 32
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_OFST 20
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LEN 8
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LO_OFST 20
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_HI_OFST 24
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LBN 160
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_WIDTH 64
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_OFST 28
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_LBN 224
+#define	MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_WIDTH 32
+
+
+/***********************************/
+/* MC_CMD_MEMCPY 
+ * Perform memory copy operation.
+ */
+#define	MC_CMD_MEMCPY  0x31
+
+/* MC_CMD_MEMCPY_IN msgrequest */
+#define	MC_CMD_MEMCPY_IN_LENMIN 32
+#define	MC_CMD_MEMCPY_IN_LENMAX 224
+#define	MC_CMD_MEMCPY_IN_LEN(num) (0+32*(num))
+#define	MC_CMD_MEMCPY_IN_RECORD_OFST 0
+#define	MC_CMD_MEMCPY_IN_RECORD_LEN 32
+#define	MC_CMD_MEMCPY_IN_RECORD_MINNUM 1
+#define	MC_CMD_MEMCPY_IN_RECORD_MAXNUM 7
+
+/* MC_CMD_MEMCPY_OUT msgresponse */
+#define	MC_CMD_MEMCPY_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_WOL_FILTER_SET 
+ * Set a WoL filter.
+ */
+#define	MC_CMD_WOL_FILTER_SET  0x32
+
+/* MC_CMD_WOL_FILTER_SET_IN msgrequest */
+#define	MC_CMD_WOL_FILTER_SET_IN_LEN 192
+#define	MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0
+#define	MC_CMD_FILTER_MODE_SIMPLE    0x0 /* enum */
+#define	MC_CMD_FILTER_MODE_STRUCTURED 0xffffffff /* enum */
+#define	MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4
+#define	MC_CMD_WOL_TYPE_MAGIC      0x0 /* enum */
+#define	MC_CMD_WOL_TYPE_WIN_MAGIC 0x2 /* enum */
+#define	MC_CMD_WOL_TYPE_IPV4_SYN   0x3 /* enum */
+#define	MC_CMD_WOL_TYPE_IPV6_SYN   0x4 /* enum */
+#define	MC_CMD_WOL_TYPE_BITMAP     0x5 /* enum */
+#define	MC_CMD_WOL_TYPE_LINK       0x6 /* enum */
+#define	MC_CMD_WOL_TYPE_MAX        0x7 /* enum */
+#define	MC_CMD_WOL_FILTER_SET_IN_DATA_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_DATA_LEN 4
+#define	MC_CMD_WOL_FILTER_SET_IN_DATA_NUM 46
+
+/* MC_CMD_WOL_FILTER_SET_IN_MAGIC msgrequest */
+#define	MC_CMD_WOL_FILTER_SET_IN_MAGIC_LEN 16
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+#define	MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LEN 8
+#define	MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_HI_OFST 12
+
+/* MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN msgrequest */
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_LEN 20
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_IP_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_IP_OFST 12
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_PORT_OFST 16
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_PORT_LEN 2
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_PORT_OFST 18
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_PORT_LEN 2
+
+/* MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN msgrequest */
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_LEN 44
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_IP_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_IP_LEN 16
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_IP_OFST 24
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_IP_LEN 16
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_PORT_OFST 40
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_PORT_LEN 2
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_PORT_OFST 42
+#define	MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_PORT_LEN 2
+
+/* MC_CMD_WOL_FILTER_SET_IN_BITMAP msgrequest */
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN 187
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_MASK_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_MASK_LEN 48
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_BITMAP_OFST 56
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_BITMAP_LEN 128
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN_OFST 184
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN_LEN 1
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER3_OFST 185
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER3_LEN 1
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER4_OFST 186
+#define	MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER4_LEN 1
+
+/* MC_CMD_WOL_FILTER_SET_IN_LINK msgrequest */
+#define	MC_CMD_WOL_FILTER_SET_IN_LINK_LEN 12
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+#define	MC_CMD_WOL_FILTER_SET_IN_LINK_MASK_OFST 8
+#define	MC_CMD_WOL_FILTER_SET_IN_LINK_UP_LBN 0
+#define	MC_CMD_WOL_FILTER_SET_IN_LINK_UP_WIDTH 1
+#define	MC_CMD_WOL_FILTER_SET_IN_LINK_DOWN_LBN 1
+#define	MC_CMD_WOL_FILTER_SET_IN_LINK_DOWN_WIDTH 1
+
+/* MC_CMD_WOL_FILTER_SET_OUT msgresponse */
+#define	MC_CMD_WOL_FILTER_SET_OUT_LEN 4
+#define	MC_CMD_WOL_FILTER_SET_OUT_FILTER_ID_OFST 0
+
+
+/***********************************/
+/* MC_CMD_WOL_FILTER_REMOVE 
+ * Remove a WoL filter.
+ */
+#define	MC_CMD_WOL_FILTER_REMOVE  0x33
+
+/* MC_CMD_WOL_FILTER_REMOVE_IN msgrequest */
+#define	MC_CMD_WOL_FILTER_REMOVE_IN_LEN 4
+#define	MC_CMD_WOL_FILTER_REMOVE_IN_FILTER_ID_OFST 0
+
+/* MC_CMD_WOL_FILTER_REMOVE_OUT msgresponse */
+#define	MC_CMD_WOL_FILTER_REMOVE_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_WOL_FILTER_RESET 
+ * Reset (i.e. remove all) WoL filters.
+ */
+#define	MC_CMD_WOL_FILTER_RESET  0x34
+
+/* MC_CMD_WOL_FILTER_RESET_IN msgrequest */
+#define	MC_CMD_WOL_FILTER_RESET_IN_LEN 4
+#define	MC_CMD_WOL_FILTER_RESET_IN_MASK_OFST 0
+#define	MC_CMD_WOL_FILTER_RESET_IN_WAKE_FILTERS 0x1 /* enum */
+#define	MC_CMD_WOL_FILTER_RESET_IN_LIGHTSOUT_OFFLOADS 0x2 /* enum */
+
+/* MC_CMD_WOL_FILTER_RESET_OUT msgresponse */
+#define	MC_CMD_WOL_FILTER_RESET_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SET_MCAST_HASH 
+ * Set the MCASH hash value.
+ */
+#define	MC_CMD_SET_MCAST_HASH  0x35
+
+/* MC_CMD_SET_MCAST_HASH_IN msgrequest */
+#define	MC_CMD_SET_MCAST_HASH_IN_LEN 32
+#define	MC_CMD_SET_MCAST_HASH_IN_HASH0_OFST 0
+#define	MC_CMD_SET_MCAST_HASH_IN_HASH0_LEN 16
+#define	MC_CMD_SET_MCAST_HASH_IN_HASH1_OFST 16
+#define	MC_CMD_SET_MCAST_HASH_IN_HASH1_LEN 16
+
+/* MC_CMD_SET_MCAST_HASH_OUT msgresponse */
+#define	MC_CMD_SET_MCAST_HASH_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_NVRAM_TYPES 
+ * Get virtual NVRAM partitions information.
+ */
+#define	MC_CMD_NVRAM_TYPES  0x36
+
+/* MC_CMD_NVRAM_TYPES_IN msgrequest */
+#define	MC_CMD_NVRAM_TYPES_IN_LEN 0
+
+/* MC_CMD_NVRAM_TYPES_OUT msgresponse */
+#define	MC_CMD_NVRAM_TYPES_OUT_LEN 4
+#define	MC_CMD_NVRAM_TYPES_OUT_TYPES_OFST 0
+#define	MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO 0x0 /* enum */
+#define	MC_CMD_NVRAM_TYPE_MC_FW 0x1 /* enum */
+#define	MC_CMD_NVRAM_TYPE_MC_FW_BACKUP 0x2 /* enum */
+#define	MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT0 0x3 /* enum */
+#define	MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT1 0x4 /* enum */
+#define	MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 0x5 /* enum */
+#define	MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1 0x6 /* enum */
+#define	MC_CMD_NVRAM_TYPE_EXP_ROM 0x7 /* enum */
+#define	MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT0 0x8 /* enum */
+#define	MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT1 0x9 /* enum */
+#define	MC_CMD_NVRAM_TYPE_PHY_PORT0 0xa /* enum */
+#define	MC_CMD_NVRAM_TYPE_PHY_PORT1 0xb /* enum */
+#define	MC_CMD_NVRAM_TYPE_LOG 0xc /* enum */
+
+
+/***********************************/
+/* MC_CMD_NVRAM_INFO 
+ * Read info about a virtual NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_INFO  0x37
+
+/* MC_CMD_NVRAM_INFO_IN msgrequest */
+#define	MC_CMD_NVRAM_INFO_IN_LEN 4
+#define	MC_CMD_NVRAM_INFO_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+
+/* MC_CMD_NVRAM_INFO_OUT msgresponse */
+#define	MC_CMD_NVRAM_INFO_OUT_LEN 24
+#define	MC_CMD_NVRAM_INFO_OUT_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define	MC_CMD_NVRAM_INFO_OUT_SIZE_OFST 4
+#define	MC_CMD_NVRAM_INFO_OUT_ERASESIZE_OFST 8
+#define	MC_CMD_NVRAM_INFO_OUT_FLAGS_OFST 12
+#define	MC_CMD_NVRAM_INFO_OUT_PROTECTED_LBN 0
+#define	MC_CMD_NVRAM_INFO_OUT_PROTECTED_WIDTH 1
+#define	MC_CMD_NVRAM_INFO_OUT_PHYSDEV_OFST 16
+#define	MC_CMD_NVRAM_INFO_OUT_PHYSADDR_OFST 20
+
+
+/***********************************/
+/* MC_CMD_NVRAM_UPDATE_START 
+ * Start a group of update operations on a virtual NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_UPDATE_START  0x38
+
+/* MC_CMD_NVRAM_UPDATE_START_IN msgrequest */
+#define	MC_CMD_NVRAM_UPDATE_START_IN_LEN 4
+#define	MC_CMD_NVRAM_UPDATE_START_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+
+/* MC_CMD_NVRAM_UPDATE_START_OUT msgresponse */
+#define	MC_CMD_NVRAM_UPDATE_START_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_NVRAM_READ 
+ * Read data from a virtual NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_READ  0x39
+
+/* MC_CMD_NVRAM_READ_IN msgrequest */
+#define	MC_CMD_NVRAM_READ_IN_LEN 12
+#define	MC_CMD_NVRAM_READ_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define	MC_CMD_NVRAM_READ_IN_OFFSET_OFST 4
+#define	MC_CMD_NVRAM_READ_IN_LENGTH_OFST 8
+
+/* MC_CMD_NVRAM_READ_OUT msgresponse */
+#define	MC_CMD_NVRAM_READ_OUT_LENMIN 1
+#define	MC_CMD_NVRAM_READ_OUT_LENMAX 255
+#define	MC_CMD_NVRAM_READ_OUT_LEN(num) (0+1*(num))
+#define	MC_CMD_NVRAM_READ_OUT_READ_BUFFER_OFST 0
+#define	MC_CMD_NVRAM_READ_OUT_READ_BUFFER_LEN 1
+#define	MC_CMD_NVRAM_READ_OUT_READ_BUFFER_MINNUM 1
+#define	MC_CMD_NVRAM_READ_OUT_READ_BUFFER_MAXNUM 255
+
+
+/***********************************/
+/* MC_CMD_NVRAM_WRITE 
+ * Write data to a virtual NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_WRITE  0x3a
+
+/* MC_CMD_NVRAM_WRITE_IN msgrequest */
+#define	MC_CMD_NVRAM_WRITE_IN_LENMIN 13
+#define	MC_CMD_NVRAM_WRITE_IN_LENMAX 255
+#define	MC_CMD_NVRAM_WRITE_IN_LEN(num) (12+1*(num))
+#define	MC_CMD_NVRAM_WRITE_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define	MC_CMD_NVRAM_WRITE_IN_OFFSET_OFST 4
+#define	MC_CMD_NVRAM_WRITE_IN_LENGTH_OFST 8
+#define	MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_OFST 12
+#define	MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_LEN 1
+#define	MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_MINNUM 1
+#define	MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_MAXNUM 243
+
+/* MC_CMD_NVRAM_WRITE_OUT msgresponse */
+#define	MC_CMD_NVRAM_WRITE_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_NVRAM_ERASE 
+ * Erase sector(s) from a virtual NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_ERASE  0x3b
+
+/* MC_CMD_NVRAM_ERASE_IN msgrequest */
+#define	MC_CMD_NVRAM_ERASE_IN_LEN 12
+#define	MC_CMD_NVRAM_ERASE_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define	MC_CMD_NVRAM_ERASE_IN_OFFSET_OFST 4
+#define	MC_CMD_NVRAM_ERASE_IN_LENGTH_OFST 8
+
+/* MC_CMD_NVRAM_ERASE_OUT msgresponse */
+#define	MC_CMD_NVRAM_ERASE_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_NVRAM_UPDATE_FINISH 
+ * Finish a group of update operations on a virtual NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_UPDATE_FINISH  0x3c
+
+/* MC_CMD_NVRAM_UPDATE_FINISH_IN msgrequest */
+#define	MC_CMD_NVRAM_UPDATE_FINISH_IN_LEN 8
+#define	MC_CMD_NVRAM_UPDATE_FINISH_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+#define	MC_CMD_NVRAM_UPDATE_FINISH_IN_REBOOT_OFST 4
+
+/* MC_CMD_NVRAM_UPDATE_FINISH_OUT msgresponse */
+#define	MC_CMD_NVRAM_UPDATE_FINISH_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_REBOOT 
+ * Reboot the MC.
+ */
+#define	MC_CMD_REBOOT  0x3d
+
+/* MC_CMD_REBOOT_IN msgrequest */
+#define	MC_CMD_REBOOT_IN_LEN 4
+#define	MC_CMD_REBOOT_IN_FLAGS_OFST 0
+#define	MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION 0x1 /* enum */
+
+/* MC_CMD_REBOOT_OUT msgresponse */
+#define	MC_CMD_REBOOT_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_SCHEDINFO 
+ * Request scheduler info.
+ */
+#define	MC_CMD_SCHEDINFO  0x3e
+
+/* MC_CMD_SCHEDINFO_IN msgrequest */
+#define	MC_CMD_SCHEDINFO_IN_LEN 0
+
+/* MC_CMD_SCHEDINFO_OUT msgresponse */
+#define	MC_CMD_SCHEDINFO_OUT_LENMIN 4
+#define	MC_CMD_SCHEDINFO_OUT_LENMAX 252
+#define	MC_CMD_SCHEDINFO_OUT_LEN(num) (0+4*(num))
+#define	MC_CMD_SCHEDINFO_OUT_DATA_OFST 0
+#define	MC_CMD_SCHEDINFO_OUT_DATA_LEN 4
+#define	MC_CMD_SCHEDINFO_OUT_DATA_MINNUM 1
+#define	MC_CMD_SCHEDINFO_OUT_DATA_MAXNUM 63
+
+
+/***********************************/
+/* MC_CMD_REBOOT_MODE 
+ */
+#define	MC_CMD_REBOOT_MODE  0x3f
+
+/* MC_CMD_REBOOT_MODE_IN msgrequest */
+#define	MC_CMD_REBOOT_MODE_IN_LEN 4
+#define	MC_CMD_REBOOT_MODE_IN_VALUE_OFST 0
+#define	MC_CMD_REBOOT_MODE_NORMAL 0x0 /* enum */
+#define	MC_CMD_REBOOT_MODE_SNAPPER 0x3 /* enum */
+
+/* MC_CMD_REBOOT_MODE_OUT msgresponse */
+#define	MC_CMD_REBOOT_MODE_OUT_LEN 4
+#define	MC_CMD_REBOOT_MODE_OUT_VALUE_OFST 0
+
+
+/***********************************/
+/* MC_CMD_SENSOR_INFO 
+ * Returns information about every available sensor.
+ */
+#define	MC_CMD_SENSOR_INFO  0x41
+
+/* MC_CMD_SENSOR_INFO_IN msgrequest */
+#define	MC_CMD_SENSOR_INFO_IN_LEN 0
+
+/* MC_CMD_SENSOR_INFO_OUT msgresponse */
+#define	MC_CMD_SENSOR_INFO_OUT_LENMIN 12
+#define	MC_CMD_SENSOR_INFO_OUT_LENMAX 252
+#define	MC_CMD_SENSOR_INFO_OUT_LEN(num) (4+8*(num))
+#define	MC_CMD_SENSOR_INFO_OUT_MASK_OFST 0
+#define	MC_CMD_SENSOR_CONTROLLER_TEMP  0x0 /* enum */
+#define	MC_CMD_SENSOR_PHY_COMMON_TEMP  0x1 /* enum */
+#define	MC_CMD_SENSOR_CONTROLLER_COOLING  0x2 /* enum */
+#define	MC_CMD_SENSOR_PHY0_TEMP  0x3 /* enum */
+#define	MC_CMD_SENSOR_PHY0_COOLING  0x4 /* enum */
+#define	MC_CMD_SENSOR_PHY1_TEMP  0x5 /* enum */
+#define	MC_CMD_SENSOR_PHY1_COOLING  0x6 /* enum */
+#define	MC_CMD_SENSOR_IN_1V0  0x7 /* enum */
+#define	MC_CMD_SENSOR_IN_1V2  0x8 /* enum */
+#define	MC_CMD_SENSOR_IN_1V8  0x9 /* enum */
+#define	MC_CMD_SENSOR_IN_2V5  0xa /* enum */
+#define	MC_CMD_SENSOR_IN_3V3  0xb /* enum */
+#define	MC_CMD_SENSOR_IN_12V0  0xc /* enum */
+#define	MC_CMD_SENSOR_ENTRY_OFST 4
+#define	MC_CMD_SENSOR_ENTRY_LEN 8
+#define	MC_CMD_SENSOR_ENTRY_LO_OFST 4
+#define	MC_CMD_SENSOR_ENTRY_HI_OFST 8
+#define	MC_CMD_SENSOR_ENTRY_MINNUM 1
+#define	MC_CMD_SENSOR_ENTRY_MAXNUM 31
+
+/* MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF structuredef */
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_LEN 8
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_OFST 0
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_LEN 2
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_LBN 0
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_WIDTH 16
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_OFST 2
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_LEN 2
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_LBN 16
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_WIDTH 16
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_OFST 4
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_LEN 2
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_LBN 32
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_WIDTH 16
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_OFST 6
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_LEN 2
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_LBN 48
+#define	MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_WIDTH 16
+
+
+/***********************************/
+/* MC_CMD_READ_SENSORS 
+ * Returns the current reading from each sensor.
+ */
+#define	MC_CMD_READ_SENSORS  0x42
+
+/* MC_CMD_READ_SENSORS_IN msgrequest */
+#define	MC_CMD_READ_SENSORS_IN_LEN 8
+#define	MC_CMD_READ_SENSORS_IN_DMA_ADDR_OFST 0
+#define	MC_CMD_READ_SENSORS_IN_DMA_ADDR_LEN 8
+#define	MC_CMD_READ_SENSORS_IN_DMA_ADDR_LO_OFST 0
+#define	MC_CMD_READ_SENSORS_IN_DMA_ADDR_HI_OFST 4
+
+/* MC_CMD_READ_SENSORS_OUT msgresponse */
+#define	MC_CMD_READ_SENSORS_OUT_LEN 0
+
+/* MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF structuredef */
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_LEN 3
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_OFST 0
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_LEN 2
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_LBN 0
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_WIDTH 16
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_OFST 2
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_LEN 1
+#define	MC_CMD_SENSOR_STATE_OK  0x0 /* enum */
+#define	MC_CMD_SENSOR_STATE_WARNING  0x1 /* enum */
+#define	MC_CMD_SENSOR_STATE_FATAL  0x2 /* enum */
+#define	MC_CMD_SENSOR_STATE_BROKEN  0x3 /* enum */
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_LBN 16
+#define	MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_WIDTH 8
+
+
+/***********************************/
+/* MC_CMD_GET_PHY_STATE 
+ * Report current state of PHY.
+ */
+#define	MC_CMD_GET_PHY_STATE  0x43
+
+/* MC_CMD_GET_PHY_STATE_IN msgrequest */
+#define	MC_CMD_GET_PHY_STATE_IN_LEN 0
+
+/* MC_CMD_GET_PHY_STATE_OUT msgresponse */
+#define	MC_CMD_GET_PHY_STATE_OUT_LEN 4
+#define	MC_CMD_GET_PHY_STATE_OUT_STATE_OFST 0
+#define	MC_CMD_PHY_STATE_OK 0x1 /* enum */
+#define	MC_CMD_PHY_STATE_ZOMBIE 0x2 /* enum */
+
+
+/***********************************/
+/* MC_CMD_SETUP_8021QBB 
+ * 802.1Qbb control.
+ */
+#define	MC_CMD_SETUP_8021QBB  0x44
+
+/* MC_CMD_SETUP_8021QBB_IN msgrequest */
+#define	MC_CMD_SETUP_8021QBB_IN_LEN 32
+#define	MC_CMD_SETUP_8021QBB_IN_TXQS_OFST 0
+#define	MC_CMD_SETUP_8021QBB_IN_TXQS_LEN 32
+
+/* MC_CMD_SETUP_8021QBB_OUT msgresponse */
+#define	MC_CMD_SETUP_8021QBB_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_WOL_FILTER_GET 
+ * Retrieve ID of any WoL filters.
+ */
+#define	MC_CMD_WOL_FILTER_GET  0x45
+
+/* MC_CMD_WOL_FILTER_GET_IN msgrequest */
+#define	MC_CMD_WOL_FILTER_GET_IN_LEN 0
+
+/* MC_CMD_WOL_FILTER_GET_OUT msgresponse */
+#define	MC_CMD_WOL_FILTER_GET_OUT_LEN 4
+#define	MC_CMD_WOL_FILTER_GET_OUT_FILTER_ID_OFST 0
+
+
+/***********************************/
+/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD 
+ * Add a protocol offload to NIC for lights-out state.
+ */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD  0x46
+
+/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN msgrequest */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LENMIN 8
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LENMAX 252
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LEN(num) (4+4*(num))
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0
+#define	MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_ARP 0x1 /* enum */
+#define	MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_NS  0x2 /* enum */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_OFST 4
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_LEN 4
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_MINNUM 1
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_MAXNUM 62
+
+/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP msgrequest */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_LEN 14
+/*            MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC_OFST 4
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC_LEN 6
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_IP_OFST 10
+
+/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS msgrequest */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_LEN 42
+/*            MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC_OFST 4
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC_LEN 6
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_SNIPV6_OFST 10
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_SNIPV6_LEN 16
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_IPV6_OFST 26
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_IPV6_LEN 16
+
+/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT msgresponse */
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_LEN 4
+#define	MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_FILTER_ID_OFST 0
+
+
+/***********************************/
+/* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD 
+ * Remove a protocol offload from NIC for lights-out state.
+ */
+#define	MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD  0x47
+
+/* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN msgrequest */
+#define	MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_LEN 8
+#define	MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0
+#define	MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_FILTER_ID_OFST 4
+
+/* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT msgresponse */
+#define	MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_MAC_RESET_RESTORE 
+ * Restore MAC after block reset.
+ */
+#define	MC_CMD_MAC_RESET_RESTORE  0x48
+
+/* MC_CMD_MAC_RESET_RESTORE_IN msgrequest */
+#define	MC_CMD_MAC_RESET_RESTORE_IN_LEN 0
+
+/* MC_CMD_MAC_RESET_RESTORE_OUT msgresponse */
+#define	MC_CMD_MAC_RESET_RESTORE_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_TESTASSERT  
+ */
+#define	MC_CMD_TESTASSERT   0x49
+
+/* MC_CMD_TESTASSERT_IN msgrequest */
+#define	MC_CMD_TESTASSERT_IN_LEN 0
+
+/* MC_CMD_TESTASSERT_OUT msgresponse */
+#define	MC_CMD_TESTASSERT_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_WORKAROUND 
+ * Enable/Disable a given workaround.
+ */
+#define	MC_CMD_WORKAROUND  0x4a
+
+/* MC_CMD_WORKAROUND_IN msgrequest */
+#define	MC_CMD_WORKAROUND_IN_LEN 8
+#define	MC_CMD_WORKAROUND_IN_TYPE_OFST 0
+#define	MC_CMD_WORKAROUND_BUG17230 0x1 /* enum */
+#define	MC_CMD_WORKAROUND_IN_ENABLED_OFST 4
+
+/* MC_CMD_WORKAROUND_OUT msgresponse */
+#define	MC_CMD_WORKAROUND_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_PHY_MEDIA_INFO 
+ * Read media-specific data from PHY.
+ */
+#define	MC_CMD_GET_PHY_MEDIA_INFO  0x4b
+
+/* MC_CMD_GET_PHY_MEDIA_INFO_IN msgrequest */
+#define	MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN 4
+#define	MC_CMD_GET_PHY_MEDIA_INFO_IN_PAGE_OFST 0
+
+/* MC_CMD_GET_PHY_MEDIA_INFO_OUT msgresponse */
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMIN 5
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMAX 255
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(num) (4+1*(num))
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATALEN_OFST 0
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_OFST 4
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_LEN 1
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_MINNUM 1
+#define	MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_MAXNUM 251
+
+
+/***********************************/
+/* MC_CMD_NVRAM_TEST 
+ * Test a particular NVRAM partition.
+ */
+#define	MC_CMD_NVRAM_TEST  0x4c
+
+/* MC_CMD_NVRAM_TEST_IN msgrequest */
+#define	MC_CMD_NVRAM_TEST_IN_LEN 4
+#define	MC_CMD_NVRAM_TEST_IN_TYPE_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
+
+/* MC_CMD_NVRAM_TEST_OUT msgresponse */
+#define	MC_CMD_NVRAM_TEST_OUT_LEN 4
+#define	MC_CMD_NVRAM_TEST_OUT_RESULT_OFST 0
+#define	MC_CMD_NVRAM_TEST_PASS 0x0 /* enum */
+#define	MC_CMD_NVRAM_TEST_FAIL 0x1 /* enum */
+#define	MC_CMD_NVRAM_TEST_NOTSUPP 0x2 /* enum */
+
+
+/***********************************/
+/* MC_CMD_MRSFP_TWEAK 
+ * Read status and/or set parameters for the 'mrsfp' driver.
+ */
+#define	MC_CMD_MRSFP_TWEAK  0x4d
+
+/* MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG msgrequest */
+#define	MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_LEN 16
+#define	MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_LEVEL_OFST 0
+#define	MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_DT_CFG_OFST 4
+#define	MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_BOOST_OFST 8
+#define	MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_DT_CFG_OFST 12
+
+/* MC_CMD_MRSFP_TWEAK_IN_READ_ONLY msgrequest */
+#define	MC_CMD_MRSFP_TWEAK_IN_READ_ONLY_LEN 0
+
+/* MC_CMD_MRSFP_TWEAK_OUT msgresponse */
+#define	MC_CMD_MRSFP_TWEAK_OUT_LEN 12
+#define	MC_CMD_MRSFP_TWEAK_OUT_IOEXP_INPUTS_OFST 0
+#define	MC_CMD_MRSFP_TWEAK_OUT_IOEXP_OUTPUTS_OFST 4
+#define	MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_OFST 8
+#define	MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_OUT 0x0 /* enum */
+#define	MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_IN 0x1 /* enum */
+
+
+/***********************************/
+/* MC_CMD_SENSOR_SET_LIMS 
+ * Adjusts the sensor limits.
+ */
+#define	MC_CMD_SENSOR_SET_LIMS  0x4e
+
+/* MC_CMD_SENSOR_SET_LIMS_IN msgrequest */
+#define	MC_CMD_SENSOR_SET_LIMS_IN_LEN 20
+#define	MC_CMD_SENSOR_SET_LIMS_IN_SENSOR_OFST 0
+/*            Enum values, see field(s): */
+/*               MC_CMD_SENSOR_INFO/MC_CMD_SENSOR_INFO_OUT/MASK */
+#define	MC_CMD_SENSOR_SET_LIMS_IN_LOW0_OFST 4
+#define	MC_CMD_SENSOR_SET_LIMS_IN_HI0_OFST 8
+#define	MC_CMD_SENSOR_SET_LIMS_IN_LOW1_OFST 12
+#define	MC_CMD_SENSOR_SET_LIMS_IN_HI1_OFST 16
+
+/* MC_CMD_SENSOR_SET_LIMS_OUT msgresponse */
+#define	MC_CMD_SENSOR_SET_LIMS_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_RESOURCE_LIMITS 
+ */
+#define	MC_CMD_GET_RESOURCE_LIMITS  0x4f
+
+/* MC_CMD_GET_RESOURCE_LIMITS_IN msgrequest */
+#define	MC_CMD_GET_RESOURCE_LIMITS_IN_LEN 0
+
+/* MC_CMD_GET_RESOURCE_LIMITS_OUT msgresponse */
+#define	MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN 16
+#define	MC_CMD_GET_RESOURCE_LIMITS_OUT_BUFTBL_OFST 0
+#define	MC_CMD_GET_RESOURCE_LIMITS_OUT_EVQ_OFST 4
+#define	MC_CMD_GET_RESOURCE_LIMITS_OUT_RXQ_OFST 8
+#define	MC_CMD_GET_RESOURCE_LIMITS_OUT_TXQ_OFST 12
+
+/* MC_CMD_RESOURCE_SPECIFIER enum */
+#define	MC_CMD_RESOURCE_INSTANCE_ANY 0xffffffff /* enum */
+#define	MC_CMD_RESOURCE_INSTANCE_NONE 0xfffffffe /* enum */
+
+
+/***********************************/
+/* MC_CMD_INIT_EVQ 
+ */
+#define	MC_CMD_INIT_EVQ  0x50
+
+/* MC_CMD_INIT_EVQ_IN msgrequest */
+#define	MC_CMD_INIT_EVQ_IN_LENMIN 36
+#define	MC_CMD_INIT_EVQ_IN_LENMAX 540
+#define	MC_CMD_INIT_EVQ_IN_LEN(num) (28+8*(num))
+#define	MC_CMD_INIT_EVQ_IN_SIZE_OFST 0
+#define	MC_CMD_INIT_EVQ_IN_INSTANCE_OFST 4
+#define	MC_CMD_INIT_EVQ_IN_TMR_LOAD_OFST 8
+#define	MC_CMD_INIT_EVQ_IN_TMR_RELOAD_OFST 12
+#define	MC_CMD_INIT_EVQ_IN_FLAGS_OFST 16
+#define	MC_CMD_INIT_EVQ_IN_FLAG_INTERRUPTING_LBN 0
+#define	MC_CMD_INIT_EVQ_IN_FLAG_INTERRUPTING_WIDTH 1
+#define	MC_CMD_INIT_EVQ_IN_FLAG_RPTR_DOS_LBN 1
+#define	MC_CMD_INIT_EVQ_IN_FLAG_RPTR_DOS_WIDTH 1
+#define	MC_CMD_INIT_EVQ_IN_TMR_MODE_OFST 20
+#define	MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS 0x0 /* enum */
+#define	MC_CMD_INIT_EVQ_IN_TMR_IMMED_START 0x1 /* enum */
+#define	MC_CMD_INIT_EVQ_IN_TMR_TRIG_START 0x2 /* enum */
+#define	MC_CMD_INIT_EVQ_IN_TMR_INT_HLDOFF 0x3 /* enum */
+#define	MC_CMD_INIT_EVQ_IN_TARGET_EVQ_OFST 24
+#define	MC_CMD_INIT_EVQ_IN_IRQ_NUM_OFST 24
+#define	MC_CMD_INIT_EVQ_IN_DMA_ADDR_OFST 28
+#define	MC_CMD_INIT_EVQ_IN_DMA_ADDR_LEN 8
+#define	MC_CMD_INIT_EVQ_IN_DMA_ADDR_LO_OFST 28
+#define	MC_CMD_INIT_EVQ_IN_DMA_ADDR_HI_OFST 32
+#define	MC_CMD_INIT_EVQ_IN_DMA_ADDR_MINNUM 1
+#define	MC_CMD_INIT_EVQ_IN_DMA_ADDR_MAXNUM 64
+
+/* MC_CMD_INIT_EVQ_OUT msgresponse */
+#define	MC_CMD_INIT_EVQ_OUT_LEN 4
+#define	MC_CMD_INIT_EVQ_OUT_IRQ_OFST 0
+
+/* QUEUE_CRC_MODE structuredef */
+#define	QUEUE_CRC_MODE_LEN 1
+#define	QUEUE_CRC_MODE_MODE_LBN 0
+#define	QUEUE_CRC_MODE_MODE_WIDTH 4
+#define	QUEUE_CRC_MODE_NONE  0x0 /* enum */
+#define	QUEUE_CRC_MODE_FCOE  0x1 /* enum */
+#define	QUEUE_CRC_MODE_ISCSI_HDR  0x2 /* enum */
+#define	QUEUE_CRC_MODE_ISCSI  0x3 /* enum */
+#define	QUEUE_CRC_MODE_FCOIPOE  0x4 /* enum */
+#define	QUEUE_CRC_MODE_MPA  0x5 /* enum */
+#define	QUEUE_CRC_MODE_SPARE_LBN 4
+#define	QUEUE_CRC_MODE_SPARE_WIDTH 4
+
+
+/***********************************/
+/* MC_CMD_INIT_RXQ 
+ */
+#define	MC_CMD_INIT_RXQ  0x51
+
+/* MC_CMD_INIT_RXQ_IN msgrequest */
+#define	MC_CMD_INIT_RXQ_IN_LENMIN 32
+#define	MC_CMD_INIT_RXQ_IN_LENMAX 248
+#define	MC_CMD_INIT_RXQ_IN_LEN(num) (24+8*(num))
+#define	MC_CMD_INIT_RXQ_IN_SIZE_OFST 0
+#define	MC_CMD_INIT_RXQ_IN_TARGET_EVQ_OFST 4
+#define	MC_CMD_INIT_RXQ_IN_LABEL_OFST 8
+#define	MC_CMD_INIT_RXQ_IN_INSTANCE_OFST 12
+#define	MC_CMD_INIT_RXQ_IN_FLAGS_OFST 16
+#define	MC_CMD_INIT_RXQ_IN_FLAG_BUFF_MODE_LBN 0
+#define	MC_CMD_INIT_RXQ_IN_FLAG_BUFF_MODE_WIDTH 1
+#define	MC_CMD_INIT_RXQ_IN_FLAG_HDR_SPLIT_LBN 1
+#define	MC_CMD_INIT_RXQ_IN_FLAG_HDR_SPLIT_WIDTH 1
+#define	MC_CMD_INIT_RXQ_IN_FLAG_PKT_EDIT_LBN 2
+#define	MC_CMD_INIT_RXQ_IN_FLAG_PKT_EDIT_WIDTH 1
+#define	MC_CMD_INIT_RXQ_IN_CRC_MODE_LBN 3
+#define	MC_CMD_INIT_RXQ_IN_CRC_MODE_WIDTH 4
+#define	MC_CMD_INIT_RXQ_IN_OWNER_ID_OFST 20
+#define	MC_CMD_INIT_RXQ_IN_DMA_ADDR_OFST 24
+#define	MC_CMD_INIT_RXQ_IN_DMA_ADDR_LEN 8
+#define	MC_CMD_INIT_RXQ_IN_DMA_ADDR_LO_OFST 24
+#define	MC_CMD_INIT_RXQ_IN_DMA_ADDR_HI_OFST 28
+#define	MC_CMD_INIT_RXQ_IN_DMA_ADDR_MINNUM 1
+#define	MC_CMD_INIT_RXQ_IN_DMA_ADDR_MAXNUM 28
+
+/* MC_CMD_INIT_RXQ_OUT msgresponse */
+#define	MC_CMD_INIT_RXQ_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_INIT_TXQ 
+ */
+#define	MC_CMD_INIT_TXQ  0x52
+
+/* MC_CMD_INIT_TXQ_IN msgrequest */
+#define	MC_CMD_INIT_TXQ_IN_LENMIN 32
+#define	MC_CMD_INIT_TXQ_IN_LENMAX 248
+#define	MC_CMD_INIT_TXQ_IN_LEN(num) (24+8*(num))
+#define	MC_CMD_INIT_TXQ_IN_SIZE_OFST 0
+#define	MC_CMD_INIT_TXQ_IN_TARGET_EVQ_OFST 4
+#define	MC_CMD_INIT_TXQ_IN_LABEL_OFST 8
+#define	MC_CMD_INIT_TXQ_IN_INSTANCE_OFST 12
+#define	MC_CMD_INIT_TXQ_IN_FLAGS_OFST 16
+#define	MC_CMD_INIT_TXQ_IN_FLAG_BUFF_MODE_LBN 0
+#define	MC_CMD_INIT_TXQ_IN_FLAG_BUFF_MODE_WIDTH 1
+#define	MC_CMD_INIT_TXQ_IN_FLAG_IP_CSUM_DIS_LBN 1
+#define	MC_CMD_INIT_TXQ_IN_FLAG_IP_CSUM_DIS_WIDTH 1
+#define	MC_CMD_INIT_TXQ_IN_FLAG_TCP_CSUM_DIS_LBN 2
+#define	MC_CMD_INIT_TXQ_IN_FLAG_TCP_CSUM_DIS_WIDTH 1
+#define	MC_CMD_INIT_TXQ_IN_CRC_MODE_LBN 4
+#define	MC_CMD_INIT_TXQ_IN_CRC_MODE_WIDTH 4
+#define	MC_CMD_INIT_TXQ_IN_OWNER_ID_OFST 20
+#define	MC_CMD_INIT_TXQ_IN_DMA_ADDR_OFST 24
+#define	MC_CMD_INIT_TXQ_IN_DMA_ADDR_LEN 8
+#define	MC_CMD_INIT_TXQ_IN_DMA_ADDR_LO_OFST 24
+#define	MC_CMD_INIT_TXQ_IN_DMA_ADDR_HI_OFST 28
+#define	MC_CMD_INIT_TXQ_IN_DMA_ADDR_MINNUM 1
+#define	MC_CMD_INIT_TXQ_IN_DMA_ADDR_MAXNUM 28
+
+/* MC_CMD_INIT_TXQ_OUT msgresponse */
+#define	MC_CMD_INIT_TXQ_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_FINI_EVQ 
+ */
+#define	MC_CMD_FINI_EVQ  0x55
+
+/* MC_CMD_FINI_EVQ_IN msgrequest */
+#define	MC_CMD_FINI_EVQ_IN_LEN 4
+#define	MC_CMD_FINI_EVQ_IN_INSTANCE_OFST 0
+
+/* MC_CMD_FINI_EVQ_OUT msgresponse */
+#define	MC_CMD_FINI_EVQ_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_FINI_RXQ 
+ */
+#define	MC_CMD_FINI_RXQ  0x56
+
+/* MC_CMD_FINI_RXQ_IN msgrequest */
+#define	MC_CMD_FINI_RXQ_IN_LEN 4
+#define	MC_CMD_FINI_RXQ_IN_INSTANCE_OFST 0
+
+/* MC_CMD_FINI_RXQ_OUT msgresponse */
+#define	MC_CMD_FINI_RXQ_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_FINI_TXQ 
+ */
+#define	MC_CMD_FINI_TXQ  0x57
+
+/* MC_CMD_FINI_TXQ_IN msgrequest */
+#define	MC_CMD_FINI_TXQ_IN_LEN 4
+#define	MC_CMD_FINI_TXQ_IN_INSTANCE_OFST 0
+
+/* MC_CMD_FINI_TXQ_OUT msgresponse */
+#define	MC_CMD_FINI_TXQ_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_DRIVER_EVENT 
+ */
+#define	MC_CMD_DRIVER_EVENT  0x5a
+
+/* MC_CMD_DRIVER_EVENT_IN msgrequest */
+#define	MC_CMD_DRIVER_EVENT_IN_LEN 12
+#define	MC_CMD_DRIVER_EVENT_IN_EVQ_OFST 0
+#define	MC_CMD_DRIVER_EVENT_IN_DATA_OFST 4
+#define	MC_CMD_DRIVER_EVENT_IN_DATA_LEN 8
+#define	MC_CMD_DRIVER_EVENT_IN_DATA_LO_OFST 4
+#define	MC_CMD_DRIVER_EVENT_IN_DATA_HI_OFST 8
+
+
+/***********************************/
+/* MC_CMD_PROXY_CMD 
+ */
+#define	MC_CMD_PROXY_CMD  0x5b
+
+/* MC_CMD_PROXY_CMD_IN msgrequest */
+#define	MC_CMD_PROXY_CMD_IN_LEN 4
+#define	MC_CMD_PROXY_CMD_IN_TARGET_OFST 0
+
+
+/***********************************/
+/* MC_CMD_ALLOC_OWNER_IDS 
+ */
+#define	MC_CMD_ALLOC_OWNER_IDS  0x54
+
+/* MC_CMD_ALLOC_OWNER_IDS_IN msgrequest */
+#define	MC_CMD_ALLOC_OWNER_IDS_IN_LEN 4
+#define	MC_CMD_ALLOC_OWNER_IDS_IN_NIDS_OFST 0
+
+/* MC_CMD_ALLOC_OWNER_IDS_OUT msgresponse */
+#define	MC_CMD_ALLOC_OWNER_IDS_OUT_LEN 12
+#define	MC_CMD_ALLOC_OWNER_IDS_OUT_HANDLE_OFST 0
+#define	MC_CMD_ALLOC_OWNER_IDS_OUT_NIDS_OFST 4
+#define	MC_CMD_ALLOC_OWNER_IDS_OUT_BASE_OFST 8
+
+
+/***********************************/
+/* MC_CMD_FREE_OWNER_IDS 
+ */
+#define	MC_CMD_FREE_OWNER_IDS  0x59
+
+/* MC_CMD_FREE_OWNER_IDS_IN msgrequest */
+#define	MC_CMD_FREE_OWNER_IDS_IN_LEN 4
+#define	MC_CMD_FREE_OWNER_IDS_IN_HANDLE_OFST 0
+
+/* MC_CMD_FREE_OWNER_IDS_OUT msgresponse */
+#define	MC_CMD_FREE_OWNER_IDS_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_ALLOC_BUFTBL_CHUNK 
+ */
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK  0x5c
+
+/* MC_CMD_ALLOC_BUFTBL_CHUNK_IN msgrequest */
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_IN_LEN 8
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_IN_OWNER_OFST 0
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_IN_PAGE_SIZE_OFST 4
+
+/* MC_CMD_ALLOC_BUFTBL_CHUNK_OUT msgresponse */
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_LEN 12
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_HANDLE_OFST 0
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_NUMENTRIES_OFST 4
+#define	MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_ID_OFST 8
+
+
+/***********************************/
+/* MC_CMD_PROGRAM_BUFTBL_ENTRIES 
+ */
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES  0x5d
+
+/* MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN msgrequest */
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LENMIN 20
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LENMAX 252
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LEN(num) (12+8*(num))
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_HANDLE_OFST 0
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_FIRSTID_OFST 4
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_NUMENTRIES_OFST 8
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_OFST 12
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LEN 8
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LO_OFST 12
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_HI_OFST 16
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MINNUM 1
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MAXNUM 30
+
+/* MC_CMD_PROGRAM_BUFTBL_ENTRIES_OUT msgresponse */
+#define	MC_CMD_PROGRAM_BUFTBL_ENTRIES_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_FREE_BUFTBL_CHUNK 
+ */
+#define	MC_CMD_FREE_BUFTBL_CHUNK  0x5e
+
+/* MC_CMD_FREE_BUFTBL_CHUNK_IN msgrequest */
+#define	MC_CMD_FREE_BUFTBL_CHUNK_IN_LEN 4
+#define	MC_CMD_FREE_BUFTBL_CHUNK_IN_HANDLE_OFST 0
+
+/* MC_CMD_FREE_BUFTBL_CHUNK_OUT msgresponse */
+#define	MC_CMD_FREE_BUFTBL_CHUNK_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_PF_COUNT 
+ */
+#define	MC_CMD_GET_PF_COUNT  0x60
+
+/* MC_CMD_GET_PF_COUNT_IN msgrequest */
+#define	MC_CMD_GET_PF_COUNT_IN_LEN 0
+
+/* MC_CMD_GET_PF_COUNT_OUT msgresponse */
+#define	MC_CMD_GET_PF_COUNT_OUT_LEN 1
+#define	MC_CMD_GET_PF_COUNT_OUT_PF_COUNT_OFST 0
+#define	MC_CMD_GET_PF_COUNT_OUT_PF_COUNT_LEN 1
+
+
+/***********************************/
+/* MC_CMD_FILTER_OP 
+ */
+#define	MC_CMD_FILTER_OP  0x61
+
+/* MC_CMD_FILTER_OP_IN msgrequest */
+#define	MC_CMD_FILTER_OP_IN_LEN 100
+#define	MC_CMD_FILTER_OP_IN_OP_OFST 0
+#define	MC_CMD_FILTER_OP_IN_OP_INSERT  0x0 /* enum */
+#define	MC_CMD_FILTER_OP_IN_OP_REMOVE  0x1 /* enum */
+#define	MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE  0x2 /* enum */
+#define	MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE  0x3 /* enum */
+#define	MC_CMD_FILTER_OP_IN_HANDLE_OFST 4
+#define	MC_CMD_FILTER_OP_IN_MATCH_FIELDS_OFST 8
+#define	MC_CMD_FILTER_OP_IN_MATCH_SRC_IP_LBN 0
+#define	MC_CMD_FILTER_OP_IN_MATCH_SRC_IP_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_DST_IP_LBN 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_DST_IP_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_SRC_MAC_LBN 2
+#define	MC_CMD_FILTER_OP_IN_MATCH_SRC_MAC_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_SRC_PORT_LBN 3
+#define	MC_CMD_FILTER_OP_IN_MATCH_SRC_PORT_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_DST_MAC_LBN 4
+#define	MC_CMD_FILTER_OP_IN_MATCH_DST_MAC_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_DST_PORT_LBN 5
+#define	MC_CMD_FILTER_OP_IN_MATCH_DST_PORT_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_ETHER_TYPE_LBN 6
+#define	MC_CMD_FILTER_OP_IN_MATCH_ETHER_TYPE_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_INNER_VLAN_LBN 7
+#define	MC_CMD_FILTER_OP_IN_MATCH_INNER_VLAN_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_OUTER_VLAN_LBN 8
+#define	MC_CMD_FILTER_OP_IN_MATCH_OUTER_VLAN_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_IP_PROTO_LBN 9
+#define	MC_CMD_FILTER_OP_IN_MATCH_IP_PROTO_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_FWDEF0_LBN 10
+#define	MC_CMD_FILTER_OP_IN_MATCH_FWDEF0_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_LBN 11
+#define	MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_RX_DEST_OFST 12
+#define	MC_CMD_FILTER_OP_IN_RX_DEST_DROP  0x0 /* enum */
+#define	MC_CMD_FILTER_OP_IN_RX_DEST_HOST  0x1 /* enum */
+#define	MC_CMD_FILTER_OP_IN_RX_DEST_MC  0x2 /* enum */
+#define	MC_CMD_FILTER_OP_IN_RX_DEST_TX0  0x3 /* enum */
+#define	MC_CMD_FILTER_OP_IN_RX_DEST_TX1  0x4 /* enum */
+#define	MC_CMD_FILTER_OP_IN_RX_QUEUE_OFST 16
+#define	MC_CMD_FILTER_OP_IN_RX_FLAGS_OFST 20
+#define	MC_CMD_FILTER_OP_IN_RX_FLAG_RSS_LBN 0
+#define	MC_CMD_FILTER_OP_IN_RX_FLAG_RSS_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_RSS_CONTEXT_OFST 24
+#define	MC_CMD_FILTER_OP_IN_TX_DOMAIN_OFST 28
+#define	MC_CMD_FILTER_OP_IN_TX_DEST_OFST 32
+#define	MC_CMD_FILTER_OP_IN_TX_DEST_MAC_LBN 0
+#define	MC_CMD_FILTER_OP_IN_TX_DEST_MAC_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_TX_DEST_PM_LBN 1
+#define	MC_CMD_FILTER_OP_IN_TX_DEST_PM_WIDTH 1
+#define	MC_CMD_FILTER_OP_IN_SRC_MAC_OFST 36
+#define	MC_CMD_FILTER_OP_IN_SRC_MAC_LEN 6
+#define	MC_CMD_FILTER_OP_IN_SRC_PORT_OFST 42
+#define	MC_CMD_FILTER_OP_IN_SRC_PORT_LEN 2
+#define	MC_CMD_FILTER_OP_IN_DST_MAC_OFST 44
+#define	MC_CMD_FILTER_OP_IN_DST_MAC_LEN 6
+#define	MC_CMD_FILTER_OP_IN_DST_PORT_OFST 50
+#define	MC_CMD_FILTER_OP_IN_DST_PORT_LEN 2
+#define	MC_CMD_FILTER_OP_IN_ETHER_TYPE_OFST 52
+#define	MC_CMD_FILTER_OP_IN_ETHER_TYPE_LEN 2
+#define	MC_CMD_FILTER_OP_IN_INNER_VLAN_OFST 54
+#define	MC_CMD_FILTER_OP_IN_INNER_VLAN_LEN 2
+#define	MC_CMD_FILTER_OP_IN_OUTER_VLAN_OFST 56
+#define	MC_CMD_FILTER_OP_IN_OUTER_VLAN_LEN 2
+#define	MC_CMD_FILTER_OP_IN_IP_PROTO_OFST 58
+#define	MC_CMD_FILTER_OP_IN_IP_PROTO_LEN 2
+#define	MC_CMD_FILTER_OP_IN_FWDEF0_OFST 60
+#define	MC_CMD_FILTER_OP_IN_FWDEF1_OFST 64
+#define	MC_CMD_FILTER_OP_IN_SRC_IP_OFST 68
+#define	MC_CMD_FILTER_OP_IN_SRC_IP_LEN 16
+#define	MC_CMD_FILTER_OP_IN_DST_IP_OFST 84
+#define	MC_CMD_FILTER_OP_IN_DST_IP_LEN 16
+
+/* MC_CMD_FILTER_OP_OUT msgresponse */
+#define	MC_CMD_FILTER_OP_OUT_LEN 8
+#define	MC_CMD_FILTER_OP_OUT_OP_OFST 0
+#define	MC_CMD_FILTER_OP_OUT_OP_INSERT  0x0 /* enum */
+#define	MC_CMD_FILTER_OP_OUT_OP_REMOVE  0x1 /* enum */
+#define	MC_CMD_FILTER_OP_OUT_OP_SUBSCRIBE  0x2 /* enum */
+#define	MC_CMD_FILTER_OP_OUT_OP_UNSUBSCRIBE  0x3 /* enum */
+#define	MC_CMD_FILTER_OP_OUT_HANDLE_OFST 4
+
+
+/***********************************/
+/* MC_CMD_SET_PF_COUNT 
+ */
+#define	MC_CMD_SET_PF_COUNT  0x62
+
+/* MC_CMD_SET_PF_COUNT_IN msgrequest */
+#define	MC_CMD_SET_PF_COUNT_IN_LEN 4
+#define	MC_CMD_SET_PF_COUNT_IN_PF_COUNT_OFST 0
+
+/* MC_CMD_SET_PF_COUNT_OUT msgresponse */
+#define	MC_CMD_SET_PF_COUNT_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_PORT_ASSIGNMENT 
+ */
+#define	MC_CMD_GET_PORT_ASSIGNMENT  0x63
+
+/* MC_CMD_GET_PORT_ASSIGNMENT_IN msgrequest */
+#define	MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN 0
+
+/* MC_CMD_GET_PORT_ASSIGNMENT_OUT msgresponse */
+#define	MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN 4
+#define	MC_CMD_GET_PORT_ASSIGNMENT_OUT_PORT_OFST 0
+
+
+/***********************************/
+/* MC_CMD_SET_PORT_ASSIGNMENT 
+ */
+#define	MC_CMD_SET_PORT_ASSIGNMENT  0x64
+
+/* MC_CMD_SET_PORT_ASSIGNMENT_IN msgrequest */
+#define	MC_CMD_SET_PORT_ASSIGNMENT_IN_LEN 4
+#define	MC_CMD_SET_PORT_ASSIGNMENT_IN_PORT_OFST 0
+
+/* MC_CMD_SET_PORT_ASSIGNMENT_OUT msgresponse */
+#define	MC_CMD_SET_PORT_ASSIGNMENT_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_ALLOC_VIS 
+ */
+#define	MC_CMD_ALLOC_VIS  0x65
+
+/* MC_CMD_ALLOC_VIS_IN msgrequest */
+#define	MC_CMD_ALLOC_VIS_IN_LEN 4
+#define	MC_CMD_ALLOC_VIS_IN_VI_COUNT_OFST 0
+
+/* MC_CMD_ALLOC_VIS_OUT msgresponse */
+#define	MC_CMD_ALLOC_VIS_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_FREE_VIS 
+ */
+#define	MC_CMD_FREE_VIS  0x66
+
+/* MC_CMD_FREE_VIS_IN msgrequest */
+#define	MC_CMD_FREE_VIS_IN_LEN 0
+
+/* MC_CMD_FREE_VIS_OUT msgresponse */
+#define	MC_CMD_FREE_VIS_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_SRIOV_CFG 
+ */
+#define	MC_CMD_GET_SRIOV_CFG  0x67
+
+/* MC_CMD_GET_SRIOV_CFG_IN msgrequest */
+#define	MC_CMD_GET_SRIOV_CFG_IN_LEN 0
+
+/* MC_CMD_GET_SRIOV_CFG_OUT msgresponse */
+#define	MC_CMD_GET_SRIOV_CFG_OUT_LEN 20
+#define	MC_CMD_GET_SRIOV_CFG_OUT_VF_CURRENT_OFST 0
+#define	MC_CMD_GET_SRIOV_CFG_OUT_VF_MAX_OFST 4
+#define	MC_CMD_GET_SRIOV_CFG_OUT_FLAGS_OFST 8
+#define	MC_CMD_GET_SRIOV_CFG_OUT_VF_ENABLED_LBN 0
+#define	MC_CMD_GET_SRIOV_CFG_OUT_VF_ENABLED_WIDTH 1
+#define	MC_CMD_GET_SRIOV_CFG_OUT_VF_OFFSET_OFST 12
+#define	MC_CMD_GET_SRIOV_CFG_OUT_VF_STRIDE_OFST 16
+
+
+/***********************************/
+/* MC_CMD_SET_SRIOV_CFG 
+ */
+#define	MC_CMD_SET_SRIOV_CFG  0x68
+
+/* MC_CMD_SET_SRIOV_CFG_IN msgrequest */
+#define	MC_CMD_SET_SRIOV_CFG_IN_LEN 20
+#define	MC_CMD_SET_SRIOV_CFG_IN_VF_CURRENT_OFST 0
+#define	MC_CMD_SET_SRIOV_CFG_IN_VF_MAX_OFST 4
+#define	MC_CMD_SET_SRIOV_CFG_IN_FLAGS_OFST 8
+#define	MC_CMD_SET_SRIOV_CFG_IN_VF_ENABLED_LBN 0
+#define	MC_CMD_SET_SRIOV_CFG_IN_VF_ENABLED_WIDTH 1
+#define	MC_CMD_SET_SRIOV_CFG_IN_VF_OFFSET_OFST 12
+#define	MC_CMD_SET_SRIOV_CFG_IN_VF_STRIDE_OFST 16
+
+/* MC_CMD_SET_SRIOV_CFG_OUT msgresponse */
+#define	MC_CMD_SET_SRIOV_CFG_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_GET_VI_COUNT 
+ */
+#define	MC_CMD_GET_VI_COUNT  0x69
+
+/* MC_CMD_GET_VI_COUNT_IN msgrequest */
+#define	MC_CMD_GET_VI_COUNT_IN_LEN 0
+
+/* MC_CMD_GET_VI_COUNT_OUT msgresponse */
+#define	MC_CMD_GET_VI_COUNT_OUT_LEN 4
+#define	MC_CMD_GET_VI_COUNT_OUT_VI_COUNT_OFST 0
+
+
+/***********************************/
+/* MC_CMD_GET_VECTOR_CFG 
+ */
+#define	MC_CMD_GET_VECTOR_CFG  0x70
+
+/* MC_CMD_GET_VECTOR_CFG_IN msgrequest */
+#define	MC_CMD_GET_VECTOR_CFG_IN_LEN 0
+
+/* MC_CMD_GET_VECTOR_CFG_OUT msgresponse */
+#define	MC_CMD_GET_VECTOR_CFG_OUT_LEN 12
+#define	MC_CMD_GET_VECTOR_CFG_OUT_VEC_BASE_OFST 0
+#define	MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_PF_OFST 4
+#define	MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_VF_OFST 8
+
+
+/***********************************/
+/* MC_CMD_SET_VECTOR_CFG 
+ */
+#define	MC_CMD_SET_VECTOR_CFG  0x71
+
+/* MC_CMD_SET_VECTOR_CFG_IN msgrequest */
+#define	MC_CMD_SET_VECTOR_CFG_IN_LEN 12
+#define	MC_CMD_SET_VECTOR_CFG_IN_VEC_BASE_OFST 0
+#define	MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_PF_OFST 4
+#define	MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_VF_OFST 8
+
+/* MC_CMD_SET_VECTOR_CFG_OUT msgresponse */
+#define	MC_CMD_SET_VECTOR_CFG_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_ALLOC_PIOBUF 
+ */
+#define	MC_CMD_ALLOC_PIOBUF  0x72
+
+/* MC_CMD_ALLOC_PIOBUF_IN msgrequest */
+#define	MC_CMD_ALLOC_PIOBUF_IN_LEN 0
+
+/* MC_CMD_ALLOC_PIOBUF_OUT msgresponse */
+#define	MC_CMD_ALLOC_PIOBUF_OUT_LEN 4
+#define	MC_CMD_ALLOC_PIOBUF_OUT_PIOBUF_HANDLE_OFST 0
+
+
+/***********************************/
+/* MC_CMD_FREE_PIOBUF 
+ */
+#define	MC_CMD_FREE_PIOBUF  0x73
+
+/* MC_CMD_FREE_PIOBUF_IN msgrequest */
+#define	MC_CMD_FREE_PIOBUF_IN_LEN 4
+#define	MC_CMD_FREE_PIOBUF_IN_PIOBUF_HANDLE_OFST 0
+
+/* MC_CMD_FREE_PIOBUF_OUT msgresponse */
+#define	MC_CMD_FREE_PIOBUF_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_V2_EXTN 
+ */
+#define	MC_CMD_V2_EXTN  0x7f
+
+/* MC_CMD_V2_EXTN_IN msgrequest */
+#define	MC_CMD_V2_EXTN_IN_LEN 4
+#define	MC_CMD_V2_EXTN_IN_EXTENDED_CMD_LBN 0
+#define	MC_CMD_V2_EXTN_IN_EXTENDED_CMD_WIDTH 15
+#define	MC_CMD_V2_EXTN_IN_UNUSED_LBN 15
+#define	MC_CMD_V2_EXTN_IN_UNUSED_WIDTH 1
+#define	MC_CMD_V2_EXTN_IN_ACTUAL_LEN_LBN 16
+#define	MC_CMD_V2_EXTN_IN_ACTUAL_LEN_WIDTH 10
+#define	MC_CMD_V2_EXTN_IN_UNUSED2_LBN 26
+#define	MC_CMD_V2_EXTN_IN_UNUSED2_WIDTH 6
+
+
+/***********************************/
+/* MC_CMD_TCM_BUCKET_ALLOC 
+ */
+#define	MC_CMD_TCM_BUCKET_ALLOC  0x80
+
+/* MC_CMD_TCM_BUCKET_ALLOC_IN msgrequest */
+#define	MC_CMD_TCM_BUCKET_ALLOC_IN_LEN 0
+
+/* MC_CMD_TCM_BUCKET_ALLOC_OUT msgresponse */
+#define	MC_CMD_TCM_BUCKET_ALLOC_OUT_LEN 4
+#define	MC_CMD_TCM_BUCKET_ALLOC_OUT_BUCKET_OFST 0
+
+
+/***********************************/
+/* MC_CMD_TCM_BUCKET_FREE 
+ */
+#define	MC_CMD_TCM_BUCKET_FREE  0x81
+
+/* MC_CMD_TCM_BUCKET_FREE_IN msgrequest */
+#define	MC_CMD_TCM_BUCKET_FREE_IN_LEN 4
+#define	MC_CMD_TCM_BUCKET_FREE_IN_BUCKET_OFST 0
+
+/* MC_CMD_TCM_BUCKET_FREE_OUT msgresponse */
+#define	MC_CMD_TCM_BUCKET_FREE_OUT_LEN 0
+
+
+/***********************************/
+/* MC_CMD_TCM_TXQ_INIT 
+ */
+#define	MC_CMD_TCM_TXQ_INIT  0x82
+
+/* MC_CMD_TCM_TXQ_INIT_IN msgrequest */
+#define	MC_CMD_TCM_TXQ_INIT_IN_LEN 28
+#define	MC_CMD_TCM_TXQ_INIT_IN_QID_OFST 0
+#define	MC_CMD_TCM_TXQ_INIT_IN_LABEL_OFST 4
+#define	MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAGS_OFST 8
+#define	MC_CMD_TCM_TXQ_INIT_IN_RP_BKT_OFST 12
+#define	MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT1_OFST 16
+#define	MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT2_OFST 20
+#define	MC_CMD_TCM_TXQ_INIT_IN_MIN_BKT_OFST 24
+
+/* MC_CMD_TCM_TXQ_INIT_OUT msgresponse */
+#define	MC_CMD_TCM_TXQ_INIT_OUT_LEN 0
+
+#endif /* _SIENA_MC_DRIVER_PCOL_H */
+/*! \cidoxg_end */
diff --git a/sys/dev/sfxge/common/efx_regs_pci.h b/sys/dev/sfxge/common/efx_regs_pci.h
new file mode 100644
index 0000000..cb5efab
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_regs_pci.h
@@ -0,0 +1,2376 @@
+/*-
+ * Copyright 2007-2010 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_SYS_EFX_REGS_PCI_H
+#define	_SYS_EFX_REGS_PCI_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * PC_VEND_ID_REG(16bit):
+ * Vendor ID register
+ */
+
+#define	PCR_AZ_VEND_ID_REG 0x00000000
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_VEND_ID_LBN 0
+#define	PCRF_AZ_VEND_ID_WIDTH 16
+
+
+/*
+ * PC_DEV_ID_REG(16bit):
+ * Device ID register
+ */
+
+#define	PCR_AZ_DEV_ID_REG 0x00000002
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_DEV_ID_LBN 0
+#define	PCRF_AZ_DEV_ID_WIDTH 16
+
+
+/*
+ * PC_CMD_REG(16bit):
+ * Command register
+ */
+
+#define	PCR_AZ_CMD_REG 0x00000004
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_INTX_DIS_LBN 10
+#define	PCRF_AZ_INTX_DIS_WIDTH 1
+#define	PCRF_AZ_FB2B_EN_LBN 9
+#define	PCRF_AZ_FB2B_EN_WIDTH 1
+#define	PCRF_AZ_SERR_EN_LBN 8
+#define	PCRF_AZ_SERR_EN_WIDTH 1
+#define	PCRF_AZ_IDSEL_CTL_LBN 7
+#define	PCRF_AZ_IDSEL_CTL_WIDTH 1
+#define	PCRF_AZ_PERR_EN_LBN 6
+#define	PCRF_AZ_PERR_EN_WIDTH 1
+#define	PCRF_AZ_VGA_PAL_SNP_LBN 5
+#define	PCRF_AZ_VGA_PAL_SNP_WIDTH 1
+#define	PCRF_AZ_MWI_EN_LBN 4
+#define	PCRF_AZ_MWI_EN_WIDTH 1
+#define	PCRF_AZ_SPEC_CYC_LBN 3
+#define	PCRF_AZ_SPEC_CYC_WIDTH 1
+#define	PCRF_AZ_MST_EN_LBN 2
+#define	PCRF_AZ_MST_EN_WIDTH 1
+#define	PCRF_AZ_MEM_EN_LBN 1
+#define	PCRF_AZ_MEM_EN_WIDTH 1
+#define	PCRF_AZ_IO_EN_LBN 0
+#define	PCRF_AZ_IO_EN_WIDTH 1
+
+
+/*
+ * PC_STAT_REG(16bit):
+ * Status register
+ */
+
+#define	PCR_AZ_STAT_REG 0x00000006
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_DET_PERR_LBN 15
+#define	PCRF_AZ_DET_PERR_WIDTH 1
+#define	PCRF_AZ_SIG_SERR_LBN 14
+#define	PCRF_AZ_SIG_SERR_WIDTH 1
+#define	PCRF_AZ_GOT_MABRT_LBN 13
+#define	PCRF_AZ_GOT_MABRT_WIDTH 1
+#define	PCRF_AZ_GOT_TABRT_LBN 12
+#define	PCRF_AZ_GOT_TABRT_WIDTH 1
+#define	PCRF_AZ_SIG_TABRT_LBN 11
+#define	PCRF_AZ_SIG_TABRT_WIDTH 1
+#define	PCRF_AZ_DEVSEL_TIM_LBN 9
+#define	PCRF_AZ_DEVSEL_TIM_WIDTH 2
+#define	PCRF_AZ_MDAT_PERR_LBN 8
+#define	PCRF_AZ_MDAT_PERR_WIDTH 1
+#define	PCRF_AZ_FB2B_CAP_LBN 7
+#define	PCRF_AZ_FB2B_CAP_WIDTH 1
+#define	PCRF_AZ_66MHZ_CAP_LBN 5
+#define	PCRF_AZ_66MHZ_CAP_WIDTH 1
+#define	PCRF_AZ_CAP_LIST_LBN 4
+#define	PCRF_AZ_CAP_LIST_WIDTH 1
+#define	PCRF_AZ_INTX_STAT_LBN 3
+#define	PCRF_AZ_INTX_STAT_WIDTH 1
+
+
+/*
+ * PC_REV_ID_REG(8bit):
+ * Class code & revision ID register
+ */
+
+#define	PCR_AZ_REV_ID_REG 0x00000008
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_REV_ID_LBN 0
+#define	PCRF_AZ_REV_ID_WIDTH 8
+
+
+/*
+ * PC_CC_REG(24bit):
+ * Class code register
+ */
+
+#define	PCR_AZ_CC_REG 0x00000009
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_BASE_CC_LBN 16
+#define	PCRF_AZ_BASE_CC_WIDTH 8
+#define	PCRF_AZ_SUB_CC_LBN 8
+#define	PCRF_AZ_SUB_CC_WIDTH 8
+#define	PCRF_AZ_PROG_IF_LBN 0
+#define	PCRF_AZ_PROG_IF_WIDTH 8
+
+
+/*
+ * PC_CACHE_LSIZE_REG(8bit):
+ * Cache line size
+ */
+
+#define	PCR_AZ_CACHE_LSIZE_REG 0x0000000c
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_CACHE_LSIZE_LBN 0
+#define	PCRF_AZ_CACHE_LSIZE_WIDTH 8
+
+
+/*
+ * PC_MST_LAT_REG(8bit):
+ * Master latency timer register
+ */
+
+#define	PCR_AZ_MST_LAT_REG 0x0000000d
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MST_LAT_LBN 0
+#define	PCRF_AZ_MST_LAT_WIDTH 8
+
+
+/*
+ * PC_HDR_TYPE_REG(8bit):
+ * Header type register
+ */
+
+#define	PCR_AZ_HDR_TYPE_REG 0x0000000e
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MULT_FUNC_LBN 7
+#define	PCRF_AZ_MULT_FUNC_WIDTH 1
+#define	PCRF_AZ_TYPE_LBN 0
+#define	PCRF_AZ_TYPE_WIDTH 7
+
+
+/*
+ * PC_BIST_REG(8bit):
+ * BIST register
+ */
+
+#define	PCR_AZ_BIST_REG 0x0000000f
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_BIST_LBN 0
+#define	PCRF_AZ_BIST_WIDTH 8
+
+
+/*
+ * PC_BAR0_REG(32bit):
+ * Primary function base address register 0
+ */
+
+#define	PCR_AZ_BAR0_REG 0x00000010
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_BAR0_LBN 4
+#define	PCRF_AZ_BAR0_WIDTH 28
+#define	PCRF_AZ_BAR0_PREF_LBN 3
+#define	PCRF_AZ_BAR0_PREF_WIDTH 1
+#define	PCRF_AZ_BAR0_TYPE_LBN 1
+#define	PCRF_AZ_BAR0_TYPE_WIDTH 2
+#define	PCRF_AZ_BAR0_IOM_LBN 0
+#define	PCRF_AZ_BAR0_IOM_WIDTH 1
+
+
+/*
+ * PC_BAR1_REG(32bit):
+ * Primary function base address register 1, BAR1 is not implemented so read only.
+ */
+
+#define	PCR_DZ_BAR1_REG 0x00000014
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_BAR1_LBN 0
+#define	PCRF_DZ_BAR1_WIDTH 32
+
+
+/*
+ * PC_BAR2_LO_REG(32bit):
+ * Primary function base address register 2 low bits
+ */
+
+#define	PCR_AZ_BAR2_LO_REG 0x00000018
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_BAR2_LO_LBN 4
+#define	PCRF_AZ_BAR2_LO_WIDTH 28
+#define	PCRF_AZ_BAR2_PREF_LBN 3
+#define	PCRF_AZ_BAR2_PREF_WIDTH 1
+#define	PCRF_AZ_BAR2_TYPE_LBN 1
+#define	PCRF_AZ_BAR2_TYPE_WIDTH 2
+#define	PCRF_AZ_BAR2_IOM_LBN 0
+#define	PCRF_AZ_BAR2_IOM_WIDTH 1
+
+
+/*
+ * PC_BAR2_HI_REG(32bit):
+ * Primary function base address register 2 high bits
+ */
+
+#define	PCR_AZ_BAR2_HI_REG 0x0000001c
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_BAR2_HI_LBN 0
+#define	PCRF_AZ_BAR2_HI_WIDTH 32
+
+
+/*
+ * PC_BAR4_LO_REG(32bit):
+ * Primary function base address register 2 low bits
+ */
+
+#define	PCR_CZ_BAR4_LO_REG 0x00000020
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_CZ_BAR4_LO_LBN 4
+#define	PCRF_CZ_BAR4_LO_WIDTH 28
+#define	PCRF_CZ_BAR4_PREF_LBN 3
+#define	PCRF_CZ_BAR4_PREF_WIDTH 1
+#define	PCRF_CZ_BAR4_TYPE_LBN 1
+#define	PCRF_CZ_BAR4_TYPE_WIDTH 2
+#define	PCRF_CZ_BAR4_IOM_LBN 0
+#define	PCRF_CZ_BAR4_IOM_WIDTH 1
+
+
+/*
+ * PC_BAR4_HI_REG(32bit):
+ * Primary function base address register 2 high bits
+ */
+
+#define	PCR_CZ_BAR4_HI_REG 0x00000024
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_CZ_BAR4_HI_LBN 0
+#define	PCRF_CZ_BAR4_HI_WIDTH 32
+
+
+/*
+ * PC_SS_VEND_ID_REG(16bit):
+ * Sub-system vendor ID register
+ */
+
+#define	PCR_AZ_SS_VEND_ID_REG 0x0000002c
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_SS_VEND_ID_LBN 0
+#define	PCRF_AZ_SS_VEND_ID_WIDTH 16
+
+
+/*
+ * PC_SS_ID_REG(16bit):
+ * Sub-system ID register
+ */
+
+#define	PCR_AZ_SS_ID_REG 0x0000002e
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_SS_ID_LBN 0
+#define	PCRF_AZ_SS_ID_WIDTH 16
+
+
+/*
+ * PC_EXPROM_BAR_REG(32bit):
+ * Expansion ROM base address register
+ */
+
+#define	PCR_AZ_EXPROM_BAR_REG 0x00000030
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_EXPROM_BAR_LBN 11
+#define	PCRF_AZ_EXPROM_BAR_WIDTH 21
+#define	PCRF_AB_EXPROM_MIN_SIZE_LBN 2
+#define	PCRF_AB_EXPROM_MIN_SIZE_WIDTH 9
+#define	PCRF_CZ_EXPROM_MIN_SIZE_LBN 1
+#define	PCRF_CZ_EXPROM_MIN_SIZE_WIDTH 10
+#define	PCRF_AB_EXPROM_FEATURE_ENABLE_LBN 1
+#define	PCRF_AB_EXPROM_FEATURE_ENABLE_WIDTH 1
+#define	PCRF_AZ_EXPROM_EN_LBN 0
+#define	PCRF_AZ_EXPROM_EN_WIDTH 1
+
+
+/*
+ * PC_CAP_PTR_REG(8bit):
+ * Capability pointer register
+ */
+
+#define	PCR_AZ_CAP_PTR_REG 0x00000034
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_CAP_PTR_LBN 0
+#define	PCRF_AZ_CAP_PTR_WIDTH 8
+
+
+/*
+ * PC_INT_LINE_REG(8bit):
+ * Interrupt line register
+ */
+
+#define	PCR_AZ_INT_LINE_REG 0x0000003c
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_INT_LINE_LBN 0
+#define	PCRF_AZ_INT_LINE_WIDTH 8
+
+
+/*
+ * PC_INT_PIN_REG(8bit):
+ * Interrupt pin register
+ */
+
+#define	PCR_AZ_INT_PIN_REG 0x0000003d
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_INT_PIN_LBN 0
+#define	PCRF_AZ_INT_PIN_WIDTH 8
+
+
+/*
+ * PC_PM_CAP_ID_REG(8bit):
+ * Power management capability ID
+ */
+
+#define	PCR_AC_PM_CAP_ID_REG 0x00000040
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_PM_CAP_ID_REG 0x00000080
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PM_CAP_ID_LBN 0
+#define	PCRF_AZ_PM_CAP_ID_WIDTH 8
+
+
+/*
+ * PC_PM_NXT_PTR_REG(8bit):
+ * Power management next item pointer
+ */
+
+#define	PCR_AC_PM_NXT_PTR_REG 0x00000041
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_PM_NXT_PTR_REG 0x00000081
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PM_NXT_PTR_LBN 0
+#define	PCRF_AZ_PM_NXT_PTR_WIDTH 8
+
+
+/*
+ * PC_PM_CAP_REG(16bit):
+ * Power management capabilities register
+ */
+
+#define	PCR_AC_PM_CAP_REG 0x00000042
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_PM_CAP_REG 0x00000082
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PM_PME_SUPT_LBN 11
+#define	PCRF_AZ_PM_PME_SUPT_WIDTH 5
+#define	PCRF_AZ_PM_D2_SUPT_LBN 10
+#define	PCRF_AZ_PM_D2_SUPT_WIDTH 1
+#define	PCRF_AZ_PM_D1_SUPT_LBN 9
+#define	PCRF_AZ_PM_D1_SUPT_WIDTH 1
+#define	PCRF_AZ_PM_AUX_CURR_LBN 6
+#define	PCRF_AZ_PM_AUX_CURR_WIDTH 3
+#define	PCRF_AZ_PM_DSI_LBN 5
+#define	PCRF_AZ_PM_DSI_WIDTH 1
+#define	PCRF_AZ_PM_PME_CLK_LBN 3
+#define	PCRF_AZ_PM_PME_CLK_WIDTH 1
+#define	PCRF_AZ_PM_PME_VER_LBN 0
+#define	PCRF_AZ_PM_PME_VER_WIDTH 3
+
+
+/*
+ * PC_PM_CS_REG(16bit):
+ * Power management control & status register
+ */
+
+#define	PCR_AC_PM_CS_REG 0x00000044
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_PM_CS_REG 0x00000084
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PM_PME_STAT_LBN 15
+#define	PCRF_AZ_PM_PME_STAT_WIDTH 1
+#define	PCRF_AZ_PM_DAT_SCALE_LBN 13
+#define	PCRF_AZ_PM_DAT_SCALE_WIDTH 2
+#define	PCRF_AZ_PM_DAT_SEL_LBN 9
+#define	PCRF_AZ_PM_DAT_SEL_WIDTH 4
+#define	PCRF_AZ_PM_PME_EN_LBN 8
+#define	PCRF_AZ_PM_PME_EN_WIDTH 1
+#define	PCRF_CZ_NO_SOFT_RESET_LBN 3
+#define	PCRF_CZ_NO_SOFT_RESET_WIDTH 1
+#define	PCRF_AZ_PM_PWR_ST_LBN 0
+#define	PCRF_AZ_PM_PWR_ST_WIDTH 2
+
+
+/*
+ * PC_MSI_CAP_ID_REG(8bit):
+ * MSI capability ID
+ */
+
+#define	PCR_AC_MSI_CAP_ID_REG 0x00000050
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_MSI_CAP_ID_REG 0x00000090
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MSI_CAP_ID_LBN 0
+#define	PCRF_AZ_MSI_CAP_ID_WIDTH 8
+
+
+/*
+ * PC_MSI_NXT_PTR_REG(8bit):
+ * MSI next item pointer
+ */
+
+#define	PCR_AC_MSI_NXT_PTR_REG 0x00000051
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_MSI_NXT_PTR_REG 0x00000091
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MSI_NXT_PTR_LBN 0
+#define	PCRF_AZ_MSI_NXT_PTR_WIDTH 8
+
+
+/*
+ * PC_MSI_CTL_REG(16bit):
+ * MSI control register
+ */
+
+#define	PCR_AC_MSI_CTL_REG 0x00000052
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_MSI_CTL_REG 0x00000092
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MSI_64_EN_LBN 7
+#define	PCRF_AZ_MSI_64_EN_WIDTH 1
+#define	PCRF_AZ_MSI_MULT_MSG_EN_LBN 4
+#define	PCRF_AZ_MSI_MULT_MSG_EN_WIDTH 3
+#define	PCRF_AZ_MSI_MULT_MSG_CAP_LBN 1
+#define	PCRF_AZ_MSI_MULT_MSG_CAP_WIDTH 3
+#define	PCRF_AZ_MSI_EN_LBN 0
+#define	PCRF_AZ_MSI_EN_WIDTH 1
+
+
+/*
+ * PC_MSI_ADR_LO_REG(32bit):
+ * MSI low 32 bits address register
+ */
+
+#define	PCR_AC_MSI_ADR_LO_REG 0x00000054
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_MSI_ADR_LO_REG 0x00000094
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MSI_ADR_LO_LBN 2
+#define	PCRF_AZ_MSI_ADR_LO_WIDTH 30
+
+
+/*
+ * PC_VPD_CAP_CTL_REG(8bit):
+ * VPD control and capabilities register
+ */
+
+#define	PCR_DZ_VPD_CAP_CTL_REG 0x00000054
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_VPD_CAP_CTL_REG 0x000000d0
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_VPD_FLAG_LBN 31
+#define	PCRF_CZ_VPD_FLAG_WIDTH 1
+#define	PCRF_CZ_VPD_ADDR_LBN 16
+#define	PCRF_CZ_VPD_ADDR_WIDTH 15
+#define	PCRF_CZ_VPD_NXT_PTR_LBN 8
+#define	PCRF_CZ_VPD_NXT_PTR_WIDTH 8
+#define	PCRF_CZ_VPD_CAP_ID_LBN 0
+#define	PCRF_CZ_VPD_CAP_ID_WIDTH 8
+
+
+/*
+ * PC_VPD_CAP_DATA_REG(32bit):
+ * documentation to be written for sum_PC_VPD_CAP_DATA_REG
+ */
+
+#define	PCR_DZ_VPD_CAP_DATA_REG 0x00000058
+/* hunta0=pci_f0_config */
+
+#define	PCR_AB_VPD_CAP_DATA_REG 0x000000b4
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_VPD_CAP_DATA_REG 0x000000d4
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_AZ_VPD_DATA_LBN 0
+#define	PCRF_AZ_VPD_DATA_WIDTH 32
+
+
+/*
+ * PC_MSI_ADR_HI_REG(32bit):
+ * MSI high 32 bits address register
+ */
+
+#define	PCR_AC_MSI_ADR_HI_REG 0x00000058
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_MSI_ADR_HI_REG 0x00000098
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MSI_ADR_HI_LBN 0
+#define	PCRF_AZ_MSI_ADR_HI_WIDTH 32
+
+
+/*
+ * PC_MSI_DAT_REG(16bit):
+ * MSI data register
+ */
+
+#define	PCR_AC_MSI_DAT_REG 0x0000005c
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCR_DZ_MSI_DAT_REG 0x0000009c
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_MSI_DAT_LBN 0
+#define	PCRF_AZ_MSI_DAT_WIDTH 16
+
+
+/*
+ * PC_PCIE_CAP_LIST_REG(16bit):
+ * PCIe capability list register
+ */
+
+#define	PCR_AB_PCIE_CAP_LIST_REG 0x00000060
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_PCIE_CAP_LIST_REG 0x00000070
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_PCIE_CAP_LIST_REG 0x000000c0
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PCIE_NXT_PTR_LBN 8
+#define	PCRF_AZ_PCIE_NXT_PTR_WIDTH 8
+#define	PCRF_AZ_PCIE_CAP_ID_LBN 0
+#define	PCRF_AZ_PCIE_CAP_ID_WIDTH 8
+
+
+/*
+ * PC_PCIE_CAP_REG(16bit):
+ * PCIe capability register
+ */
+
+#define	PCR_AB_PCIE_CAP_REG 0x00000062
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_PCIE_CAP_REG 0x00000072
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_PCIE_CAP_REG 0x000000c2
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PCIE_INT_MSG_NUM_LBN 9
+#define	PCRF_AZ_PCIE_INT_MSG_NUM_WIDTH 5
+#define	PCRF_AZ_PCIE_SLOT_IMP_LBN 8
+#define	PCRF_AZ_PCIE_SLOT_IMP_WIDTH 1
+#define	PCRF_AZ_PCIE_DEV_PORT_TYPE_LBN 4
+#define	PCRF_AZ_PCIE_DEV_PORT_TYPE_WIDTH 4
+#define	PCRF_AZ_PCIE_CAP_VER_LBN 0
+#define	PCRF_AZ_PCIE_CAP_VER_WIDTH 4
+
+
+/*
+ * PC_DEV_CAP_REG(32bit):
+ * PCIe device capabilities register
+ */
+
+#define	PCR_AB_DEV_CAP_REG 0x00000064
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_DEV_CAP_REG 0x00000074
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_DEV_CAP_REG 0x000000c4
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_CAP_FN_LEVEL_RESET_LBN 28
+#define	PCRF_CZ_CAP_FN_LEVEL_RESET_WIDTH 1
+#define	PCRF_AZ_CAP_SLOT_PWR_SCL_LBN 26
+#define	PCRF_AZ_CAP_SLOT_PWR_SCL_WIDTH 2
+#define	PCRF_AZ_CAP_SLOT_PWR_VAL_LBN 18
+#define	PCRF_AZ_CAP_SLOT_PWR_VAL_WIDTH 8
+#define	PCRF_CZ_ROLE_BASE_ERR_REPORTING_LBN 15
+#define	PCRF_CZ_ROLE_BASE_ERR_REPORTING_WIDTH 1
+#define	PCRF_AB_PWR_IND_LBN 14
+#define	PCRF_AB_PWR_IND_WIDTH 1
+#define	PCRF_AB_ATTN_IND_LBN 13
+#define	PCRF_AB_ATTN_IND_WIDTH 1
+#define	PCRF_AB_ATTN_BUTTON_LBN 12
+#define	PCRF_AB_ATTN_BUTTON_WIDTH 1
+#define	PCRF_AZ_ENDPT_L1_LAT_LBN 9
+#define	PCRF_AZ_ENDPT_L1_LAT_WIDTH 3
+#define	PCRF_AZ_ENDPT_L0_LAT_LBN 6
+#define	PCRF_AZ_ENDPT_L0_LAT_WIDTH 3
+#define	PCRF_AZ_TAG_FIELD_LBN 5
+#define	PCRF_AZ_TAG_FIELD_WIDTH 1
+#define	PCRF_AZ_PHAN_FUNC_LBN 3
+#define	PCRF_AZ_PHAN_FUNC_WIDTH 2
+#define	PCRF_AZ_MAX_PAYL_SIZE_SUPT_LBN 0
+#define	PCRF_AZ_MAX_PAYL_SIZE_SUPT_WIDTH 3
+
+
+/*
+ * PC_DEV_CTL_REG(16bit):
+ * PCIe device control register
+ */
+
+#define	PCR_AB_DEV_CTL_REG 0x00000068
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_DEV_CTL_REG 0x00000078
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_DEV_CTL_REG 0x000000c8
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_FN_LEVEL_RESET_LBN 15
+#define	PCRF_CZ_FN_LEVEL_RESET_WIDTH 1
+#define	PCRF_AZ_MAX_RD_REQ_SIZE_LBN 12
+#define	PCRF_AZ_MAX_RD_REQ_SIZE_WIDTH 3
+#define	PCFE_AZ_MAX_RD_REQ_SIZE_4096 5
+#define	PCFE_AZ_MAX_RD_REQ_SIZE_2048 4
+#define	PCFE_AZ_MAX_RD_REQ_SIZE_1024 3
+#define	PCFE_AZ_MAX_RD_REQ_SIZE_512 2
+#define	PCFE_AZ_MAX_RD_REQ_SIZE_256 1
+#define	PCFE_AZ_MAX_RD_REQ_SIZE_128 0
+#define	PCFE_DZ_OTHER other
+#define	PCRF_AZ_EN_NO_SNOOP_LBN 11
+#define	PCRF_AZ_EN_NO_SNOOP_WIDTH 1
+#define	PCRF_AZ_AUX_PWR_PM_EN_LBN 10
+#define	PCRF_AZ_AUX_PWR_PM_EN_WIDTH 1
+#define	PCRF_AZ_PHAN_FUNC_EN_LBN 9
+#define	PCRF_AZ_PHAN_FUNC_EN_WIDTH 1
+#define	PCRF_AB_DEV_CAP_REG_RSVD0_LBN 8
+#define	PCRF_AB_DEV_CAP_REG_RSVD0_WIDTH 1
+#define	PCRF_CZ_EXTENDED_TAG_EN_LBN 8
+#define	PCRF_CZ_EXTENDED_TAG_EN_WIDTH 1
+#define	PCRF_AZ_MAX_PAYL_SIZE_LBN 5
+#define	PCRF_AZ_MAX_PAYL_SIZE_WIDTH 3
+#define	PCFE_AZ_MAX_PAYL_SIZE_4096 5
+#define	PCFE_AZ_MAX_PAYL_SIZE_2048 4
+#define	PCFE_AZ_MAX_PAYL_SIZE_1024 3
+#define	PCFE_AZ_MAX_PAYL_SIZE_512 2
+#define	PCFE_AZ_MAX_PAYL_SIZE_256 1
+#define	PCFE_AZ_MAX_PAYL_SIZE_128 0
+#define	PCFE_DZ_OTHER other
+#define	PCRF_AZ_EN_RELAX_ORDER_LBN 4
+#define	PCRF_AZ_EN_RELAX_ORDER_WIDTH 1
+#define	PCRF_AZ_UNSUP_REQ_RPT_EN_LBN 3
+#define	PCRF_AZ_UNSUP_REQ_RPT_EN_WIDTH 1
+#define	PCRF_AZ_FATAL_ERR_RPT_EN_LBN 2
+#define	PCRF_AZ_FATAL_ERR_RPT_EN_WIDTH 1
+#define	PCRF_AZ_NONFATAL_ERR_RPT_EN_LBN 1
+#define	PCRF_AZ_NONFATAL_ERR_RPT_EN_WIDTH 1
+#define	PCRF_AZ_CORR_ERR_RPT_EN_LBN 0
+#define	PCRF_AZ_CORR_ERR_RPT_EN_WIDTH 1
+
+
+/*
+ * PC_DEV_STAT_REG(16bit):
+ * PCIe device status register
+ */
+
+#define	PCR_AB_DEV_STAT_REG 0x0000006a
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_DEV_STAT_REG 0x0000007a
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_DEV_STAT_REG 0x000000ca
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_TRNS_PEND_LBN 5
+#define	PCRF_AZ_TRNS_PEND_WIDTH 1
+#define	PCRF_AZ_AUX_PWR_DET_LBN 4
+#define	PCRF_AZ_AUX_PWR_DET_WIDTH 1
+#define	PCRF_AZ_UNSUP_REQ_DET_LBN 3
+#define	PCRF_AZ_UNSUP_REQ_DET_WIDTH 1
+#define	PCRF_AZ_FATAL_ERR_DET_LBN 2
+#define	PCRF_AZ_FATAL_ERR_DET_WIDTH 1
+#define	PCRF_AZ_NONFATAL_ERR_DET_LBN 1
+#define	PCRF_AZ_NONFATAL_ERR_DET_WIDTH 1
+#define	PCRF_AZ_CORR_ERR_DET_LBN 0
+#define	PCRF_AZ_CORR_ERR_DET_WIDTH 1
+
+
+/*
+ * PC_LNK_CAP_REG(32bit):
+ * PCIe link capabilities register
+ */
+
+#define	PCR_AB_LNK_CAP_REG 0x0000006c
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_LNK_CAP_REG 0x0000007c
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_LNK_CAP_REG 0x000000cc
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_PORT_NUM_LBN 24
+#define	PCRF_AZ_PORT_NUM_WIDTH 8
+#define	PCRF_CZ_LINK_BWDITH_NOTIF_CAP_LBN 21
+#define	PCRF_CZ_LINK_BWDITH_NOTIF_CAP_WIDTH 1
+#define	PCRF_CZ_DATA_LINK_ACTIVE_RPT_CAP_LBN 20
+#define	PCRF_CZ_DATA_LINK_ACTIVE_RPT_CAP_WIDTH 1
+#define	PCRF_CZ_SURPISE_DOWN_RPT_CAP_LBN 19
+#define	PCRF_CZ_SURPISE_DOWN_RPT_CAP_WIDTH 1
+#define	PCRF_CZ_CLOCK_PWR_MNGMNT_CAP_LBN 18
+#define	PCRF_CZ_CLOCK_PWR_MNGMNT_CAP_WIDTH 1
+#define	PCRF_AZ_DEF_L1_EXIT_LAT_LBN 15
+#define	PCRF_AZ_DEF_L1_EXIT_LAT_WIDTH 3
+#define	PCRF_AZ_DEF_L0_EXIT_LATPORT_NUM_LBN 12
+#define	PCRF_AZ_DEF_L0_EXIT_LATPORT_NUM_WIDTH 3
+#define	PCRF_AZ_AS_LNK_PM_SUPT_LBN 10
+#define	PCRF_AZ_AS_LNK_PM_SUPT_WIDTH 2
+#define	PCRF_AZ_MAX_LNK_WIDTH_LBN 4
+#define	PCRF_AZ_MAX_LNK_WIDTH_WIDTH 6
+#define	PCRF_AZ_MAX_LNK_SP_LBN 0
+#define	PCRF_AZ_MAX_LNK_SP_WIDTH 4
+
+
+/*
+ * PC_LNK_CTL_REG(16bit):
+ * PCIe link control register
+ */
+
+#define	PCR_AB_LNK_CTL_REG 0x00000070
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_LNK_CTL_REG 0x00000080
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_LNK_CTL_REG 0x000000d0
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_EXT_SYNC_LBN 7
+#define	PCRF_AZ_EXT_SYNC_WIDTH 1
+#define	PCRF_AZ_COMM_CLK_CFG_LBN 6
+#define	PCRF_AZ_COMM_CLK_CFG_WIDTH 1
+#define	PCRF_AB_LNK_CTL_REG_RSVD0_LBN 5
+#define	PCRF_AB_LNK_CTL_REG_RSVD0_WIDTH 1
+#define	PCRF_CZ_LNK_RETRAIN_LBN 5
+#define	PCRF_CZ_LNK_RETRAIN_WIDTH 1
+#define	PCRF_AZ_LNK_DIS_LBN 4
+#define	PCRF_AZ_LNK_DIS_WIDTH 1
+#define	PCRF_AZ_RD_COM_BDRY_LBN 3
+#define	PCRF_AZ_RD_COM_BDRY_WIDTH 1
+#define	PCRF_AZ_ACT_ST_LNK_PM_CTL_LBN 0
+#define	PCRF_AZ_ACT_ST_LNK_PM_CTL_WIDTH 2
+
+
+/*
+ * PC_LNK_STAT_REG(16bit):
+ * PCIe link status register
+ */
+
+#define	PCR_AB_LNK_STAT_REG 0x00000072
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_LNK_STAT_REG 0x00000082
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_LNK_STAT_REG 0x000000d2
+/* hunta0=pci_f0_config */
+
+#define	PCRF_AZ_SLOT_CLK_CFG_LBN 12
+#define	PCRF_AZ_SLOT_CLK_CFG_WIDTH 1
+#define	PCRF_AZ_LNK_TRAIN_LBN 11
+#define	PCRF_AZ_LNK_TRAIN_WIDTH 1
+#define	PCRF_AB_TRAIN_ERR_LBN 10
+#define	PCRF_AB_TRAIN_ERR_WIDTH 1
+#define	PCRF_AZ_LNK_WIDTH_LBN 4
+#define	PCRF_AZ_LNK_WIDTH_WIDTH 6
+#define	PCRF_AZ_LNK_SP_LBN 0
+#define	PCRF_AZ_LNK_SP_WIDTH 4
+
+
+/*
+ * PC_SLOT_CAP_REG(32bit):
+ * PCIe slot capabilities register
+ */
+
+#define	PCR_AB_SLOT_CAP_REG 0x00000074
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_SLOT_NUM_LBN 19
+#define	PCRF_AB_SLOT_NUM_WIDTH 13
+#define	PCRF_AB_SLOT_PWR_LIM_SCL_LBN 15
+#define	PCRF_AB_SLOT_PWR_LIM_SCL_WIDTH 2
+#define	PCRF_AB_SLOT_PWR_LIM_VAL_LBN 7
+#define	PCRF_AB_SLOT_PWR_LIM_VAL_WIDTH 8
+#define	PCRF_AB_SLOT_HP_CAP_LBN 6
+#define	PCRF_AB_SLOT_HP_CAP_WIDTH 1
+#define	PCRF_AB_SLOT_HP_SURP_LBN 5
+#define	PCRF_AB_SLOT_HP_SURP_WIDTH 1
+#define	PCRF_AB_SLOT_PWR_IND_PRST_LBN 4
+#define	PCRF_AB_SLOT_PWR_IND_PRST_WIDTH 1
+#define	PCRF_AB_SLOT_ATTN_IND_PRST_LBN 3
+#define	PCRF_AB_SLOT_ATTN_IND_PRST_WIDTH 1
+#define	PCRF_AB_SLOT_MRL_SENS_PRST_LBN 2
+#define	PCRF_AB_SLOT_MRL_SENS_PRST_WIDTH 1
+#define	PCRF_AB_SLOT_PWR_CTL_PRST_LBN 1
+#define	PCRF_AB_SLOT_PWR_CTL_PRST_WIDTH 1
+#define	PCRF_AB_SLOT_ATTN_BUT_PRST_LBN 0
+#define	PCRF_AB_SLOT_ATTN_BUT_PRST_WIDTH 1
+
+
+/*
+ * PC_SLOT_CTL_REG(16bit):
+ * PCIe slot control register
+ */
+
+#define	PCR_AB_SLOT_CTL_REG 0x00000078
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_SLOT_PWR_CTLR_CTL_LBN 10
+#define	PCRF_AB_SLOT_PWR_CTLR_CTL_WIDTH 1
+#define	PCRF_AB_SLOT_PWR_IND_CTL_LBN 8
+#define	PCRF_AB_SLOT_PWR_IND_CTL_WIDTH 2
+#define	PCRF_AB_SLOT_ATT_IND_CTL_LBN 6
+#define	PCRF_AB_SLOT_ATT_IND_CTL_WIDTH 2
+#define	PCRF_AB_SLOT_HP_INT_EN_LBN 5
+#define	PCRF_AB_SLOT_HP_INT_EN_WIDTH 1
+#define	PCRF_AB_SLOT_CMD_COMP_INT_EN_LBN 4
+#define	PCRF_AB_SLOT_CMD_COMP_INT_EN_WIDTH 1
+#define	PCRF_AB_SLOT_PRES_DET_CHG_EN_LBN 3
+#define	PCRF_AB_SLOT_PRES_DET_CHG_EN_WIDTH 1
+#define	PCRF_AB_SLOT_MRL_SENS_CHG_EN_LBN 2
+#define	PCRF_AB_SLOT_MRL_SENS_CHG_EN_WIDTH 1
+#define	PCRF_AB_SLOT_PWR_FLTDET_EN_LBN 1
+#define	PCRF_AB_SLOT_PWR_FLTDET_EN_WIDTH 1
+#define	PCRF_AB_SLOT_ATTN_BUT_EN_LBN 0
+#define	PCRF_AB_SLOT_ATTN_BUT_EN_WIDTH 1
+
+
+/*
+ * PC_SLOT_STAT_REG(16bit):
+ * PCIe slot status register
+ */
+
+#define	PCR_AB_SLOT_STAT_REG 0x0000007a
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_PRES_DET_ST_LBN 6
+#define	PCRF_AB_PRES_DET_ST_WIDTH 1
+#define	PCRF_AB_MRL_SENS_ST_LBN 5
+#define	PCRF_AB_MRL_SENS_ST_WIDTH 1
+#define	PCRF_AB_SLOT_PWR_IND_LBN 4
+#define	PCRF_AB_SLOT_PWR_IND_WIDTH 1
+#define	PCRF_AB_SLOT_ATTN_IND_LBN 3
+#define	PCRF_AB_SLOT_ATTN_IND_WIDTH 1
+#define	PCRF_AB_SLOT_MRL_SENS_LBN 2
+#define	PCRF_AB_SLOT_MRL_SENS_WIDTH 1
+#define	PCRF_AB_PWR_FLTDET_LBN 1
+#define	PCRF_AB_PWR_FLTDET_WIDTH 1
+#define	PCRF_AB_ATTN_BUTDET_LBN 0
+#define	PCRF_AB_ATTN_BUTDET_WIDTH 1
+
+
+/*
+ * PC_MSIX_CAP_ID_REG(8bit):
+ * MSIX Capability ID
+ */
+
+#define	PCR_BB_MSIX_CAP_ID_REG 0x00000090
+/* falconb0=pci_f0_config */
+
+#define	PCR_CZ_MSIX_CAP_ID_REG 0x000000b0
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_BZ_MSIX_CAP_ID_LBN 0
+#define	PCRF_BZ_MSIX_CAP_ID_WIDTH 8
+
+
+/*
+ * PC_MSIX_NXT_PTR_REG(8bit):
+ * MSIX Capability Next Capability Ptr
+ */
+
+#define	PCR_BB_MSIX_NXT_PTR_REG 0x00000091
+/* falconb0=pci_f0_config */
+
+#define	PCR_CZ_MSIX_NXT_PTR_REG 0x000000b1
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_BZ_MSIX_NXT_PTR_LBN 0
+#define	PCRF_BZ_MSIX_NXT_PTR_WIDTH 8
+
+
+/*
+ * PC_MSIX_CTL_REG(16bit):
+ * MSIX control register
+ */
+
+#define	PCR_BB_MSIX_CTL_REG 0x00000092
+/* falconb0=pci_f0_config */
+
+#define	PCR_CZ_MSIX_CTL_REG 0x000000b2
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_BZ_MSIX_EN_LBN 15
+#define	PCRF_BZ_MSIX_EN_WIDTH 1
+#define	PCRF_BZ_MSIX_FUNC_MASK_LBN 14
+#define	PCRF_BZ_MSIX_FUNC_MASK_WIDTH 1
+#define	PCRF_BZ_MSIX_TBL_SIZE_LBN 0
+#define	PCRF_BZ_MSIX_TBL_SIZE_WIDTH 11
+
+
+/*
+ * PC_DEV_CAP2_REG(16bit):
+ * PCIe Device Capabilities 2
+ */
+
+#define	PCR_CC_DEV_CAP2_REG 0x00000094
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_DEV_CAP2_REG 0x000000e4
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_CMPL_TIMEOUT_DIS_LBN 4
+#define	PCRF_CZ_CMPL_TIMEOUT_DIS_WIDTH 1
+#define	PCRF_CZ_CMPL_TIMEOUT_LBN 0
+#define	PCRF_CZ_CMPL_TIMEOUT_WIDTH 4
+#define	PCFE_CZ_CMPL_TIMEOUT_17000_TO_6400MS 14
+#define	PCFE_CZ_CMPL_TIMEOUT_4000_TO_1300MS 13
+#define	PCFE_CZ_CMPL_TIMEOUT_1000_TO_3500MS 10
+#define	PCFE_CZ_CMPL_TIMEOUT_260_TO_900MS 9
+#define	PCFE_CZ_CMPL_TIMEOUT_65_TO_210MS 6
+#define	PCFE_CZ_CMPL_TIMEOUT_16_TO_55MS 5
+#define	PCFE_CZ_CMPL_TIMEOUT_1_TO_10MS 2
+#define	PCFE_CZ_CMPL_TIMEOUT_50_TO_100US 1
+#define	PCFE_CZ_CMPL_TIMEOUT_DEFAULT 0
+
+
+/*
+ * PC_MSIX_TBL_BASE_REG(32bit):
+ * MSIX Capability Vector Table Base
+ */
+
+#define	PCR_BB_MSIX_TBL_BASE_REG 0x00000094
+/* falconb0=pci_f0_config */
+
+#define	PCR_CZ_MSIX_TBL_BASE_REG 0x000000b4
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_BZ_MSIX_TBL_OFF_LBN 3
+#define	PCRF_BZ_MSIX_TBL_OFF_WIDTH 29
+#define	PCRF_BZ_MSIX_TBL_BIR_LBN 0
+#define	PCRF_BZ_MSIX_TBL_BIR_WIDTH 3
+
+
+/*
+ * PC_DEV_CTL2_REG(16bit):
+ * PCIe Device Control 2
+ */
+
+#define	PCR_CC_DEV_CTL2_REG 0x00000098
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_DEV_CTL2_REG 0x000000e8
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_CMPL_TIMEOUT_DIS_CTL_LBN 4
+#define	PCRF_CZ_CMPL_TIMEOUT_DIS_CTL_WIDTH 1
+#define	PCRF_CZ_CMPL_TIMEOUT_CTL_LBN 0
+#define	PCRF_CZ_CMPL_TIMEOUT_CTL_WIDTH 4
+
+
+/*
+ * PC_MSIX_PBA_BASE_REG(32bit):
+ * MSIX Capability PBA Base
+ */
+
+#define	PCR_BB_MSIX_PBA_BASE_REG 0x00000098
+/* falconb0=pci_f0_config */
+
+#define	PCR_CZ_MSIX_PBA_BASE_REG 0x000000b8
+/* sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_BZ_MSIX_PBA_OFF_LBN 3
+#define	PCRF_BZ_MSIX_PBA_OFF_WIDTH 29
+#define	PCRF_BZ_MSIX_PBA_BIR_LBN 0
+#define	PCRF_BZ_MSIX_PBA_BIR_WIDTH 3
+
+
+/*
+ * PC_LNK_CTL2_REG(16bit):
+ * PCIe Link Control 2
+ */
+
+#define	PCR_CC_LNK_CTL2_REG 0x000000a0
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_LNK_CTL2_REG 0x000000f0
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_POLLING_DEEMPH_LVL_LBN 12
+#define	PCRF_CZ_POLLING_DEEMPH_LVL_WIDTH 1
+#define	PCRF_CZ_COMPLIANCE_SOS_CTL_LBN 11
+#define	PCRF_CZ_COMPLIANCE_SOS_CTL_WIDTH 1
+#define	PCRF_CZ_ENTER_MODIFIED_COMPLIANCE_CTL_LBN 10
+#define	PCRF_CZ_ENTER_MODIFIED_COMPLIANCE_CTL_WIDTH 1
+#define	PCRF_CZ_TRANSMIT_MARGIN_LBN 7
+#define	PCRF_CZ_TRANSMIT_MARGIN_WIDTH 3
+#define	PCRF_CZ_SELECT_DEEMPH_LBN 6
+#define	PCRF_CZ_SELECT_DEEMPH_WIDTH 1
+#define	PCRF_CZ_HW_AUTONOMOUS_SPEED_DIS_LBN 5
+#define	PCRF_CZ_HW_AUTONOMOUS_SPEED_DIS_WIDTH 1
+#define	PCRF_CZ_ENTER_COMPLIANCE_CTL_LBN 4
+#define	PCRF_CZ_ENTER_COMPLIANCE_CTL_WIDTH 1
+#define	PCRF_CZ_TGT_LNK_SPEED_CTL_LBN 0
+#define	PCRF_CZ_TGT_LNK_SPEED_CTL_WIDTH 4
+
+
+/*
+ * PC_LNK_STAT2_REG(16bit):
+ * PCIe Link Status 2
+ */
+
+#define	PCR_CC_LNK_STAT2_REG 0x000000a2
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_LNK_STAT2_REG 0x000000f2
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_CURRENT_DEEMPH_LBN 0
+#define	PCRF_CZ_CURRENT_DEEMPH_WIDTH 1
+
+
+/*
+ * PC_VPD_CAP_ID_REG(8bit):
+ * VPD data register
+ */
+
+#define	PCR_AB_VPD_CAP_ID_REG 0x000000b0
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_VPD_CAP_ID_LBN 0
+#define	PCRF_AB_VPD_CAP_ID_WIDTH 8
+
+
+/*
+ * PC_VPD_NXT_PTR_REG(8bit):
+ * VPD next item pointer
+ */
+
+#define	PCR_AB_VPD_NXT_PTR_REG 0x000000b1
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_VPD_NXT_PTR_LBN 0
+#define	PCRF_AB_VPD_NXT_PTR_WIDTH 8
+
+
+/*
+ * PC_VPD_ADDR_REG(16bit):
+ * VPD address register
+ */
+
+#define	PCR_AB_VPD_ADDR_REG 0x000000b2
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_VPD_FLAG_LBN 15
+#define	PCRF_AB_VPD_FLAG_WIDTH 1
+#define	PCRF_AB_VPD_ADDR_LBN 0
+#define	PCRF_AB_VPD_ADDR_WIDTH 15
+
+
+/*
+ * PC_AER_CAP_HDR_REG(32bit):
+ * AER capability header register
+ */
+
+#define	PCR_AZ_AER_CAP_HDR_REG 0x00000100
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_AERCAPHDR_NXT_PTR_LBN 20
+#define	PCRF_AZ_AERCAPHDR_NXT_PTR_WIDTH 12
+#define	PCRF_AZ_AERCAPHDR_VER_LBN 16
+#define	PCRF_AZ_AERCAPHDR_VER_WIDTH 4
+#define	PCRF_AZ_AERCAPHDR_ID_LBN 0
+#define	PCRF_AZ_AERCAPHDR_ID_WIDTH 16
+
+
+/*
+ * PC_AER_UNCORR_ERR_STAT_REG(32bit):
+ * AER Uncorrectable error status register
+ */
+
+#define	PCR_AZ_AER_UNCORR_ERR_STAT_REG 0x00000104
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_UNSUPT_REQ_ERR_STAT_LBN 20
+#define	PCRF_AZ_UNSUPT_REQ_ERR_STAT_WIDTH 1
+#define	PCRF_AZ_ECRC_ERR_STAT_LBN 19
+#define	PCRF_AZ_ECRC_ERR_STAT_WIDTH 1
+#define	PCRF_AZ_MALF_TLP_STAT_LBN 18
+#define	PCRF_AZ_MALF_TLP_STAT_WIDTH 1
+#define	PCRF_AZ_RX_OVF_STAT_LBN 17
+#define	PCRF_AZ_RX_OVF_STAT_WIDTH 1
+#define	PCRF_AZ_UNEXP_COMP_STAT_LBN 16
+#define	PCRF_AZ_UNEXP_COMP_STAT_WIDTH 1
+#define	PCRF_AZ_COMP_ABRT_STAT_LBN 15
+#define	PCRF_AZ_COMP_ABRT_STAT_WIDTH 1
+#define	PCRF_AZ_COMP_TIMEOUT_STAT_LBN 14
+#define	PCRF_AZ_COMP_TIMEOUT_STAT_WIDTH 1
+#define	PCRF_AZ_FC_PROTO_ERR_STAT_LBN 13
+#define	PCRF_AZ_FC_PROTO_ERR_STAT_WIDTH 1
+#define	PCRF_AZ_PSON_TLP_STAT_LBN 12
+#define	PCRF_AZ_PSON_TLP_STAT_WIDTH 1
+#define	PCRF_AZ_DL_PROTO_ERR_STAT_LBN 4
+#define	PCRF_AZ_DL_PROTO_ERR_STAT_WIDTH 1
+#define	PCRF_AB_TRAIN_ERR_STAT_LBN 0
+#define	PCRF_AB_TRAIN_ERR_STAT_WIDTH 1
+
+
+/*
+ * PC_AER_UNCORR_ERR_MASK_REG(32bit):
+ * AER Uncorrectable error mask register
+ */
+
+#define	PCR_AZ_AER_UNCORR_ERR_MASK_REG 0x00000108
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_UNSUPT_REQ_ERR_MASK_LBN 20
+#define	PCRF_AZ_UNSUPT_REQ_ERR_MASK_WIDTH 1
+#define	PCRF_AZ_ECRC_ERR_MASK_LBN 19
+#define	PCRF_AZ_ECRC_ERR_MASK_WIDTH 1
+#define	PCRF_AZ_MALF_TLP_MASK_LBN 18
+#define	PCRF_AZ_MALF_TLP_MASK_WIDTH 1
+#define	PCRF_AZ_RX_OVF_MASK_LBN 17
+#define	PCRF_AZ_RX_OVF_MASK_WIDTH 1
+#define	PCRF_AZ_UNEXP_COMP_MASK_LBN 16
+#define	PCRF_AZ_UNEXP_COMP_MASK_WIDTH 1
+#define	PCRF_AZ_COMP_ABRT_MASK_LBN 15
+#define	PCRF_AZ_COMP_ABRT_MASK_WIDTH 1
+#define	PCRF_AZ_COMP_TIMEOUT_MASK_LBN 14
+#define	PCRF_AZ_COMP_TIMEOUT_MASK_WIDTH 1
+#define	PCRF_AZ_FC_PROTO_ERR_MASK_LBN 13
+#define	PCRF_AZ_FC_PROTO_ERR_MASK_WIDTH 1
+#define	PCRF_AZ_PSON_TLP_MASK_LBN 12
+#define	PCRF_AZ_PSON_TLP_MASK_WIDTH 1
+#define	PCRF_AZ_DL_PROTO_ERR_MASK_LBN 4
+#define	PCRF_AZ_DL_PROTO_ERR_MASK_WIDTH 1
+#define	PCRF_AB_TRAIN_ERR_MASK_LBN 0
+#define	PCRF_AB_TRAIN_ERR_MASK_WIDTH 1
+
+
+/*
+ * PC_AER_UNCORR_ERR_SEV_REG(32bit):
+ * AER Uncorrectable error severity register
+ */
+
+#define	PCR_AZ_AER_UNCORR_ERR_SEV_REG 0x0000010c
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_UNSUPT_REQ_ERR_SEV_LBN 20
+#define	PCRF_AZ_UNSUPT_REQ_ERR_SEV_WIDTH 1
+#define	PCRF_AZ_ECRC_ERR_SEV_LBN 19
+#define	PCRF_AZ_ECRC_ERR_SEV_WIDTH 1
+#define	PCRF_AZ_MALF_TLP_SEV_LBN 18
+#define	PCRF_AZ_MALF_TLP_SEV_WIDTH 1
+#define	PCRF_AZ_RX_OVF_SEV_LBN 17
+#define	PCRF_AZ_RX_OVF_SEV_WIDTH 1
+#define	PCRF_AZ_UNEXP_COMP_SEV_LBN 16
+#define	PCRF_AZ_UNEXP_COMP_SEV_WIDTH 1
+#define	PCRF_AZ_COMP_ABRT_SEV_LBN 15
+#define	PCRF_AZ_COMP_ABRT_SEV_WIDTH 1
+#define	PCRF_AZ_COMP_TIMEOUT_SEV_LBN 14
+#define	PCRF_AZ_COMP_TIMEOUT_SEV_WIDTH 1
+#define	PCRF_AZ_FC_PROTO_ERR_SEV_LBN 13
+#define	PCRF_AZ_FC_PROTO_ERR_SEV_WIDTH 1
+#define	PCRF_AZ_PSON_TLP_SEV_LBN 12
+#define	PCRF_AZ_PSON_TLP_SEV_WIDTH 1
+#define	PCRF_AZ_DL_PROTO_ERR_SEV_LBN 4
+#define	PCRF_AZ_DL_PROTO_ERR_SEV_WIDTH 1
+#define	PCRF_AB_TRAIN_ERR_SEV_LBN 0
+#define	PCRF_AB_TRAIN_ERR_SEV_WIDTH 1
+
+
+/*
+ * PC_AER_CORR_ERR_STAT_REG(32bit):
+ * AER Correctable error status register
+ */
+
+#define	PCR_AZ_AER_CORR_ERR_STAT_REG 0x00000110
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_CZ_ADVSY_NON_FATAL_STAT_LBN 13
+#define	PCRF_CZ_ADVSY_NON_FATAL_STAT_WIDTH 1
+#define	PCRF_AZ_RPLY_TMR_TOUT_STAT_LBN 12
+#define	PCRF_AZ_RPLY_TMR_TOUT_STAT_WIDTH 1
+#define	PCRF_AZ_RPLAY_NUM_RO_STAT_LBN 8
+#define	PCRF_AZ_RPLAY_NUM_RO_STAT_WIDTH 1
+#define	PCRF_AZ_BAD_DLLP_STAT_LBN 7
+#define	PCRF_AZ_BAD_DLLP_STAT_WIDTH 1
+#define	PCRF_AZ_BAD_TLP_STAT_LBN 6
+#define	PCRF_AZ_BAD_TLP_STAT_WIDTH 1
+#define	PCRF_AZ_RX_ERR_STAT_LBN 0
+#define	PCRF_AZ_RX_ERR_STAT_WIDTH 1
+
+
+/*
+ * PC_AER_CORR_ERR_MASK_REG(32bit):
+ * AER Correctable error status register
+ */
+
+#define	PCR_AZ_AER_CORR_ERR_MASK_REG 0x00000114
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_CZ_ADVSY_NON_FATAL_MASK_LBN 13
+#define	PCRF_CZ_ADVSY_NON_FATAL_MASK_WIDTH 1
+#define	PCRF_AZ_RPLY_TMR_TOUT_MASK_LBN 12
+#define	PCRF_AZ_RPLY_TMR_TOUT_MASK_WIDTH 1
+#define	PCRF_AZ_RPLAY_NUM_RO_MASK_LBN 8
+#define	PCRF_AZ_RPLAY_NUM_RO_MASK_WIDTH 1
+#define	PCRF_AZ_BAD_DLLP_MASK_LBN 7
+#define	PCRF_AZ_BAD_DLLP_MASK_WIDTH 1
+#define	PCRF_AZ_BAD_TLP_MASK_LBN 6
+#define	PCRF_AZ_BAD_TLP_MASK_WIDTH 1
+#define	PCRF_AZ_RX_ERR_MASK_LBN 0
+#define	PCRF_AZ_RX_ERR_MASK_WIDTH 1
+
+
+/*
+ * PC_AER_CAP_CTL_REG(32bit):
+ * AER capability and control register
+ */
+
+#define	PCR_AZ_AER_CAP_CTL_REG 0x00000118
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_ECRC_CHK_EN_LBN 8
+#define	PCRF_AZ_ECRC_CHK_EN_WIDTH 1
+#define	PCRF_AZ_ECRC_CHK_CAP_LBN 7
+#define	PCRF_AZ_ECRC_CHK_CAP_WIDTH 1
+#define	PCRF_AZ_ECRC_GEN_EN_LBN 6
+#define	PCRF_AZ_ECRC_GEN_EN_WIDTH 1
+#define	PCRF_AZ_ECRC_GEN_CAP_LBN 5
+#define	PCRF_AZ_ECRC_GEN_CAP_WIDTH 1
+#define	PCRF_AZ_1ST_ERR_PTR_LBN 0
+#define	PCRF_AZ_1ST_ERR_PTR_WIDTH 5
+
+
+/*
+ * PC_AER_HDR_LOG_REG(128bit):
+ * AER Header log register
+ */
+
+#define	PCR_AZ_AER_HDR_LOG_REG 0x0000011c
+/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */
+
+#define	PCRF_AZ_HDR_LOG_LBN 0
+#define	PCRF_AZ_HDR_LOG_WIDTH 128
+
+
+/*
+ * PC_DEVSN_CAP_HDR_REG(32bit):
+ * Device serial number capability header register
+ */
+
+#define	PCR_DZ_DEVSN_CAP_HDR_REG 0x00000130
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_DEVSN_CAP_HDR_REG 0x00000140
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_DEVSNCAPHDR_NXT_PTR_LBN 20
+#define	PCRF_CZ_DEVSNCAPHDR_NXT_PTR_WIDTH 12
+#define	PCRF_CZ_DEVSNCAPHDR_VER_LBN 16
+#define	PCRF_CZ_DEVSNCAPHDR_VER_WIDTH 4
+#define	PCRF_CZ_DEVSNCAPHDR_ID_LBN 0
+#define	PCRF_CZ_DEVSNCAPHDR_ID_WIDTH 16
+
+
+/*
+ * PC_DEVSN_DWORD0_REG(32bit):
+ * Device serial number DWORD0
+ */
+
+#define	PCR_DZ_DEVSN_DWORD0_REG 0x00000134
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_DEVSN_DWORD0_REG 0x00000144
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_DEVSN_DWORD0_LBN 0
+#define	PCRF_CZ_DEVSN_DWORD0_WIDTH 32
+
+
+/*
+ * PC_DEVSN_DWORD1_REG(32bit):
+ * Device serial number DWORD0
+ */
+
+#define	PCR_DZ_DEVSN_DWORD1_REG 0x00000138
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_DEVSN_DWORD1_REG 0x00000148
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_DEVSN_DWORD1_LBN 0
+#define	PCRF_CZ_DEVSN_DWORD1_WIDTH 32
+
+
+/*
+ * PC_ARI_CAP_HDR_REG(32bit):
+ * ARI capability header register
+ */
+
+#define	PCR_DZ_ARI_CAP_HDR_REG 0x00000140
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_ARI_CAP_HDR_REG 0x00000150
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_ARICAPHDR_NXT_PTR_LBN 20
+#define	PCRF_CZ_ARICAPHDR_NXT_PTR_WIDTH 12
+#define	PCRF_CZ_ARICAPHDR_VER_LBN 16
+#define	PCRF_CZ_ARICAPHDR_VER_WIDTH 4
+#define	PCRF_CZ_ARICAPHDR_ID_LBN 0
+#define	PCRF_CZ_ARICAPHDR_ID_WIDTH 16
+
+
+/*
+ * PC_ARI_CAP_REG(16bit):
+ * ARI Capabilities
+ */
+
+#define	PCR_DZ_ARI_CAP_REG 0x00000144
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_ARI_CAP_REG 0x00000154
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_ARI_NXT_FN_NUM_LBN 8
+#define	PCRF_CZ_ARI_NXT_FN_NUM_WIDTH 8
+#define	PCRF_CZ_ARI_ACS_FNGRP_CAP_LBN 1
+#define	PCRF_CZ_ARI_ACS_FNGRP_CAP_WIDTH 1
+#define	PCRF_CZ_ARI_MFVC_FNGRP_CAP_LBN 0
+#define	PCRF_CZ_ARI_MFVC_FNGRP_CAP_WIDTH 1
+
+
+/*
+ * PC_ARI_CTL_REG(16bit):
+ * ARI Control
+ */
+
+#define	PCR_DZ_ARI_CTL_REG 0x00000146
+/* hunta0=pci_f0_config */
+
+#define	PCR_CC_ARI_CTL_REG 0x00000156
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CZ_ARI_FN_GRP_LBN 4
+#define	PCRF_CZ_ARI_FN_GRP_WIDTH 3
+#define	PCRF_CZ_ARI_ACS_FNGRP_EN_LBN 1
+#define	PCRF_CZ_ARI_ACS_FNGRP_EN_WIDTH 1
+#define	PCRF_CZ_ARI_MFVC_FNGRP_EN_LBN 0
+#define	PCRF_CZ_ARI_MFVC_FNGRP_EN_WIDTH 1
+
+
+/*
+ * PC_SRIOV_CAP_HDR_REG(32bit):
+ * SRIOV capability header register
+ */
+
+#define	PCR_CC_SRIOV_CAP_HDR_REG 0x00000160
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_CAP_HDR_REG 0x00000200
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_SRIOVCAPHDR_NXT_PTR_LBN 20
+#define	PCRF_CZ_SRIOVCAPHDR_NXT_PTR_WIDTH 12
+#define	PCRF_CZ_SRIOVCAPHDR_VER_LBN 16
+#define	PCRF_CZ_SRIOVCAPHDR_VER_WIDTH 4
+#define	PCRF_CZ_SRIOVCAPHDR_ID_LBN 0
+#define	PCRF_CZ_SRIOVCAPHDR_ID_WIDTH 16
+
+
+/*
+ * PC_SRIOV_CAP_REG(32bit):
+ * SRIOV Capabilities
+ */
+
+#define	PCR_CC_SRIOV_CAP_REG 0x00000164
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_CAP_REG 0x00000204
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_MIGR_INT_MSG_NUM_LBN 21
+#define	PCRF_CZ_VF_MIGR_INT_MSG_NUM_WIDTH 11
+#define	PCRF_CZ_VF_MIGR_CAP_LBN 0
+#define	PCRF_CZ_VF_MIGR_CAP_WIDTH 1
+
+
+/*
+ * PC_SRIOV_CTL_REG(16bit):
+ * SRIOV Control
+ */
+
+#define	PCR_CC_SRIOV_CTL_REG 0x00000168
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_CTL_REG 0x00000208
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_ARI_CAP_HRCHY_LBN 4
+#define	PCRF_CZ_VF_ARI_CAP_HRCHY_WIDTH 1
+#define	PCRF_CZ_VF_MSE_LBN 3
+#define	PCRF_CZ_VF_MSE_WIDTH 1
+#define	PCRF_CZ_VF_MIGR_INT_EN_LBN 2
+#define	PCRF_CZ_VF_MIGR_INT_EN_WIDTH 1
+#define	PCRF_CZ_VF_MIGR_EN_LBN 1
+#define	PCRF_CZ_VF_MIGR_EN_WIDTH 1
+#define	PCRF_CZ_VF_EN_LBN 0
+#define	PCRF_CZ_VF_EN_WIDTH 1
+
+
+/*
+ * PC_SRIOV_STAT_REG(16bit):
+ * SRIOV Status
+ */
+
+#define	PCR_CC_SRIOV_STAT_REG 0x0000016a
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_STAT_REG 0x0000020a
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_MIGR_STAT_LBN 0
+#define	PCRF_CZ_VF_MIGR_STAT_WIDTH 1
+
+
+/*
+ * PC_SRIOV_INITIALVFS_REG(16bit):
+ * SRIOV Initial VFs
+ */
+
+#define	PCR_CC_SRIOV_INITIALVFS_REG 0x0000016c
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_INITIALVFS_REG 0x0000020c
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_INITIALVFS_LBN 0
+#define	PCRF_CZ_VF_INITIALVFS_WIDTH 16
+
+
+/*
+ * PC_SRIOV_TOTALVFS_REG(10bit):
+ * SRIOV Total VFs
+ */
+
+#define	PCR_CC_SRIOV_TOTALVFS_REG 0x0000016e
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_TOTALVFS_REG 0x0000020e
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_TOTALVFS_LBN 0
+#define	PCRF_CZ_VF_TOTALVFS_WIDTH 16
+
+
+/*
+ * PC_SRIOV_NUMVFS_REG(16bit):
+ * SRIOV Number of VFs
+ */
+
+#define	PCR_CC_SRIOV_NUMVFS_REG 0x00000170
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_NUMVFS_REG 0x00000210
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_NUMVFS_LBN 0
+#define	PCRF_CZ_VF_NUMVFS_WIDTH 16
+
+
+/*
+ * PC_SRIOV_FN_DPND_LNK_REG(16bit):
+ * SRIOV Function dependency link
+ */
+
+#define	PCR_CC_SRIOV_FN_DPND_LNK_REG 0x00000172
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_FN_DPND_LNK_REG 0x00000212
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_SRIOV_FN_DPND_LNK_LBN 0
+#define	PCRF_CZ_SRIOV_FN_DPND_LNK_WIDTH 8
+
+
+/*
+ * PC_SRIOV_1STVF_OFFSET_REG(16bit):
+ * SRIOV First VF Offset
+ */
+
+#define	PCR_CC_SRIOV_1STVF_OFFSET_REG 0x00000174
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_1STVF_OFFSET_REG 0x00000214
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_1STVF_OFFSET_LBN 0
+#define	PCRF_CZ_VF_1STVF_OFFSET_WIDTH 16
+
+
+/*
+ * PC_SRIOV_VFSTRIDE_REG(16bit):
+ * SRIOV VF Stride
+ */
+
+#define	PCR_CC_SRIOV_VFSTRIDE_REG 0x00000176
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_VFSTRIDE_REG 0x00000216
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_VFSTRIDE_LBN 0
+#define	PCRF_CZ_VF_VFSTRIDE_WIDTH 16
+
+
+/*
+ * PC_SRIOV_DEVID_REG(16bit):
+ * SRIOV VF Device ID
+ */
+
+#define	PCR_CC_SRIOV_DEVID_REG 0x0000017a
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_DEVID_REG 0x0000021a
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_DEVID_LBN 0
+#define	PCRF_CZ_VF_DEVID_WIDTH 16
+
+
+/*
+ * PC_SRIOV_SUP_PAGESZ_REG(16bit):
+ * SRIOV Supported Page Sizes
+ */
+
+#define	PCR_CC_SRIOV_SUP_PAGESZ_REG 0x0000017c
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_SUP_PAGESZ_REG 0x0000021c
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_SUP_PAGESZ_LBN 0
+#define	PCRF_CZ_VF_SUP_PAGESZ_WIDTH 16
+
+
+/*
+ * PC_SRIOV_SYS_PAGESZ_REG(32bit):
+ * SRIOV System Page Size
+ */
+
+#define	PCR_CC_SRIOV_SYS_PAGESZ_REG 0x00000180
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_SYS_PAGESZ_REG 0x00000220
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_SYS_PAGESZ_LBN 0
+#define	PCRF_CZ_VF_SYS_PAGESZ_WIDTH 16
+
+
+/*
+ * PC_SRIOV_BAR0_REG(32bit):
+ * SRIOV VF Bar0
+ */
+
+#define	PCR_CC_SRIOV_BAR0_REG 0x00000184
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_BAR0_REG 0x00000224
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CC_VF_BAR_ADDRESS_LBN 0
+#define	PCRF_CC_VF_BAR_ADDRESS_WIDTH 32
+#define	PCRF_DZ_VF_BAR0_ADDRESS_LBN 0
+#define	PCRF_DZ_VF_BAR0_ADDRESS_WIDTH 32
+
+
+/*
+ * PC_SRIOV_BAR1_REG(32bit):
+ * SRIOV Bar1
+ */
+
+#define	PCR_CC_SRIOV_BAR1_REG 0x00000188
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_BAR1_REG 0x00000228
+/* hunta0=pci_f0_config */
+
+/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */
+/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */
+#define	PCRF_DZ_VF_BAR1_ADDRESS_LBN 0
+#define	PCRF_DZ_VF_BAR1_ADDRESS_WIDTH 32
+
+
+/*
+ * PC_SRIOV_BAR2_REG(32bit):
+ * SRIOV Bar2
+ */
+
+#define	PCR_CC_SRIOV_BAR2_REG 0x0000018c
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_BAR2_REG 0x0000022c
+/* hunta0=pci_f0_config */
+
+/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */
+/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */
+#define	PCRF_DZ_VF_BAR2_ADDRESS_LBN 0
+#define	PCRF_DZ_VF_BAR2_ADDRESS_WIDTH 32
+
+
+/*
+ * PC_SRIOV_BAR3_REG(32bit):
+ * SRIOV Bar3
+ */
+
+#define	PCR_CC_SRIOV_BAR3_REG 0x00000190
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_BAR3_REG 0x00000230
+/* hunta0=pci_f0_config */
+
+/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */
+/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */
+#define	PCRF_DZ_VF_BAR3_ADDRESS_LBN 0
+#define	PCRF_DZ_VF_BAR3_ADDRESS_WIDTH 32
+
+
+/*
+ * PC_SRIOV_BAR4_REG(32bit):
+ * SRIOV Bar4
+ */
+
+#define	PCR_CC_SRIOV_BAR4_REG 0x00000194
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_BAR4_REG 0x00000234
+/* hunta0=pci_f0_config */
+
+/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */
+/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */
+#define	PCRF_DZ_VF_BAR4_ADDRESS_LBN 0
+#define	PCRF_DZ_VF_BAR4_ADDRESS_WIDTH 32
+
+
+/*
+ * PC_SRIOV_BAR5_REG(32bit):
+ * SRIOV Bar5
+ */
+
+#define	PCR_CC_SRIOV_BAR5_REG 0x00000198
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_BAR5_REG 0x00000238
+/* hunta0=pci_f0_config */
+
+/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */
+/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */
+#define	PCRF_DZ_VF_BAR5_ADDRESS_LBN 0
+#define	PCRF_DZ_VF_BAR5_ADDRESS_WIDTH 32
+
+
+/*
+ * PC_SRIOV_MIBR_SARRAY_OFFSET_REG(32bit):
+ * SRIOV VF Migration State Array Offset
+ */
+
+#define	PCR_CC_SRIOV_MIBR_SARRAY_OFFSET_REG 0x0000019c
+/* sienaa0=pci_f0_config */
+
+#define	PCR_DZ_SRIOV_MIBR_SARRAY_OFFSET_REG 0x0000023c
+/* hunta0=pci_f0_config */
+
+#define	PCRF_CZ_VF_MIGR_OFFSET_LBN 3
+#define	PCRF_CZ_VF_MIGR_OFFSET_WIDTH 29
+#define	PCRF_CZ_VF_MIGR_BIR_LBN 0
+#define	PCRF_CZ_VF_MIGR_BIR_WIDTH 3
+
+
+/*
+ * PC_LTR_CAP_HDR_REG(32bit):
+ * Latency Tolerance Reporting Cap Header Reg
+ */
+
+#define	PCR_DZ_LTR_CAP_HDR_REG 0x00000240
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LTR_NXT_PTR_LBN 20
+#define	PCRF_DZ_LTR_NXT_PTR_WIDTH 12
+#define	PCRF_DZ_LTR_VERSION_LBN 16
+#define	PCRF_DZ_LTR_VERSION_WIDTH 4
+#define	PCRF_DZ_LTR_EXT_CAP_ID_LBN 0
+#define	PCRF_DZ_LTR_EXT_CAP_ID_WIDTH 16
+
+
+/*
+ * PC_LTR_MAX_SNOOP_REG(32bit):
+ * LTR Maximum Snoop/No Snoop Register
+ */
+
+#define	PCR_DZ_LTR_MAX_SNOOP_REG 0x00000244
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LTR_MAX_NOSNOOP_SCALE_LBN 26
+#define	PCRF_DZ_LTR_MAX_NOSNOOP_SCALE_WIDTH 3
+#define	PCRF_DZ_LTR_MAX_NOSNOOP_LAT_LBN 16
+#define	PCRF_DZ_LTR_MAX_NOSNOOP_LAT_WIDTH 10
+#define	PCRF_DZ_LTR_MAX_SNOOP_SCALE_LBN 10
+#define	PCRF_DZ_LTR_MAX_SNOOP_SCALE_WIDTH 3
+#define	PCRF_DZ_LTR_MAX_SNOOP_LAT_LBN 0
+#define	PCRF_DZ_LTR_MAX_SNOOP_LAT_WIDTH 10
+
+
+/*
+ * PC_TPH_CAP_HDR_REG(32bit):
+ * TPH Capability Header Register
+ */
+
+#define	PCR_DZ_TPH_CAP_HDR_REG 0x00000274
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_TPH_NXT_PTR_LBN 20
+#define	PCRF_DZ_TPH_NXT_PTR_WIDTH 12
+#define	PCRF_DZ_TPH_VERSION_LBN 16
+#define	PCRF_DZ_TPH_VERSION_WIDTH 4
+#define	PCRF_DZ_TPH_EXT_CAP_ID_LBN 0
+#define	PCRF_DZ_TPH_EXT_CAP_ID_WIDTH 16
+
+
+/*
+ * PC_TPH_REQ_CAP_REG(32bit):
+ * TPH Requester Capability Register
+ */
+
+#define	PCR_DZ_TPH_REQ_CAP_REG 0x00000278
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_ST_TBLE_SIZE_LBN 16
+#define	PCRF_DZ_ST_TBLE_SIZE_WIDTH 11
+#define	PCRF_DZ_ST_TBLE_LOC_LBN 9
+#define	PCRF_DZ_ST_TBLE_LOC_WIDTH 2
+#define	PCRF_DZ_EXT_TPH_MODE_SUP_LBN 8
+#define	PCRF_DZ_EXT_TPH_MODE_SUP_WIDTH 1
+#define	PCRF_DZ_TPH_DEV_MODE_SUP_LBN 2
+#define	PCRF_DZ_TPH_DEV_MODE_SUP_WIDTH 1
+#define	PCRF_DZ_TPH_INT_MODE_SUP_LBN 1
+#define	PCRF_DZ_TPH_INT_MODE_SUP_WIDTH 1
+#define	PCRF_DZ_TPH_NOST_MODE_SUP_LBN 0
+#define	PCRF_DZ_TPH_NOST_MODE_SUP_WIDTH 1
+
+
+/*
+ * PC_TPH_REQ_CTL_REG(32bit):
+ * TPH Requester Control Register
+ */
+
+#define	PCR_DZ_TPH_REQ_CTL_REG 0x0000027c
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_TPH_REQ_ENABLE_LBN 8
+#define	PCRF_DZ_TPH_REQ_ENABLE_WIDTH 2
+#define	PCRF_DZ_TPH_ST_MODE_LBN 0
+#define	PCRF_DZ_TPH_ST_MODE_WIDTH 3
+
+
+/*
+ * PC_SEC_PCIE_CAP_REG(32bit):
+ * Secondary PCIE Capability Register
+ */
+
+#define	PCR_DZ_SEC_PCIE_CAP_REG 0x00000300
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_SEC_NXT_PTR_LBN 20
+#define	PCRF_DZ_SEC_NXT_PTR_WIDTH 12
+#define	PCRF_DZ_SEC_VERSION_LBN 16
+#define	PCRF_DZ_SEC_VERSION_WIDTH 4
+#define	PCRF_DZ_SEC_EXT_CAP_ID_LBN 0
+#define	PCRF_DZ_SEC_EXT_CAP_ID_WIDTH 16
+
+
+/*
+ * PC_LINK_CONTROL3_REG(32bit):
+ * Link Control 3.
+ */
+
+#define	PCR_DZ_LINK_CONTROL3_REG 0x00000304
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LINK_EQ_INT_EN_LBN 1
+#define	PCRF_DZ_LINK_EQ_INT_EN_WIDTH 1
+#define	PCRF_DZ_PERFORM_EQL_LBN 0
+#define	PCRF_DZ_PERFORM_EQL_WIDTH 1
+
+
+/*
+ * PC_LANE_ERROR_STAT_REG(32bit):
+ * Lane Error Status Register.
+ */
+
+#define	PCR_DZ_LANE_ERROR_STAT_REG 0x00000308
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LANE_STATUS_LBN 0
+#define	PCRF_DZ_LANE_STATUS_WIDTH 8
+
+
+/*
+ * PC_LANE01_EQU_CONTROL_REG(32bit):
+ * Lanes 0,1 Equalization Control Register.
+ */
+
+#define	PCR_DZ_LANE01_EQU_CONTROL_REG 0x0000030c
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LANE1_EQ_CTRL_LBN 16
+#define	PCRF_DZ_LANE1_EQ_CTRL_WIDTH 16
+#define	PCRF_DZ_LANE0_EQ_CTRL_LBN 0
+#define	PCRF_DZ_LANE0_EQ_CTRL_WIDTH 16
+
+
+/*
+ * PC_LANE23_EQU_CONTROL_REG(32bit):
+ * Lanes 2,3 Equalization Control Register.
+ */
+
+#define	PCR_DZ_LANE23_EQU_CONTROL_REG 0x00000310
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LANE3_EQ_CTRL_LBN 16
+#define	PCRF_DZ_LANE3_EQ_CTRL_WIDTH 16
+#define	PCRF_DZ_LANE2_EQ_CTRL_LBN 0
+#define	PCRF_DZ_LANE2_EQ_CTRL_WIDTH 16
+
+
+/*
+ * PC_LANE45_EQU_CONTROL_REG(32bit):
+ * Lanes 4,5 Equalization Control Register.
+ */
+
+#define	PCR_DZ_LANE45_EQU_CONTROL_REG 0x00000314
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LANE5_EQ_CTRL_LBN 16
+#define	PCRF_DZ_LANE5_EQ_CTRL_WIDTH 16
+#define	PCRF_DZ_LANE4_EQ_CTRL_LBN 0
+#define	PCRF_DZ_LANE4_EQ_CTRL_WIDTH 16
+
+
+/*
+ * PC_LANE67_EQU_CONTROL_REG(32bit):
+ * Lanes 6,7 Equalization Control Register.
+ */
+
+#define	PCR_DZ_LANE67_EQU_CONTROL_REG 0x00000318
+/* hunta0=pci_f0_config */
+
+#define	PCRF_DZ_LANE7_EQ_CTRL_LBN 16
+#define	PCRF_DZ_LANE7_EQ_CTRL_WIDTH 16
+#define	PCRF_DZ_LANE6_EQ_CTRL_LBN 0
+#define	PCRF_DZ_LANE6_EQ_CTRL_WIDTH 16
+
+
+/*
+ * PC_ACK_LAT_TMR_REG(32bit):
+ * ACK latency timer & replay timer register
+ */
+
+#define	PCR_AC_ACK_LAT_TMR_REG 0x00000700
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_RT_LBN 16
+#define	PCRF_AC_RT_WIDTH 16
+#define	PCRF_AC_ALT_LBN 0
+#define	PCRF_AC_ALT_WIDTH 16
+
+
+/*
+ * PC_OTHER_MSG_REG(32bit):
+ * Other message register
+ */
+
+#define	PCR_AC_OTHER_MSG_REG 0x00000704
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_OM_CRPT3_LBN 24
+#define	PCRF_AC_OM_CRPT3_WIDTH 8
+#define	PCRF_AC_OM_CRPT2_LBN 16
+#define	PCRF_AC_OM_CRPT2_WIDTH 8
+#define	PCRF_AC_OM_CRPT1_LBN 8
+#define	PCRF_AC_OM_CRPT1_WIDTH 8
+#define	PCRF_AC_OM_CRPT0_LBN 0
+#define	PCRF_AC_OM_CRPT0_WIDTH 8
+
+
+/*
+ * PC_FORCE_LNK_REG(24bit):
+ * Port force link register
+ */
+
+#define	PCR_AC_FORCE_LNK_REG 0x00000708
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_LFS_LBN 16
+#define	PCRF_AC_LFS_WIDTH 6
+#define	PCRF_AC_FL_LBN 15
+#define	PCRF_AC_FL_WIDTH 1
+#define	PCRF_AC_LN_LBN 0
+#define	PCRF_AC_LN_WIDTH 8
+
+
+/*
+ * PC_ACK_FREQ_REG(32bit):
+ * ACK frequency register
+ */
+
+#define	PCR_AC_ACK_FREQ_REG 0x0000070c
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_CC_ALLOW_L1_WITHOUT_L0S_LBN 30
+#define	PCRF_CC_ALLOW_L1_WITHOUT_L0S_WIDTH 1
+#define	PCRF_AC_L1_ENTR_LAT_LBN 27
+#define	PCRF_AC_L1_ENTR_LAT_WIDTH 3
+#define	PCRF_AC_L0_ENTR_LAT_LBN 24
+#define	PCRF_AC_L0_ENTR_LAT_WIDTH 3
+#define	PCRF_CC_COMM_NFTS_LBN 16
+#define	PCRF_CC_COMM_NFTS_WIDTH 8
+#define	PCRF_AB_ACK_FREQ_REG_RSVD0_LBN 16
+#define	PCRF_AB_ACK_FREQ_REG_RSVD0_WIDTH 3
+#define	PCRF_AC_MAX_FTS_LBN 8
+#define	PCRF_AC_MAX_FTS_WIDTH 8
+#define	PCRF_AC_ACK_FREQ_LBN 0
+#define	PCRF_AC_ACK_FREQ_WIDTH 8
+
+
+/*
+ * PC_PORT_LNK_CTL_REG(32bit):
+ * Port link control register
+ */
+
+#define	PCR_AC_PORT_LNK_CTL_REG 0x00000710
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AB_LRE_LBN 27
+#define	PCRF_AB_LRE_WIDTH 1
+#define	PCRF_AB_ESYNC_LBN 26
+#define	PCRF_AB_ESYNC_WIDTH 1
+#define	PCRF_AB_CRPT_LBN 25
+#define	PCRF_AB_CRPT_WIDTH 1
+#define	PCRF_AB_XB_LBN 24
+#define	PCRF_AB_XB_WIDTH 1
+#define	PCRF_AC_LC_LBN 16
+#define	PCRF_AC_LC_WIDTH 6
+#define	PCRF_AC_LDR_LBN 8
+#define	PCRF_AC_LDR_WIDTH 4
+#define	PCRF_AC_FLM_LBN 7
+#define	PCRF_AC_FLM_WIDTH 1
+#define	PCRF_AC_LKD_LBN 6
+#define	PCRF_AC_LKD_WIDTH 1
+#define	PCRF_AC_DLE_LBN 5
+#define	PCRF_AC_DLE_WIDTH 1
+#define	PCRF_AB_PORT_LNK_CTL_REG_RSVD0_LBN 4
+#define	PCRF_AB_PORT_LNK_CTL_REG_RSVD0_WIDTH 1
+#define	PCRF_AC_RA_LBN 3
+#define	PCRF_AC_RA_WIDTH 1
+#define	PCRF_AC_LE_LBN 2
+#define	PCRF_AC_LE_WIDTH 1
+#define	PCRF_AC_SD_LBN 1
+#define	PCRF_AC_SD_WIDTH 1
+#define	PCRF_AC_OMR_LBN 0
+#define	PCRF_AC_OMR_WIDTH 1
+
+
+/*
+ * PC_LN_SKEW_REG(32bit):
+ * Lane skew register
+ */
+
+#define	PCR_AC_LN_SKEW_REG 0x00000714
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_DIS_LBN 31
+#define	PCRF_AC_DIS_WIDTH 1
+#define	PCRF_AB_RST_LBN 30
+#define	PCRF_AB_RST_WIDTH 1
+#define	PCRF_AC_AD_LBN 25
+#define	PCRF_AC_AD_WIDTH 1
+#define	PCRF_AC_FCD_LBN 24
+#define	PCRF_AC_FCD_WIDTH 1
+#define	PCRF_AC_LS2_LBN 16
+#define	PCRF_AC_LS2_WIDTH 8
+#define	PCRF_AC_LS1_LBN 8
+#define	PCRF_AC_LS1_WIDTH 8
+#define	PCRF_AC_LS0_LBN 0
+#define	PCRF_AC_LS0_WIDTH 8
+
+
+/*
+ * PC_SYM_NUM_REG(16bit):
+ * Symbol number register
+ */
+
+#define	PCR_AC_SYM_NUM_REG 0x00000718
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_CC_MAX_FUNCTIONS_LBN 29
+#define	PCRF_CC_MAX_FUNCTIONS_WIDTH 3
+#define	PCRF_CC_FC_WATCHDOG_TMR_LBN 24
+#define	PCRF_CC_FC_WATCHDOG_TMR_WIDTH 5
+#define	PCRF_CC_ACK_NAK_TMR_MOD_LBN 19
+#define	PCRF_CC_ACK_NAK_TMR_MOD_WIDTH 5
+#define	PCRF_CC_REPLAY_TMR_MOD_LBN 14
+#define	PCRF_CC_REPLAY_TMR_MOD_WIDTH 5
+#define	PCRF_AB_ES_LBN 12
+#define	PCRF_AB_ES_WIDTH 3
+#define	PCRF_AB_SYM_NUM_REG_RSVD0_LBN 11
+#define	PCRF_AB_SYM_NUM_REG_RSVD0_WIDTH 1
+#define	PCRF_CC_NUM_SKP_SYMS_LBN 8
+#define	PCRF_CC_NUM_SKP_SYMS_WIDTH 3
+#define	PCRF_AB_TS2_LBN 4
+#define	PCRF_AB_TS2_WIDTH 4
+#define	PCRF_AC_TS1_LBN 0
+#define	PCRF_AC_TS1_WIDTH 4
+
+
+/*
+ * PC_SYM_TMR_FLT_MSK_REG(16bit):
+ * Symbol timer and Filter Mask Register
+ */
+
+#define	PCR_CC_SYM_TMR_FLT_MSK_REG 0x0000071c
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CC_DEFAULT_FLT_MSK1_LBN 16
+#define	PCRF_CC_DEFAULT_FLT_MSK1_WIDTH 16
+#define	PCRF_CC_FC_WDOG_TMR_DIS_LBN 15
+#define	PCRF_CC_FC_WDOG_TMR_DIS_WIDTH 1
+#define	PCRF_CC_SI1_LBN 8
+#define	PCRF_CC_SI1_WIDTH 3
+#define	PCRF_CC_SKIP_INT_VAL_LBN 0
+#define	PCRF_CC_SKIP_INT_VAL_WIDTH 11
+#define	PCRF_CC_SI0_LBN 0
+#define	PCRF_CC_SI0_WIDTH 8
+
+
+/*
+ * PC_SYM_TMR_REG(16bit):
+ * Symbol timer register
+ */
+
+#define	PCR_AB_SYM_TMR_REG 0x0000071c
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCRF_AB_ET_LBN 11
+#define	PCRF_AB_ET_WIDTH 4
+#define	PCRF_AB_SI1_LBN 8
+#define	PCRF_AB_SI1_WIDTH 3
+#define	PCRF_AB_SI0_LBN 0
+#define	PCRF_AB_SI0_WIDTH 8
+
+
+/*
+ * PC_PHY_STAT_REG(32bit):
+ * PHY status register
+ */
+
+#define	PCR_AB_PHY_STAT_REG 0x00000720
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_PHY_STAT_REG 0x00000810
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_AC_SSL_LBN 3
+#define	PCRF_AC_SSL_WIDTH 1
+#define	PCRF_AC_SSR_LBN 2
+#define	PCRF_AC_SSR_WIDTH 1
+#define	PCRF_AC_SSCL_LBN 1
+#define	PCRF_AC_SSCL_WIDTH 1
+#define	PCRF_AC_SSCD_LBN 0
+#define	PCRF_AC_SSCD_WIDTH 1
+
+
+/*
+ * PC_FLT_MSK_REG(32bit):
+ * Filter Mask Register 2
+ */
+
+#define	PCR_CC_FLT_MSK_REG 0x00000720
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CC_DEFAULT_FLT_MSK2_LBN 0
+#define	PCRF_CC_DEFAULT_FLT_MSK2_WIDTH 32
+
+
+/*
+ * PC_PHY_CTL_REG(32bit):
+ * PHY control register
+ */
+
+#define	PCR_AB_PHY_CTL_REG 0x00000724
+/* falcona0,falconb0=pci_f0_config */
+
+#define	PCR_CC_PHY_CTL_REG 0x00000814
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_AC_BD_LBN 31
+#define	PCRF_AC_BD_WIDTH 1
+#define	PCRF_AC_CDS_LBN 30
+#define	PCRF_AC_CDS_WIDTH 1
+#define	PCRF_AC_DWRAP_LB_LBN 29
+#define	PCRF_AC_DWRAP_LB_WIDTH 1
+#define	PCRF_AC_EBD_LBN 28
+#define	PCRF_AC_EBD_WIDTH 1
+#define	PCRF_AC_SNR_LBN 27
+#define	PCRF_AC_SNR_WIDTH 1
+#define	PCRF_AC_RX_NOT_DET_LBN 2
+#define	PCRF_AC_RX_NOT_DET_WIDTH 1
+#define	PCRF_AC_FORCE_LOS_VAL_LBN 1
+#define	PCRF_AC_FORCE_LOS_VAL_WIDTH 1
+#define	PCRF_AC_FORCE_LOS_EN_LBN 0
+#define	PCRF_AC_FORCE_LOS_EN_WIDTH 1
+
+
+/*
+ * PC_DEBUG0_REG(32bit):
+ * Debug register 0
+ */
+
+#define	PCR_AC_DEBUG0_REG 0x00000728
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_CDI03_LBN 24
+#define	PCRF_AC_CDI03_WIDTH 8
+#define	PCRF_AC_CDI0_LBN 0
+#define	PCRF_AC_CDI0_WIDTH 32
+#define	PCRF_AC_CDI02_LBN 16
+#define	PCRF_AC_CDI02_WIDTH 8
+#define	PCRF_AC_CDI01_LBN 8
+#define	PCRF_AC_CDI01_WIDTH 8
+#define	PCRF_AC_CDI00_LBN 0
+#define	PCRF_AC_CDI00_WIDTH 8
+
+
+/*
+ * PC_DEBUG1_REG(32bit):
+ * Debug register 1
+ */
+
+#define	PCR_AC_DEBUG1_REG 0x0000072c
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_CDI13_LBN 24
+#define	PCRF_AC_CDI13_WIDTH 8
+#define	PCRF_AC_CDI1_LBN 0
+#define	PCRF_AC_CDI1_WIDTH 32
+#define	PCRF_AC_CDI12_LBN 16
+#define	PCRF_AC_CDI12_WIDTH 8
+#define	PCRF_AC_CDI11_LBN 8
+#define	PCRF_AC_CDI11_WIDTH 8
+#define	PCRF_AC_CDI10_LBN 0
+#define	PCRF_AC_CDI10_WIDTH 8
+
+
+/*
+ * PC_XPFCC_STAT_REG(24bit):
+ * documentation to be written for sum_PC_XPFCC_STAT_REG
+ */
+
+#define	PCR_AC_XPFCC_STAT_REG 0x00000730
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_XPDC_LBN 12
+#define	PCRF_AC_XPDC_WIDTH 8
+#define	PCRF_AC_XPHC_LBN 0
+#define	PCRF_AC_XPHC_WIDTH 12
+
+
+/*
+ * PC_XNPFCC_STAT_REG(24bit):
+ * documentation to be written for sum_PC_XNPFCC_STAT_REG
+ */
+
+#define	PCR_AC_XNPFCC_STAT_REG 0x00000734
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_XNPDC_LBN 12
+#define	PCRF_AC_XNPDC_WIDTH 8
+#define	PCRF_AC_XNPHC_LBN 0
+#define	PCRF_AC_XNPHC_WIDTH 12
+
+
+/*
+ * PC_XCFCC_STAT_REG(24bit):
+ * documentation to be written for sum_PC_XCFCC_STAT_REG
+ */
+
+#define	PCR_AC_XCFCC_STAT_REG 0x00000738
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_XCDC_LBN 12
+#define	PCRF_AC_XCDC_WIDTH 8
+#define	PCRF_AC_XCHC_LBN 0
+#define	PCRF_AC_XCHC_WIDTH 12
+
+
+/*
+ * PC_Q_STAT_REG(8bit):
+ * documentation to be written for sum_PC_Q_STAT_REG
+ */
+
+#define	PCR_AC_Q_STAT_REG 0x0000073c
+/* falcona0,falconb0,sienaa0=pci_f0_config */
+
+#define	PCRF_AC_RQNE_LBN 2
+#define	PCRF_AC_RQNE_WIDTH 1
+#define	PCRF_AC_XRNE_LBN 1
+#define	PCRF_AC_XRNE_WIDTH 1
+#define	PCRF_AC_RCNR_LBN 0
+#define	PCRF_AC_RCNR_WIDTH 1
+
+
+/*
+ * PC_VC_XMIT_ARB1_REG(32bit):
+ * VC Transmit Arbitration Register 1
+ */
+
+#define	PCR_CC_VC_XMIT_ARB1_REG 0x00000740
+/* sienaa0=pci_f0_config */
+
+
+
+/*
+ * PC_VC_XMIT_ARB2_REG(32bit):
+ * VC Transmit Arbitration Register 2
+ */
+
+#define	PCR_CC_VC_XMIT_ARB2_REG 0x00000744
+/* sienaa0=pci_f0_config */
+
+
+
+/*
+ * PC_VC0_P_RQ_CTL_REG(32bit):
+ * VC0 Posted Receive Queue Control
+ */
+
+#define	PCR_CC_VC0_P_RQ_CTL_REG 0x00000748
+/* sienaa0=pci_f0_config */
+
+
+
+/*
+ * PC_VC0_NP_RQ_CTL_REG(32bit):
+ * VC0 Non-Posted Receive Queue Control
+ */
+
+#define	PCR_CC_VC0_NP_RQ_CTL_REG 0x0000074c
+/* sienaa0=pci_f0_config */
+
+
+
+/*
+ * PC_VC0_C_RQ_CTL_REG(32bit):
+ * VC0 Completion Receive Queue Control
+ */
+
+#define	PCR_CC_VC0_C_RQ_CTL_REG 0x00000750
+/* sienaa0=pci_f0_config */
+
+
+
+/*
+ * PC_GEN2_REG(32bit):
+ * Gen2 Register
+ */
+
+#define	PCR_CC_GEN2_REG 0x0000080c
+/* sienaa0=pci_f0_config */
+
+#define	PCRF_CC_SET_DE_EMPHASIS_LBN 20
+#define	PCRF_CC_SET_DE_EMPHASIS_WIDTH 1
+#define	PCRF_CC_CFG_TX_COMPLIANCE_LBN 19
+#define	PCRF_CC_CFG_TX_COMPLIANCE_WIDTH 1
+#define	PCRF_CC_CFG_TX_SWING_LBN 18
+#define	PCRF_CC_CFG_TX_SWING_WIDTH 1
+#define	PCRF_CC_DIR_SPEED_CHANGE_LBN 17
+#define	PCRF_CC_DIR_SPEED_CHANGE_WIDTH 1
+#define	PCRF_CC_LANE_ENABLE_LBN 8
+#define	PCRF_CC_LANE_ENABLE_WIDTH 9
+#define	PCRF_CC_NUM_FTS_LBN 0
+#define	PCRF_CC_NUM_FTS_WIDTH 8
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_EFX_REGS_PCI_H */
diff --git a/sys/dev/sfxge/common/efx_rx.c b/sys/dev/sfxge/common/efx_rx.c
new file mode 100644
index 0000000..7d3d7c1
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_rx.c
@@ -0,0 +1,816 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+	__checkReturn	int
+efx_rx_init(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+	unsigned int index;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (enp->en_mod_flags & EFX_MOD_RX) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
+
+	/* Zero the RSS table */
+	for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
+	    index++) {
+		EFX_ZERO_OWORD(oword);
+		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
+				    index, &oword);
+	}
+
+	enp->en_mod_flags |= EFX_MOD_RX;
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_RX_HDR_SPLIT
+	__checkReturn	int
+efx_rx_hdr_split_enable(
+	__in		efx_nic_t *enp,
+	__in		unsigned int hdr_buf_size,
+	__in		unsigned int pld_buf_size)
+{
+	unsigned int nhdr32;
+	unsigned int npld32;
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
+
+	nhdr32 = hdr_buf_size / 32;
+	if ((nhdr32 == 0) ||
+	    (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
+	    ((hdr_buf_size % 32) != 0)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	npld32 = pld_buf_size / 32;
+	if ((npld32 == 0) ||
+	    (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
+	    ((pld_buf_size % 32) != 0)) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	if (enp->en_rx_qcount > 0) {
+		rc = EBUSY;
+		goto fail3;
+	}
+
+	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
+
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
+
+
+#if EFSYS_OPT_RX_SCATTER
+	__checkReturn	int
+efx_rx_scatter_enable(
+	__in		efx_nic_t *enp,
+	__in		unsigned int buf_size)
+{
+	unsigned int nbuf32;
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
+
+	nbuf32 = buf_size / 32;
+	if ((nbuf32 == 0) ||
+	    (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
+	    ((buf_size % 32) != 0)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (enp->en_rx_qcount > 0) {
+		rc = EBUSY;
+		goto fail2;
+	}
+
+	/* Set scatter buffer size */
+	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
+
+	/* Enable scatter for packets not matching a filter */
+	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+#endif	/* EFSYS_OPT_RX_SCATTER */
+
+
+#define	EFX_RX_LFSR_HASH(_enp, _insert)					\
+	do {                                    			\
+		efx_oword_t oword;					\
+									\
+		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);	\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);	\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);	\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
+		    (_insert) ? 1 : 0);					\
+		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
+									\
+		if ((_enp)->en_family == EFX_FAMILY_SIENA) {		\
+			EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
+			    &oword);					\
+			EFX_SET_OWORD_FIELD(oword,			\
+			    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);	\
+			EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
+			    &oword);					\
+		}							\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+	} while (B_FALSE)
+
+#define	EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)		\
+	do {                                    			\
+		efx_oword_t oword;					\
+									\
+		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG,	&oword);	\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);	\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,		\
+		    (_ip) ? 1 : 0);					\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,		\
+		    (_tcp) ? 0 : 1);					\
+		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
+		    (_insert) ? 1 : 0);					\
+		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+	} while (B_FALSE)
+
+#define	EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)			\
+	do {                                    			\
+		efx_oword_t oword;					\
+									\
+		if ((_enp)->en_family == EFX_FAMILY_FALCON) {		\
+			(_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;	\
+			break;						\
+		}							\
+									\
+		EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
+		EFX_SET_OWORD_FIELD(oword,				\
+		    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);		\
+		EFX_SET_OWORD_FIELD(oword,				\
+		    FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0);	\
+		EFX_SET_OWORD_FIELD(oword,				\
+		    FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);	\
+		EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
+									\
+		(_rc) = 0;						\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+	} while (B_FALSE)
+
+
+#if EFSYS_OPT_RX_SCALE
+	__checkReturn	int
+efx_rx_scale_mode_set(
+	__in		efx_nic_t *enp,
+	__in		efx_rx_hash_alg_t alg,
+	__in		efx_rx_hash_type_t type,
+	__in		boolean_t insert)
+{
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
+
+	switch (alg) {
+	case EFX_RX_HASHALG_LFSR:
+		EFX_RX_LFSR_HASH(enp, insert);
+		break;
+
+	case EFX_RX_HASHALG_TOEPLITZ:
+		EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
+		    type & (1 << EFX_RX_HASH_IPV4),
+		    type & (1 << EFX_RX_HASH_TCPIPV4));
+
+		EFX_RX_TOEPLITZ_IPV6_HASH(enp,
+		    type & (1 << EFX_RX_HASH_IPV6),
+		    type & (1 << EFX_RX_HASH_TCPIPV6),
+		    rc);
+		if (rc != 0)
+			goto fail1;
+
+		break;
+
+	default:
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	EFX_RX_LFSR_HASH(enp, B_FALSE);
+
+	return (rc);
+}
+#endif
+
+#if EFSYS_OPT_RX_SCALE
+	__checkReturn	int
+efx_rx_scale_toeplitz_ipv4_key_set(
+	__in		efx_nic_t *enp,
+	__in_ecount(n)	uint8_t *key,
+	__in		size_t n)
+{
+	efx_oword_t oword;
+	unsigned int byte;
+	unsigned int offset;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+
+	byte = 0;
+
+	/* Write toeplitz hash key */
+	EFX_ZERO_OWORD(oword);
+	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset)
+		oword.eo_u8[offset - 1] = key[byte++];
+
+	EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
+
+	byte = 0;
+
+	/* Verify toeplitz hash key */
+	EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
+	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset) {
+		if (oword.eo_u8[offset - 1] != key[byte++]) {
+			rc = EFAULT;
+			goto fail1;
+		}
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+#endif
+
+#if EFSYS_OPT_RX_SCALE
+	__checkReturn	int
+efx_rx_scale_toeplitz_ipv6_key_set(
+	__in		efx_nic_t *enp,
+	__in_ecount(n)	uint8_t *key,
+	__in		size_t n)
+{
+	efx_oword_t oword;
+	unsigned int byte;
+	int offset;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+
+	byte = 0;
+
+	/* Write toeplitz hash key 3 */
+	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
+	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
+	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset)
+		oword.eo_u8[offset - 1] = key[byte++];
+
+	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
+
+	/* Write toeplitz hash key 2 */
+	EFX_ZERO_OWORD(oword);
+	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
+	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset)
+		oword.eo_u8[offset - 1] = key[byte++];
+
+	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
+
+	/* Write toeplitz hash key 1 */
+	EFX_ZERO_OWORD(oword);
+	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
+	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset)
+		oword.eo_u8[offset - 1] = key[byte++];
+
+	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
+
+	byte = 0;
+
+	/* Verify toeplitz hash key 3 */
+	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
+	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
+	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset) {
+		if (oword.eo_u8[offset - 1] != key[byte++]) {
+			rc = EFAULT;
+			goto fail1;
+		}
+	}
+
+	/* Verify toeplitz hash key 2 */
+	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
+	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
+	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset) {
+		if (oword.eo_u8[offset - 1] != key[byte++]) {
+			rc = EFAULT;
+			goto fail2;
+		}
+	}
+
+	/* Verify toeplitz hash key 1 */
+	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
+	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
+	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
+	    offset > 0 && byte < n;
+	    --offset) {
+		if (oword.eo_u8[offset - 1] != key[byte++]) {
+			rc = EFAULT;
+			goto fail3;
+		}
+	}
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+#endif
+
+#if EFSYS_OPT_RX_SCALE
+	__checkReturn	int
+efx_rx_scale_tbl_set(
+	__in		efx_nic_t *enp,
+	__in_ecount(n)	unsigned int *table,
+	__in		size_t n)
+{
+	efx_oword_t oword;
+	int index;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+
+	EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
+	EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
+
+	if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
+		uint32_t byte;
+
+		/* Calculate the entry to place in the table */
+		byte = (uint32_t)table[index % n];
+
+		EFSYS_PROBE2(table, int, index, uint32_t, byte);
+
+		EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
+
+		/* Write the table */
+		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
+				    index, &oword);
+	}
+
+	for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
+		uint32_t byte;
+
+		/* Determine if we're starting a new batch */
+		byte = (uint32_t)table[index % n];
+
+		/* Read the table */
+		EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
+				    index, &oword);
+
+		/* Verify the entry */
+		if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
+			rc = EFAULT;
+			goto fail2;
+		}
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+#endif
+
+#if EFSYS_OPT_FILTER
+extern	__checkReturn	int
+efx_rx_filter_insert(
+	__in		efx_rxq_t *erp,
+	__inout		efx_filter_spec_t *spec)
+{
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_dmaq_id = (uint16_t)erp->er_index;
+	return efx_filter_insert_filter(erp->er_enp, spec, B_FALSE);
+}
+#endif
+
+#if EFSYS_OPT_FILTER
+extern	__checkReturn	int
+efx_rx_filter_remove(
+	__in		efx_rxq_t *erp,
+	__inout		efx_filter_spec_t *spec)
+{
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_dmaq_id = (uint16_t)erp->er_index;
+	return efx_filter_remove_filter(erp->er_enp, spec);
+}
+#endif
+
+extern			void
+efx_rx_qpost(
+	__in		efx_rxq_t *erp,
+	__in_ecount(n)	efsys_dma_addr_t *addrp,
+	__in		size_t size,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__in		unsigned int added)
+{
+	efx_qword_t qword;
+	unsigned int i;
+	unsigned int offset;
+	unsigned int id;
+
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+	/* The client driver must not overfill the queue */
+	EFSYS_ASSERT3U(added - completed + n, <=,
+	    EFX_RXQ_LIMIT(erp->er_mask + 1));
+
+	id = added & (erp->er_mask);
+	for (i = 0; i < n; i++) {
+		EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
+		    unsigned int, id, efsys_dma_addr_t, addrp[i],
+		    size_t, size);
+
+		EFX_POPULATE_QWORD_3(qword,
+		    FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
+		    FSF_AZ_RX_KER_BUF_ADDR_DW0,
+		    (uint32_t)(addrp[i] & 0xffffffff),
+		    FSF_AZ_RX_KER_BUF_ADDR_DW1,
+		    (uint32_t)(addrp[i] >> 32));
+
+		offset = id * sizeof (efx_qword_t);
+		EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
+
+		id = (id + 1) & (erp->er_mask);
+	}
+}
+
+		void
+efx_rx_qpush(
+	__in	efx_rxq_t *erp,
+	__in	unsigned int added)
+{
+	efx_nic_t *enp = erp->er_enp;
+	uint32_t wptr;
+	efx_oword_t oword;
+	efx_dword_t dword;
+
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
+	EFSYS_PIO_WRITE_BARRIER();
+
+	/* Push the populated descriptors out */
+	wptr = added & erp->er_mask;
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
+
+	/* Only write the third DWORD */
+	EFX_POPULATE_DWORD_1(dword,
+	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
+	EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
+			    erp->er_index, &dword, B_FALSE);
+}
+
+		void
+efx_rx_qflush(
+	__in	efx_rxq_t *erp)
+{
+	efx_nic_t *enp = erp->er_enp;
+	efx_oword_t oword;
+	uint32_t label;
+
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+	label = erp->er_index;
+
+	/* Flush the queue */
+	EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
+	    FRF_AZ_RX_FLUSH_DESCQ, label);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
+}
+
+		void
+efx_rx_qenable(
+	__in	efx_rxq_t *erp)
+{
+	efx_nic_t *enp = erp->er_enp;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+	EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
+			    erp->er_index, &oword);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
+			    erp->er_index, &oword);
+}
+
+	__checkReturn	int
+efx_rx_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		unsigned int label,
+	__in		efx_rxq_type_t type,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__in		efx_evq_t *eep,
+	__deref_out	efx_rxq_t **erpp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_rxq_t *erp;
+	efx_oword_t oword;
+	uint32_t size;
+	boolean_t split;
+	boolean_t jumbo;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+
+	EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
+	EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
+	EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
+
+	if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+	if (index >= encp->enc_rxq_limit) {
+		rc = EINVAL;
+		goto fail2;
+	}
+	for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
+	    size++)
+		if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
+			break;
+	if (id + (1 << size) >= encp->enc_buftbl_limit) {
+		rc = EINVAL;
+		goto fail3;
+	}
+
+	switch (type) {
+	case EFX_RXQ_TYPE_DEFAULT:
+		split = B_FALSE;
+		jumbo = B_FALSE;
+		break;
+
+#if EFSYS_OPT_RX_HDR_SPLIT
+	case EFX_RXQ_TYPE_SPLIT_HEADER:
+		if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
+			rc = EINVAL;
+			goto fail4;
+		}
+		split = B_TRUE;
+		jumbo = B_TRUE;
+		break;
+
+	case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
+		if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
+			rc = EINVAL;
+			goto fail4;
+		}
+		split = B_FALSE;
+		jumbo = B_TRUE;
+		break;
+#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
+
+#if EFSYS_OPT_RX_SCATTER
+	case EFX_RXQ_TYPE_SCATTER:
+		if (enp->en_family < EFX_FAMILY_SIENA) {
+			rc = EINVAL;
+			goto fail4;
+		}
+		split = B_FALSE;
+		jumbo = B_TRUE;
+		break;
+#endif	/* EFSYS_OPT_RX_SCATTER */
+
+	default:
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	/* Allocate an RXQ object */
+	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
+
+	if (erp == NULL) {
+		rc = ENOMEM;
+		goto fail5;
+	}
+
+	erp->er_magic = EFX_RXQ_MAGIC;
+	erp->er_enp = enp;
+	erp->er_index = index;
+	erp->er_mask = n - 1;
+	erp->er_esmp = esmp;
+
+	/* Set up the new descriptor queue */
+	EFX_POPULATE_OWORD_10(oword,
+	    FRF_CZ_RX_HDR_SPLIT, split,
+	    FRF_AZ_RX_ISCSI_DDIG_EN, 0,
+	    FRF_AZ_RX_ISCSI_HDIG_EN, 0,
+	    FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
+	    FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
+	    FRF_AZ_RX_DESCQ_OWNER_ID, 0,
+	    FRF_AZ_RX_DESCQ_LABEL, label,
+	    FRF_AZ_RX_DESCQ_SIZE, size,
+	    FRF_AZ_RX_DESCQ_TYPE, 0,
+	    FRF_AZ_RX_DESCQ_JUMBO, jumbo);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
+			    erp->er_index, &oword);
+
+	enp->en_rx_qcount++;
+	*erpp = erp;
+	return (0);
+
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+		void
+efx_rx_qdestroy(
+	__in	efx_rxq_t *erp)
+{
+	efx_nic_t *enp = erp->er_enp;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+	EFSYS_ASSERT(enp->en_rx_qcount != 0);
+	--enp->en_rx_qcount;
+
+	/* Purge descriptor queue */
+	EFX_ZERO_OWORD(oword);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
+			    erp->er_index, &oword);
+
+	/* Free the RXQ object */
+	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
+}
+
+		void
+efx_rx_fini(
+	__in	efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
+	EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
+
+	enp->en_mod_flags &= ~EFX_MOD_RX;
+}
diff --git a/sys/dev/sfxge/common/efx_sram.c b/sys/dev/sfxge/common/efx_sram.c
new file mode 100644
index 0000000..16a3229
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_sram.c
@@ -0,0 +1,294 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+	__checkReturn	int
+efx_sram_buf_tbl_set(
+	__in		efx_nic_t *enp,
+	__in		uint32_t id,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n)
+{
+	efx_qword_t qword;
+	uint32_t start = id;
+	uint32_t stop = start + n;
+	efsys_dma_addr_t addr;
+	efx_oword_t oword;
+	unsigned int count;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	if (stop >= EFX_BUF_TBL_SIZE) {
+		rc = EFBIG;
+		goto fail1;
+	}
+
+	/* Add the entries into the buffer table */
+	addr = EFSYS_MEM_ADDR(esmp);
+	for (id = start; id != stop; id++) {
+		EFX_POPULATE_QWORD_5(qword,
+		    FRF_AZ_IP_DAT_BUF_SIZE, 0, FRF_AZ_BUF_ADR_REGION, 0,
+		    FRF_AZ_BUF_ADR_FBUF_DW0,
+		    (uint32_t)((addr >> 12) & 0xffffffff),
+		    FRF_AZ_BUF_ADR_FBUF_DW1,
+		    (uint32_t)((addr >> 12) >> 32),
+		    FRF_AZ_BUF_OWNER_ID_FBUF, 0);
+
+		EFX_BAR_TBL_WRITEQ(enp, FR_AZ_BUF_FULL_TBL,
+				    id, &qword);
+
+		addr += EFX_BUF_SIZE;
+	}
+
+	EFSYS_PROBE2(buf, uint32_t, start, uint32_t, stop - 1);
+
+	/* Flush the write buffer */
+	EFX_POPULATE_OWORD_2(oword, FRF_AZ_BUF_UPD_CMD, 1,
+	    FRF_AZ_BUF_CLR_CMD, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_UPD_REG, &oword);
+
+	/* Poll for the last entry being written to the buffer table */
+	EFSYS_ASSERT3U(id, ==, stop);
+	addr -= EFX_BUF_SIZE;
+
+	count = 0;
+	do {
+		EFSYS_PROBE1(wait, unsigned int, count);
+
+		/* Spin for 1 ms */
+		EFSYS_SPIN(1000);
+
+		EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_FULL_TBL,
+				    id - 1, &qword);
+
+		if (EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW0) ==
+		    (uint32_t)((addr >> 12) & 0xffffffff) &&
+		    EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW1) ==
+		    (uint32_t)((addr >> 12) >> 32))
+			goto verify;
+
+	} while (++count < 100);
+
+	rc = ETIMEDOUT;
+	goto fail2;
+
+verify:
+	/* Verify the rest of the entries in the buffer table */
+	while (--id != start) {
+		addr -= EFX_BUF_SIZE;
+
+		/* Read the buffer table entry */
+		EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_FULL_TBL,
+				    id - 1, &qword);
+
+		if (EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW0) !=
+		    (uint32_t)((addr >> 12) & 0xffffffff) ||
+		    EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW1) !=
+		    (uint32_t)((addr >> 12) >> 32)) {
+			rc = EFAULT;
+			goto fail3;
+		}
+	}
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+
+	id = stop;
+
+fail2:
+	EFSYS_PROBE(fail2);
+
+	EFX_POPULATE_OWORD_4(oword, FRF_AZ_BUF_UPD_CMD, 0,
+	    FRF_AZ_BUF_CLR_CMD, 1, FRF_AZ_BUF_CLR_END_ID, id - 1,
+	    FRF_AZ_BUF_CLR_START_ID, start);
+	EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_UPD_REG, &oword);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+		void
+efx_sram_buf_tbl_clear(
+	__in	efx_nic_t *enp,
+	__in	uint32_t id,
+	__in	size_t n)
+{
+	efx_oword_t oword;
+	uint32_t start = id;
+	uint32_t stop = start + n;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	EFSYS_ASSERT3U(stop, <, EFX_BUF_TBL_SIZE);
+
+	EFSYS_PROBE2(buf, uint32_t, start, uint32_t, stop - 1);
+
+	EFX_POPULATE_OWORD_4(oword, FRF_AZ_BUF_UPD_CMD, 0,
+	    FRF_AZ_BUF_CLR_CMD, 1, FRF_AZ_BUF_CLR_END_ID, stop - 1,
+	    FRF_AZ_BUF_CLR_START_ID, start);
+	EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_UPD_REG, &oword);
+}
+
+
+#if EFSYS_OPT_DIAG
+
+static			void
+efx_sram_byte_increment_set(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp)
+{
+	size_t offset = row * FR_AZ_SRM_DBG_REG_STEP;
+	unsigned int index;
+
+	_NOTE(ARGUNUSED(negate))
+
+	for (index = 0; index < sizeof (efx_qword_t); index++)
+		eqp->eq_u8[index] = offset + index;
+}
+
+static			void
+efx_sram_all_the_same_set(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp)
+{
+	_NOTE(ARGUNUSED(row))
+
+	if (negate)
+		EFX_SET_QWORD(*eqp);
+	else
+		EFX_ZERO_QWORD(*eqp);
+}
+
+static			void
+efx_sram_bit_alternate_set(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp)
+{
+	_NOTE(ARGUNUSED(row))
+
+	EFX_POPULATE_QWORD_2(*eqp,
+	    EFX_DWORD_0, (negate) ? 0x55555555 : 0xaaaaaaaa,
+	    EFX_DWORD_1, (negate) ? 0x55555555 : 0xaaaaaaaa);
+}
+
+static			void
+efx_sram_byte_alternate_set(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp)
+{
+	_NOTE(ARGUNUSED(row))
+
+	EFX_POPULATE_QWORD_2(*eqp,
+	    EFX_DWORD_0, (negate) ? 0x00ff00ff : 0xff00ff00,
+	    EFX_DWORD_1, (negate) ? 0x00ff00ff : 0xff00ff00);
+}
+
+static			void
+efx_sram_byte_changing_set(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp)
+{
+	size_t offset = row * FR_AZ_SRM_DBG_REG_STEP;
+	unsigned int index;
+
+	for (index = 0; index < sizeof (efx_qword_t); index++) {
+		uint8_t byte;
+
+		if (offset / 256 == 0)
+			byte = (uint8_t)((offset % 257) % 256);
+		else
+			byte = (uint8_t)(~((offset - 8) % 257) % 256);
+
+		eqp->eq_u8[index] = (negate) ? ~byte : byte;
+	}
+}
+
+static			void
+efx_sram_bit_sweep_set(
+	__in		size_t row,
+	__in		boolean_t negate,
+	__out		efx_qword_t *eqp)
+{
+	size_t offset = row * FR_AZ_SRM_DBG_REG_STEP;
+
+	if (negate) {
+		EFX_SET_QWORD(*eqp);
+		EFX_CLEAR_QWORD_BIT(*eqp, (offset / sizeof (efx_qword_t)) % 64);
+	} else {
+		EFX_ZERO_QWORD(*eqp);
+		EFX_SET_QWORD_BIT(*eqp, (offset / sizeof (efx_qword_t)) % 64);
+	}
+}
+
+efx_sram_pattern_fn_t	__cs __efx_sram_pattern_fns[] = {
+	efx_sram_byte_increment_set,
+	efx_sram_all_the_same_set,
+	efx_sram_bit_alternate_set,
+	efx_sram_byte_alternate_set,
+	efx_sram_byte_changing_set,
+	efx_sram_bit_sweep_set
+};
+
+	__checkReturn	int
+efx_sram_test(
+	__in		efx_nic_t *enp,
+	__in		efx_pattern_type_t type)
+{
+	efx_nic_ops_t *enop = enp->en_enop;
+	efx_sram_pattern_fn_t func;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
+
+	/* Select pattern generator */
+	EFSYS_ASSERT3U(type, <, EFX_PATTERN_NTYPES);
+	func = __efx_sram_pattern_fns[type];
+
+	return (enop->eno_sram_test(enp, func));
+}
+
+#endif	/* EFSYS_OPT_DIAG */
diff --git a/sys/dev/sfxge/common/efx_tx.c b/sys/dev/sfxge/common/efx_tx.c
new file mode 100644
index 0000000..0dc347c
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_tx.c
@@ -0,0 +1,430 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_QSTATS
+#define	EFX_TX_QSTAT_INCR(_etp, _stat)					\
+	do {								\
+		(_etp)->et_stat[_stat]++;				\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+#else
+#define	EFX_TX_QSTAT_INCR(_etp, _stat)
+#endif
+
+	__checkReturn	int
+efx_tx_init(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+
+	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (enp->en_mod_flags & EFX_MOD_TX) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
+
+	/*
+	 * Disable the timer-based TX DMA backoff and allow TX DMA to be
+	 * controlled by the RX FIFO fill level (although always allow a
+	 * minimal trickle).
+	 */
+	EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
+
+	/*
+	 * Filter all packets less than 14 bytes to avoid parsing
+	 * errors.
+	 */
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
+
+	/*
+	 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
+	 * descriptors (which is bad).
+	 */
+	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
+
+	enp->en_mod_flags |= EFX_MOD_TX;
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_FILTER
+extern	__checkReturn	int
+efx_tx_filter_insert(
+	__in		efx_txq_t *etp,
+	__inout		efx_filter_spec_t *spec)
+{
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_dmaq_id = (uint16_t)etp->et_index;
+	return efx_filter_insert_filter(etp->et_enp, spec, B_FALSE);
+}
+#endif
+
+#if EFSYS_OPT_FILTER
+extern	__checkReturn	int
+efx_tx_filter_remove(
+	__in		efx_txq_t *etp,
+	__inout		efx_filter_spec_t *spec)
+{
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_dmaq_id = (uint16_t)etp->et_index;
+	return efx_filter_remove_filter(etp->et_enp, spec);
+}
+#endif
+
+#define	EFX_TX_DESC(_etp, _addr, _size, _eop, _added)			\
+	do {								\
+		unsigned int id;					\
+		size_t offset;						\
+		efx_qword_t qword;					\
+									\
+		id = (_added)++ & (_etp)->et_mask;			\
+		offset = id * sizeof (efx_qword_t);			\
+									\
+		EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index,	\
+		    unsigned int, id, efsys_dma_addr_t, (_addr),	\
+		    size_t, (_size), boolean_t, (_eop));		\
+									\
+		EFX_POPULATE_QWORD_4(qword,				\
+		    FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1,			\
+		    FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size),	\
+		    FSF_AZ_TX_KER_BUF_ADDR_DW0,				\
+		    (uint32_t)((_addr) & 0xffffffff),			\
+		    FSF_AZ_TX_KER_BUF_ADDR_DW1,				\
+		    (uint32_t)((_addr) >> 32));				\
+		EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword);	\
+									\
+		_NOTE(CONSTANTCONDITION)				\
+	} while (B_FALSE)
+
+	__checkReturn	int
+efx_tx_qpost(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_buffer_t *eb,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp)
+{
+	unsigned int added = *addedp;
+	unsigned int i;
+	int rc = ENOSPC;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
+		goto fail1;
+
+	for (i = 0; i < n; i++) {
+		efx_buffer_t *ebp = &eb[i];
+		efsys_dma_addr_t start = ebp->eb_addr;
+		size_t size = ebp->eb_size;
+		efsys_dma_addr_t end = start + size;
+
+		/* Fragments must not span 4k boundaries. */
+		EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
+
+		EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
+	}
+
+	EFX_TX_QSTAT_INCR(etp, TX_POST);
+
+	*addedp = added;
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+		void
+efx_tx_qpush(
+	__in	efx_txq_t *etp,
+	__in	unsigned int added)
+{
+	efx_nic_t *enp = etp->et_enp;
+	uint32_t wptr;
+	efx_dword_t dword;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
+	EFSYS_PIO_WRITE_BARRIER();
+
+	/* Push the populated descriptors out */
+	wptr = added & etp->et_mask;
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
+
+	/* Only write the third DWORD */
+	EFX_POPULATE_DWORD_1(dword,
+	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
+	EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
+			    etp->et_index, &dword, B_FALSE);
+}
+
+		void
+efx_tx_qflush(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_oword_t oword;
+	uint32_t label;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	label = etp->et_index;
+
+	/* Flush the queue */
+	EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
+	    FRF_AZ_TX_FLUSH_DESCQ, label);
+	EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
+}
+
+		void
+efx_tx_qenable(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
+			    etp->et_index, &oword);
+
+	EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
+	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
+	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
+	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
+	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
+
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
+	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
+			    etp->et_index, &oword);
+}
+
+	__checkReturn	int
+efx_tx_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		unsigned int label,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__in		uint16_t flags,
+	__in		efx_evq_t *eep,
+	__deref_out	efx_txq_t **etpp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_txq_t *etp;
+	efx_oword_t oword;
+	uint32_t size;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
+
+	EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
+	EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
+	EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
+
+	if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+	if (index >= encp->enc_txq_limit) {
+		rc = EINVAL;
+		goto fail2;
+	}
+	for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
+	    size++)
+		if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
+			break;
+	if (id + (1 << size) >= encp->enc_buftbl_limit) {
+		rc = EINVAL;
+		goto fail3;
+	}
+
+	/* Allocate an TXQ object */
+	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
+
+	if (etp == NULL) {
+		rc = ENOMEM;
+		goto fail4;
+	}
+
+	etp->et_magic = EFX_TXQ_MAGIC;
+	etp->et_enp = enp;
+	etp->et_index = index;
+	etp->et_mask = n - 1;
+	etp->et_esmp = esmp;
+
+	/* Set up the new descriptor queue */
+	EFX_POPULATE_OWORD_6(oword,
+	    FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
+	    FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
+	    FRF_AZ_TX_DESCQ_OWNER_ID, 0,
+	    FRF_AZ_TX_DESCQ_LABEL, label,
+	    FRF_AZ_TX_DESCQ_SIZE, size,
+	    FRF_AZ_TX_DESCQ_TYPE, 0);
+
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
+	    (flags & EFX_CKSUM_IPV4) ? 0 : 1);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
+	    (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
+	    etp->et_index, &oword);
+
+	enp->en_tx_qcount++;
+	*etpp = etp;
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_NAMES
+/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
+static const char 	__cs * __cs __efx_tx_qstat_name[] = {
+	"post",
+	"unaligned_split",
+};
+/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
+
+		const char __cs *
+efx_tx_qstat_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(id, <, TX_NQSTATS);
+
+	return (__efx_tx_qstat_name[id]);
+}
+#endif	/* EFSYS_OPT_NAMES */
+
+#if EFSYS_OPT_QSTATS
+					void
+efx_tx_qstats_update(
+	__in				efx_txq_t *etp,
+	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
+{
+	unsigned int id;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	for (id = 0; id < TX_NQSTATS; id++) {
+		efsys_stat_t *essp = &stat[id];
+
+		EFSYS_STAT_INCR(essp, etp->et_stat[id]);
+		etp->et_stat[id] = 0;
+	}
+}
+#endif	/* EFSYS_OPT_QSTATS */
+
+		void
+efx_tx_qdestroy(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_oword_t oword;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	EFSYS_ASSERT(enp->en_tx_qcount != 0);
+	--enp->en_tx_qcount;
+
+	/* Purge descriptor queue */
+	EFX_ZERO_OWORD(oword);
+
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
+			    etp->et_index, &oword);
+
+	/* Free the TXQ object */
+	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
+}
+
+		void
+efx_tx_fini(
+	__in	efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
+	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
+
+	enp->en_mod_flags &= ~EFX_MOD_TX;
+}
diff --git a/sys/dev/sfxge/common/efx_types.h b/sys/dev/sfxge/common/efx_types.h
new file mode 100644
index 0000000..d691482
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_types.h
@@ -0,0 +1,1605 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Ackowledgement to Fen Systems Ltd.
+ */
+
+#ifndef	_SYS_EFX_TYPES_H
+#define	_SYS_EFX_TYPES_H
+
+#include "efsys.h"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * Bitfield access
+ *
+ * Solarflare NICs make extensive use of bitfields up to 128 bits
+ * wide.  Since there is no native 128-bit datatype on most systems,
+ * and since 64-bit datatypes are inefficient on 32-bit systems and
+ * vice versa, we wrap accesses in a way that uses the most efficient
+ * datatype.
+ *
+ * The NICs are PCI devices and therefore little-endian.  Since most
+ * of the quantities that we deal with are DMAed to/from host memory,
+ * we define	our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
+ * to be little-endian.
+ *
+ * In the less common case of using PIO for individual register
+ * writes, we construct the little-endian datatype in host memory and
+ * then use non-swapping register access primitives, rather than
+ * constructing a native-endian datatype and relying on implicit
+ * byte-swapping.  (We use a similar strategy for register reads.)
+ */
+
+/*
+ * NOTE: Field definitions here and elsewhere are done in terms of a lowest
+ *       bit number (LBN) and a width.
+ */
+
+#define	EFX_DUMMY_FIELD_LBN 0
+#define	EFX_DUMMY_FIELD_WIDTH 0
+
+#define	EFX_BYTE_0_LBN 0
+#define	EFX_BYTE_0_WIDTH 8
+
+#define	EFX_BYTE_1_LBN 8
+#define	EFX_BYTE_1_WIDTH 8
+
+#define	EFX_BYTE_2_LBN 16
+#define	EFX_BYTE_2_WIDTH 8
+
+#define	EFX_BYTE_3_LBN 24
+#define	EFX_BYTE_3_WIDTH 8
+
+#define	EFX_BYTE_4_LBN 32
+#define	EFX_BYTE_4_WIDTH 8
+
+#define	EFX_BYTE_5_LBN 40
+#define	EFX_BYTE_5_WIDTH 8
+
+#define	EFX_BYTE_6_LBN 48
+#define	EFX_BYTE_6_WIDTH 8
+
+#define	EFX_BYTE_7_LBN 56
+#define	EFX_BYTE_7_WIDTH 8
+
+#define	EFX_WORD_0_LBN 0
+#define	EFX_WORD_0_WIDTH 16
+
+#define	EFX_WORD_1_LBN 16
+#define	EFX_WORD_1_WIDTH 16
+
+#define	EFX_WORD_2_LBN 32
+#define	EFX_WORD_2_WIDTH 16
+
+#define	EFX_WORD_3_LBN 48
+#define	EFX_WORD_3_WIDTH 16
+
+#define	EFX_DWORD_0_LBN 0
+#define	EFX_DWORD_0_WIDTH 32
+
+#define	EFX_DWORD_1_LBN 32
+#define	EFX_DWORD_1_WIDTH 32
+
+#define	EFX_DWORD_2_LBN 64
+#define	EFX_DWORD_2_WIDTH 32
+
+#define	EFX_DWORD_3_LBN 96
+#define	EFX_DWORD_3_WIDTH 32
+
+#define	EFX_QWORD_0_LBN 0
+#define	EFX_QWORD_0_WIDTH 64
+
+#define	EFX_QWORD_1_LBN 64
+#define	EFX_QWORD_1_WIDTH 64
+
+/* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
+#define	EFX_VAL(_field, _attribute)					\
+	_field ## _ ## _attribute
+
+/* Lowest bit number of the specified field */
+#define	EFX_LOW_BIT(_field)						\
+	EFX_VAL(_field, LBN)
+
+/* Width of the specified field */
+#define	EFX_WIDTH(_field)						\
+	EFX_VAL(_field, WIDTH)
+
+/* Highest bit number of the specified field */
+#define	EFX_HIGH_BIT(_field)						\
+	(EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
+
+/*
+ * 64-bit mask equal in width to the specified field.
+ *
+ * For example, a field with width 5 would have a mask of 0x000000000000001f.
+ */
+#define	EFX_MASK64(_field)						\
+	((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :			\
+	    (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
+/*
+ * 32-bit mask equal in width to the specified field.
+ *
+ * For example, a field with width 5 would have a mask of 0x0000001f.
+ */
+#define	EFX_MASK32(_field)						\
+	((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :			\
+	    (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
+
+/*
+ * 16-bit mask equal in width to the specified field.
+ *
+ * For example, a field with width 5 would have a mask of 0x001f.
+ */
+#define	EFX_MASK16(_field)						\
+	((EFX_WIDTH(_field) == 16) ? 0xffffu :				\
+	    (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
+
+/*
+ * 8-bit mask equal in width to the specified field.
+ *
+ * For example, a field with width 5 would have a mask of 0x1f.
+ */
+#define	EFX_MASK8(_field)						\
+	((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
+
+#pragma pack(1)
+
+/*
+ * A byte (i.e. 8-bit) datatype
+ */
+typedef union efx_byte_u {
+	uint8_t eb_u8[1];
+} efx_byte_t;
+
+/*
+ * A word (i.e. 16-bit) datatype
+ *
+ * This datatype is defined to be little-endian.
+ */
+typedef union efx_word_u {
+	efx_byte_t ew_byte[2];
+	uint16_t ew_u16[1];
+	uint8_t ew_u8[2];
+} efx_word_t;
+
+/*
+ * A doubleword (i.e. 32-bit) datatype
+ *
+ * This datatype is defined to be little-endian.
+ */
+typedef union efx_dword_u {
+	efx_byte_t ed_byte[4];
+	efx_word_t ed_word[2];
+	uint32_t ed_u32[1];
+	uint16_t ed_u16[2];
+	uint8_t ed_u8[4];
+} efx_dword_t;
+
+/*
+ * A quadword (i.e. 64-bit) datatype
+ *
+ * This datatype is defined to be little-endian.
+ */
+typedef union efx_qword_u {
+	efx_byte_t eq_byte[8];
+	efx_word_t eq_word[4];
+	efx_dword_t eq_dword[2];
+#if EFSYS_HAS_UINT64
+	uint64_t eq_u64[1];
+#endif
+	uint32_t eq_u32[2];
+	uint16_t eq_u16[4];
+	uint8_t eq_u8[8];
+} efx_qword_t;
+
+/*
+ * An octword (i.e. 128-bit) datatype
+ *
+ * This datatype is defined to be little-endian.
+ */
+typedef union efx_oword_u {
+	efx_byte_t eo_byte[16];
+	efx_word_t eo_word[8];
+	efx_dword_t eo_dword[4];
+	efx_qword_t eo_qword[2];
+#if EFSYS_HAS_UINT64
+	uint64_t eo_u64[2];
+#endif	
+	uint32_t eo_u32[4];
+	uint16_t eo_u16[8];
+	uint8_t eo_u8[16];
+} efx_oword_t;
+
+#pragma pack()
+
+#define	__SWAP16(_x) 				\
+	((((_x) & 0xff) << 8) |			\
+	(((_x) >> 8) & 0xff))
+
+#define	__SWAP32(_x)				\
+	((__SWAP16((_x) & 0xffff) << 16) |	\
+	__SWAP16(((_x) >> 16) & 0xffff))
+
+#define	__SWAP64(_x)				\
+	((__SWAP32((_x) & 0xffffffff) << 32) |	\
+	__SWAP32(((_x) >> 32) & 0xffffffff))
+
+#define	__NOSWAP16(_x)		(_x)
+#define	__NOSWAP32(_x)		(_x)
+#define	__NOSWAP64(_x)		(_x)
+
+#if EFSYS_IS_BIG_ENDIAN
+
+#define	__CPU_TO_LE_16(_x)	(uint16_t)__SWAP16(_x)
+#define	__LE_TO_CPU_16(_x)	(uint16_t)__SWAP16(_x)
+#define	__CPU_TO_BE_16(_x)	(uint16_t)__NOSWAP16(_x)
+#define	__BE_TO_CPU_16(_x)	(uint16_t)__NOSWAP16(_x)
+
+#define	__CPU_TO_LE_32(_x)	(uint32_t)__SWAP32(_x)
+#define	__LE_TO_CPU_32(_x)	(uint32_t)__SWAP32(_x)
+#define	__CPU_TO_BE_32(_x)	(uint32_t)__NOSWAP32(_x)
+#define	__BE_TO_CPU_32(_x)	(uint32_t)__NOSWAP32(_x)
+
+#define	__CPU_TO_LE_64(_x)	(uint64_t)__SWAP64(_x)
+#define	__LE_TO_CPU_64(_x)	(uint64_t)__SWAP64(_x)
+#define	__CPU_TO_BE_64(_x)	(uint64_t)__NOSWAP64(_x)
+#define	__BE_TO_CPU_64(_x)	(uint64_t)__NOSWAP64(_x)
+
+#elif EFSYS_IS_LITTLE_ENDIAN
+
+#define	__CPU_TO_LE_16(_x)	(uint16_t)__NOSWAP16(_x)
+#define	__LE_TO_CPU_16(_x)	(uint16_t)__NOSWAP16(_x)
+#define	__CPU_TO_BE_16(_x)	(uint16_t)__SWAP16(_x)
+#define	__BE_TO_CPU_16(_x)	(uint16_t)__SWAP16(_x)
+
+#define	__CPU_TO_LE_32(_x)	(uint32_t)__NOSWAP32(_x)
+#define	__LE_TO_CPU_32(_x)	(uint32_t)__NOSWAP32(_x)
+#define	__CPU_TO_BE_32(_x)	(uint32_t)__SWAP32(_x)
+#define	__BE_TO_CPU_32(_x)	(uint32_t)__SWAP32(_x)
+
+#define	__CPU_TO_LE_64(_x)	(uint64_t)__NOSWAP64(_x)
+#define	__LE_TO_CPU_64(_x)	(uint64_t)__NOSWAP64(_x)
+#define	__CPU_TO_BE_64(_x)	(uint64_t)__SWAP64(_x)
+#define	__BE_TO_CPU_64(_x)	(uint64_t)__SWAP64(_x)
+
+#else
+
+#error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
+
+#endif
+
+#define	__NATIVE_8(_x)	(uint8_t)(_x)
+
+/* Format string for printing an efx_byte_t */
+#define	EFX_BYTE_FMT "0x%02x"
+
+/* Format string for printing an efx_word_t */
+#define	EFX_WORD_FMT "0x%04x"
+
+/* Format string for printing an efx_dword_t */
+#define	EFX_DWORD_FMT "0x%08x"
+
+/* Format string for printing an efx_qword_t */
+#define	EFX_QWORD_FMT "0x%08x:%08x"
+
+/* Format string for printing an efx_oword_t */
+#define	EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
+
+/* Parameters for printing an efx_byte_t */
+#define	EFX_BYTE_VAL(_byte)					\
+	((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
+
+/* Parameters for printing an efx_word_t */
+#define	EFX_WORD_VAL(_word)					\
+	((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
+
+/* Parameters for printing an efx_dword_t */
+#define	EFX_DWORD_VAL(_dword)					\
+	((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
+
+/* Parameters for printing an efx_qword_t */
+#define	EFX_QWORD_VAL(_qword)					\
+	((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),	\
+	((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
+
+/* Parameters for printing an efx_oword_t */
+#define	EFX_OWORD_VAL(_oword)					\
+	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),	\
+	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),	\
+	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),	\
+	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
+
+/*
+ * Stop lint complaining about some shifts.
+ */
+#ifdef	__lint
+extern int fix_lint;
+#define	FIX_LINT(_x)	(_x + fix_lint)
+#else
+#define	FIX_LINT(_x)	(_x)
+#endif
+
+/*
+ * Extract bit field portion [low,high) from the native-endian element
+ * which contains bits [min,max).
+ *
+ * For example, suppose "element" represents the high 32 bits of a
+ * 64-bit value, and we wish to extract the bits belonging to the bit
+ * field occupying bits 28-45 of this 64-bit value.
+ *
+ * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
+ *
+ *   (_element) << 4
+ *
+ * The result will contain the relevant bits filled in in the range
+ * [0,high-low), with garbage in bits [high-low+1,...).
+ */
+#define	EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)		\
+	((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?		\
+		0U :							\
+		((_low > _min) ?					\
+			((_element) >> (_low - _min)) :			\
+			((_element) << (_min - _low))))
+
+/*
+ * Extract bit field portion [low,high) from the 64-bit little-endian
+ * element which contains bits [min,max)
+ */
+#define	EFX_EXTRACT64(_element, _min, _max, _low, _high)		\
+	EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
+
+/*
+ * Extract bit field portion [low,high) from the 32-bit little-endian
+ * element which contains bits [min,max)
+ */
+#define	EFX_EXTRACT32(_element, _min, _max, _low, _high)		\
+	EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
+
+/*
+ * Extract bit field portion [low,high) from the 16-bit little-endian
+ * element which contains bits [min,max)
+ */
+#define	EFX_EXTRACT16(_element, _min, _max, _low, _high)		\
+	EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
+
+/*
+ * Extract bit field portion [low,high) from the 8-bit
+ * element which contains bits [min,max)
+ */
+#define	EFX_EXTRACT8(_element, _min, _max, _low, _high)			\
+	EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
+
+#define	EFX_EXTRACT_OWORD64(_oword, _low, _high)			\
+	(EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),	\
+	    _low, _high) |						\
+	EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),	\
+	    _low, _high))
+
+#define	EFX_EXTRACT_OWORD32(_oword, _low, _high)			\
+	(EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),	\
+	    _low, _high) |						\
+	EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),	\
+	    _low, _high) |						\
+	EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),	\
+	    _low, _high) |						\
+	EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),	\
+	    _low, _high))
+
+#define	EFX_EXTRACT_QWORD64(_qword, _low, _high)			\
+	(EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),	\
+	    _low, _high))
+
+#define	EFX_EXTRACT_QWORD32(_qword, _low, _high)			\
+	(EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),	\
+	    _low, _high) |						\
+	EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),	\
+	    _low, _high))
+
+#define	EFX_EXTRACT_DWORD(_dword, _low, _high)				\
+	(EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),	\
+	    _low, _high))
+
+#define	EFX_EXTRACT_WORD(_word, _low, _high)				\
+	(EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),	\
+	    _low, _high))
+
+#define	EFX_EXTRACT_BYTE(_byte, _low, _high)				\
+	(EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),	\
+	    _low, _high))
+
+
+#define	EFX_OWORD_FIELD64(_oword, _field)				\
+	((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),	\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
+
+#define	EFX_OWORD_FIELD32(_oword, _field)				\
+	(EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),		\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
+
+#define	EFX_QWORD_FIELD64(_qword, _field)				\
+	((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),	\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
+
+#define	EFX_QWORD_FIELD32(_qword, _field)				\
+	(EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),		\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
+
+#define	EFX_DWORD_FIELD(_dword, _field)					\
+	(EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),			\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
+
+#define	EFX_WORD_FIELD(_word, _field)					\
+	(EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),			\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
+
+#define	EFX_BYTE_FIELD(_byte, _field)					\
+	(EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),			\
+	    EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
+
+
+#define	EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)			\
+	((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&		\
+	    (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
+
+#define	EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)			\
+	((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&		\
+	    (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&		\
+	    (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&		\
+	    (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
+
+#define	EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)			\
+	((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
+
+#define	EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)			\
+	((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&		\
+	    (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
+
+#define	EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)				\
+	((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
+
+#define	EFX_WORD_IS_EQUAL(_word_a, _word_b)				\
+	((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
+
+#define	EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)				\
+	((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
+
+
+#define	EFX_OWORD_IS_ZERO64(_oword)					\
+	(((_oword).eo_u64[0] |						\
+	    (_oword).eo_u64[1]) == 0)
+
+#define	EFX_OWORD_IS_ZERO32(_oword)					\
+	(((_oword).eo_u32[0] |						\
+	    (_oword).eo_u32[1] |					\
+	    (_oword).eo_u32[2] |					\
+	    (_oword).eo_u32[3]) == 0)
+
+#define	EFX_QWORD_IS_ZERO64(_qword)					\
+	(((_qword).eq_u64[0]) == 0)
+
+#define	EFX_QWORD_IS_ZERO32(_qword)					\
+	(((_qword).eq_u32[0] |						\
+	    (_qword).eq_u32[1]) == 0)
+
+#define	EFX_DWORD_IS_ZERO(_dword)					\
+	(((_dword).ed_u32[0]) == 0)
+
+#define	EFX_WORD_IS_ZERO(_word)						\
+	(((_word).ew_u16[0]) == 0)
+
+#define	EFX_BYTE_IS_ZERO(_byte)						\
+	(((_byte).eb_u8[0]) == 0)
+
+
+#define	EFX_OWORD_IS_SET64(_oword)					\
+	(((_oword).eo_u64[0] &						\
+	    (_oword).eo_u64[1]) == ~((uint64_t)0))
+
+#define	EFX_OWORD_IS_SET32(_oword)					\
+	(((_oword).eo_u32[0] &						\
+	    (_oword).eo_u32[1] &					\
+	    (_oword).eo_u32[2] &					\
+	    (_oword).eo_u32[3]) == ~((uint32_t)0))
+
+#define	EFX_QWORD_IS_SET64(_qword)					\
+	(((_qword).eq_u64[0]) == ~((uint32_t)0))
+
+#define	EFX_QWORD_IS_SET32(_qword)					\
+	(((_qword).eq_u32[0] &						\
+	    (_qword).eq_u32[1]) == ~((uint32_t)0))
+
+#define	EFX_DWORD_IS_SET(_dword)					\
+	((_dword).ed_u32[0] == ~((uint32_t)0))
+
+#define	EFX_WORD_IS_SET(_word)						\
+	((_word).ew_u16[0] == ~((uint16_t)0))
+
+#define	EFX_BYTE_IS_SET(_byte)						\
+	((_byte).eb_u8[0] == ~((uint8_t)0))
+
+/*
+ * Construct bit field portion
+ *
+ * Creates the portion of the bit field [low,high) that lies within
+ * the range [min,max).
+ */
+
+#define	EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)		\
+	(((_low > _max) || (_high < _min)) ?				\
+		0U :							\
+		((_low > _min) ?					\
+			(((uint64_t)(_value)) << (_low - _min)) :	\
+			(((uint64_t)(_value)) >> (_min - _low))))
+
+#define	EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)		\
+	(((_low > _max) || (_high < _min)) ?				\
+		0U :							\
+		((_low > _min) ?					\
+			(((uint32_t)(_value)) << (_low - _min)) :	\
+			(((uint32_t)(_value)) >> (_min - _low))))
+
+#define	EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)		\
+	(((_low > _max) || (_high < _min)) ?				\
+		0U :							\
+		(uint16_t)((_low > _min) ?				\
+				((_value) << (_low - _min)) :		\
+				((_value) >> (_min - _low))))
+
+#define	EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)		\
+	(((_low > _max) || (_high < _min)) ?				\
+		0U :							\
+		(uint8_t)((_low > _min) ?				\
+				((_value) << (_low - _min)) :	\
+				((_value) >> (_min - _low))))
+
+/*
+ * Construct bit field portion
+ *
+ * Creates the portion of the named bit field that lies within the
+ * range [min,max).
+ */
+#define	EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)		\
+	EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),		\
+	    EFX_HIGH_BIT(_field), _value)
+
+#define	EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)		\
+	EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),		\
+	    EFX_HIGH_BIT(_field), _value)
+
+#define	EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)		\
+	EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),		\
+	    EFX_HIGH_BIT(_field), _value)
+
+#define	EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)		\
+	EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),		\
+	    EFX_HIGH_BIT(_field), _value)
+
+/*
+ * Construct bit field
+ *
+ * Creates the portion of the named bit fields that lie within the
+ * range [min,max).
+ */
+#define	EFX_INSERT_FIELDS64(_min, _max,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	__CPU_TO_LE_64(							\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |	\
+	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
+
+#define	EFX_INSERT_FIELDS32(_min, _max,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	__CPU_TO_LE_32(							\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |	\
+	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
+
+#define	EFX_INSERT_FIELDS16(_min, _max,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	__CPU_TO_LE_16(							\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |	\
+	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
+
+#define	EFX_INSERT_FIELDS8(_min, _max,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	__NATIVE_8(							\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |	\
+	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
+
+#define	EFX_POPULATE_OWORD64(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,	\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_POPULATE_OWORD32(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,	\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,	\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,	\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_POPULATE_QWORD64(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_POPULATE_QWORD32(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,	\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_POPULATE_DWORD(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_POPULATE_WORD(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+#define	EFX_POPULATE_BYTE(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
+	    _field10, _value10)						\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,		\
+		    _field1, _value1, _field2, _value2,			\
+		    _field3, _value3, _field4, _value4,			\
+		    _field5, _value5, _field6, _value6,			\
+		    _field7, _value7, _field8, _value8,			\
+		    _field9, _value9, _field10, _value10);		\
+	_NOTE(CONSTANTCONDITION)					\
+	} while (B_FALSE)
+
+/* Populate an octword field with various numbers of arguments */
+#define	EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
+
+#define	EFX_POPULATE_OWORD_9(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
+	EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)
+
+#define	EFX_POPULATE_OWORD_8(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)				\
+	EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)
+
+#define	EFX_POPULATE_OWORD_7(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)						\
+	EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)
+
+#define	EFX_POPULATE_OWORD_6(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
+	EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)
+
+#define	EFX_POPULATE_OWORD_5(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)				\
+	EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)
+
+#define	EFX_POPULATE_OWORD_4(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)						\
+	EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)
+
+#define	EFX_POPULATE_OWORD_3(_oword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3)	\
+	EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3)
+
+#define	EFX_POPULATE_OWORD_2(_oword,					\
+	    _field1, _value1, _field2, _value2)				\
+	EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2)
+
+#define	EFX_POPULATE_OWORD_1(_oword,					\
+	    _field1, _value1)						\
+	EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1)
+
+#define	EFX_ZERO_OWORD(_oword)						\
+	EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
+
+#define	EFX_SET_OWORD64(_oword)						\
+	EFX_POPULATE_OWORD_2(_oword,					\
+	    EFX_QWORD_0, (uint64_t)-1, EFX_QWORD_1, (uint64_t)-1)
+
+#define	EFX_SET_OWORD32(_oword)						\
+	EFX_POPULATE_OWORD_4(_oword,					\
+	    EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,		\
+	    EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
+
+/* Populate a quadword field with various numbers of arguments */
+#define	EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
+
+#define	EFX_POPULATE_QWORD_9(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
+	EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)
+
+#define	EFX_POPULATE_QWORD_8(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)				\
+	EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)
+
+#define	EFX_POPULATE_QWORD_7(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)						\
+	EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)
+
+#define	EFX_POPULATE_QWORD_6(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
+	EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)
+
+#define	EFX_POPULATE_QWORD_5(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)				\
+	EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)
+
+#define	EFX_POPULATE_QWORD_4(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)						\
+	EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)
+
+#define	EFX_POPULATE_QWORD_3(_qword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3)	\
+	EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3)
+
+#define	EFX_POPULATE_QWORD_2(_qword,					\
+	    _field1, _value1, _field2, _value2)				\
+	EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2)
+
+#define	EFX_POPULATE_QWORD_1(_qword,					\
+	    _field1, _value1)						\
+	EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1)
+
+#define	EFX_ZERO_QWORD(_qword)						\
+	EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
+
+#define	EFX_SET_QWORD64(_qword)						\
+	EFX_POPULATE_QWORD_1(_qword,					\
+	    EFX_QWORD_0, (uint64_t)-1)
+
+#define	EFX_SET_QWORD32(_qword)						\
+	EFX_POPULATE_QWORD_2(_qword,					\
+	    EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
+
+/* Populate a dword field with various numbers of arguments */
+#define	EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
+
+#define	EFX_POPULATE_DWORD_9(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
+	EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)
+
+#define	EFX_POPULATE_DWORD_8(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)				\
+	EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)
+
+#define	EFX_POPULATE_DWORD_7(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)						\
+	EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)
+
+#define	EFX_POPULATE_DWORD_6(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
+	EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)
+
+#define	EFX_POPULATE_DWORD_5(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)				\
+	EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)
+
+#define	EFX_POPULATE_DWORD_4(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)						\
+	EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)
+
+#define	EFX_POPULATE_DWORD_3(_dword,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3)	\
+	EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2, _field3, _value3)
+
+#define	EFX_POPULATE_DWORD_2(_dword,					\
+	    _field1, _value1, _field2, _value2)				\
+	EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1, _field2, _value2)
+
+#define	EFX_POPULATE_DWORD_1(_dword,					\
+	    _field1, _value1)						\
+	EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,		\
+	    _field1, _value1)
+
+#define	EFX_ZERO_DWORD(_dword)						\
+	EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
+
+#define	EFX_SET_DWORD(_dword)						\
+	EFX_POPULATE_DWORD_1(_dword,					\
+	    EFX_DWORD_0, 0xffffffff)
+
+/* Populate a word field with various numbers of arguments */
+#define	EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
+
+#define	EFX_POPULATE_WORD_9(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
+	EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)
+
+#define	EFX_POPULATE_WORD_8(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)				\
+	EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)
+
+#define	EFX_POPULATE_WORD_7(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)						\
+	EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)
+
+#define	EFX_POPULATE_WORD_6(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
+	EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)
+
+#define	EFX_POPULATE_WORD_5(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)				\
+	EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)
+
+#define	EFX_POPULATE_WORD_4(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)						\
+	EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)
+
+#define	EFX_POPULATE_WORD_3(_word,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3)	\
+	EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3)
+
+#define	EFX_POPULATE_WORD_2(_word,					\
+	    _field1, _value1, _field2, _value2)				\
+	EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2)
+
+#define	EFX_POPULATE_WORD_1(_word,					\
+	    _field1, _value1)						\
+	EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1)
+
+#define	EFX_ZERO_WORD(_word)						\
+	EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
+
+#define	EFX_SET_WORD(_word)						\
+	EFX_POPULATE_WORD_1(_word,					\
+	    EFX_WORD_0, 0xffff)
+
+/* Populate a byte field with various numbers of arguments */
+#define	EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
+
+#define	EFX_POPULATE_BYTE_9(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
+	EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8,	_field9, _value9)
+
+#define	EFX_POPULATE_BYTE_8(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)				\
+	EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7, _field8, _value8)
+
+#define	EFX_POPULATE_BYTE_7(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)						\
+	EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
+	    _field7, _value7)
+
+#define	EFX_POPULATE_BYTE_6(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
+	EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5,	_field6, _value6)
+
+#define	EFX_POPULATE_BYTE_5(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)				\
+	EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4, _field5, _value5)
+
+#define	EFX_POPULATE_BYTE_4(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)						\
+	EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3,	\
+	    _field4, _value4)
+
+#define	EFX_POPULATE_BYTE_3(_byte,					\
+	    _field1, _value1, _field2, _value2, _field3, _value3)	\
+	EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2, _field3, _value3)
+
+#define	EFX_POPULATE_BYTE_2(_byte,					\
+	    _field1, _value1, _field2, _value2)				\
+	EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1, _field2, _value2)
+
+#define	EFX_POPULATE_BYTE_1(_byte,					\
+	    _field1, _value1)						\
+	EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,			\
+	    _field1, _value1)
+
+#define	EFX_ZERO_BYTE(_byte)						\
+	EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
+
+#define	EFX_SET_BYTE(_byte)						\
+	EFX_POPULATE_BYTE_1(_byte,					\
+	    EFX_BYTE_0, 0xff)
+
+/*
+ * Modify a named field within an already-populated structure.  Used
+ * for read-modify-write operations.
+ */
+
+#define	EFX_INSERT_FIELD64(_min, _max, _field, _value)			\
+	__CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
+
+#define	EFX_INSERT_FIELD32(_min, _max, _field, _value)			\
+	__CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
+
+#define	EFX_INSERT_FIELD16(_min, _max, _field, _value)			\
+	__CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
+
+#define	EFX_INSERT_FIELD8(_min, _max, _field, _value)			\
+	__NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
+
+#define	EFX_INPLACE_MASK64(_min, _max, _field)			       	\
+	EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
+
+#define	EFX_INPLACE_MASK32(_min, _max, _field)			       	\
+	EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
+
+#define	EFX_INPLACE_MASK16(_min, _max, _field)			       	\
+	EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
+
+#define	EFX_INPLACE_MASK8(_min, _max, _field)			       	\
+	EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
+
+#define	EFX_SET_OWORD_FIELD64(_oword, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u64[0] = (((_oword).eo_u64[0] &		\
+		    ~EFX_INPLACE_MASK64(0, 63, _field)) |		\
+		    EFX_INSERT_FIELD64(0, 63, _field, _value));		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u64[1] = (((_oword).eo_u64[1] &		\
+		    ~EFX_INPLACE_MASK64(64, 127, _field)) |		\
+		    EFX_INSERT_FIELD64(64, 127, _field, _value));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_OWORD_FIELD32(_oword, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[0] = (((_oword).eo_u32[0] &		\
+		    ~EFX_INPLACE_MASK32(0, 31, _field)) |		\
+		    EFX_INSERT_FIELD32(0, 31, _field, _value));		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[1] = (((_oword).eo_u32[1] &		\
+		    ~EFX_INPLACE_MASK32(32, 63, _field)) |		\
+		    EFX_INSERT_FIELD32(32, 63, _field, _value));	\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[2] = (((_oword).eo_u32[2] &		\
+		    ~EFX_INPLACE_MASK32(64, 95, _field)) |		\
+		    EFX_INSERT_FIELD32(64, 95, _field, _value));	\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[3] = (((_oword).eo_u32[3] &		\
+		    ~EFX_INPLACE_MASK32(96, 127, _field)) |		\
+		    EFX_INSERT_FIELD32(96, 127, _field, _value));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_QWORD_FIELD64(_qword, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u64[0] = (((_qword).eq_u64[0] &		\
+		    ~EFX_INPLACE_MASK64(0, 63, _field)) |		\
+		    EFX_INSERT_FIELD64(0, 63, _field, _value));		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_QWORD_FIELD32(_qword, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u32[0] = (((_qword).eq_u32[0] &		\
+		    ~EFX_INPLACE_MASK32(0, 31, _field)) |		\
+		    EFX_INSERT_FIELD32(0, 31, _field, _value));		\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u32[1] = (((_qword).eq_u32[1] &		\
+		    ~EFX_INPLACE_MASK32(32, 63, _field)) |		\
+		    EFX_INSERT_FIELD32(32, 63, _field, _value));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_DWORD_FIELD(_dword, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_dword).ed_u32[0] = (((_dword).ed_u32[0] &		\
+		    ~EFX_INPLACE_MASK32(0, 31, _field)) |		\
+		    EFX_INSERT_FIELD32(0, 31, _field, _value));		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_WORD_FIELD(_word, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_word).ew_u16[0] = (((_word).ew_u16[0] &		\
+		    ~EFX_INPLACE_MASK16(0, 15, _field)) |		\
+		    EFX_INSERT_FIELD16(0, 15, _field, _value));		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_BYTE_FIELD(_byte, _field, _value)			\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_byte).eb_u8[0] = (((_byte).eb_u8[0] &			\
+		    ~EFX_INPLACE_MASK8(0, 7, _field)) |			\
+		    EFX_INSERT_FIELD8(0, 7, _field, _value));		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+/*
+ * Set or clear a numbered bit within an octword.
+ */
+ 
+#define	EFX_SHIFT64(_bit, _base)					\
+	(((_bit) >= (_base) && (_bit) < (_base) + 64) ?			\
+		((uint64_t)1 << ((_bit) - (_base))) :			\
+		0U)
+
+#define	EFX_SHIFT32(_bit, _base)					\
+	(((_bit) >= (_base) && (_bit) < (_base) + 32) ?			\
+		((uint32_t)1 << ((_bit) - (_base))) :			\
+		0U)
+
+#define	EFX_SHIFT16(_bit, _base)					\
+	(((_bit) >= (_base) && (_bit) < (_base) + 16) ?			\
+		(uint16_t)(1 << ((_bit) - (_base))) :			\
+		0U)
+
+#define	EFX_SHIFT8(_bit, _base)						\
+	(((_bit) >= (_base) && (_bit) < (_base) + 8) ?			\
+		(uint8_t)(1 << ((_bit) - (_base))) :			\
+		0U)
+
+#define	EFX_SET_OWORD_BIT64(_oword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u64[0] |=					\
+		    __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));	\
+		(_oword).eo_u64[1] |=					\
+		    __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_OWORD_BIT32(_oword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[0] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));	\
+		(_oword).eo_u32[1] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));	\
+		(_oword).eo_u32[2] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));	\
+		(_oword).eo_u32[3] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_OWORD_BIT64(_oword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u64[0] &=					\
+		    __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));	\
+		(_oword).eo_u64[1] &=					\
+		    __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_OWORD_BIT32(_oword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_oword).eo_u32[0] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));	\
+		(_oword).eo_u32[1] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));	\
+		(_oword).eo_u32[2] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));	\
+		(_oword).eo_u32[3] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_QWORD_BIT64(_qword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u64[0] |=					\
+		    __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_QWORD_BIT32(_qword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u32[0] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));	\
+		(_qword).eq_u32[1] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_QWORD_BIT64(_qword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u64[0] &=					\
+		    __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_QWORD_BIT32(_qword, _bit)				\
+	do {								\
+		_NOTE(CONSTANTCONDITION) 				\
+		(_qword).eq_u32[0] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));	\
+		(_qword).eq_u32[1] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_DWORD_BIT(_dword, _bit)					\
+	do {								\
+		(_dword).ed_u32[0] |=					\
+		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_DWORD_BIT(_dword, _bit)				\
+	do {								\
+		(_dword).ed_u32[0] &=					\
+		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_WORD_BIT(_word, _bit)					\
+	do {								\
+		(_word).ew_u16[0] |=					\
+		    __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_WORD_BIT(_word, _bit)					\
+	do {								\
+		(_word).ew_u32[0] &=					\
+		    __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));	\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_SET_BYTE_BIT(_byte, _bit)					\
+	do {								\
+		(_byte).eb_u8[0] |=					\
+		    __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_CLEAR_BYTE_BIT(_byte, _bit)					\
+	do {								\
+		(_byte).eb_u8[0] &=					\
+		    __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_OWORD64(_oword1, _oword2)				\
+	do {								\
+		(_oword1).eo_u64[0] |= (_oword2).eo_u64[0];		\
+		(_oword1).eo_u64[1] |= (_oword2).eo_u64[1];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_OWORD32(_oword1, _oword2)				\
+	do {								\
+		(_oword1).eo_u32[0] |= (_oword2).eo_u32[0];		\
+		(_oword1).eo_u32[1] |= (_oword2).eo_u32[1];		\
+		(_oword1).eo_u32[2] |= (_oword2).eo_u32[2];		\
+		(_oword1).eo_u32[3] |= (_oword2).eo_u32[3];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_OWORD64(_oword1, _oword2)				\
+	do {								\
+		(_oword1).eo_u64[0] &= (_oword2).eo_u64[0];		\
+		(_oword1).eo_u64[1] &= (_oword2).eo_u64[1];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_OWORD32(_oword1, _oword2)				\
+	do {								\
+		(_oword1).eo_u32[0] &= (_oword2).eo_u32[0];		\
+		(_oword1).eo_u32[1] &= (_oword2).eo_u32[1];		\
+		(_oword1).eo_u32[2] &= (_oword2).eo_u32[2];		\
+		(_oword1).eo_u32[3] &= (_oword2).eo_u32[3];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_QWORD64(_qword1, _qword2)				\
+	do {								\
+		(_qword1).eq_u64[0] |= (_qword2).eq_u64[0];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_QWORD32(_qword1, _qword2)				\
+	do {								\
+		(_qword1).eq_u32[0] |= (_qword2).eq_u32[0];		\
+		(_qword1).eq_u32[1] |= (_qword2).eq_u32[1];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_QWORD64(_qword1, _qword2)				\
+	do {								\
+		(_qword1).eq_u64[0] &= (_qword2).eq_u64[0];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_QWORD32(_qword1, _qword2)				\
+	do {								\
+		(_qword1).eq_u32[0] &= (_qword2).eq_u32[0];		\
+		(_qword1).eq_u32[1] &= (_qword2).eq_u32[1];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_DWORD(_dword1, _dword2)					\
+	do {								\
+		(_dword1).ed_u32[0] |= (_dword2).ed_u32[0];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_DWORD(_dword1, _dword2)					\
+	do {								\
+		(_dword1).ed_u32[0] &= (_dword2).ed_u32[0];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_WORD(_word1, _word2)					\
+	do {								\
+		(_word1).ew_u16[0] |= (_word2).ew_u16[0];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_WORD(_word1, _word2)					\
+	do {								\
+		(_word1).ew_u16[0] &= (_word2).ew_u16[0];		\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_OR_BYTE(_byte1, _byte2)					\
+	do {								\
+		(_byte1).eb_u8[0] &= (_byte2).eb_u8[0];			\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#define	EFX_AND_BYTE(_byte1, _byte2)					\
+	do {								\
+		(_byte1).eb_u8[0] &= (_byte2).eb_u8[0];			\
+	_NOTE(CONSTANTCONDITION) 					\
+	} while (B_FALSE)
+
+#if EFSYS_USE_UINT64
+#define	EFX_OWORD_FIELD		EFX_OWORD_FIELD64
+#define	EFX_QWORD_FIELD		EFX_QWORD_FIELD64
+#define	EFX_OWORD_IS_EQUAL	EFX_OWORD_IS_EQUAL64
+#define	EFX_QWORD_IS_EQUAL	EFX_QWORD_IS_EQUAL64
+#define	EFX_OWORD_IS_ZERO	EFX_OWORD_IS_ZERO64
+#define	EFX_QWORD_IS_ZERO	EFX_QWORD_IS_ZERO64
+#define	EFX_OWORD_IS_SET	EFX_OWORD_IS_SET64
+#define	EFX_QWORD_IS_SET	EFX_QWORD_IS_SET64
+#define	EFX_POPULATE_OWORD	EFX_POPULATE_OWORD64
+#define	EFX_POPULATE_QWORD	EFX_POPULATE_QWORD64
+#define	EFX_SET_OWORD		EFX_SET_OWORD64
+#define	EFX_SET_QWORD		EFX_SET_QWORD64
+#define	EFX_SET_OWORD_FIELD	EFX_SET_OWORD_FIELD64
+#define	EFX_SET_QWORD_FIELD	EFX_SET_QWORD_FIELD64
+#define	EFX_SET_OWORD_BIT	EFX_SET_OWORD_BIT64
+#define	EFX_CLEAR_OWORD_BIT	EFX_CLEAR_OWORD_BIT64
+#define	EFX_SET_QWORD_BIT	EFX_SET_QWORD_BIT64
+#define	EFX_CLEAR_QWORD_BIT	EFX_CLEAR_QWORD_BIT64
+#define	EFX_OR_OWORD		EFX_OR_OWORD64
+#define	EFX_AND_OWORD		EFX_AND_OWORD64
+#define	EFX_OR_QWORD		EFX_OR_QWORD64
+#define	EFX_AND_QWORD		EFX_OR_QWORD64
+#else
+#define	EFX_OWORD_FIELD		EFX_OWORD_FIELD32
+#define	EFX_QWORD_FIELD		EFX_QWORD_FIELD32
+#define	EFX_OWORD_IS_EQUAL	EFX_OWORD_IS_EQUAL32
+#define	EFX_QWORD_IS_EQUAL	EFX_QWORD_IS_EQUAL32
+#define	EFX_OWORD_IS_ZERO	EFX_OWORD_IS_ZERO32
+#define	EFX_QWORD_IS_ZERO	EFX_QWORD_IS_ZERO32
+#define	EFX_OWORD_IS_SET	EFX_OWORD_IS_SET32
+#define	EFX_QWORD_IS_SET	EFX_QWORD_IS_SET32
+#define	EFX_POPULATE_OWORD	EFX_POPULATE_OWORD32
+#define	EFX_POPULATE_QWORD	EFX_POPULATE_QWORD32
+#define	EFX_SET_OWORD		EFX_SET_OWORD32
+#define	EFX_SET_QWORD		EFX_SET_QWORD32
+#define	EFX_SET_OWORD_FIELD	EFX_SET_OWORD_FIELD32
+#define	EFX_SET_QWORD_FIELD	EFX_SET_QWORD_FIELD32
+#define	EFX_SET_OWORD_BIT	EFX_SET_OWORD_BIT32
+#define	EFX_CLEAR_OWORD_BIT	EFX_CLEAR_OWORD_BIT32
+#define	EFX_SET_QWORD_BIT	EFX_SET_QWORD_BIT32
+#define	EFX_CLEAR_QWORD_BIT	EFX_CLEAR_QWORD_BIT32
+#define	EFX_OR_OWORD		EFX_OR_OWORD32
+#define	EFX_AND_OWORD		EFX_AND_OWORD32
+#define	EFX_OR_QWORD		EFX_OR_QWORD32
+#define	EFX_AND_QWORD		EFX_OR_QWORD32
+#endif
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_EFX_TYPES_H */
diff --git a/sys/dev/sfxge/common/efx_vpd.c b/sys/dev/sfxge/common/efx_vpd.c
new file mode 100644
index 0000000..699e890
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_vpd.c
@@ -0,0 +1,999 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_VPD
+
+#define	TAG_TYPE_LBN 7
+#define	TAG_TYPE_WIDTH 1
+#define	TAG_TYPE_LARGE_ITEM_DECODE 1
+#define	TAG_TYPE_SMALL_ITEM_DECODE 0
+
+#define	TAG_SMALL_ITEM_NAME_LBN 3
+#define	TAG_SMALL_ITEM_NAME_WIDTH 4
+#define	TAG_SMALL_ITEM_SIZE_LBN 0
+#define	TAG_SMALL_ITEM_SIZE_WIDTH 3
+
+#define	TAG_LARGE_ITEM_NAME_LBN 0
+#define	TAG_LARGE_ITEM_NAME_WIDTH 7
+
+#define	TAG_NAME_END_DECODE 0x0f
+#define	TAG_NAME_ID_STRING_DECODE 0x02
+#define	TAG_NAME_VPD_R_DECODE 0x10
+#define	TAG_NAME_VPD_W_DECODE 0x11
+
+#if EFSYS_OPT_FALCON
+
+static efx_vpd_ops_t	__cs	__efx_vpd_falcon_ops = {
+	NULL,			/* evpdo_init */
+	falcon_vpd_size,	/* evpdo_size */
+	falcon_vpd_read,	/* evpdo_read */
+	falcon_vpd_verify,	/* evpdo_verify */
+	NULL,			/* evpdo_reinit */
+	falcon_vpd_get,		/* evpdo_get */
+	falcon_vpd_set,		/* evpdo_set */
+	falcon_vpd_next,	/* evpdo_next */
+	falcon_vpd_write,	/* evpdo_write */
+	NULL,			/* evpdo_fini */
+};
+
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+
+static efx_vpd_ops_t	__cs	__efx_vpd_siena_ops = {
+	siena_vpd_init,		/* evpdo_init */
+	siena_vpd_size,		/* evpdo_size */
+	siena_vpd_read,		/* evpdo_read */
+	siena_vpd_verify,	/* evpdo_verify */
+	siena_vpd_reinit,	/* evpdo_reinit */
+	siena_vpd_get,		/* evpdo_get */
+	siena_vpd_set,		/* evpdo_set */
+	siena_vpd_next,		/* evpdo_next */
+	siena_vpd_write,	/* evpdo_write */
+	siena_vpd_fini,		/* evpdo_fini */
+};
+
+#endif	/* EFSYS_OPT_SIENA */
+
+	__checkReturn		int
+efx_vpd_init(
+	__in			efx_nic_t *enp)
+{
+	efx_vpd_ops_t *evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_VPD));
+
+	switch (enp->en_family) {
+#if EFSYS_OPT_FALCON
+	case EFX_FAMILY_FALCON:
+		evpdop = (efx_vpd_ops_t *)&__efx_vpd_falcon_ops;
+		break;
+#endif	/* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+	case EFX_FAMILY_SIENA:
+		evpdop = (efx_vpd_ops_t *)&__efx_vpd_siena_ops;
+		break;
+#endif	/* EFSYS_OPT_SIENA */
+
+	default:
+		EFSYS_ASSERT(0);
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if (evpdop->evpdo_init != NULL) {
+		if ((rc = evpdop->evpdo_init(enp)) != 0)
+			goto fail2;
+	}
+
+	enp->en_evpdop = evpdop;
+	enp->en_mod_flags |= EFX_MOD_VPD;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_size(
+	__in			efx_nic_t *enp,
+	__out			size_t *sizep)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_size(enp, sizep)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_read(
+	__in			efx_nic_t *enp,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_read(enp, data, size)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_verify(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_verify(enp, data, size)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_reinit(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if (evpdop->evpdo_reinit == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = evpdop->evpdo_reinit(enp, data, size)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_get(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__inout			efx_vpd_value_t *evvp)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_get(enp, data, size, evvp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_set(
+	__in			efx_nic_t *enp,
+	__inout_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_value_t *evvp)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_set(enp, data, size, evvp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_next(
+	__in			efx_nic_t *enp,
+	__inout_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			efx_vpd_value_t *evvp,
+	__inout			unsigned int *contp)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_next(enp, data, size, evvp, contp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_write(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if ((rc = evpdop->evpdo_write(enp, data, size)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn		int
+efx_vpd_next_tag(
+	__in			caddr_t data,
+	__in			size_t size,
+	__inout			unsigned int *offsetp,
+	__out			efx_vpd_tag_t *tagp,
+	__out			uint16_t *lengthp)
+{
+	efx_byte_t byte;
+	efx_word_t word;
+	uint8_t name;
+	uint16_t length;
+	size_t headlen;
+	int rc;
+
+	if (*offsetp >= size) {
+		rc = EFAULT;
+		goto fail1;
+	}
+
+	EFX_POPULATE_BYTE_1(byte, EFX_BYTE_0, data[*offsetp]);
+
+	switch (EFX_BYTE_FIELD(byte, TAG_TYPE)) {
+	case TAG_TYPE_SMALL_ITEM_DECODE:
+		headlen = 1;
+
+		name = EFX_BYTE_FIELD(byte, TAG_SMALL_ITEM_NAME);
+		length = (uint16_t)EFX_BYTE_FIELD(byte, TAG_SMALL_ITEM_SIZE);
+
+		break;
+
+	case TAG_TYPE_LARGE_ITEM_DECODE:
+		headlen = 3;
+
+		if (*offsetp + headlen > size) {
+			rc = EFAULT;
+			goto fail2;
+		}
+
+		name = EFX_BYTE_FIELD(byte, TAG_LARGE_ITEM_NAME);
+		EFX_POPULATE_WORD_2(word,
+				    EFX_BYTE_0, data[*offsetp + 1],
+				    EFX_BYTE_1, data[*offsetp + 2]);
+		length = EFX_WORD_FIELD(word, EFX_WORD_0);
+
+		break;
+
+	default:
+		rc = EFAULT;
+		goto fail2;
+	}
+
+	if (*offsetp + headlen + length > size) {
+		rc = EFAULT;
+		goto fail3;
+	}
+
+	EFX_STATIC_ASSERT(TAG_NAME_END_DECODE == EFX_VPD_END);
+	EFX_STATIC_ASSERT(TAG_NAME_ID_STRING_DECODE == EFX_VPD_ID);
+	EFX_STATIC_ASSERT(TAG_NAME_VPD_R_DECODE == EFX_VPD_RO);
+	EFX_STATIC_ASSERT(TAG_NAME_VPD_W_DECODE == EFX_VPD_RW);
+	if (name != EFX_VPD_END && name != EFX_VPD_ID &&
+	    name != EFX_VPD_RO) {
+		rc = EFAULT;
+		goto fail4;
+	}
+
+	*tagp = name;
+	*lengthp = length;
+	*offsetp += headlen;
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn		int
+efx_vpd_next_keyword(
+	__in_bcount(size)	caddr_t tag,
+	__in			size_t size,
+	__in			unsigned int pos,
+	__out			efx_vpd_keyword_t *keywordp,
+	__out			uint8_t *lengthp)
+{
+	efx_vpd_keyword_t keyword;
+	uint8_t length;
+	int rc;
+
+	if (pos + 3U > size) {
+		rc = EFAULT;
+		goto fail1;
+	}
+
+	keyword = EFX_VPD_KEYWORD(tag[pos], tag[pos + 1]);
+	length = tag[pos + 2];
+
+	if (length == 0 || pos + 3U + length > size) {
+		rc = EFAULT;
+		goto fail2;
+	}
+
+	*keywordp = keyword;
+	*lengthp = length;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_hunk_length(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			size_t *lengthp)
+{
+	efx_vpd_tag_t tag;
+	unsigned int offset;
+	uint16_t taglen;
+	int rc;
+
+	offset = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		if ((rc = efx_vpd_next_tag(data, size, &offset,
+		    &tag, &taglen)) != 0)
+			goto fail1;
+		offset += taglen;
+		if (tag == EFX_VPD_END)
+			break;
+	}
+
+	*lengthp = offset;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_hunk_verify(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out_opt		boolean_t *cksummedp)
+{
+	efx_vpd_tag_t tag;
+	efx_vpd_keyword_t keyword;
+	unsigned int offset;
+	unsigned int pos;
+	unsigned int i;
+	uint16_t taglen;
+	uint8_t keylen;
+	uint8_t cksum;
+	boolean_t cksummed = B_FALSE;
+	int rc;
+
+	/*
+	 * Parse every tag,keyword in the existing VPD. If the csum is present,
+	 * the assert it is correct, and is the final keyword in the RO block.
+	 */
+	offset = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		if ((rc = efx_vpd_next_tag(data, size, &offset,
+		    &tag, &taglen)) != 0)
+			goto fail1;
+		if (tag == EFX_VPD_END)
+			break;
+		else if (tag == EFX_VPD_ID)
+			goto done;
+
+		for (pos = 0; pos != taglen; pos += 3 + keylen) {
+			/* RV keyword must be the last in the block */
+			if (cksummed)
+				goto fail2;
+
+			if ((rc = efx_vpd_next_keyword(data + offset,
+			    taglen, pos, &keyword, &keylen)) != 0)
+				goto fail3;
+
+			if (keyword == EFX_VPD_KEYWORD('R', 'V')) {
+				cksum = 0;
+				for (i = 0; i < offset + pos + 4; i++)
+					cksum += data[i];
+
+				if (cksum != 0) {
+					rc = EFAULT;
+					goto fail4;
+				}
+
+				cksummed = B_TRUE;
+			}
+		}
+
+	done:
+		offset += taglen;
+	}
+
+	if (!cksummed) {
+		rc = EFAULT;
+		goto fail5;
+	}
+
+	if (cksummedp != NULL)
+		*cksummedp = cksummed;
+
+	return (0);
+
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	uint8_t	__cs	__efx_vpd_blank_pid[] = {
+	/* Large resource type ID length 1 */
+	0x82, 0x01, 0x00,
+	/* Product name ' ' */
+	0x32,
+};
+
+static uint8_t __cs	__efx_vpd_blank_r[] = {
+	/* Large resource type VPD-R length 4 */
+	0x90, 0x04, 0x00,
+	/* RV keyword length 1 */
+	'R', 'V', 0x01,
+	/* RV payload checksum */
+	0x00,
+};
+
+	__checkReturn		int
+efx_vpd_hunk_reinit(
+	__in			caddr_t data,
+	__in			size_t size,
+	__in			boolean_t wantpid)
+{
+	unsigned int offset = 0;
+	unsigned int pos;
+	efx_byte_t byte;
+	uint8_t cksum;
+	int rc;
+
+	if (size < 0x100) {
+		rc = ENOSPC;
+		goto fail1;
+	}
+
+	if (wantpid) {
+		memcpy(data + offset, __efx_vpd_blank_pid,
+		    sizeof (__efx_vpd_blank_pid));
+		offset += sizeof (__efx_vpd_blank_pid);
+	}
+
+	memcpy(data + offset, __efx_vpd_blank_r, sizeof (__efx_vpd_blank_r));
+	offset += sizeof (__efx_vpd_blank_r);
+
+	/* Update checksum */
+	cksum = 0;
+	for (pos = 0; pos < offset; pos++)
+		cksum += data[pos];
+	data[offset - 1] -= cksum;
+
+	/* Append trailing tag */
+	EFX_POPULATE_BYTE_3(byte,
+			    TAG_TYPE, TAG_TYPE_SMALL_ITEM_DECODE,
+			    TAG_SMALL_ITEM_NAME, TAG_NAME_END_DECODE,
+			    TAG_SMALL_ITEM_SIZE, 0);
+	data[offset] = EFX_BYTE_FIELD(byte, EFX_BYTE_0);
+	offset++;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+efx_vpd_hunk_next(
+	__in_bcount(size)		caddr_t data,
+	__in				size_t size,
+	__out				efx_vpd_tag_t *tagp,
+	__out				efx_vpd_keyword_t *keywordp,
+	__out_bcount_opt(*paylenp)	unsigned int *payloadp,
+	__out_opt			uint8_t *paylenp,
+	__inout				unsigned int *contp)
+{
+	efx_vpd_tag_t tag;
+	efx_vpd_keyword_t keyword = 0;
+	unsigned int offset;
+	unsigned int pos;
+	unsigned int index;
+	uint16_t taglen;
+	uint8_t keylen;
+	uint8_t paylen;
+	int rc;
+
+	offset = index = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		if ((rc = efx_vpd_next_tag(data, size, &offset,
+		    &tag, &taglen)) != 0)
+			goto fail1;
+		if (tag == EFX_VPD_END)
+			break;
+
+		if (tag == EFX_VPD_ID) {
+			if (index == *contp) {
+				EFSYS_ASSERT3U(taglen, <, 0x100);
+				paylen = (uint8_t)MIN(taglen, 0xff);
+
+				goto done;
+			}
+		} else {
+			for (pos = 0; pos != taglen; pos += 3 + keylen) {
+				if ((rc = efx_vpd_next_keyword(data + offset,
+				    taglen, pos, &keyword, &keylen)) != 0)
+					goto fail2;
+
+				if (index == *contp) {
+					offset += pos + 3;
+					paylen = keylen;
+
+					goto done;
+				}
+			}
+		}
+
+		offset += taglen;
+	}
+
+	*contp = 0;
+	return (0);
+
+done:
+	*tagp = tag;
+	*keywordp = keyword;
+	if (payloadp != NULL)
+		*payloadp = offset;
+	if (paylenp != NULL)
+		*paylenp = paylen;
+
+	++(*contp);
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_hunk_get(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_tag_t tag,
+	__in			efx_vpd_keyword_t keyword,
+	__out			unsigned int *payloadp,
+	__out			uint8_t *paylenp)
+{
+	efx_vpd_tag_t itag;
+	efx_vpd_keyword_t ikeyword;
+	unsigned int offset;
+	unsigned int pos;
+	uint16_t taglen;
+	uint8_t keylen;
+	int rc;
+
+	offset = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		if ((rc = efx_vpd_next_tag(data, size, &offset,
+		    &itag, &taglen)) != 0)
+			goto fail1;
+		if (itag == EFX_VPD_END)
+			break;
+
+		if (itag == tag) {
+			if (itag == EFX_VPD_ID) {
+				EFSYS_ASSERT3U(taglen, <, 0x100);
+
+				*paylenp = (uint8_t)MIN(taglen, 0xff);
+				*payloadp = offset;
+				return (0);
+			}
+
+			for (pos = 0; pos != taglen; pos += 3 + keylen) {
+				if ((rc = efx_vpd_next_keyword(data + offset,
+				    taglen, pos, &ikeyword, &keylen)) != 0)
+					goto fail2;
+
+				if (ikeyword == keyword) {
+					*paylenp = keylen;
+					*payloadp = offset + pos + 3;
+					return (0);
+				}
+			}
+		}
+
+		offset += taglen;
+	}
+
+	/* Not an error */
+	return (ENOENT);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+efx_vpd_hunk_set(
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_value_t *evvp)
+{
+	efx_word_t word;
+	efx_vpd_tag_t tag;
+	efx_vpd_keyword_t keyword;
+	unsigned int offset;
+	unsigned int pos;
+	unsigned int taghead;
+	unsigned int source;
+	unsigned int dest;
+	unsigned int i;
+	uint16_t taglen;
+	uint8_t keylen;
+	uint8_t cksum;
+	size_t used;
+	int rc;
+
+	switch (evvp->evv_tag) {
+	case EFX_VPD_ID:
+		if (evvp->evv_keyword != 0) {
+			rc = EINVAL;
+			goto fail1;
+		}
+
+		/* Can't delete the ID keyword */
+		if (evvp->evv_length == 0) {
+			rc = EINVAL;
+			goto fail1;
+		}
+		break;
+
+	case EFX_VPD_RO:
+		if (evvp->evv_keyword == EFX_VPD_KEYWORD('R', 'V')) {
+			rc = EINVAL;
+			goto fail1;
+		}
+		break;
+
+	default:
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	/* Determine total size of all current tags */
+	if ((rc = efx_vpd_hunk_length(data, size, &used)) != 0)
+		goto fail2;
+
+	offset = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		taghead = offset;
+		if ((rc = efx_vpd_next_tag(data, size, &offset,
+		    &tag, &taglen)) != 0)
+			goto fail3;
+		if (tag == EFX_VPD_END)
+			break;
+		else if (tag != evvp->evv_tag) {
+			offset += taglen;
+			continue;
+		}
+
+		/* We only support modifying large resource tags */
+		if (offset - taghead != 3) {
+			rc = EINVAL;
+			goto fail4;
+		}
+
+		/*
+		 * Work out the offset of the byte immediately after the
+		 * old (=source) and new (=dest) new keyword/tag
+		 */
+		pos = 0;
+		if (tag == EFX_VPD_ID) {
+			source = offset + taglen;
+			dest = offset + evvp->evv_length;
+			goto check_space;
+		}
+
+		EFSYS_ASSERT3U(tag, ==, EFX_VPD_RO);
+		source = dest = 0;
+		for (pos = 0; pos != taglen; pos += 3 + keylen) {
+			if ((rc = efx_vpd_next_keyword(data + offset,
+			    taglen, pos, &keyword, &keylen)) != 0)
+				goto fail5;
+
+			if (keyword == evvp->evv_keyword &&
+			    evvp->evv_length == 0) {
+				/* Deleting this keyword */
+				source = offset + pos + 3 + keylen;
+				dest = offset + pos;
+				break;
+
+			} else if (keyword == evvp->evv_keyword) {
+				/* Adjusting this keyword */
+				source = offset + pos + 3 + keylen;
+				dest = offset + pos + 3 + evvp->evv_length;
+				break;
+
+			} else if (keyword == EFX_VPD_KEYWORD('R', 'V')) {
+				/* The RV keyword must be at the end */
+				EFSYS_ASSERT3U(pos + 3 + keylen, ==, taglen);
+
+				/*
+				 * The keyword doesn't already exist. If the
+				 * user deleting a non-existant keyword then
+				 * this is a no-op.
+				 */
+				if (evvp->evv_length == 0)
+					return (0);
+
+				/* Insert this keyword before the RV keyword */
+				source = offset + pos;
+				dest = offset + pos + 3 + evvp->evv_length;
+				break;
+			}
+		}
+
+	check_space:
+		if (used + dest > size + source) {
+			rc = ENOSPC;
+			goto fail6;
+		}
+
+		/* Move trailing data */
+		(void) memmove(data + dest, data + source, used - source);
+
+		/* Copy contents */
+		memcpy(data + dest - evvp->evv_length, evvp->evv_value,
+		    evvp->evv_length);
+
+		/* Insert new keyword header if required */
+		if (tag != EFX_VPD_ID && evvp->evv_length > 0) {
+			EFX_POPULATE_WORD_1(word, EFX_WORD_0,
+					    evvp->evv_keyword);
+			data[offset + pos + 0] =
+			    EFX_WORD_FIELD(word, EFX_BYTE_0);
+			data[offset + pos + 1] =
+			    EFX_WORD_FIELD(word, EFX_BYTE_1);
+			data[offset + pos + 2] = evvp->evv_length;
+		}
+
+		/* Modify tag length (large resource type) */
+		taglen += (dest - source);
+		EFX_POPULATE_WORD_1(word, EFX_WORD_0, taglen);
+		data[offset - 2] = EFX_WORD_FIELD(word, EFX_BYTE_0);
+		data[offset - 1] = EFX_WORD_FIELD(word, EFX_BYTE_1);
+
+		goto checksum;
+	}
+
+	/* Unable to find the matching tag */
+	rc = ENOENT;
+	goto fail7;
+
+checksum:
+	/* Find the RV tag, and update the checksum */
+	offset = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		if ((rc = efx_vpd_next_tag(data, size, &offset,
+		    &tag, &taglen)) != 0)
+			goto fail8;
+		if (tag == EFX_VPD_END)
+			break;
+		if (tag == EFX_VPD_RO) {
+			for (pos = 0; pos != taglen; pos += 3 + keylen) {
+				if ((rc = efx_vpd_next_keyword(data + offset,
+				    taglen, pos, &keyword, &keylen)) != 0)
+					goto fail9;
+
+				if (keyword == EFX_VPD_KEYWORD('R', 'V')) {
+					cksum = 0;
+					for (i = 0; i < offset + pos + 3; i++)
+						cksum += data[i];
+					data[i] = -cksum;
+					break;
+				}
+			}
+		}
+
+		offset += taglen;
+	}
+
+	/* Zero out the unused portion */
+	(void) memset(data + offset + taglen, 0xff, size - offset - taglen);
+
+	return (0);
+
+fail9:
+	EFSYS_PROBE(fail9);
+fail8:
+	EFSYS_PROBE(fail8);
+fail7:
+	EFSYS_PROBE(fail7);
+fail6:
+	EFSYS_PROBE(fail6);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				void
+efx_vpd_fini(
+	__in			efx_nic_t *enp)
+{
+	efx_vpd_ops_t *evpdop = enp->en_evpdop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD);
+
+	if (evpdop->evpdo_fini != NULL)
+		evpdop->evpdo_fini(enp);
+
+	enp->en_evpdop = NULL;
+	enp->en_mod_flags &= ~EFX_MOD_VPD;
+}
+
+#endif	/* EFSYS_OPT_VPD */
diff --git a/sys/dev/sfxge/common/efx_wol.c b/sys/dev/sfxge/common/efx_wol.c
new file mode 100644
index 0000000..74f11de
--- /dev/null
+++ b/sys/dev/sfxge/common/efx_wol.c
@@ -0,0 +1,396 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_WOL
+
+	__checkReturn	int
+efx_wol_init(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_WOL));
+
+	if (~(encp->enc_features) & EFX_FEATURE_WOL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	/* Current implementation is Siena specific */
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+
+	enp->en_mod_flags |= EFX_MOD_WOL;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_wol_filter_clear(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_WOL_FILTER_RESET_IN_LEN];
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_WOL);
+
+	req.emr_cmd = MC_CMD_WOL_FILTER_RESET;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_WOL_FILTER_RESET_IN_LEN;
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, WOL_FILTER_RESET_IN_MASK,
+			    MC_CMD_WOL_FILTER_RESET_IN_WAKE_FILTERS |
+			    MC_CMD_WOL_FILTER_RESET_IN_LIGHTSOUT_OFFLOADS);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_wol_filter_add(
+	__in		efx_nic_t *enp,
+	__in		efx_wol_type_t type,
+	__in		efx_wol_param_t *paramp,
+	__out		uint32_t *filter_idp)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MC_CMD_WOL_FILTER_SET_IN_LEN,
+			    MC_CMD_WOL_FILTER_SET_OUT_LEN)];
+	efx_byte_t link_mask;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_WOL);
+
+	req.emr_cmd = MC_CMD_WOL_FILTER_SET;
+	(void) memset(payload, '\0', sizeof (payload));
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_WOL_FILTER_SET_IN_LEN;
+	req.emr_out_buf = payload;
+	req.emr_out_length = MC_CMD_WOL_FILTER_SET_OUT_LEN;
+
+	switch (type) {
+	case EFX_WOL_TYPE_MAGIC:
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_FILTER_MODE,
+				    MC_CMD_FILTER_MODE_SIMPLE);
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_WOL_TYPE,
+				    MC_CMD_WOL_TYPE_MAGIC);
+		EFX_MAC_ADDR_COPY(
+			MCDI_IN2(req, uint8_t, WOL_FILTER_SET_IN_MAGIC_MAC),
+			paramp->ewp_magic.mac_addr);
+		break;
+
+	case EFX_WOL_TYPE_BITMAP: {
+		uint32_t swapped = 0;
+		efx_dword_t *dwordp;
+		unsigned int pos, bit;
+
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_FILTER_MODE,
+				    MC_CMD_FILTER_MODE_SIMPLE);
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_WOL_TYPE,
+				    MC_CMD_WOL_TYPE_BITMAP);
+
+		/*
+		 * MC bitmask is supposed to be bit swapped
+		 * amongst 32 bit words(!)
+		 */
+
+		dwordp = MCDI_IN2(req, efx_dword_t,
+				    WOL_FILTER_SET_IN_BITMAP_MASK);
+
+		EFSYS_ASSERT3U(EFX_WOL_BITMAP_MASK_SIZE % 4, ==, 0);
+
+		for (pos = 0; pos < EFX_WOL_BITMAP_MASK_SIZE; ++pos) {
+			uint8_t native = paramp->ewp_bitmap.mask[pos];
+
+			for (bit = 0; bit < 8; ++bit) {
+				swapped <<= 1;
+				swapped |= (native & 0x1);
+				native >>= 1;
+			}
+
+			if ((pos & 3) == 3) {
+				EFX_POPULATE_DWORD_1(dwordp[pos >> 2],
+				    EFX_DWORD_0, swapped);
+				swapped = 0;
+			}
+		}
+
+		memcpy(MCDI_IN2(req, uint8_t, WOL_FILTER_SET_IN_BITMAP_BITMAP),
+		    paramp->ewp_bitmap.value,
+		    sizeof (paramp->ewp_bitmap.value));
+
+		EFSYS_ASSERT3U(paramp->ewp_bitmap.value_len, <=,
+				    sizeof (paramp->ewp_bitmap.value));
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_BITMAP_LEN,
+				    paramp->ewp_bitmap.value_len);
+		}
+		break;
+
+	case EFX_WOL_TYPE_LINK:
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_FILTER_MODE,
+				    MC_CMD_FILTER_MODE_SIMPLE);
+		MCDI_IN_SET_DWORD(req, WOL_FILTER_SET_IN_WOL_TYPE,
+				    MC_CMD_WOL_TYPE_LINK);
+
+		EFX_ZERO_BYTE(link_mask);
+		EFX_SET_BYTE_FIELD(link_mask, MC_CMD_WOL_FILTER_SET_IN_LINK_UP,
+		    1);
+		MCDI_IN_SET_BYTE(req, WOL_FILTER_SET_IN_LINK_MASK,
+		    link_mask.eb_u8[0]);
+		break;
+
+	default:
+		EFSYS_ASSERT3U(type, !=, type);
+	}
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_WOL_FILTER_SET_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	*filter_idp = MCDI_OUT_DWORD(req, WOL_FILTER_SET_OUT_FILTER_ID);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+efx_wol_filter_remove(
+	__in		efx_nic_t *enp,
+	__in		uint32_t filter_id)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_WOL_FILTER_REMOVE_IN_LEN];
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_WOL);
+
+	req.emr_cmd = MC_CMD_WOL_FILTER_REMOVE;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_WOL_FILTER_REMOVE_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_WOL_FILTER_REMOVE_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, WOL_FILTER_REMOVE_IN_FILTER_ID, filter_id);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+
+	__checkReturn	int
+efx_lightsout_offload_add(
+	__in		efx_nic_t *enp,
+	__in		efx_lightsout_offload_type_t type,
+	__in		efx_lightsout_offload_param_t *paramp,
+	__out		uint32_t *filter_idp)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MAX(MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_LEN,
+				MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_LEN),
+			    MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_LEN)];
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_WOL);
+
+	req.emr_cmd = MC_CMD_ADD_LIGHTSOUT_OFFLOAD;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (type);
+	req.emr_out_buf = payload;
+	req.emr_out_length = MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_LEN;
+
+	switch (type) {
+	case EFX_LIGHTSOUT_OFFLOAD_TYPE_ARP:
+		req.emr_in_length = MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_LEN;
+		MCDI_IN_SET_DWORD(req, ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL,
+				    MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_ARP);
+		EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t,
+					    ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC),
+				    paramp->elop_arp.mac_addr);
+		MCDI_IN_SET_DWORD(req, ADD_LIGHTSOUT_OFFLOAD_IN_ARP_IP,
+				    paramp->elop_arp.ip);
+		break;
+	case EFX_LIGHTSOUT_OFFLOAD_TYPE_NS:
+		req.emr_in_length = MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_LEN;
+		MCDI_IN_SET_DWORD(req, ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL,
+				    MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_NS);
+		EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t,
+					    ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC),
+				    paramp->elop_ns.mac_addr);
+		memcpy(MCDI_IN2(req, uint8_t,
+		    ADD_LIGHTSOUT_OFFLOAD_IN_NS_SNIPV6),
+		    paramp->elop_ns.solicited_node,
+		    sizeof (paramp->elop_ns.solicited_node));
+		memcpy(MCDI_IN2(req, uint8_t, ADD_LIGHTSOUT_OFFLOAD_IN_NS_IPV6),
+		    paramp->elop_ns.ip, sizeof (paramp->elop_ns.ip));
+		break;
+	default:
+		EFSYS_ASSERT3U(type, !=, type);
+	}
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	*filter_idp = MCDI_OUT_DWORD(req, ADD_LIGHTSOUT_OFFLOAD_OUT_FILTER_ID);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+
+	__checkReturn	int
+efx_lightsout_offload_remove(
+	__in		efx_nic_t *enp,
+	__in		efx_lightsout_offload_type_t type,
+	__in		uint32_t filter_id)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_LEN];
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_WOL);
+
+	req.emr_cmd = MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	switch (type) {
+	case EFX_LIGHTSOUT_OFFLOAD_TYPE_ARP:
+		MCDI_IN_SET_DWORD(req, REMOVE_LIGHTSOUT_OFFLOAD_IN_PROTOCOL,
+				    MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_ARP);
+		break;
+	case EFX_LIGHTSOUT_OFFLOAD_TYPE_NS:
+		MCDI_IN_SET_DWORD(req, REMOVE_LIGHTSOUT_OFFLOAD_IN_PROTOCOL,
+				    MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_NS);
+		break;
+	default:
+		EFSYS_ASSERT3U(type, !=, type);
+	}
+
+	MCDI_IN_SET_DWORD(req, REMOVE_LIGHTSOUT_OFFLOAD_IN_FILTER_ID,
+			    filter_id);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+
+			void
+efx_wol_fini(
+	__in		efx_nic_t *enp)
+{
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_WOL);
+
+	enp->en_mod_flags &= ~EFX_MOD_WOL;
+}
+
+#endif	/* EFSYS_OPT_WOL */
diff --git a/sys/dev/sfxge/common/siena_flash.h b/sys/dev/sfxge/common/siena_flash.h
new file mode 100644
index 0000000..7df3995
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_flash.h
@@ -0,0 +1,132 @@
+/*-
+ * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+#ifndef	_SYS_SIENA_FLASH_H
+#define	_SYS_SIENA_FLASH_H
+
+#pragma pack(1)
+
+/* Fixed locations near the start of flash (which may be in the internal PHY
+ * firmware header) point to the boot header.
+ *
+ * - parsed by MC boot ROM and firmware
+ * - reserved (but not parsed) by PHY firmware
+ * - opaque to driver
+ */
+
+#define	SIENA_MC_BOOT_PHY_FW_HDR_LEN (0x20)
+
+#define	SIENA_MC_BOOT_PTR_LOCATION (0x18)      /* First thing we try to boot */
+#define	SIENA_MC_BOOT_ALT_PTR_LOCATION (0x1c)  /* Alternative if that fails */
+
+#define	SIENA_MC_BOOT_HDR_LEN (0x200)
+
+#define	SIENA_MC_BOOT_MAGIC (0x51E4A001)
+#define	SIENA_MC_BOOT_VERSION (1)
+
+typedef struct siena_mc_boot_hdr_s {		/* GENERATED BY scripts/genfwdef */
+	efx_dword_t	magic;			/* = SIENA_MC_BOOT_MAGIC */
+	efx_word_t	hdr_version;		/* this structure definition is version 1 */
+	efx_byte_t	board_type;
+	efx_byte_t	firmware_version_a;
+	efx_byte_t	firmware_version_b;
+	efx_byte_t	firmware_version_c;
+	efx_word_t	checksum;		/* of whole header area + firmware image */
+	efx_word_t	firmware_version_d;
+	efx_word_t	reserved_a[1];		/* (set to 0) */
+	efx_dword_t	firmware_text_offset;	/* offset to firmware .text */
+	efx_dword_t	firmware_text_size;	/* length of firmware .text, in bytes */
+	efx_dword_t	firmware_data_offset;	/* offset to firmware .data */
+	efx_dword_t	firmware_data_size;	/* length of firmware .data, in bytes */
+	efx_dword_t	reserved_b[8];		/* (set to 0) */
+} siena_mc_boot_hdr_t;
+
+#define	SIENA_MC_STATIC_CONFIG_MAGIC (0xBDCF5555)
+#define	SIENA_MC_STATIC_CONFIG_VERSION (0)
+
+typedef struct siena_mc_static_config_hdr_s {	/* GENERATED BY scripts/genfwdef */
+	efx_dword_t	magic;			/* = SIENA_MC_STATIC_CONFIG_MAGIC */
+	efx_word_t	length;			/* of header area (i.e. not including VPD) */
+	efx_byte_t	version;
+	efx_byte_t	csum;			/* over header area (i.e. not including VPD) */
+	efx_dword_t	static_vpd_offset;
+	efx_dword_t	static_vpd_length;
+	efx_dword_t	capabilities;
+	efx_byte_t	mac_addr_base[6];
+	efx_byte_t	green_mode_cal;		/* Green mode calibration result */
+	efx_byte_t	green_mode_valid;	/* Whether cal holds a valid value */
+	efx_word_t	mac_addr_count;
+	efx_word_t	mac_addr_stride;
+	efx_dword_t	reserved2[2];		/* (write as zero) */
+	efx_dword_t	num_dbi_items;
+	struct {
+		efx_word_t	addr;
+		efx_word_t	byte_enables;
+		efx_dword_t	value;
+	} dbi[];
+} siena_mc_static_config_hdr_t;
+
+#define	SIENA_MC_DYNAMIC_CONFIG_MAGIC (0xBDCFDDDD)
+#define	SIENA_MC_DYNAMIC_CONFIG_VERSION (0)
+
+typedef struct siena_mc_fw_version_s {		/* GENERATED BY scripts/genfwdef */
+	efx_dword_t	fw_subtype;
+	efx_word_t	version_w;
+	efx_word_t	version_x;
+	efx_word_t	version_y;
+	efx_word_t	version_z;
+} siena_mc_fw_version_t;
+
+typedef struct siena_mc_dynamic_config_hdr_s {	/* GENERATED BY scripts/genfwdef */
+	efx_dword_t	magic;			/* = SIENA_MC_DYNAMIC_CONFIG_MAGIC */
+	efx_word_t	length;			/* of header area (i.e. not including VPD) */
+	efx_byte_t	version;
+	efx_byte_t	csum;			/* over header area (i.e. not including VPD) */
+	efx_dword_t	dynamic_vpd_offset;
+	efx_dword_t	dynamic_vpd_length;
+	efx_dword_t	num_fw_version_items;
+	siena_mc_fw_version_t	fw_version[];
+} siena_mc_dynamic_config_hdr_t;
+
+#define	SIENA_MC_EXPROM_SINGLE_MAGIC (0xAA55)  /* little-endian uint16_t */
+
+#define	SIENA_MC_EXPROM_COMBO_MAGIC (0xB0070102)  /* little-endian uint32_t */
+
+typedef struct siena_mc_combo_rom_hdr_s {	/* GENERATED BY scripts/genfwdef */
+	efx_dword_t	magic;			/* = SIENA_MC_EXPROM_COMBO_MAGIC */
+	efx_dword_t	len1;			/* length of first image */
+	efx_dword_t	len2;			/* length of second image */
+	efx_dword_t	off1;			/* offset of first byte to edit to combine images */
+	efx_dword_t	off2;			/* offset of second byte to edit to combine images */
+	efx_word_t	infoblk0_off;		/* infoblk offset */
+	efx_word_t	infoblk1_off;		/* infoblk offset */
+	efx_byte_t	infoblk_len;		/* length of space reserved for infoblk structures */
+	efx_byte_t	reserved[7];		/* (set to 0) */
+} siena_mc_combo_rom_hdr_t;
+
+#pragma pack()
+
+#endif	/* _SYS_SIENA_FLASH_H */
diff --git a/sys/dev/sfxge/common/siena_impl.h b/sys/dev/sfxge/common/siena_impl.h
new file mode 100644
index 0000000..b566adb
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_impl.h
@@ -0,0 +1,477 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SYS_SIENA_IMPL_H
+#define	_SYS_SIENA_IMPL_H
+
+#include "efx.h"
+#include "efx_regs.h"
+#include "efx_mcdi.h"
+#include "siena_flash.h"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if EFSYS_OPT_PHY_PROPS
+
+/* START MKCONFIG GENERATED SienaPhyHeaderPropsBlock a8db1f8eb5106efd */
+typedef enum siena_phy_prop_e {
+	SIENA_PHY_NPROPS
+} siena_phy_prop_t;
+
+/* END MKCONFIG GENERATED SienaPhyHeaderPropsBlock */
+
+#endif  /* EFSYS_OPT_PHY_PROPS */
+
+#define	SIENA_NVRAM_CHUNK 0x80
+
+extern	__checkReturn	int
+siena_nic_probe(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_PCIE_TUNE
+
+extern	__checkReturn	int
+siena_nic_pcie_extended_sync(
+	__in		efx_nic_t *enp);
+
+#endif
+
+extern	__checkReturn	int
+siena_nic_reset(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+siena_nic_init(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_DIAG
+
+extern	__checkReturn	int
+siena_nic_register_test(
+	__in		efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+extern			void
+siena_nic_fini(
+	__in		efx_nic_t *enp);
+
+extern			void
+siena_nic_unprobe(
+	__in		efx_nic_t *enp);
+
+#define	SIENA_SRAM_ROWS	0x12000
+
+extern			void
+siena_sram_init(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_DIAG
+
+extern	__checkReturn	int
+siena_sram_test(
+	__in		efx_nic_t *enp,
+	__in		efx_sram_pattern_fn_t func);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+
+#if EFSYS_OPT_NVRAM || EFSYS_OPT_VPD
+
+extern	__checkReturn		int
+siena_nvram_partn_size(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__out			size_t *sizep);
+
+extern	__checkReturn		int
+siena_nvram_partn_lock(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn);
+
+extern	__checkReturn		int
+siena_nvram_partn_read(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+siena_nvram_partn_erase(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			unsigned int offset,
+	__in			size_t size);
+
+extern	__checkReturn		int
+siena_nvram_partn_write(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern				void
+siena_nvram_partn_unlock(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn);
+
+extern	__checkReturn		int
+siena_nvram_get_dynamic_cfg(
+	__in			efx_nic_t *enp,
+	__in			unsigned int index,
+	__in			boolean_t vpd,
+	__out			siena_mc_dynamic_config_hdr_t **dcfgp,
+	__out			size_t *sizep);
+
+#endif	/* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */
+
+#if EFSYS_OPT_NVRAM
+
+#if EFSYS_OPT_DIAG
+
+extern	__checkReturn		int
+siena_nvram_test(
+	__in			efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_DIAG */
+
+extern	__checkReturn		int
+siena_nvram_size(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			size_t *sizep);
+
+extern	__checkReturn		int
+siena_nvram_get_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint32_t *subtypep,
+	__out_ecount(4)		uint16_t version[4]);
+
+extern	__checkReturn		int
+siena_nvram_rw_start(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			size_t *pref_chunkp);
+
+extern	__checkReturn		int
+siena_nvram_read_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	 __checkReturn		int
+siena_nvram_erase(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type);
+
+extern	__checkReturn		int
+siena_nvram_write_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern				void
+siena_nvram_rw_finish(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type);
+
+extern	__checkReturn		int
+siena_nvram_set_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint16_t version[4]);
+
+#endif	/* EFSYS_OPT_NVRAM */
+
+#if EFSYS_OPT_VPD
+
+extern	__checkReturn		int
+siena_vpd_init(
+	__in			efx_nic_t *enp);
+
+extern	__checkReturn		int
+siena_vpd_size(
+	__in			efx_nic_t *enp,
+	__out			size_t *sizep);
+
+extern	__checkReturn		int
+siena_vpd_read(
+	__in			efx_nic_t *enp,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+siena_vpd_verify(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+siena_vpd_reinit(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern	__checkReturn		int
+siena_vpd_get(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__inout			efx_vpd_value_t *evvp);
+
+extern	__checkReturn		int
+siena_vpd_set(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_value_t *evvp);
+
+extern	__checkReturn		int
+siena_vpd_next(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			efx_vpd_value_t *evvp,
+	__inout			unsigned int *contp);
+
+extern __checkReturn		int
+siena_vpd_write(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size);
+
+extern				void
+siena_vpd_fini(
+	__in			efx_nic_t *enp);
+
+#endif	/* EFSYS_OPT_VPD */
+
+typedef struct siena_link_state_s {
+	uint32_t		sls_adv_cap_mask;
+	uint32_t		sls_lp_cap_mask;
+	unsigned int 		sls_fcntl;
+	efx_link_mode_t		sls_link_mode;
+#if EFSYS_OPT_LOOPBACK
+	efx_loopback_type_t	sls_loopback;
+#endif
+	boolean_t		sls_mac_up;
+} siena_link_state_t;
+
+extern			void
+siena_phy_link_ev(
+	__in		efx_nic_t *enp,
+	__in		efx_qword_t *eqp,
+	__out		efx_link_mode_t *link_modep);
+
+extern	__checkReturn	int
+siena_phy_get_link(
+	__in		efx_nic_t *enp,
+	__out		siena_link_state_t *slsp);
+
+extern	__checkReturn	int
+siena_phy_power(
+	__in		efx_nic_t *enp,
+	__in		boolean_t on);
+
+extern	__checkReturn	int
+siena_phy_reconfigure(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+siena_phy_verify(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+siena_phy_oui_get(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *ouip);
+
+#if EFSYS_OPT_PHY_STATS
+
+extern					void
+siena_phy_decode_stats(
+	__in				efx_nic_t *enp,
+	__in				uint32_t vmask,
+	__in_opt			efsys_mem_t *esmp,
+	__out_opt			uint64_t *smaskp,
+	__out_ecount_opt(EFX_PHY_NSTATS)	uint32_t *stat);
+
+extern	__checkReturn			int
+siena_phy_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_PHY_NSTATS)	uint32_t *stat);
+
+#endif	/* EFSYS_OPT_PHY_STATS */
+
+#if EFSYS_OPT_PHY_PROPS
+
+#if EFSYS_OPT_NAMES
+
+extern		const char __cs *
+siena_phy_prop_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id);
+
+#endif	/* EFSYS_OPT_NAMES */
+
+extern	__checkReturn	int
+siena_phy_prop_get(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t flags,
+	__out		uint32_t *valp);
+
+extern	__checkReturn	int
+siena_phy_prop_set(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t val);
+
+#endif	/* EFSYS_OPT_PHY_PROPS */
+
+#if EFSYS_OPT_PHY_BIST
+
+extern	__checkReturn		int
+siena_phy_bist_start(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type);
+
+extern	__checkReturn		int
+siena_phy_bist_poll(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type,
+	__out			efx_phy_bist_result_t *resultp,
+	__out_opt __drv_when(count > 0, __notnull)
+	uint32_t 	*value_maskp,
+	__out_ecount_opt(count)	__drv_when(count > 0, __notnull)
+	unsigned long	*valuesp,
+	__in			size_t count);
+
+extern				void
+siena_phy_bist_stop(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type);
+
+#endif	/* EFSYS_OPT_PHY_BIST */
+
+extern	__checkReturn	int
+siena_mac_poll(
+	__in		efx_nic_t *enp,
+	__out		efx_link_mode_t *link_modep);
+
+extern	__checkReturn	int
+siena_mac_up(
+	__in		efx_nic_t *enp,
+	__out		boolean_t *mac_upp);
+
+extern	__checkReturn	int
+siena_mac_reconfigure(
+	__in	efx_nic_t *enp);
+
+#if EFSYS_OPT_LOOPBACK
+
+extern	__checkReturn	int
+siena_mac_loopback_set(
+	__in		efx_nic_t *enp,
+	__in		efx_link_mode_t link_mode,
+	__in		efx_loopback_type_t loopback_type);
+
+#endif	/* EFSYS_OPT_LOOPBACK */
+
+#if EFSYS_OPT_MAC_STATS
+
+extern	__checkReturn			int
+siena_mac_stats_clear(
+	__in				efx_nic_t *enp);
+
+extern	__checkReturn			int
+siena_mac_stats_upload(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp);
+
+extern	__checkReturn			int
+siena_mac_stats_periodic(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__in				uint16_t period_ms,
+	__in				boolean_t events);
+
+extern	__checkReturn			int
+siena_mac_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_MAC_NSTATS)	efsys_stat_t *stat,
+	__out_opt			uint32_t *generationp);
+
+#endif	/* EFSYS_OPT_MAC_STATS */
+
+extern	__checkReturn	int
+siena_mon_reset(
+	__in		efx_nic_t *enp);
+
+extern	__checkReturn	int
+siena_mon_reconfigure(
+	__in		efx_nic_t *enp);
+
+#if EFSYS_OPT_MON_STATS
+
+extern					void
+siena_mon_decode_stats(
+	__in				efx_nic_t *enp,
+	__in				uint32_t dmask,
+	__in_opt			efsys_mem_t *esmp,
+	__out_opt				uint32_t *vmaskp,
+	__out_ecount_opt(EFX_MON_NSTATS)	efx_mon_stat_value_t *value);
+
+extern	__checkReturn			int
+siena_mon_ev(
+	__in				efx_nic_t *enp,
+	__in				efx_qword_t *eqp,
+	__out				efx_mon_stat_t *idp,
+	__out				efx_mon_stat_value_t *valuep);
+
+extern	__checkReturn			int
+siena_mon_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values);
+
+#endif	/* EFSYS_OPT_MON_STATS */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_SIENA_IMPL_H */
diff --git a/sys/dev/sfxge/common/siena_mac.c b/sys/dev/sfxge/common/siena_mac.c
new file mode 100644
index 0000000..7facf1c
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_mac.c
@@ -0,0 +1,545 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "efsys.h"
+#include "efx.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_SIENA
+
+	__checkReturn	int
+siena_mac_poll(
+	__in		efx_nic_t *enp,
+	__out		efx_link_mode_t *link_modep)
+{
+	efx_port_t *epp = &(enp->en_port);
+	siena_link_state_t sls;
+	int rc;
+
+	if ((rc = siena_phy_get_link(enp, &sls)) != 0)
+		goto fail1;
+
+	epp->ep_adv_cap_mask = sls.sls_adv_cap_mask;
+	epp->ep_fcntl = sls.sls_fcntl;
+
+	*link_modep = sls.sls_link_mode;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	*link_modep = EFX_LINK_UNKNOWN;
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_mac_up(
+	__in		efx_nic_t *enp,
+	__out		boolean_t *mac_upp)
+{
+	siena_link_state_t sls;
+	int rc;
+
+	/*
+	 * Because Siena doesn't *require* polling, we can't rely on
+	 * siena_mac_poll() being executed to populate epp->ep_mac_up.
+	 */
+	if ((rc = siena_phy_get_link(enp, &sls)) != 0)
+		goto fail1;
+
+	*mac_upp = sls.sls_mac_up;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_mac_reconfigure(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	uint8_t payload[MAX(MC_CMD_SET_MAC_IN_LEN,
+			    MC_CMD_SET_MCAST_HASH_IN_LEN)];
+	efx_mcdi_req_t req;
+	unsigned int fcntl;
+	int rc;
+
+	req.emr_cmd = MC_CMD_SET_MAC;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_SET_MAC_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_SET_MAC_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, SET_MAC_IN_MTU, epp->ep_mac_pdu);
+	MCDI_IN_SET_DWORD(req, SET_MAC_IN_DRAIN, epp->ep_mac_drain ? 1 : 0);
+	EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, SET_MAC_IN_ADDR),
+			    epp->ep_mac_addr);
+	MCDI_IN_POPULATE_DWORD_2(req, SET_MAC_IN_REJECT,
+				    SET_MAC_IN_REJECT_UNCST, !epp->ep_unicst,
+				    SET_MAC_IN_REJECT_BRDCST, !epp->ep_brdcst);
+
+	if (epp->ep_fcntl_autoneg)
+		/* efx_fcntl_set() has already set the phy capabilities */
+		fcntl = MC_CMD_FCNTL_AUTO;
+	else if (epp->ep_fcntl & EFX_FCNTL_RESPOND)
+		fcntl = (epp->ep_fcntl & EFX_FCNTL_GENERATE)
+			? MC_CMD_FCNTL_BIDIR
+			: MC_CMD_FCNTL_RESPOND;
+	else
+		fcntl = MC_CMD_FCNTL_OFF;
+
+	MCDI_IN_SET_DWORD(req, SET_MAC_IN_FCNTL, fcntl);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	/* Push multicast hash. Set the broadcast bit (0xff) appropriately */
+	req.emr_cmd = MC_CMD_SET_MCAST_HASH;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_SET_MCAST_HASH_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_SET_MCAST_HASH_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	memcpy(MCDI_IN2(req, uint8_t, SET_MCAST_HASH_IN_HASH0),
+	    epp->ep_multicst_hash, sizeof (epp->ep_multicst_hash));
+	if (epp->ep_brdcst)
+		EFX_SET_OWORD_BIT(*MCDI_IN2(req, efx_oword_t,
+		    SET_MCAST_HASH_IN_HASH1), 0x7f);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_LOOPBACK
+
+	__checkReturn	int
+siena_mac_loopback_set(
+	__in		efx_nic_t *enp,
+	__in		efx_link_mode_t link_mode,
+	__in		efx_loopback_type_t loopback_type)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_phy_ops_t *epop = epp->ep_epop;
+	efx_loopback_type_t old_loopback_type;
+	efx_link_mode_t old_loopback_link_mode;
+	int rc;
+
+	/* The PHY object handles this on Siena */
+	old_loopback_type = epp->ep_loopback_type;
+	old_loopback_link_mode = epp->ep_loopback_link_mode;
+	epp->ep_loopback_type = loopback_type;
+	epp->ep_loopback_link_mode = link_mode;
+
+	if ((rc = epop->epo_reconfigure(enp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE(fail2);
+
+	epp->ep_loopback_type = old_loopback_type;
+	epp->ep_loopback_link_mode = old_loopback_link_mode;
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_LOOPBACK */
+
+#if EFSYS_OPT_MAC_STATS
+
+	__checkReturn			int
+siena_mac_stats_clear(
+	__in				efx_nic_t *enp)
+{
+	uint8_t payload[MC_CMD_MAC_STATS_IN_LEN];
+	efx_mcdi_req_t req;
+	int rc;
+
+	req.emr_cmd = MC_CMD_MAC_STATS;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_MAC_STATS_OUT_DMA_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_POPULATE_DWORD_3(req, MAC_STATS_IN_CMD,
+				    MAC_STATS_IN_DMA, 0,
+				    MAC_STATS_IN_CLEAR, 1,
+				    MAC_STATS_IN_PERIODIC_CHANGE, 0);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+siena_mac_stats_upload(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp)
+{
+	uint8_t payload[MC_CMD_MAC_STATS_IN_LEN];
+	efx_mcdi_req_t req;
+	size_t bytes;
+	int rc;
+
+	EFX_STATIC_ASSERT(MC_CMD_MAC_NSTATS * sizeof (uint64_t) <=
+	    EFX_MAC_STATS_SIZE);
+
+	bytes = MC_CMD_MAC_NSTATS * sizeof (uint64_t);
+
+	req.emr_cmd = MC_CMD_MAC_STATS;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_MAC_STATS_OUT_DMA_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_LO,
+			    EFSYS_MEM_ADDR(esmp) & 0xffffffff);
+	MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_HI,
+			    EFSYS_MEM_ADDR(esmp) >> 32);
+	MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_LEN, bytes);
+
+	/*
+	 * The MC DMAs aggregate statistics for our convinience, so we can
+	 * avoid having to pull the statistics buffer into the cache to
+	 * maintain cumulative statistics.
+	 */
+	MCDI_IN_POPULATE_DWORD_3(req, MAC_STATS_IN_CMD,
+				    MAC_STATS_IN_DMA, 1,
+				    MAC_STATS_IN_CLEAR, 0,
+				    MAC_STATS_IN_PERIODIC_CHANGE, 0);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+siena_mac_stats_periodic(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__in				uint16_t period,
+	__in				boolean_t events)
+{
+	uint8_t payload[MC_CMD_MAC_STATS_IN_LEN];
+	efx_mcdi_req_t req;
+	size_t bytes;
+	int rc;
+
+	bytes = MC_CMD_MAC_NSTATS * sizeof (uint64_t);
+
+	req.emr_cmd = MC_CMD_MAC_STATS;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_MAC_STATS_OUT_DMA_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_LO,
+			    EFSYS_MEM_ADDR(esmp) & 0xffffffff);
+	MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_HI,
+			    EFSYS_MEM_ADDR(esmp) >> 32);
+	MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_LEN, bytes);
+
+	/*
+	 * The MC DMAs aggregate statistics for our convinience, so we can
+	 * avoid having to pull the statistics buffer into the cache to
+	 * maintain cumulative statistics.
+	 */
+	MCDI_IN_POPULATE_DWORD_6(req, MAC_STATS_IN_CMD,
+			    MAC_STATS_IN_DMA, 0,
+			    MAC_STATS_IN_CLEAR, 0,
+			    MAC_STATS_IN_PERIODIC_CHANGE, 1,
+			    MAC_STATS_IN_PERIODIC_ENABLE, period ? 1 : 0,
+			    MAC_STATS_IN_PERIODIC_NOEVENT, events ? 0 : 1,
+			    MAC_STATS_IN_PERIOD_MS, period);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+
+#define	SIENA_MAC_STAT_READ(_esmp, _field, _eqp)			\
+	EFSYS_MEM_READQ((_esmp), (_field) * sizeof (efx_qword_t), _eqp)
+
+	__checkReturn			int
+siena_mac_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_MAC_NSTATS)	efsys_stat_t *stat,
+	__out_opt			uint32_t *generationp)
+{
+	efx_qword_t rx_pkts;
+	efx_qword_t value;
+	efx_qword_t generation_start;
+	efx_qword_t generation_end;
+
+	_NOTE(ARGUNUSED(enp))
+
+	/* Read END first so we don't race with the MC */
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_END,
+			    &generation_end);
+	EFSYS_MEM_READ_BARRIER();
+
+	/* TX */
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value);
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_CONTROL_PKTS, &value);
+	EFSYS_STAT_SUBR_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PAUSE_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PAUSE_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_UNICAST_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_UNICST_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTICAST_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULTICST_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BROADCAST_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_BRDCST_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BYTES, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_OCTETS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LT64_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_64_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_65_TO_127_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_65_TO_127_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_128_TO_255_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_128_TO_255_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_256_TO_511_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_256_TO_511_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_512_TO_1023_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_512_TO_1023_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_1024_TO_15XX_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_1024_TO_15XX_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value);
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_GTJUMBO_PKTS, &value);
+	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BAD_FCS_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_ERRORS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_SGL_COL_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS,
+			    &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULT_COL_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS,
+			    &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_COL_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LATE_COLLISION_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LATE_COL_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_DEFERRED_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_DEF_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS,
+	    &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_DEF_PKTS]), &value);
+
+	/* RX */
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BYTES, &rx_pkts);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_OCTETS]), &rx_pkts);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNICAST_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_UNICST_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MULTICAST_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MULTICST_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BROADCAST_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_BRDCST_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PAUSE_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PAUSE_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNDERSIZE_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value);
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_64_PKTS, &value);
+	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_65_TO_127_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_65_TO_127_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_128_TO_255_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_128_TO_255_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_256_TO_511_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_256_TO_511_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_512_TO_1023_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_512_TO_1023_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_1024_TO_15XX_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_1024_TO_15XX_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value);
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_GTJUMBO_PKTS, &value);
+	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BAD_FCS_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FCS_ERRORS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_OVERFLOW_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_DROP_EVENTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_FALSE_CARRIER_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FALSE_CARRIER_ERRORS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_SYMBOL_ERRORS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_ALIGN_ERROR_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_ALIGN_ERRORS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_INTERNAL_ERRORS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_JABBER_PKTS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_JABBER_PKTS]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_CHAR_ERR, &value);
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_CHAR_ERR]),
+			    &(value.eq_dword[0]));
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_CHAR_ERR]),
+			    &(value.eq_dword[1]));
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_CHAR_ERR, &value);
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_CHAR_ERR]),
+			    &(value.eq_dword[0]));
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_CHAR_ERR]),
+			    &(value.eq_dword[1]));
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_DISP_ERR, &value);
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_DISP_ERR]),
+			    &(value.eq_dword[0]));
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_DISP_ERR]),
+			    &(value.eq_dword[1]));
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_DISP_ERR, &value);
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_DISP_ERR]),
+			    &(value.eq_dword[0]));
+	EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_DISP_ERR]),
+			    &(value.eq_dword[1]));
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MATCH_FAULT, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MATCH_FAULT]), &value);
+
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_NODESC_DROPS, &value);
+	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_NODESC_DROP_CNT]), &value);
+
+	EFSYS_MEM_READ_BARRIER();
+	SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_START,
+			    &generation_start);
+
+	/* Check that we didn't read the stats in the middle of a DMA */
+	if (memcmp(&generation_start, &generation_end,
+	    sizeof (generation_start)))
+		return (EAGAIN);
+
+	if (generationp)
+		*generationp = EFX_QWORD_FIELD(generation_start, EFX_DWORD_0);
+
+	return (0);
+}
+
+#endif	/* EFSYS_OPT_MAC_STATS */
+
+#endif	/* EFSYS_OPT_SIENA */
diff --git a/sys/dev/sfxge/common/siena_mon.c b/sys/dev/sfxge/common/siena_mon.c
new file mode 100644
index 0000000..de9a1df
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_mon.c
@@ -0,0 +1,248 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "efsys.h"
+#include "efx.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_MON_SIENA
+
+	__checkReturn	int
+siena_mon_reset(
+	__in		efx_nic_t *enp)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	return (0);
+}
+
+	__checkReturn	int
+siena_mon_reconfigure(
+	__in		efx_nic_t *enp)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	return (0);
+}
+
+#if EFSYS_OPT_MON_STATS
+
+#define	SIENA_MON_WRONG_PORT (uint16_t)0xffff
+
+static __cs uint16_t __siena_mon_port0_map[] = {
+	EFX_MON_STAT_INT_TEMP,		/* MC_CMD_SENSOR_CONTROLLER_TEMP */
+	EFX_MON_STAT_EXT_TEMP,		/* MC_CMD_SENSOR_PHY_COMMON_TEMP */
+	EFX_MON_STAT_INT_COOLING,	/* MC_CMD_SENSOR_CONTROLLER_COOLING */
+	EFX_MON_STAT_EXT_TEMP,		/* MC_CMD_SENSOR_PHY0_TEMP */
+	EFX_MON_STAT_EXT_COOLING,	/* MC_CMD_SENSOR_PHY0_COOLING */
+	SIENA_MON_WRONG_PORT,		/* MC_CMD_SENSOR_PHY1_TEMP */
+	SIENA_MON_WRONG_PORT,		/* MC_CMD_SENSOR_PHY1_COOLING */
+	EFX_MON_STAT_1V,		/* MC_CMD_SENSOR_IN_1V0 */
+	EFX_MON_STAT_1_2V,		/* MC_CMD_SENSOR_IN_1V2 */
+	EFX_MON_STAT_1_8V,		/* MC_CMD_SENSOR_IN_1V8 */
+	EFX_MON_STAT_2_5V,		/* MC_CMD_SENSOR_IN_2V5 */
+	EFX_MON_STAT_3_3V,		/* MC_CMD_SENSOR_IN_3V3 */
+	EFX_MON_STAT_12V,		/* MC_CMD_SENSOR_IN_12V0 */
+};
+
+static __cs uint16_t __siena_mon_port1_map[] = {
+	EFX_MON_STAT_INT_TEMP,		/* MC_CMD_SENSOR_CONTROLLER_TEMP */
+	EFX_MON_STAT_EXT_TEMP,		/* MC_CMD_SENSOR_PHY_COMMON_TEMP */
+	EFX_MON_STAT_INT_COOLING,	/* MC_CMD_SENSOR_CONTROLLER_COOLING */
+	SIENA_MON_WRONG_PORT,		/* MC_CMD_SENSOR_PHY0_TEMP */
+	SIENA_MON_WRONG_PORT,		/* MC_CMD_SENSOR_PHY0_COOLING */
+	EFX_MON_STAT_EXT_TEMP,		/* MC_CMD_SENSOR_PHY1_TEMP */
+	EFX_MON_STAT_EXT_COOLING,	/* MC_CMD_SENSOR_PHY1_COOLING */
+	EFX_MON_STAT_1V,		/* MC_CMD_SENSOR_IN_1V0 */
+	EFX_MON_STAT_1_2V,		/* MC_CMD_SENSOR_IN_1V2 */
+	EFX_MON_STAT_1_8V,		/* MC_CMD_SENSOR_IN_1V8 */
+	EFX_MON_STAT_2_5V,		/* MC_CMD_SENSOR_IN_2V5 */
+	EFX_MON_STAT_3_3V,		/* MC_CMD_SENSOR_IN_3V3 */
+	EFX_MON_STAT_12V,		/* MC_CMD_SENSOR_IN_12V0 */
+};
+
+#define	SIENA_STATIC_SENSOR_ASSERT(_field)				\
+	EFX_STATIC_ASSERT(MC_CMD_SENSOR_STATE_ ## _field		\
+			    == EFX_MON_STAT_STATE_ ## _field)
+
+					void
+siena_mon_decode_stats(
+	__in				efx_nic_t *enp,
+	__in				uint32_t dmask,
+	__in_opt			efsys_mem_t *esmp,
+	__out_opt			uint32_t *vmaskp,
+	__out_ecount_opt(EFX_MON_NSTATS)	efx_mon_stat_value_t *value)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	uint16_t *sensor_map;
+	uint16_t mc_sensor;
+	size_t mc_sensor_max;
+	uint32_t vmask = 0;
+
+	/* Assert the MC_CMD_SENSOR and EFX_MON_STATE namespaces agree */
+	SIENA_STATIC_SENSOR_ASSERT(OK);
+	SIENA_STATIC_SENSOR_ASSERT(WARNING);
+	SIENA_STATIC_SENSOR_ASSERT(FATAL);
+	SIENA_STATIC_SENSOR_ASSERT(BROKEN);
+
+	EFX_STATIC_ASSERT(sizeof (__siena_mon_port1_map)
+			    == sizeof (__siena_mon_port0_map));
+	mc_sensor_max = EFX_ARRAY_SIZE(__siena_mon_port0_map);
+	sensor_map = (emip->emi_port == 1)
+		? __siena_mon_port0_map
+		: __siena_mon_port1_map;
+
+	/*
+	 * dmask may legitimately contain sensors not understood by the driver
+	 */
+	for (mc_sensor = 0; mc_sensor < mc_sensor_max; ++mc_sensor) {
+		uint16_t efx_sensor = sensor_map[mc_sensor];
+
+		if (efx_sensor == SIENA_MON_WRONG_PORT)
+			continue;
+		EFSYS_ASSERT(efx_sensor < EFX_MON_NSTATS);
+
+		if (~dmask & (1 << mc_sensor))
+			continue;
+
+		vmask |= (1 << efx_sensor);
+		if (value != NULL && esmp != NULL && !EFSYS_MEM_IS_NULL(esmp)) {
+			efx_mon_stat_value_t *emsvp = value + efx_sensor;
+			efx_dword_t dword;
+			EFSYS_MEM_READD(esmp, 4 * mc_sensor, &dword);
+			emsvp->emsv_value =
+				(uint16_t)EFX_DWORD_FIELD(
+					dword,
+					MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE);
+			emsvp->emsv_state =
+				(uint16_t)EFX_DWORD_FIELD(
+					dword,
+					MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE);
+		}
+	}
+
+	if (vmaskp != NULL)
+		*vmaskp = vmask;
+}
+
+	__checkReturn			int
+siena_mon_ev(
+	__in				efx_nic_t *enp,
+	__in				efx_qword_t *eqp,
+	__out				efx_mon_stat_t *idp,
+	__out				efx_mon_stat_value_t *valuep)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	uint16_t ev_monitor;
+	uint16_t ev_state;
+	uint16_t ev_value;
+	uint16_t *sensor_map;
+	efx_mon_stat_t id;
+	int rc;
+
+	sensor_map = (emip->emi_port == 1)
+		? __siena_mon_port0_map
+		: __siena_mon_port1_map;
+
+	ev_monitor = (uint16_t)MCDI_EV_FIELD(eqp, SENSOREVT_MONITOR);
+	ev_state = (uint16_t)MCDI_EV_FIELD(eqp, SENSOREVT_STATE);
+	ev_value = (uint16_t)MCDI_EV_FIELD(eqp, SENSOREVT_VALUE);
+
+	/* Hardware must support this statistic */
+	EFSYS_ASSERT((1 << ev_monitor) & encp->enc_siena_mon_stat_mask);
+
+	/* But we don't have to understand it */
+	if (ev_monitor >= EFX_ARRAY_SIZE(__siena_mon_port0_map)) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	id = sensor_map[ev_monitor];
+	if (id == SIENA_MON_WRONG_PORT)
+		return (ENODEV);
+	EFSYS_ASSERT(id < EFX_MON_NSTATS);
+
+	*idp = id;
+	valuep->emsv_value = ev_value;
+	valuep->emsv_state = ev_state;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn			int
+siena_mon_stats_update(
+	__in				efx_nic_t *enp,
+	__in				efsys_mem_t *esmp,
+	__out_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	uint32_t dmask = encp->enc_siena_mon_stat_mask;
+	uint32_t vmask;
+	uint8_t payload[MC_CMD_READ_SENSORS_IN_LEN];
+	efx_mcdi_req_t req;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	req.emr_cmd = MC_CMD_READ_SENSORS;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_READ_SENSORS_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, READ_SENSORS_IN_DMA_ADDR_LO,
+			    EFSYS_MEM_ADDR(esmp) & 0xffffffff);
+	MCDI_IN_SET_DWORD(req, READ_SENSORS_IN_DMA_ADDR_HI,
+			    EFSYS_MEM_ADDR(esmp) >> 32);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	siena_mon_decode_stats(enp, dmask, esmp, &vmask, values);
+	EFSYS_ASSERT(vmask == encp->enc_mon_stat_mask);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_MON_STATS */
+
+#endif	/* EFSYS_OPT_MON_SIENA */
diff --git a/sys/dev/sfxge/common/siena_nic.c b/sys/dev/sfxge/common/siena_nic.c
new file mode 100644
index 0000000..9eb0db2
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_nic.c
@@ -0,0 +1,964 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "efsys.h"
+#include "efx.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_SIENA
+
+static	__checkReturn		int
+siena_nic_get_partn_mask(
+	__in			efx_nic_t *enp,
+	__out			unsigned int *maskp)
+{
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MC_CMD_NVRAM_TYPES_OUT_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_NVRAM_TYPES;
+	EFX_STATIC_ASSERT(MC_CMD_NVRAM_TYPES_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_NVRAM_TYPES_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	*maskp = MCDI_OUT_DWORD(req, NVRAM_TYPES_OUT_TYPES);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn	int
+siena_nic_exit_assertion_handler(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_REBOOT_IN_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_REBOOT;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_REBOOT_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_REBOOT_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, REBOOT_IN_FLAGS,
+			    MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0 && req.emr_rc != EIO) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn	int
+siena_nic_read_assertion(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MC_CMD_GET_ASSERTS_IN_LEN,
+			    MC_CMD_GET_ASSERTS_OUT_LEN)];
+	const char *reason;
+	unsigned int flags;
+	unsigned int index;
+	unsigned int ofst;
+	int retry;
+	int rc;
+
+	/*
+	 * Before we attempt to chat to the MC, we should verify that the MC
+	 * isn't in it's assertion handler, either due to a previous reboot,
+	 * or because we're reinitializing due to an eec_exception().
+	 *
+	 * Use GET_ASSERTS to read any assertion state that may be present.
+	 * Retry this command twice. Once because a boot-time assertion failure
+	 * might cause the 1st MCDI request to fail. And once again because
+	 * we might race with siena_nic_exit_assertion_handler() running on the
+	 * other port.
+	 */
+	retry = 2;
+	do {
+		req.emr_cmd = MC_CMD_GET_ASSERTS;
+		req.emr_in_buf = payload;
+		req.emr_in_length = MC_CMD_GET_ASSERTS_IN_LEN;
+		req.emr_out_buf = payload;
+		req.emr_out_length = MC_CMD_GET_ASSERTS_OUT_LEN;
+
+		MCDI_IN_SET_DWORD(req, GET_ASSERTS_IN_CLEAR, 1);
+		efx_mcdi_execute(enp, &req);
+
+	} while ((req.emr_rc == EINTR || req.emr_rc == EIO) && retry-- > 0);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_ASSERTS_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	/* Print out any assertion state recorded */
+	flags = MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_GLOBAL_FLAGS);
+	if (flags == MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS)
+		return (0);
+
+	reason = (flags == MC_CMD_GET_ASSERTS_FLAGS_SYS_FAIL)
+		? "system-level assertion"
+		: (flags == MC_CMD_GET_ASSERTS_FLAGS_THR_FAIL)
+		? "thread-level assertion"
+		: (flags == MC_CMD_GET_ASSERTS_FLAGS_WDOG_FIRED)
+		? "watchdog reset"
+		: "unknown assertion";
+	EFSYS_PROBE3(mcpu_assertion,
+	    const char *, reason, unsigned int,
+	    MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_SAVED_PC_OFFS),
+	    unsigned int,
+	    MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_THREAD_OFFS));
+
+	/* Print out the registers */
+	ofst = MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST;
+	for (index = 1; index < 32; index++) {
+		EFSYS_PROBE2(mcpu_register, unsigned int, index, unsigned int,
+			    EFX_DWORD_FIELD(*MCDI_OUT(req, efx_dword_t, ofst),
+					    EFX_DWORD_0));
+		ofst += sizeof (efx_dword_t);
+	}
+	EFSYS_ASSERT(ofst <= MC_CMD_GET_ASSERTS_OUT_LEN);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn	int
+siena_nic_attach(
+	__in		efx_nic_t *enp,
+	__in		boolean_t attach)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_DRV_ATTACH_IN_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_DRV_ATTACH;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_DRV_ATTACH_IN_LEN;
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_NEW_STATE, attach ? 1 : 0);
+	MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_UPDATE, 1);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_DRV_ATTACH_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_PCIE_TUNE
+
+	__checkReturn	int
+siena_nic_pcie_extended_sync(
+	__in		efx_nic_t *enp)
+{
+	uint8_t inbuf[MC_CMD_WORKAROUND_IN_LEN];
+	efx_mcdi_req_t req;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+
+	req.emr_cmd = MC_CMD_WORKAROUND;
+	req.emr_in_buf = inbuf;
+	req.emr_in_length = sizeof (inbuf);
+	EFX_STATIC_ASSERT(MC_CMD_WORKAROUND_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, WORKAROUND_IN_TYPE, MC_CMD_WORKAROUND_BUG17230);
+	MCDI_IN_SET_DWORD(req, WORKAROUND_IN_ENABLED, 1);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_PCIE_TUNE */
+
+static	__checkReturn	int
+siena_board_cfg(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	uint8_t outbuf[MAX(MC_CMD_GET_BOARD_CFG_OUT_LEN,
+		    MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN)];
+	efx_mcdi_req_t req;
+	uint8_t *src;
+	int rc;
+
+	/* Board configuration */
+	req.emr_cmd = MC_CMD_GET_BOARD_CFG;
+	EFX_STATIC_ASSERT(MC_CMD_GET_BOARD_CFG_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = MC_CMD_GET_BOARD_CFG_OUT_LEN;
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_BOARD_CFG_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	if (emip->emi_port == 1)
+		src = MCDI_OUT2(req, uint8_t,
+			    GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0);
+	else
+		src = MCDI_OUT2(req, uint8_t,
+			    GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1);
+	EFX_MAC_ADDR_COPY(encp->enc_mac_addr, src);
+
+	encp->enc_board_type = MCDI_OUT_DWORD(req,
+				    GET_BOARD_CFG_OUT_BOARD_TYPE);
+
+	/* Resource limits */
+	req.emr_cmd = MC_CMD_GET_RESOURCE_LIMITS;
+	EFX_STATIC_ASSERT(MC_CMD_GET_RESOURCE_LIMITS_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN;
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc == 0) {
+		if (req.emr_out_length_used < MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN) {
+			rc = EMSGSIZE;
+			goto fail3;
+		}
+
+		encp->enc_evq_limit = MCDI_OUT_DWORD(req,
+		    GET_RESOURCE_LIMITS_OUT_EVQ);
+		encp->enc_txq_limit = MIN(EFX_TXQ_LIMIT_TARGET,
+		    MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_TXQ));
+		encp->enc_rxq_limit = MIN(EFX_RXQ_LIMIT_TARGET,
+		    MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_RXQ));
+	} else if (req.emr_rc == ENOTSUP) {
+		encp->enc_evq_limit = 1024;
+		encp->enc_txq_limit = EFX_TXQ_LIMIT_TARGET;
+		encp->enc_rxq_limit = EFX_RXQ_LIMIT_TARGET;
+	} else {
+		rc = req.emr_rc;
+		goto fail4;
+	}
+
+	encp->enc_buftbl_limit = SIENA_SRAM_ROWS -
+	    (encp->enc_txq_limit * 16) - (encp->enc_rxq_limit * 64);
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn	int
+siena_phy_cfg(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MC_CMD_GET_PHY_CFG_OUT_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_GET_PHY_CFG;
+	EFX_STATIC_ASSERT(MC_CMD_GET_PHY_CFG_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_PHY_CFG_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	encp->enc_phy_type = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_TYPE);
+#if EFSYS_OPT_NAMES
+	(void) strncpy(encp->enc_phy_name,
+		MCDI_OUT2(req, char, GET_PHY_CFG_OUT_NAME),
+		MIN(sizeof (encp->enc_phy_name) - 1,
+		    MC_CMD_GET_PHY_CFG_OUT_NAME_LEN));
+#endif	/* EFSYS_OPT_NAMES */
+	(void) memset(encp->enc_phy_revision, 0,
+	    sizeof (encp->enc_phy_revision));
+	memcpy(encp->enc_phy_revision,
+		MCDI_OUT2(req, char, GET_PHY_CFG_OUT_REVISION),
+		MIN(sizeof (encp->enc_phy_revision) - 1,
+		    MC_CMD_GET_PHY_CFG_OUT_REVISION_LEN));
+#if EFSYS_OPT_PHY_LED_CONTROL
+	encp->enc_led_mask = ((1 << EFX_PHY_LED_DEFAULT) |
+			    (1 << EFX_PHY_LED_OFF) |
+			    (1 << EFX_PHY_LED_ON));
+#endif	/* EFSYS_OPT_PHY_LED_CONTROL */
+
+#if EFSYS_OPT_PHY_PROPS
+	encp->enc_phy_nprops  = 0;
+#endif	/* EFSYS_OPT_PHY_PROPS */
+
+	/* Get the media type of the fixed port, if recognised. */
+	EFX_STATIC_ASSERT(MC_CMD_MEDIA_XAUI == EFX_PHY_MEDIA_XAUI);
+	EFX_STATIC_ASSERT(MC_CMD_MEDIA_CX4 == EFX_PHY_MEDIA_CX4);
+	EFX_STATIC_ASSERT(MC_CMD_MEDIA_KX4 == EFX_PHY_MEDIA_KX4);
+	EFX_STATIC_ASSERT(MC_CMD_MEDIA_XFP == EFX_PHY_MEDIA_XFP);
+	EFX_STATIC_ASSERT(MC_CMD_MEDIA_SFP_PLUS == EFX_PHY_MEDIA_SFP_PLUS);
+	EFX_STATIC_ASSERT(MC_CMD_MEDIA_BASE_T == EFX_PHY_MEDIA_BASE_T);
+	epp->ep_fixed_port_type =
+		MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_MEDIA_TYPE);
+	if (epp->ep_fixed_port_type >= EFX_PHY_MEDIA_NTYPES)
+		epp->ep_fixed_port_type = EFX_PHY_MEDIA_INVALID;
+
+	epp->ep_phy_cap_mask =
+		MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_SUPPORTED_CAP);
+#if EFSYS_OPT_PHY_FLAGS
+	encp->enc_phy_flags_mask = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_FLAGS);
+#endif	/* EFSYS_OPT_PHY_FLAGS */
+
+	encp->enc_port = (uint8_t)MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_PRT);
+
+	/* Populate internal state */
+	encp->enc_siena_channel =
+		(uint8_t)MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_CHANNEL);
+
+#if EFSYS_OPT_PHY_STATS
+	encp->enc_siena_phy_stat_mask =
+		MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_STATS_MASK);
+
+	/* Convert the MCDI statistic mask into the EFX_PHY_STAT mask */
+	siena_phy_decode_stats(enp, encp->enc_siena_phy_stat_mask,
+			    NULL, &encp->enc_phy_stat_mask, NULL);
+#endif	/* EFSYS_OPT_PHY_STATS */
+
+#if EFSYS_OPT_PHY_BIST
+	encp->enc_bist_mask = 0;
+	if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS,
+	    GET_PHY_CFG_OUT_BIST_CABLE_SHORT))
+		encp->enc_bist_mask |= (1 << EFX_PHY_BIST_TYPE_CABLE_SHORT);
+	if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS,
+	    GET_PHY_CFG_OUT_BIST_CABLE_LONG))
+		encp->enc_bist_mask |= (1 << EFX_PHY_BIST_TYPE_CABLE_LONG);
+	if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS,
+	    GET_PHY_CFG_OUT_BIST))
+		encp->enc_bist_mask |= (1 << EFX_PHY_BIST_TYPE_NORMAL);
+#endif	/* EFSYS_OPT_BIST */
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#if EFSYS_OPT_LOOPBACK
+
+static	__checkReturn	int
+siena_loopback_cfg(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MC_CMD_GET_LOOPBACK_MODES_OUT_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_GET_LOOPBACK_MODES;
+	EFX_STATIC_ASSERT(MC_CMD_GET_LOOPBACK_MODES_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_LOOPBACK_MODES_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	/*
+	 * We assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespaces agree
+	 * in siena_phy.c:siena_phy_get_link()
+	 */
+	encp->enc_loopback_types[EFX_LINK_100FDX] = EFX_LOOPBACK_MASK &
+	    MCDI_OUT_DWORD(req, GET_LOOPBACK_MODES_OUT_100M) &
+	    MCDI_OUT_DWORD(req, GET_LOOPBACK_MODES_OUT_SUGGESTED);
+	encp->enc_loopback_types[EFX_LINK_1000FDX] = EFX_LOOPBACK_MASK &
+	    MCDI_OUT_DWORD(req, GET_LOOPBACK_MODES_OUT_1G) &
+	    MCDI_OUT_DWORD(req, GET_LOOPBACK_MODES_OUT_SUGGESTED);
+	encp->enc_loopback_types[EFX_LINK_10000FDX] = EFX_LOOPBACK_MASK &
+	    MCDI_OUT_DWORD(req, GET_LOOPBACK_MODES_OUT_10G) &
+	    MCDI_OUT_DWORD(req, GET_LOOPBACK_MODES_OUT_SUGGESTED);
+	encp->enc_loopback_types[EFX_LINK_UNKNOWN] =
+	    (1 << EFX_LOOPBACK_OFF) |
+	    encp->enc_loopback_types[EFX_LINK_100FDX] |
+	    encp->enc_loopback_types[EFX_LINK_1000FDX] |
+	    encp->enc_loopback_types[EFX_LINK_10000FDX];
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_LOOPBACK */
+
+#if EFSYS_OPT_MON_STATS
+
+static	__checkReturn	int
+siena_monitor_cfg(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MCDI_CTL_SDU_LEN_MAX];
+	int rc;
+
+	req.emr_cmd = MC_CMD_SENSOR_INFO;
+	EFX_STATIC_ASSERT(MC_CMD_SENSOR_INFO_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_SENSOR_INFO_OUT_MASK_OFST + 4) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	encp->enc_siena_mon_stat_mask =
+		MCDI_OUT_DWORD(req, SENSOR_INFO_OUT_MASK);
+	encp->enc_mon_type = EFX_MON_SFC90X0;
+
+	siena_mon_decode_stats(enp, encp->enc_siena_mon_stat_mask,
+			    NULL, &(encp->enc_mon_stat_mask), NULL);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_MON_STATS */
+
+	__checkReturn	int
+siena_nic_probe(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	siena_link_state_t sls;
+	unsigned int mask;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+
+	/* Read clear any assertion state */
+	if ((rc = siena_nic_read_assertion(enp)) != 0)
+		goto fail1;
+
+	/* Exit the assertion handler */
+	if ((rc = siena_nic_exit_assertion_handler(enp)) != 0)
+		goto fail2;
+
+	/* Wrestle control from the BMC */
+	if ((rc = siena_nic_attach(enp, B_TRUE)) != 0)
+		goto fail3;
+
+	if ((rc = siena_board_cfg(enp)) != 0)
+		goto fail4;
+
+	encp->enc_evq_moderation_max =
+		EFX_EV_TIMER_QUANTUM << FRF_CZ_TIMER_VAL_WIDTH;
+
+	if ((rc = siena_phy_cfg(enp)) != 0)
+		goto fail5;
+
+	/* Obtain the default PHY advertised capabilities */
+	if ((rc = siena_nic_reset(enp)) != 0)
+		goto fail6;
+	if ((rc = siena_phy_get_link(enp, &sls)) != 0)
+		goto fail7;
+	epp->ep_default_adv_cap_mask = sls.sls_adv_cap_mask;
+	epp->ep_adv_cap_mask = sls.sls_adv_cap_mask;
+
+#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM
+	if ((rc = siena_nic_get_partn_mask(enp, &mask)) != 0)
+		goto fail8;
+	enp->en_u.siena.enu_partn_mask = mask;
+#endif
+
+#if EFSYS_OPT_MAC_STATS
+	/* Wipe the MAC statistics */
+	if ((rc = siena_mac_stats_clear(enp)) != 0)
+		goto fail9;
+#endif
+
+#if EFSYS_OPT_LOOPBACK
+	if ((rc = siena_loopback_cfg(enp)) != 0)
+		goto fail10;
+#endif
+
+#if EFSYS_OPT_MON_STATS
+	if ((rc = siena_monitor_cfg(enp)) != 0)
+		goto fail11;
+#endif
+
+	encp->enc_features = enp->en_features;
+
+	return (0);
+
+#if EFSYS_OPT_MON_STATS
+fail11:
+	EFSYS_PROBE(fail11);
+#endif
+#if EFSYS_OPT_LOOPBACK
+fail10:
+	EFSYS_PROBE(fail10);
+#endif
+#if EFSYS_OPT_MAC_STATS
+fail9:
+	EFSYS_PROBE(fail9);
+#endif
+#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM
+fail8:
+	EFSYS_PROBE(fail8);
+#endif
+fail7:
+	EFSYS_PROBE(fail7);
+fail6:
+	EFSYS_PROBE(fail6);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_nic_reset(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_req_t req;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+
+	/* siena_nic_reset() is called to recover from BADASSERT failures. */
+	if ((rc = siena_nic_read_assertion(enp)) != 0)
+		goto fail1;
+	if ((rc = siena_nic_exit_assertion_handler(enp)) != 0)
+		goto fail2;
+
+	req.emr_cmd = MC_CMD_PORT_RESET;
+	EFX_STATIC_ASSERT(MC_CMD_PORT_RESET_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	EFX_STATIC_ASSERT(MC_CMD_PORT_RESET_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail3;
+	}
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (0);
+}
+
+static	__checkReturn	int
+siena_nic_logging(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_LOG_CTRL_IN_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_LOG_CTRL;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_LOG_CTRL_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_LOG_CTRL_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, LOG_CTRL_IN_LOG_DEST,
+		    MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ);
+	MCDI_IN_SET_DWORD(req, LOG_CTRL_IN_LOG_DEST_EVQ, 0);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static			void
+siena_nic_rx_cfg(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+
+	/*
+	 * RX_INGR_EN is always enabled on Siena, because we rely on
+	 * the RX parser to be resiliant to missing SOP/EOP.
+	 */
+	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_INGR_EN, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
+
+	/* Disable parsing of additional 802.1Q in Q packets */
+	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
+	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
+}
+
+static			void
+siena_nic_usrev_dis(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t	oword;
+
+	EFX_POPULATE_OWORD_1(oword, FRF_CZ_USREV_DIS, 1);
+	EFX_BAR_WRITEO(enp, FR_CZ_USR_EV_CFG, &oword);
+}
+
+	__checkReturn	int
+siena_nic_init(
+	__in		efx_nic_t *enp)
+{
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
+
+	if ((rc = siena_nic_logging(enp)) != 0)
+		goto fail1;
+
+	siena_sram_init(enp);
+
+	/* Configure Siena's RX block */
+	siena_nic_rx_cfg(enp);
+
+	/* Disable USR_EVents for now */
+	siena_nic_usrev_dis(enp);
+
+	/* bug17057: Ensure set_link is called */
+	if ((rc = siena_phy_reconfigure(enp)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			void
+siena_nic_fini(
+	__in		efx_nic_t *enp)
+{
+	_NOTE(ARGUNUSED(enp))
+}
+
+			void
+siena_nic_unprobe(
+	__in		efx_nic_t *enp)
+{
+	(void) siena_nic_attach(enp, B_FALSE);
+}
+
+#if EFSYS_OPT_DIAG
+
+static efx_register_set_t __cs	__siena_registers[] = {
+	{ FR_AZ_ADR_REGION_REG_OFST, 0, 1 },
+	{ FR_CZ_USR_EV_CFG_OFST, 0, 1 },
+	{ FR_AZ_RX_CFG_REG_OFST, 0, 1 },
+	{ FR_AZ_TX_CFG_REG_OFST, 0, 1 },
+	{ FR_AZ_TX_RESERVED_REG_OFST, 0, 1 },
+	{ FR_AZ_SRM_TX_DC_CFG_REG_OFST, 0, 1 },
+	{ FR_AZ_RX_DC_CFG_REG_OFST, 0, 1 },
+	{ FR_AZ_RX_DC_PF_WM_REG_OFST, 0, 1 },
+	{ FR_AZ_DP_CTRL_REG_OFST, 0, 1 },
+	{ FR_BZ_RX_RSS_TKEY_REG_OFST, 0, 1},
+	{ FR_CZ_RX_RSS_IPV6_REG1_OFST, 0, 1},
+	{ FR_CZ_RX_RSS_IPV6_REG2_OFST, 0, 1},
+	{ FR_CZ_RX_RSS_IPV6_REG3_OFST, 0, 1}
+};
+
+static const uint32_t __cs	__siena_register_masks[] = {
+	0x0003FFFF, 0x0003FFFF, 0x0003FFFF, 0x0003FFFF,
+	0x000103FF, 0x00000000, 0x00000000, 0x00000000,
+	0xFFFFFFFE, 0xFFFFFFFF, 0x0003FFFF, 0x00000000,
+	0x7FFF0037, 0xFFFF8000, 0xFFFFFFFF, 0x03FFFFFF,
+	0xFFFEFE80, 0x1FFFFFFF, 0x020000FE, 0x007FFFFF,
+	0x001FFFFF, 0x00000000, 0x00000000, 0x00000000,
+	0x00000003, 0x00000000, 0x00000000, 0x00000000,
+	0x000003FF, 0x00000000, 0x00000000, 0x00000000,
+	0x00000FFF, 0x00000000, 0x00000000, 0x00000000,
+	0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+	0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+	0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+	0xFFFFFFFF, 0xFFFFFFFF, 0x00000007, 0x00000000
+};
+
+static efx_register_set_t __cs	__siena_tables[] = {
+	{ FR_AZ_RX_FILTER_TBL0_OFST, FR_AZ_RX_FILTER_TBL0_STEP,
+	    FR_AZ_RX_FILTER_TBL0_ROWS },
+	{ FR_CZ_RX_MAC_FILTER_TBL0_OFST, FR_CZ_RX_MAC_FILTER_TBL0_STEP,
+	    FR_CZ_RX_MAC_FILTER_TBL0_ROWS },
+	{ FR_AZ_RX_DESC_PTR_TBL_OFST,
+	    FR_AZ_RX_DESC_PTR_TBL_STEP, FR_CZ_RX_DESC_PTR_TBL_ROWS },
+	{ FR_AZ_TX_DESC_PTR_TBL_OFST,
+	    FR_AZ_TX_DESC_PTR_TBL_STEP, FR_CZ_TX_DESC_PTR_TBL_ROWS },
+	{ FR_AZ_TIMER_TBL_OFST, FR_AZ_TIMER_TBL_STEP, FR_CZ_TIMER_TBL_ROWS },
+	{ FR_CZ_TX_FILTER_TBL0_OFST,
+	    FR_CZ_TX_FILTER_TBL0_STEP, FR_CZ_TX_FILTER_TBL0_ROWS },
+	{ FR_CZ_TX_MAC_FILTER_TBL0_OFST,
+	    FR_CZ_TX_MAC_FILTER_TBL0_STEP, FR_CZ_TX_MAC_FILTER_TBL0_ROWS }
+};
+
+static const uint32_t __cs	__siena_table_masks[] = {
+	0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x000003FF,
+	0xFFFF0FFF, 0xFFFFFFFF, 0x00000E7F, 0x00000000,
+	0xFFFFFFFF, 0x0FFFFFFF, 0x01800000, 0x00000000,
+	0xFFFFFFFE, 0x0FFFFFFF, 0x0C000000, 0x00000000,
+	0x3FFFFFFF, 0x00000000, 0x00000000, 0x00000000,
+	0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x000013FF,
+	0xFFFF07FF, 0xFFFFFFFF, 0x0000007F, 0x00000000,
+};
+
+	__checkReturn	int
+siena_nic_register_test(
+	__in		efx_nic_t *enp)
+{
+	efx_register_set_t *rsp;
+	const uint32_t *dwordp;
+	unsigned int nitems;
+	unsigned int count;
+	int rc;
+
+	/* Fill out the register mask entries */
+	EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(__siena_register_masks)
+		    == EFX_ARRAY_SIZE(__siena_registers) * 4);
+
+	nitems = EFX_ARRAY_SIZE(__siena_registers);
+	dwordp = __siena_register_masks;
+	for (count = 0; count < nitems; ++count) {
+		rsp = __siena_registers + count;
+		rsp->mask.eo_u32[0] = *dwordp++;
+		rsp->mask.eo_u32[1] = *dwordp++;
+		rsp->mask.eo_u32[2] = *dwordp++;
+		rsp->mask.eo_u32[3] = *dwordp++;
+	}
+
+	/* Fill out the register table entries */
+	EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(__siena_table_masks)
+		    == EFX_ARRAY_SIZE(__siena_tables) * 4);
+
+	nitems = EFX_ARRAY_SIZE(__siena_tables);
+	dwordp = __siena_table_masks;
+	for (count = 0; count < nitems; ++count) {
+		rsp = __siena_tables + count;
+		rsp->mask.eo_u32[0] = *dwordp++;
+		rsp->mask.eo_u32[1] = *dwordp++;
+		rsp->mask.eo_u32[2] = *dwordp++;
+		rsp->mask.eo_u32[3] = *dwordp++;
+	}
+
+	if ((rc = efx_nic_test_registers(enp, __siena_registers,
+	    EFX_ARRAY_SIZE(__siena_registers))) != 0)
+		goto fail1;
+
+	if ((rc = efx_nic_test_tables(enp, __siena_tables,
+	    EFX_PATTERN_BYTE_ALTERNATE,
+	    EFX_ARRAY_SIZE(__siena_tables))) != 0)
+		goto fail2;
+
+	if ((rc = efx_nic_test_tables(enp, __siena_tables,
+	    EFX_PATTERN_BYTE_CHANGING,
+	    EFX_ARRAY_SIZE(__siena_tables))) != 0)
+		goto fail3;
+
+	if ((rc = efx_nic_test_tables(enp, __siena_tables,
+	    EFX_PATTERN_BIT_SWEEP, EFX_ARRAY_SIZE(__siena_tables))) != 0)
+		goto fail4;
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_DIAG */
+
+#endif	/* EFSYS_OPT_SIENA */
diff --git a/sys/dev/sfxge/common/siena_nvram.c b/sys/dev/sfxge/common/siena_nvram.c
new file mode 100644
index 0000000..9cdd3ad
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_nvram.c
@@ -0,0 +1,985 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_SIENA
+
+#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM
+
+	__checkReturn		int
+siena_nvram_partn_size(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__out			size_t *sizep)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MC_CMD_NVRAM_INFO_IN_LEN,
+			    MC_CMD_NVRAM_INFO_OUT_LEN)];
+	int rc;
+
+	if ((1 << partn) & ~enp->en_u.siena.enu_partn_mask) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	req.emr_cmd = MC_CMD_NVRAM_INFO;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_NVRAM_INFO_IN_LEN;
+	req.emr_out_buf = payload;
+	req.emr_out_length = MC_CMD_NVRAM_INFO_OUT_LEN;
+
+	MCDI_IN_SET_DWORD(req, NVRAM_INFO_IN_TYPE, partn);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail2;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_NVRAM_INFO_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail3;
+	}
+
+	*sizep = MCDI_OUT_DWORD(req, NVRAM_INFO_OUT_SIZE);
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_partn_lock(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_NVRAM_UPDATE_START_IN_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_NVRAM_UPDATE_START;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_NVRAM_UPDATE_START_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_NVRAM_UPDATE_START_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, NVRAM_UPDATE_START_IN_TYPE, partn);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_partn_read(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MC_CMD_NVRAM_READ_IN_LEN,
+			    MC_CMD_NVRAM_READ_OUT_LEN(SIENA_NVRAM_CHUNK))];
+	size_t chunk;
+	int rc;
+
+	while (size > 0) {
+		chunk = MIN(size, SIENA_NVRAM_CHUNK);
+
+		req.emr_cmd = MC_CMD_NVRAM_READ;
+		req.emr_in_buf = payload;
+		req.emr_in_length = MC_CMD_NVRAM_READ_IN_LEN;
+		req.emr_out_buf = payload;
+		req.emr_out_length =
+			MC_CMD_NVRAM_READ_OUT_LEN(SIENA_NVRAM_CHUNK);
+
+		MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_TYPE, partn);
+		MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_OFFSET, offset);
+		MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_LENGTH, chunk);
+
+		efx_mcdi_execute(enp, &req);
+
+		if (req.emr_rc != 0) {
+			rc = req.emr_rc;
+			goto fail1;
+		}
+
+		if (req.emr_out_length_used <
+		    MC_CMD_NVRAM_READ_OUT_LEN(chunk)) {
+			rc = EMSGSIZE;
+			goto fail2;
+		}
+
+		memcpy(data,
+		    MCDI_OUT2(req, uint8_t, NVRAM_READ_OUT_READ_BUFFER),
+		    chunk);
+
+		size -= chunk;
+		data += chunk;
+		offset += chunk;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_partn_erase(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			unsigned int offset,
+	__in			size_t size)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_NVRAM_ERASE_IN_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_NVRAM_ERASE;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_NVRAM_ERASE_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_NVRAM_ERASE_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, NVRAM_ERASE_IN_TYPE, partn);
+	MCDI_IN_SET_DWORD(req, NVRAM_ERASE_IN_OFFSET, offset);
+	MCDI_IN_SET_DWORD(req, NVRAM_ERASE_IN_LENGTH, size);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_partn_write(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_NVRAM_WRITE_IN_LEN(SIENA_NVRAM_CHUNK)];
+	size_t chunk;
+	int rc;
+
+	while (size > 0) {
+		chunk = MIN(size, SIENA_NVRAM_CHUNK);
+
+		req.emr_cmd = MC_CMD_NVRAM_WRITE;
+		req.emr_in_buf = payload;
+		req.emr_in_length = MC_CMD_NVRAM_WRITE_IN_LEN(chunk);
+		EFX_STATIC_ASSERT(MC_CMD_NVRAM_WRITE_OUT_LEN == 0);
+		req.emr_out_buf = NULL;
+		req.emr_out_length = 0;
+
+		MCDI_IN_SET_DWORD(req, NVRAM_WRITE_IN_TYPE, partn);
+		MCDI_IN_SET_DWORD(req, NVRAM_WRITE_IN_OFFSET, offset);
+		MCDI_IN_SET_DWORD(req, NVRAM_WRITE_IN_LENGTH, chunk);
+
+		memcpy(MCDI_IN2(req, uint8_t, NVRAM_WRITE_IN_WRITE_BUFFER),
+		    data, chunk);
+
+		efx_mcdi_execute(enp, &req);
+
+		if (req.emr_rc != 0) {
+			rc = req.emr_rc;
+			goto fail1;
+		}
+
+		size -= chunk;
+		data += chunk;
+		offset += chunk;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				void
+siena_nvram_partn_unlock(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn)
+{
+	efx_mcdi_req_t req;
+	uint8_t payload[MC_CMD_NVRAM_UPDATE_FINISH_IN_LEN];
+	uint32_t reboot;
+	int rc;
+
+	req.emr_cmd = MC_CMD_NVRAM_UPDATE_FINISH;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_NVRAM_UPDATE_FINISH_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_NVRAM_UPDATE_FINISH_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	/*
+	 * Reboot into the new image only for PHYs. The driver has to
+	 * explicitly cope with an MC reboot after a firmware update.
+	 */
+	reboot = (partn == MC_CMD_NVRAM_TYPE_PHY_PORT0 ||
+		    partn == MC_CMD_NVRAM_TYPE_PHY_PORT1 ||
+		    partn == MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO);
+
+	MCDI_IN_SET_DWORD(req, NVRAM_UPDATE_FINISH_IN_TYPE, partn);
+	MCDI_IN_SET_DWORD(req, NVRAM_UPDATE_FINISH_IN_REBOOT, reboot);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return;
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+}
+
+#endif	/* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */
+
+#if EFSYS_OPT_NVRAM
+
+typedef struct siena_parttbl_entry_s {
+	unsigned int		partn;
+	unsigned int		port;
+	efx_nvram_type_t	nvtype;
+} siena_parttbl_entry_t;
+
+static siena_parttbl_entry_t siena_parttbl[] = {
+	{MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO,	1, EFX_NVRAM_NULLPHY},
+	{MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO,	2, EFX_NVRAM_NULLPHY},
+	{MC_CMD_NVRAM_TYPE_MC_FW,		1, EFX_NVRAM_MC_FIRMWARE},
+	{MC_CMD_NVRAM_TYPE_MC_FW,		2, EFX_NVRAM_MC_FIRMWARE},
+	{MC_CMD_NVRAM_TYPE_MC_FW_BACKUP,	1, EFX_NVRAM_MC_GOLDEN},
+	{MC_CMD_NVRAM_TYPE_MC_FW_BACKUP,	2, EFX_NVRAM_MC_GOLDEN},
+	{MC_CMD_NVRAM_TYPE_EXP_ROM,		1, EFX_NVRAM_BOOTROM},
+	{MC_CMD_NVRAM_TYPE_EXP_ROM,		2, EFX_NVRAM_BOOTROM},
+	{MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT0,	1, EFX_NVRAM_BOOTROM_CFG},
+	{MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT1,	2, EFX_NVRAM_BOOTROM_CFG},
+	{MC_CMD_NVRAM_TYPE_PHY_PORT0,		1, EFX_NVRAM_PHY},
+	{MC_CMD_NVRAM_TYPE_PHY_PORT1,		2, EFX_NVRAM_PHY},
+	{0, 0, 0},
+};
+
+static	__checkReturn		siena_parttbl_entry_t *
+siena_parttbl_entry(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	siena_parttbl_entry_t *entry;
+
+	EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES);
+
+	for (entry = siena_parttbl; entry->port > 0; ++entry) {
+		if (entry->port == emip->emi_port && entry->nvtype == type)
+			return (entry);
+	}
+
+	return (NULL);
+}
+
+#if EFSYS_OPT_DIAG
+
+	__checkReturn		int
+siena_nvram_test(
+	__in			efx_nic_t *enp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	siena_parttbl_entry_t *entry;
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MC_CMD_NVRAM_TEST_IN_LEN,
+			    MC_CMD_NVRAM_TEST_OUT_LEN)];
+	int result;
+	int rc;
+
+	req.emr_cmd = MC_CMD_NVRAM_TEST;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_NVRAM_TEST_IN_LEN;
+	req.emr_out_buf = payload;
+	req.emr_out_length = MC_CMD_NVRAM_TEST_OUT_LEN;
+
+	/*
+	 * Iterate over the list of supported partition types
+	 * applicable to *this* port
+	 */
+	for (entry = siena_parttbl; entry->port > 0; ++entry) {
+		if (entry->port != emip->emi_port ||
+		    !(enp->en_u.siena.enu_partn_mask & (1 << entry->partn)))
+			continue;
+
+		MCDI_IN_SET_DWORD(req, NVRAM_TEST_IN_TYPE, entry->partn);
+
+		efx_mcdi_execute(enp, &req);
+
+		if (req.emr_rc != 0) {
+			rc = req.emr_rc;
+			goto fail1;
+		}
+
+		if (req.emr_out_length_used < MC_CMD_NVRAM_TEST_OUT_LEN) {
+			rc = EMSGSIZE;
+			goto fail2;
+		}
+
+		result = MCDI_OUT_DWORD(req, NVRAM_TEST_OUT_RESULT);
+		if (result == MC_CMD_NVRAM_TEST_FAIL) {
+
+			EFSYS_PROBE1(nvram_test_failure, int, entry->partn);
+
+			rc = (EINVAL);
+			goto fail3;
+		}
+	}
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_DIAG */
+
+	__checkReturn		int
+siena_nvram_size(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			size_t *sizep)
+{
+	siena_parttbl_entry_t *entry;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = siena_nvram_partn_size(enp, entry->partn, sizep)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	*sizep = 0;
+
+	return (rc);
+}
+
+#define	SIENA_DYNAMIC_CFG_SIZE(_nitems)					\
+	(sizeof (siena_mc_dynamic_config_hdr_t) + ((_nitems) *		\
+	sizeof (((siena_mc_dynamic_config_hdr_t *)NULL)->fw_version[0])))
+
+	__checkReturn		int
+siena_nvram_get_dynamic_cfg(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__in			boolean_t vpd,
+	__out			siena_mc_dynamic_config_hdr_t **dcfgp,
+	__out			size_t *sizep)
+{
+	siena_mc_dynamic_config_hdr_t *dcfg;
+	size_t size;
+	uint8_t cksum;
+	unsigned int vpd_offset;
+	unsigned int vpd_length;
+	unsigned int hdr_length;
+	unsigned int nversions;
+	unsigned int pos;
+	unsigned int region;
+	int rc;
+
+	EFSYS_ASSERT(partn == MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 ||
+		    partn == MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1);
+
+	/*
+	 * Allocate sufficient memory for the entire dynamiccfg area, even
+	 * if we're not actually going to read in the VPD.
+	 */
+	if ((rc = siena_nvram_partn_size(enp, partn, &size)) != 0)
+		goto fail1;
+
+	EFSYS_KMEM_ALLOC(enp->en_esip, size, dcfg);
+	if (dcfg == NULL) {
+		rc = ENOMEM;
+		goto fail2;
+	}
+
+	if ((rc = siena_nvram_partn_read(enp, partn, 0,
+	    (caddr_t)dcfg, SIENA_NVRAM_CHUNK)) != 0)
+		goto fail3;
+
+	/* Verify the magic */
+	if (EFX_DWORD_FIELD(dcfg->magic, EFX_DWORD_0)
+	    != SIENA_MC_DYNAMIC_CONFIG_MAGIC)
+		goto invalid1;
+
+	/* All future versions of the structure must be backwards compatable */
+	EFX_STATIC_ASSERT(SIENA_MC_DYNAMIC_CONFIG_VERSION == 0);
+
+	hdr_length = EFX_WORD_FIELD(dcfg->length, EFX_WORD_0);
+	nversions = EFX_DWORD_FIELD(dcfg->num_fw_version_items, EFX_DWORD_0);
+	vpd_offset = EFX_DWORD_FIELD(dcfg->dynamic_vpd_offset, EFX_DWORD_0);
+	vpd_length = EFX_DWORD_FIELD(dcfg->dynamic_vpd_length, EFX_DWORD_0);
+
+	/* Verify the hdr doesn't overflow the partn size */
+	if (hdr_length > size || vpd_offset > size || vpd_length > size ||
+	    vpd_length + vpd_offset > size)
+		goto invalid2;
+
+	/* Verify the header has room for all it's versions */
+	if (hdr_length < SIENA_DYNAMIC_CFG_SIZE(0) ||
+	    hdr_length < SIENA_DYNAMIC_CFG_SIZE(nversions))
+		goto invalid3;
+
+	/*
+	 * Read the remaining portion of the dcfg, either including
+	 * the whole of VPD (there is no vpd length in this structure,
+	 * so we have to parse each tag), or just the dcfg header itself
+	 */
+	region = vpd ? vpd_offset + vpd_length : hdr_length;
+	if (region > SIENA_NVRAM_CHUNK) {
+		if ((rc = siena_nvram_partn_read(enp, partn, SIENA_NVRAM_CHUNK,
+		    (caddr_t)dcfg + SIENA_NVRAM_CHUNK,
+		    region - SIENA_NVRAM_CHUNK)) != 0)
+			goto fail4;
+	}
+
+	/* Verify checksum */
+	cksum = 0;
+	for (pos = 0; pos < hdr_length; pos++)
+		cksum += ((uint8_t *)dcfg)[pos];
+	if (cksum != 0)
+		goto invalid4;
+
+	goto done;
+
+invalid4:
+	EFSYS_PROBE(invalid4);
+invalid3:
+	EFSYS_PROBE(invalid3);
+invalid2:
+	EFSYS_PROBE(invalid2);
+invalid1:
+	EFSYS_PROBE(invalid1);
+
+	/*
+	 * Construct a new "null" dcfg, with an empty version vector,
+	 * and an empty VPD chunk trailing. This has the neat side effect
+	 * of testing the exception paths in the write path.
+	 */
+	EFX_POPULATE_DWORD_1(dcfg->magic,
+			    EFX_DWORD_0, SIENA_MC_DYNAMIC_CONFIG_MAGIC);
+	EFX_POPULATE_WORD_1(dcfg->length, EFX_WORD_0, sizeof (*dcfg));
+	EFX_POPULATE_BYTE_1(dcfg->version, EFX_BYTE_0,
+			    SIENA_MC_DYNAMIC_CONFIG_VERSION);
+	EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_offset,
+			    EFX_DWORD_0, sizeof (*dcfg));
+	EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_length, EFX_DWORD_0, 0);
+	EFX_POPULATE_DWORD_1(dcfg->num_fw_version_items, EFX_DWORD_0, 0);
+
+done:
+	*dcfgp = dcfg;
+	*sizep = size;
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+
+	EFSYS_KMEM_FREE(enp->en_esip, size, dcfg);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn		int
+siena_nvram_get_subtype(
+	__in			efx_nic_t *enp,
+	__in			unsigned int partn,
+	__out			uint32_t *subtypep)
+{
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MC_CMD_GET_BOARD_CFG_OUT_LEN];
+	efx_word_t *fw_list;
+	int rc;
+
+	req.emr_cmd = MC_CMD_GET_BOARD_CFG;
+	EFX_STATIC_ASSERT(MC_CMD_GET_BOARD_CFG_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_BOARD_CFG_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	fw_list = MCDI_OUT2(req, efx_word_t,
+			    GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST);
+	*subtypep = EFX_WORD_FIELD(fw_list[partn], EFX_WORD_0);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_get_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint32_t *subtypep,
+	__out_ecount(4)		uint16_t version[4])
+{
+	siena_mc_dynamic_config_hdr_t *dcfg;
+	siena_parttbl_entry_t *entry;
+	unsigned int dcfg_partn;
+	unsigned int partn;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+	partn = entry->partn;
+
+	if ((1 << partn) & ~enp->en_u.siena.enu_partn_mask) {
+		rc = ENOTSUP;
+		goto fail2;
+	}
+
+	if ((rc = siena_nvram_get_subtype(enp, partn, subtypep)) != 0)
+		goto fail3;
+
+	/*
+	 * Some partitions are accessible from both ports (for instance BOOTROM)
+	 * Find the highest version reported by all dcfg structures on ports
+	 * that have access to this partition.
+	 */
+	version[0] = version[1] = version[2] = version[3] = 0;
+	for (entry = siena_parttbl; entry->port > 0; ++entry) {
+		unsigned int nitems;
+		uint16_t temp[4];
+		size_t length;
+
+		if (entry->partn != partn)
+			continue;
+
+		dcfg_partn = (entry->port == 1)
+			? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0
+			: MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1;
+		/*
+		 * Ingore missing partitions on port 2, assuming they're due
+		 * to to running on a single port part.
+		 */
+		if ((1 << dcfg_partn) &  ~enp->en_u.siena.enu_partn_mask) {
+			if (entry->port == 2)
+				continue;
+		}
+
+		if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn,
+		    B_FALSE, &dcfg, &length)) != 0)
+			goto fail4;
+
+		nitems = EFX_DWORD_FIELD(dcfg->num_fw_version_items,
+			    EFX_DWORD_0);
+		if (nitems < entry->partn)
+			goto done;
+
+		temp[0] = EFX_WORD_FIELD(dcfg->fw_version[partn].version_w,
+			    EFX_WORD_0);
+		temp[1] = EFX_WORD_FIELD(dcfg->fw_version[partn].version_x,
+			    EFX_WORD_0);
+		temp[2] = EFX_WORD_FIELD(dcfg->fw_version[partn].version_y,
+			    EFX_WORD_0);
+		temp[3] = EFX_WORD_FIELD(dcfg->fw_version[partn].version_z,
+			    EFX_WORD_0);
+		if (memcmp(version, temp, sizeof (temp)) < 0)
+			memcpy(version, temp, sizeof (temp));
+
+	done:
+		EFSYS_KMEM_FREE(enp->en_esip, length, dcfg);
+	}
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_rw_start(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			size_t *chunk_sizep)
+{
+	siena_parttbl_entry_t *entry;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = siena_nvram_partn_lock(enp, entry->partn)) != 0)
+		goto fail2;
+
+	if (chunk_sizep != NULL)
+		*chunk_sizep = SIENA_NVRAM_CHUNK;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_read_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	siena_parttbl_entry_t *entry;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = siena_nvram_partn_read(enp, entry->partn,
+	    offset, data, size)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_erase(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type)
+{
+	siena_parttbl_entry_t *entry;
+	size_t size;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = siena_nvram_partn_size(enp, entry->partn, &size)) != 0)
+		goto fail2;
+
+	if ((rc = siena_nvram_partn_erase(enp, entry->partn, 0, size)) != 0)
+		goto fail3;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_nvram_write_chunk(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__in			unsigned int offset,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	siena_parttbl_entry_t *entry;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = siena_nvram_partn_write(enp, entry->partn,
+	    offset, data, size)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				void
+siena_nvram_rw_finish(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type)
+{
+	siena_parttbl_entry_t *entry;
+
+	if ((entry = siena_parttbl_entry(enp, type)) != NULL)
+		siena_nvram_partn_unlock(enp, entry->partn);
+}
+
+	__checkReturn		int
+siena_nvram_set_version(
+	__in			efx_nic_t *enp,
+	__in			efx_nvram_type_t type,
+	__out			uint16_t version[4])
+{
+	siena_mc_dynamic_config_hdr_t *dcfg = NULL;
+	siena_parttbl_entry_t *entry;
+	unsigned int dcfg_partn;
+	size_t partn_size;
+	unsigned int hdr_length;
+	unsigned int vpd_length;
+	unsigned int vpd_offset;
+	unsigned int nitems;
+	unsigned int required_hdr_length;
+	unsigned int pos;
+	uint8_t cksum;
+	uint32_t subtype;
+	size_t length;
+	int rc;
+
+	if ((entry = siena_parttbl_entry(enp, type)) == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	dcfg_partn = (entry->port == 1)
+		? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0
+		: MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1;
+
+	if ((rc = siena_nvram_partn_size(enp, dcfg_partn, &partn_size)) != 0)
+		goto fail2;
+
+	if ((rc = siena_nvram_partn_lock(enp, dcfg_partn)) != 0)
+		goto fail2;
+
+	if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn,
+	    B_TRUE, &dcfg, &length)) != 0)
+		goto fail3;
+
+	hdr_length = EFX_WORD_FIELD(dcfg->length, EFX_WORD_0);
+	nitems = EFX_DWORD_FIELD(dcfg->num_fw_version_items, EFX_DWORD_0);
+	vpd_length = EFX_DWORD_FIELD(dcfg->dynamic_vpd_length, EFX_DWORD_0);
+	vpd_offset = EFX_DWORD_FIELD(dcfg->dynamic_vpd_offset, EFX_DWORD_0);
+
+	/*
+	 * NOTE: This function will blatt any fields trailing the version
+	 * vector, or the VPD chunk.
+	 */
+	required_hdr_length = SIENA_DYNAMIC_CFG_SIZE(entry->partn + 1);
+	if (required_hdr_length + vpd_length > length) {
+		rc = ENOSPC;
+		goto fail4;
+	}
+
+	if (vpd_offset < required_hdr_length) {
+		(void) memmove((caddr_t)dcfg + required_hdr_length,
+			(caddr_t)dcfg + vpd_offset, vpd_length);
+		vpd_offset = required_hdr_length;
+		EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_offset,
+				    EFX_DWORD_0, vpd_offset);
+	}
+
+	if (hdr_length < required_hdr_length) {
+		(void) memset((caddr_t)dcfg + hdr_length, 0,
+			required_hdr_length - hdr_length);
+		hdr_length = required_hdr_length;
+		EFX_POPULATE_WORD_1(dcfg->length,
+				    EFX_WORD_0, hdr_length);
+	}
+
+	/* Get the subtype to insert into the fw_subtype array */
+	if ((rc = siena_nvram_get_subtype(enp, entry->partn, &subtype)) != 0)
+		goto fail5;
+
+	/* Fill out the new version */
+	EFX_POPULATE_DWORD_1(dcfg->fw_version[entry->partn].fw_subtype,
+			    EFX_DWORD_0, subtype);
+	EFX_POPULATE_WORD_1(dcfg->fw_version[entry->partn].version_w,
+			    EFX_WORD_0, version[0]);
+	EFX_POPULATE_WORD_1(dcfg->fw_version[entry->partn].version_x,
+			    EFX_WORD_0, version[1]);
+	EFX_POPULATE_WORD_1(dcfg->fw_version[entry->partn].version_y,
+			    EFX_WORD_0, version[2]);
+	EFX_POPULATE_WORD_1(dcfg->fw_version[entry->partn].version_z,
+			    EFX_WORD_0, version[3]);
+
+	/* Update the version count */
+	if (nitems < entry->partn + 1) {
+		nitems = entry->partn + 1;
+		EFX_POPULATE_DWORD_1(dcfg->num_fw_version_items,
+				    EFX_DWORD_0, nitems);
+	}
+
+	/* Update the checksum */
+	cksum = 0;
+	for (pos = 0; pos < hdr_length; pos++)
+		cksum += ((uint8_t *)dcfg)[pos];
+	dcfg->csum.eb_u8[0] -= cksum;
+
+	/* Erase and write the new partition */
+	if ((rc = siena_nvram_partn_erase(enp, dcfg_partn, 0, partn_size)) != 0)
+		goto fail6;
+
+	/* Write out the new structure to nvram */
+	if ((rc = siena_nvram_partn_write(enp, dcfg_partn, 0,
+	    (caddr_t)dcfg, vpd_offset + vpd_length)) != 0)
+		goto fail7;
+
+	EFSYS_KMEM_FREE(enp->en_esip, length, dcfg);
+
+	siena_nvram_partn_unlock(enp, dcfg_partn);
+
+	return (0);
+
+fail7:
+	EFSYS_PROBE(fail7);
+fail6:
+	EFSYS_PROBE(fail6);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+
+	EFSYS_KMEM_FREE(enp->en_esip, length, dcfg);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_NVRAM */
+
+#endif	/* EFSYS_OPT_SIENA */
diff --git a/sys/dev/sfxge/common/siena_phy.c b/sys/dev/sfxge/common/siena_phy.c
new file mode 100644
index 0000000..7eb8468
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_phy.c
@@ -0,0 +1,857 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "efsys.h"
+#include "efx.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_SIENA
+
+static			void
+siena_phy_decode_cap(
+	__in		uint32_t mcdi_cap,
+	__out		uint32_t *maskp)
+{
+	uint32_t mask;
+
+	mask = 0;
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_10HDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10FDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_10FDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100HDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_100HDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100FDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_100FDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000HDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_1000HDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_1000FDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN))
+		mask |= (1 << EFX_PHY_CAP_10000FDX);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_PAUSE_LBN))
+		mask |= (1 << EFX_PHY_CAP_PAUSE);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_ASYM_LBN))
+		mask |= (1 << EFX_PHY_CAP_ASYM);
+	if (mcdi_cap & (1 << MC_CMD_PHY_CAP_AN_LBN))
+		mask |= (1 << EFX_PHY_CAP_AN);
+
+	*maskp = mask;
+}
+
+static			void
+siena_phy_decode_link_mode(
+	__in		efx_nic_t *enp,
+	__in		uint32_t link_flags,
+	__in		unsigned int speed,
+	__in		unsigned int fcntl,
+	__out		efx_link_mode_t *link_modep,
+	__out		unsigned int *fcntlp)
+{
+	boolean_t fd = !!(link_flags &
+		    (1 << MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN));
+	boolean_t up = !!(link_flags &
+		    (1 << MC_CMD_GET_LINK_OUT_LINK_UP_LBN));
+
+	_NOTE(ARGUNUSED(enp))
+
+	if (!up)
+		*link_modep = EFX_LINK_DOWN;
+	else if (speed == 10000 && fd)
+		*link_modep = EFX_LINK_10000FDX;
+	else if (speed == 1000)
+		*link_modep = fd ? EFX_LINK_1000FDX : EFX_LINK_1000HDX;
+	else if (speed == 100)
+		*link_modep = fd ? EFX_LINK_100FDX : EFX_LINK_100HDX;
+	else if (speed == 10)
+		*link_modep = fd ? EFX_LINK_10FDX : EFX_LINK_10HDX;
+	else
+		*link_modep = EFX_LINK_UNKNOWN;
+
+	if (fcntl == MC_CMD_FCNTL_OFF)
+		*fcntlp = 0;
+	else if (fcntl == MC_CMD_FCNTL_RESPOND)
+		*fcntlp = EFX_FCNTL_RESPOND;
+	else if (fcntl == MC_CMD_FCNTL_BIDIR)
+		*fcntlp = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
+	else {
+		EFSYS_PROBE1(mc_pcol_error, int, fcntl);
+		*fcntlp = 0;
+	}
+}
+
+			void
+siena_phy_link_ev(
+	__in		efx_nic_t *enp,
+	__in		efx_qword_t *eqp,
+	__out		efx_link_mode_t *link_modep)
+{
+	efx_port_t *epp = &(enp->en_port);
+	unsigned int link_flags;
+	unsigned int speed;
+	unsigned int fcntl;
+	efx_link_mode_t link_mode;
+	uint32_t lp_cap_mask;
+
+	/*
+	 * Convert the LINKCHANGE speed enumeration into mbit/s, in the
+	 * same way as GET_LINK encodes the speed
+	 */
+	switch (MCDI_EV_FIELD(*eqp, LINKCHANGE_SPEED)) {
+	case MCDI_EVENT_LINKCHANGE_SPEED_100M:
+		speed = 100;
+		break;
+	case MCDI_EVENT_LINKCHANGE_SPEED_1G:
+		speed = 1000;
+		break;
+	case MCDI_EVENT_LINKCHANGE_SPEED_10G:
+		speed = 10000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	link_flags = MCDI_EV_FIELD(*eqp, LINKCHANGE_LINK_FLAGS);
+	siena_phy_decode_link_mode(enp, link_flags, speed,
+				    MCDI_EV_FIELD(*eqp, LINKCHANGE_FCNTL),
+				    &link_mode, &fcntl);
+	siena_phy_decode_cap(MCDI_EV_FIELD(*eqp, LINKCHANGE_LP_CAP),
+			    &lp_cap_mask);
+
+	/*
+	 * It's safe to update ep_lp_cap_mask without the driver's port lock
+	 * because presumably any concurrently running efx_port_poll() is
+	 * only going to arrive at the same value.
+	 *
+	 * ep_fcntl has two meanings. It's either the link common fcntl
+	 * (if the PHY supports AN), or it's the forced link state. If
+	 * the former, it's safe to update the value for the same reason as
+	 * for ep_lp_cap_mask. If the latter, then just ignore the value,
+	 * because we can race with efx_mac_fcntl_set().
+	 */
+	epp->ep_lp_cap_mask = lp_cap_mask;
+	if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN))
+		epp->ep_fcntl = fcntl;
+
+	*link_modep = link_mode;
+}
+
+	__checkReturn	int
+siena_phy_power(
+	__in		efx_nic_t *enp,
+	__in		boolean_t power)
+{
+	int rc;
+
+	if (!power)
+		return (0);
+
+	/* Check if the PHY is a zombie */
+	if ((rc = siena_phy_verify(enp)) != 0)
+		goto fail1;
+
+	enp->en_reset_flags |= EFX_RESET_PHY;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_phy_get_link(
+	__in		efx_nic_t *enp,
+	__out		siena_link_state_t *slsp)
+{
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MC_CMD_GET_LINK_OUT_LEN];
+	int rc;
+
+	req.emr_cmd = MC_CMD_GET_LINK;
+	EFX_STATIC_ASSERT(MC_CMD_GET_LINK_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_LINK_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	siena_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_CAP),
+			    &slsp->sls_adv_cap_mask);
+	siena_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_LP_CAP),
+			    &slsp->sls_lp_cap_mask);
+
+	siena_phy_decode_link_mode(enp, MCDI_OUT_DWORD(req, GET_LINK_OUT_FLAGS),
+			    MCDI_OUT_DWORD(req, GET_LINK_OUT_LINK_SPEED),
+			    MCDI_OUT_DWORD(req, GET_LINK_OUT_FCNTL),
+			    &slsp->sls_link_mode, &slsp->sls_fcntl);
+
+#if EFSYS_OPT_LOOPBACK
+	/* Assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespace agree */
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_NONE == EFX_LOOPBACK_OFF);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_DATA == EFX_LOOPBACK_DATA);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMAC == EFX_LOOPBACK_GMAC);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII == EFX_LOOPBACK_XGMII);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGXS == EFX_LOOPBACK_XGXS);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI == EFX_LOOPBACK_XAUI);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII == EFX_LOOPBACK_GMII);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII == EFX_LOOPBACK_SGMII);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGBR == EFX_LOOPBACK_XGBR);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI == EFX_LOOPBACK_XFI);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_FAR == EFX_LOOPBACK_XAUI_FAR);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_FAR == EFX_LOOPBACK_GMII_FAR);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII_FAR == EFX_LOOPBACK_SGMII_FAR);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_FAR == EFX_LOOPBACK_XFI_FAR);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GPHY == EFX_LOOPBACK_GPHY);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS == EFX_LOOPBACK_PHY_XS);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PCS == EFX_LOOPBACK_PCS);
+	EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMAPMD == EFX_LOOPBACK_PMA_PMD);
+
+	slsp->sls_loopback = MCDI_OUT_DWORD(req, GET_LINK_OUT_LOOPBACK_MODE);
+#endif	/* EFSYS_OPT_LOOPBACK */
+
+	slsp->sls_mac_up = MCDI_OUT_DWORD(req, GET_LINK_OUT_MAC_FAULT) == 0;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_phy_reconfigure(
+	__in		efx_nic_t *enp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	efx_mcdi_req_t req;
+	uint8_t payload[MAX(MC_CMD_SET_ID_LED_IN_LEN,
+			    MC_CMD_SET_LINK_IN_LEN)];
+	uint32_t cap_mask;
+	unsigned int led_mode;
+	unsigned int speed;
+	int rc;
+
+	req.emr_cmd = MC_CMD_SET_LINK;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_SET_LINK_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_SET_LINK_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	cap_mask = epp->ep_adv_cap_mask;
+	MCDI_IN_POPULATE_DWORD_10(req, SET_LINK_IN_CAP,
+		PHY_CAP_10HDX, (cap_mask >> EFX_PHY_CAP_10HDX) & 0x1,
+		PHY_CAP_10FDX, (cap_mask >> EFX_PHY_CAP_10FDX) & 0x1,
+		PHY_CAP_100HDX, (cap_mask >> EFX_PHY_CAP_100HDX) & 0x1,
+		PHY_CAP_100FDX, (cap_mask >> EFX_PHY_CAP_100FDX) & 0x1,
+		PHY_CAP_1000HDX, (cap_mask >> EFX_PHY_CAP_1000HDX) & 0x1,
+		PHY_CAP_1000FDX, (cap_mask >> EFX_PHY_CAP_1000FDX) & 0x1,
+		PHY_CAP_10000FDX, (cap_mask >> EFX_PHY_CAP_10000FDX) & 0x1,
+		PHY_CAP_PAUSE, (cap_mask >> EFX_PHY_CAP_PAUSE) & 0x1,
+		PHY_CAP_ASYM, (cap_mask >> EFX_PHY_CAP_ASYM) & 0x1,
+		PHY_CAP_AN, (cap_mask >> EFX_PHY_CAP_AN) & 0x1);
+
+#if EFSYS_OPT_LOOPBACK
+	MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE,
+		    epp->ep_loopback_type);
+	switch (epp->ep_loopback_link_mode) {
+	case EFX_LINK_100FDX:
+		speed = 100;
+		break;
+	case EFX_LINK_1000FDX:
+		speed = 1000;
+		break;
+	case EFX_LINK_10000FDX:
+		speed = 10000;
+		break;
+	default:
+		speed = 0;
+	}
+#else
+	MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, MC_CMD_LOOPBACK_NONE);
+	speed = 0;
+#endif	/* EFSYS_OPT_LOOPBACK */
+	MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_SPEED, speed);
+
+#if EFSYS_OPT_PHY_FLAGS
+	MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, epp->ep_phy_flags);
+#else
+	MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, 0);
+#endif	/* EFSYS_OPT_PHY_FLAGS */
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	/* And set the blink mode */
+	req.emr_cmd = MC_CMD_SET_ID_LED;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_SET_ID_LED_IN_LEN;
+	EFX_STATIC_ASSERT(MC_CMD_SET_ID_LED_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+#if EFSYS_OPT_PHY_LED_CONTROL
+	switch (epp->ep_phy_led_mode) {
+	case EFX_PHY_LED_DEFAULT:
+		led_mode = MC_CMD_LED_DEFAULT;
+		break;
+	case EFX_PHY_LED_OFF:
+		led_mode = MC_CMD_LED_OFF;
+		break;
+	case EFX_PHY_LED_ON:
+		led_mode = MC_CMD_LED_ON;
+		break;
+	default:
+		EFSYS_ASSERT(0);
+		led_mode = MC_CMD_LED_DEFAULT;
+	}
+
+	MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, led_mode);
+#else
+	MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, MC_CMD_LED_DEFAULT);
+#endif	/* EFSYS_OPT_PHY_LED_CONTROL */
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_phy_verify(
+	__in		efx_nic_t *enp)
+{
+	efx_mcdi_req_t req;
+	uint8_t outbuf[MC_CMD_GET_PHY_STATE_OUT_LEN];
+	uint32_t state;
+	int rc;
+
+	req.emr_cmd = MC_CMD_GET_PHY_STATE;
+	EFX_STATIC_ASSERT(MC_CMD_GET_PHY_STATE_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = outbuf;
+	req.emr_out_length = sizeof (outbuf);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_GET_PHY_STATE_OUT_LEN) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	state = MCDI_OUT_DWORD(req, GET_PHY_STATE_OUT_STATE);
+	if (state != MC_CMD_PHY_STATE_OK) {
+		if (state != MC_CMD_PHY_STATE_ZOMBIE)
+			EFSYS_PROBE1(mc_pcol_error, int, state);
+		rc = ENOTACTIVE;
+		goto fail3;
+	}
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn	int
+siena_phy_oui_get(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *ouip)
+{
+	_NOTE(ARGUNUSED(enp, ouip))
+
+	return (ENOTSUP);
+}
+
+#if EFSYS_OPT_PHY_STATS
+
+#define	SIENA_SIMPLE_STAT_SET(_vmask, _esmp, _smask, _stat,		\
+			    _mc_record, _efx_record)			\
+	if ((_vmask) & (1ULL << (_mc_record))) {			\
+		(_smask) |= (1ULL << (_efx_record));			\
+		if ((_stat) != NULL && !EFSYS_MEM_IS_NULL(_esmp)) {	\
+			efx_dword_t dword;				\
+			EFSYS_MEM_READD(_esmp, (_mc_record) * 4, &dword);\
+			(_stat)[_efx_record] =				\
+				EFX_DWORD_FIELD(dword, EFX_DWORD_0);	\
+		}							\
+	}
+
+#define	SIENA_SIMPLE_STAT_SET2(_vmask, _esmp, _smask, _stat, _record)	\
+	SIENA_SIMPLE_STAT_SET(_vmask, _esmp, _smask, _stat, 		\
+			    MC_CMD_ ## _record,				\
+			    EFX_PHY_STAT_ ## _record)
+
+						void
+siena_phy_decode_stats(
+	__in					efx_nic_t *enp,
+	__in					uint32_t vmask,
+	__in_opt				efsys_mem_t *esmp,
+	__out_opt				uint64_t *smaskp,
+	__out_ecount_opt(EFX_PHY_NSTATS)	uint32_t *stat)
+{
+	uint64_t smask = 0;
+
+	_NOTE(ARGUNUSED(enp))
+
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, OUI);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_LINK_UP);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_RX_FAULT);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_TX_FAULT);
+
+	if (vmask & (1 << MC_CMD_PMA_PMD_SIGNAL)) {
+		smask |=   ((1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_A) |
+			    (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_B) |
+			    (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_C) |
+			    (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_D));
+		if (stat != NULL && esmp != NULL && !EFSYS_MEM_IS_NULL(esmp)) {
+			efx_dword_t dword;
+			uint32_t sig;
+			EFSYS_MEM_READD(esmp, 4 * MC_CMD_PMA_PMD_SIGNAL,
+					&dword);
+			sig = EFX_DWORD_FIELD(dword, EFX_DWORD_0);
+			stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_A] = (sig >> 1) & 1;
+			stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_B] = (sig >> 2) & 1;
+			stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_C] = (sig >> 3) & 1;
+			stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_D] = (sig >> 4) & 1;
+		}
+	}
+
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_A,
+			    EFX_PHY_STAT_SNR_A);
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_B,
+			    EFX_PHY_STAT_SNR_B);
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_C,
+			    EFX_PHY_STAT_SNR_C);
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_D,
+			    EFX_PHY_STAT_SNR_D);
+
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_LINK_UP);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_RX_FAULT);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_TX_FAULT);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_BER);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_BLOCK_ERRORS);
+
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_LINK_UP,
+			    EFX_PHY_STAT_PHY_XS_LINK_UP);
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_RX_FAULT,
+			    EFX_PHY_STAT_PHY_XS_RX_FAULT);
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_TX_FAULT,
+			    EFX_PHY_STAT_PHY_XS_TX_FAULT);
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_ALIGN,
+			    EFX_PHY_STAT_PHY_XS_ALIGN);
+
+	if (vmask & (1 << MC_CMD_PHYXS_SYNC)) {
+		smask |=   ((1 << EFX_PHY_STAT_PHY_XS_SYNC_A) |
+			    (1 << EFX_PHY_STAT_PHY_XS_SYNC_B) |
+			    (1 << EFX_PHY_STAT_PHY_XS_SYNC_C) |
+			    (1 << EFX_PHY_STAT_PHY_XS_SYNC_D));
+		if (stat != NULL && !EFSYS_MEM_IS_NULL(esmp)) {
+			efx_dword_t dword;
+			uint32_t sync;
+			EFSYS_MEM_READD(esmp, 4 * MC_CMD_PHYXS_SYNC, &dword);
+			sync = EFX_DWORD_FIELD(dword, EFX_DWORD_0);
+			stat[EFX_PHY_STAT_PHY_XS_SYNC_A] = (sync >> 0) & 1;
+			stat[EFX_PHY_STAT_PHY_XS_SYNC_B] = (sync >> 1) & 1;
+			stat[EFX_PHY_STAT_PHY_XS_SYNC_C] = (sync >> 2) & 1;
+			stat[EFX_PHY_STAT_PHY_XS_SYNC_D] = (sync >> 3) & 1;
+		}
+	}
+
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, AN_LINK_UP);
+	SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, AN_COMPLETE);
+
+	SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_CL22_LINK_UP,
+			    EFX_PHY_STAT_CL22EXT_LINK_UP);
+
+	if (smaskp != NULL)
+		*smaskp = smask;
+}
+
+	__checkReturn				int
+siena_phy_stats_update(
+	__in					efx_nic_t *enp,
+	__in					efsys_mem_t *esmp,
+	__out_ecount(EFX_PHY_NSTATS)		uint32_t *stat)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	uint32_t vmask = encp->enc_siena_phy_stat_mask;
+	uint8_t payload[MC_CMD_PHY_STATS_IN_LEN];
+	uint64_t smask;
+	efx_mcdi_req_t req;
+	int rc;
+
+	req.emr_cmd = MC_CMD_PHY_STATS;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_PHY_STATS_OUT_DMA_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	MCDI_IN_SET_DWORD(req, PHY_STATS_IN_DMA_ADDR_LO,
+			    EFSYS_MEM_ADDR(esmp) & 0xffffffff);
+	MCDI_IN_SET_DWORD(req, PHY_STATS_IN_DMA_ADDR_HI,
+			    EFSYS_MEM_ADDR(esmp) >> 32);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+	EFSYS_ASSERT3U(req.emr_out_length, ==, MC_CMD_PHY_STATS_OUT_DMA_LEN);
+
+	siena_phy_decode_stats(enp, vmask, esmp, &smask, stat);
+	EFSYS_ASSERT(smask == encp->enc_phy_stat_mask);
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (0);
+}
+
+#endif	/* EFSYS_OPT_PHY_STATS */
+
+#if EFSYS_OPT_PHY_PROPS
+
+#if EFSYS_OPT_NAMES
+
+extern		const char __cs *
+siena_phy_prop_name(
+	__in	efx_nic_t *enp,
+	__in	unsigned int id)
+{
+	_NOTE(ARGUNUSED(enp, id))
+
+	return (NULL);
+}
+
+#endif	/* EFSYS_OPT_NAMES */
+
+extern	__checkReturn	int
+siena_phy_prop_get(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t flags,
+	__out		uint32_t *valp)
+{
+	_NOTE(ARGUNUSED(enp, id, flags, valp))
+
+	return (ENOTSUP);
+}
+
+extern	__checkReturn	int
+siena_phy_prop_set(
+	__in		efx_nic_t *enp,
+	__in		unsigned int id,
+	__in		uint32_t val)
+{
+	_NOTE(ARGUNUSED(enp, id, val))
+
+	return (ENOTSUP);
+}
+
+#endif	/* EFSYS_OPT_PHY_PROPS */
+
+#if EFSYS_OPT_PHY_BIST
+
+	__checkReturn		int
+siena_phy_bist_start(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type)
+{
+	uint8_t payload[MC_CMD_START_BIST_IN_LEN];
+	efx_mcdi_req_t req;
+	int rc;
+
+	req.emr_cmd = MC_CMD_START_BIST;
+	req.emr_in_buf = payload;
+	req.emr_in_length = sizeof (payload);
+	EFX_STATIC_ASSERT(MC_CMD_START_BIST_OUT_LEN == 0);
+	req.emr_out_buf = NULL;
+	req.emr_out_length = 0;
+
+	switch (type) {
+	case EFX_PHY_BIST_TYPE_NORMAL:
+		MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_PHY_BIST);
+		break;
+	case EFX_PHY_BIST_TYPE_CABLE_SHORT:
+		MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE,
+		    MC_CMD_PHY_BIST_CABLE_SHORT);
+		break;
+	case EFX_PHY_BIST_TYPE_CABLE_LONG:
+		MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE,
+		    MC_CMD_PHY_BIST_CABLE_LONG);
+		break;
+	default:
+		EFSYS_ASSERT(0);
+	}
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+static	__checkReturn		unsigned long
+siena_phy_sft9001_bist_status(
+	__in			uint16_t code)
+{
+	switch (code) {
+	case MC_CMD_POLL_BIST_SFT9001_PAIR_BUSY:
+		return (EFX_PHY_CABLE_STATUS_BUSY);
+	case MC_CMD_POLL_BIST_SFT9001_INTER_PAIR_SHORT:
+		return (EFX_PHY_CABLE_STATUS_INTERPAIRSHORT);
+	case MC_CMD_POLL_BIST_SFT9001_INTRA_PAIR_SHORT:
+		return (EFX_PHY_CABLE_STATUS_INTRAPAIRSHORT);
+	case MC_CMD_POLL_BIST_SFT9001_PAIR_OPEN:
+		return (EFX_PHY_CABLE_STATUS_OPEN);
+	case MC_CMD_POLL_BIST_SFT9001_PAIR_OK:
+		return (EFX_PHY_CABLE_STATUS_OK);
+	default:
+		return (EFX_PHY_CABLE_STATUS_INVALID);
+	}
+}
+
+	__checkReturn		int
+siena_phy_bist_poll(
+	__in			efx_nic_t *enp,
+	__in			efx_phy_bist_type_t type,
+	__out			efx_phy_bist_result_t *resultp,
+	__out_opt __drv_when(count > 0, __notnull)
+	uint32_t *value_maskp,
+	__out_ecount_opt(count)	__drv_when(count > 0, __notnull)
+	unsigned long *valuesp,
+	__in			size_t count)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	uint8_t payload[MCDI_CTL_SDU_LEN_MAX];
+	uint32_t value_mask = 0;
+	efx_mcdi_req_t req;
+	uint32_t result;
+	int rc;
+
+	req.emr_cmd = MC_CMD_POLL_BIST;
+	_NOTE(CONSTANTCONDITION)
+	EFSYS_ASSERT(MC_CMD_POLL_BIST_IN_LEN == 0);
+	req.emr_in_buf = NULL;
+	req.emr_in_length = 0;
+	req.emr_out_buf = payload;
+	req.emr_out_length = sizeof (payload);
+
+	efx_mcdi_execute(enp, &req);
+
+	if (req.emr_rc != 0) {
+		rc = req.emr_rc;
+		goto fail1;
+	}
+
+	if (req.emr_out_length_used < MC_CMD_POLL_BIST_OUT_RESULT_OFST + 4) {
+		rc = EMSGSIZE;
+		goto fail2;
+	}
+
+	if (count > 0)
+		(void) memset(valuesp, '\0', count * sizeof (unsigned long));
+
+	result = MCDI_OUT_DWORD(req, POLL_BIST_OUT_RESULT);
+
+	/* Extract PHY specific results */
+	if (result == MC_CMD_POLL_BIST_PASSED &&
+	    encp->enc_phy_type == EFX_PHY_SFT9001B &&
+	    req.emr_out_length_used >= MC_CMD_POLL_BIST_OUT_SFT9001_LEN &&
+	    (type == EFX_PHY_BIST_TYPE_CABLE_SHORT ||
+	    type == EFX_PHY_BIST_TYPE_CABLE_LONG)) {
+		uint16_t word;
+
+		if (count > EFX_PHY_BIST_CABLE_LENGTH_A) {
+			if (valuesp != NULL)
+				valuesp[EFX_PHY_BIST_CABLE_LENGTH_A] =
+				    MCDI_OUT_DWORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A);
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_LENGTH_A);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_LENGTH_B) {
+			if (valuesp != NULL)
+				valuesp[EFX_PHY_BIST_CABLE_LENGTH_B] =
+				    MCDI_OUT_DWORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B);
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_LENGTH_B);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_LENGTH_C) {
+			if (valuesp != NULL)
+				valuesp[EFX_PHY_BIST_CABLE_LENGTH_C] =
+				    MCDI_OUT_DWORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C);
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_LENGTH_C);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_LENGTH_D) {
+			if (valuesp != NULL)
+				valuesp[EFX_PHY_BIST_CABLE_LENGTH_D] =
+				    MCDI_OUT_DWORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D);
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_LENGTH_D);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_STATUS_A) {
+			if (valuesp != NULL) {
+				word = MCDI_OUT_WORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_STATUS_A);
+				valuesp[EFX_PHY_BIST_CABLE_STATUS_A] =
+				    siena_phy_sft9001_bist_status(word);
+			}
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_STATUS_A);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_STATUS_B) {
+			if (valuesp != NULL) {
+				word = MCDI_OUT_WORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_STATUS_B);
+				valuesp[EFX_PHY_BIST_CABLE_STATUS_B] =
+				    siena_phy_sft9001_bist_status(word);
+			}
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_STATUS_B);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_STATUS_C) {
+			if (valuesp != NULL) {
+				word = MCDI_OUT_WORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_STATUS_C);
+				valuesp[EFX_PHY_BIST_CABLE_STATUS_C] =
+				    siena_phy_sft9001_bist_status(word);
+			}
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_STATUS_C);
+		}
+
+		if (count > EFX_PHY_BIST_CABLE_STATUS_D) {
+			if (valuesp != NULL) {
+				word = MCDI_OUT_WORD(req,
+				    POLL_BIST_OUT_SFT9001_CABLE_STATUS_D);
+				valuesp[EFX_PHY_BIST_CABLE_STATUS_D] =
+				    siena_phy_sft9001_bist_status(word);
+			}
+			value_mask |= (1 << EFX_PHY_BIST_CABLE_STATUS_D);
+		}
+
+	} else if (result == MC_CMD_POLL_BIST_FAILED &&
+		    encp->enc_phy_type == EFX_PHY_QLX111V &&
+		    req.emr_out_length >= MC_CMD_POLL_BIST_OUT_MRSFP_LEN &&
+		    count > EFX_PHY_BIST_FAULT_CODE) {
+		if (valuesp != NULL)
+			valuesp[EFX_PHY_BIST_FAULT_CODE] =
+			    MCDI_OUT_DWORD(req, POLL_BIST_OUT_MRSFP_TEST);
+		value_mask |= 1 << EFX_PHY_BIST_FAULT_CODE;
+	}
+
+	if (value_maskp != NULL)
+		*value_maskp = value_mask;
+
+	EFSYS_ASSERT(resultp != NULL);
+	if (result == MC_CMD_POLL_BIST_RUNNING)
+		*resultp = EFX_PHY_BIST_RESULT_RUNNING;
+	else if (result == MC_CMD_POLL_BIST_PASSED)
+		*resultp = EFX_PHY_BIST_RESULT_PASSED;
+	else
+		*resultp = EFX_PHY_BIST_RESULT_FAILED;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+			void
+siena_phy_bist_stop(
+	__in		efx_nic_t *enp,
+	__in		efx_phy_bist_type_t type)
+{
+	/* There is no way to stop BIST on Siena */
+	_NOTE(ARGUNUSED(enp, type))
+}
+
+#endif	/* EFSYS_OPT_PHY_BIST */
+
+#endif	/* EFSYS_OPT_SIENA */
diff --git a/sys/dev/sfxge/common/siena_sram.c b/sys/dev/sfxge/common/siena_sram.c
new file mode 100644
index 0000000..64fce98
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_sram.c
@@ -0,0 +1,172 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include "efsys.h"
+#include "efx.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_SIENA
+
+			void
+siena_sram_init(
+	__in		efx_nic_t *enp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_oword_t oword;
+	uint32_t rx_base, tx_base;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	rx_base = encp->enc_buftbl_limit;
+	tx_base = rx_base + (encp->enc_rxq_limit * 64);
+
+	/* Initialize the transmit descriptor cache */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_TX_DC_BASE_ADR, tx_base);
+	EFX_BAR_WRITEO(enp, FR_AZ_SRM_TX_DC_CFG_REG, &oword);
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DC_SIZE, 1); /* 16 descriptors */
+	EFX_BAR_WRITEO(enp, FR_AZ_TX_DC_CFG_REG, &oword);
+
+	/* Initialize the receive descriptor cache */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_RX_DC_BASE_ADR, rx_base);
+	EFX_BAR_WRITEO(enp, FR_AZ_SRM_RX_DC_CFG_REG, &oword);
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DC_SIZE, 3); /* 64 descriptors */
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_DC_CFG_REG, &oword);
+
+	/* Set receive descriptor pre-fetch low water mark */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DC_PF_LWM, 56);
+	EFX_BAR_WRITEO(enp, FR_AZ_RX_DC_PF_WM_REG, &oword);
+
+	/* Set the event queue to use for SRAM updates */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_UPD_EVQ_ID, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_SRM_UPD_EVQ_REG, &oword);
+}
+
+#if EFSYS_OPT_DIAG
+
+	__checkReturn	int
+siena_sram_test(
+	__in		efx_nic_t *enp,
+	__in		efx_sram_pattern_fn_t func)
+{
+	efx_oword_t oword;
+	efx_qword_t qword;
+	efx_qword_t verify;
+	size_t rows;
+	unsigned int wptr;
+	unsigned int rptr;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	/* Reconfigure into HALF buffer table mode */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_BUF_TBL_MODE, 0);
+	EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_CFG_REG, &oword);
+
+	/*
+	 * Move the descriptor caches up to the top of SRAM, and test
+	 * all of SRAM below them. We only miss out one row here.
+	 */
+	rows = SIENA_SRAM_ROWS - 1;
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_RX_DC_BASE_ADR, rows);
+	EFX_BAR_WRITEO(enp, FR_AZ_SRM_RX_DC_CFG_REG, &oword);
+
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_TX_DC_BASE_ADR, rows + 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_SRM_TX_DC_CFG_REG, &oword);
+
+	/*
+	 * Write the pattern through BUF_HALF_TBL. Write
+	 * in 64 entry batches, waiting 1us in between each batch
+	 * to guarantee not to overflow the SRAM fifo
+	 */
+	for (wptr = 0, rptr = 0; wptr < rows; ++wptr) {
+		func(wptr, B_FALSE, &qword);
+		EFX_BAR_TBL_WRITEQ(enp, FR_AZ_BUF_HALF_TBL, wptr, &qword);
+
+		if ((wptr - rptr) < 64 && wptr < rows - 1)
+			continue;
+
+		EFSYS_SPIN(1);
+
+		for (; rptr <= wptr; ++rptr) {
+			func(rptr, B_FALSE, &qword);
+			EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_HALF_TBL, rptr,
+			    &verify);
+
+			if (!EFX_QWORD_IS_EQUAL(verify, qword)) {
+				rc = EFAULT;
+				goto fail1;
+			}
+		}
+	}
+
+	/* And do the same negated */
+	for (wptr = 0, rptr = 0; wptr < rows; ++wptr) {
+		func(wptr, B_TRUE, &qword);
+		EFX_BAR_TBL_WRITEQ(enp, FR_AZ_BUF_HALF_TBL, wptr, &qword);
+
+		if ((wptr - rptr) < 64 && wptr < rows - 1)
+			continue;
+
+		EFSYS_SPIN(1);
+
+		for (; rptr <= wptr; ++rptr) {
+			func(rptr, B_TRUE, &qword);
+			EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_HALF_TBL, rptr,
+			    &verify);
+
+			if (!EFX_QWORD_IS_EQUAL(verify, qword)) {
+				rc = EFAULT;
+				goto fail2;
+			}
+		}
+	}
+
+	/* Restore back to FULL buffer table mode */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_BUF_TBL_MODE, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_CFG_REG, &oword);
+
+	/*
+	 * We don't need to reconfigure SRAM again because the API
+	 * requires efx_nic_fini() to be called after an sram test.
+	 */
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	/* Restore back to FULL buffer table mode */
+	EFX_POPULATE_OWORD_1(oword, FRF_AZ_BUF_TBL_MODE, 1);
+	EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_CFG_REG, &oword);
+
+	return (rc);
+}
+
+#endif	/* EFSYS_OPT_DIAG */
+
+#endif	/* EFSYS_OPT_SIENA */
diff --git a/sys/dev/sfxge/common/siena_vpd.c b/sys/dev/sfxge/common/siena_vpd.c
new file mode 100644
index 0000000..5c44608
--- /dev/null
+++ b/sys/dev/sfxge/common/siena_vpd.c
@@ -0,0 +1,603 @@
+/*-
+ * Copyright 2009 Solarflare Communications Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "efsys.h"
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_impl.h"
+
+#if EFSYS_OPT_VPD
+
+#if EFSYS_OPT_SIENA
+
+static	__checkReturn			int
+siena_vpd_get_static(
+	__in				efx_nic_t *enp,
+	__in				unsigned int partn,
+	__deref_out_bcount_opt(*sizep)	caddr_t *svpdp,
+	__out				size_t *sizep)
+{
+	siena_mc_static_config_hdr_t *scfg;
+	caddr_t svpd;
+	size_t size;
+	uint8_t cksum;
+	unsigned int vpd_offset;
+	unsigned int vpd_length;
+	unsigned int hdr_length;
+	unsigned int pos;
+	unsigned int region;
+	int rc;
+
+	EFSYS_ASSERT(partn == MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT0 ||
+		    partn == MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT1);
+
+	/* Allocate sufficient memory for the entire static cfg area */
+	if ((rc = siena_nvram_partn_size(enp, partn, &size)) != 0)
+		goto fail1;
+
+	EFSYS_KMEM_ALLOC(enp->en_esip, size, scfg);
+	if (scfg == NULL) {
+		rc = ENOMEM;
+		goto fail2;
+	}
+
+	if ((rc = siena_nvram_partn_read(enp, partn, 0,
+	    (caddr_t)scfg, SIENA_NVRAM_CHUNK)) != 0)
+		goto fail3;
+
+	/* Verify the magic number */
+	if (EFX_DWORD_FIELD(scfg->magic, EFX_DWORD_0) !=
+	    SIENA_MC_STATIC_CONFIG_MAGIC) {
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	/* All future versions of the structure must be backwards compatable */
+	EFX_STATIC_ASSERT(SIENA_MC_STATIC_CONFIG_VERSION == 0);
+
+	hdr_length = EFX_WORD_FIELD(scfg->length, EFX_WORD_0);
+	vpd_offset = EFX_DWORD_FIELD(scfg->static_vpd_offset, EFX_DWORD_0);
+	vpd_length = EFX_DWORD_FIELD(scfg->static_vpd_length, EFX_DWORD_0);
+
+	/* Verify the hdr doesn't overflow the sector size */
+	if (hdr_length > size || vpd_offset > size || vpd_length > size ||
+	    vpd_length + vpd_offset > size) {
+		rc = EINVAL;
+		goto fail5;
+	}
+
+	/* Read the remainder of scfg + static vpd */
+	region = vpd_offset + vpd_length;
+	if (region > SIENA_NVRAM_CHUNK) {
+		if ((rc = siena_nvram_partn_read(enp, partn, SIENA_NVRAM_CHUNK,
+		    (caddr_t)scfg + SIENA_NVRAM_CHUNK,
+		    region - SIENA_NVRAM_CHUNK)) != 0)
+			goto fail6;
+	}
+
+	/* Verify checksum */
+	cksum = 0;
+	for (pos = 0; pos < hdr_length; pos++)
+		cksum += ((uint8_t *)scfg)[pos];
+	if (cksum != 0) {
+		rc = EINVAL;
+		goto fail7;
+	}
+
+	if (vpd_length == 0)
+		svpd = NULL;
+	else {
+		/* Copy the vpd data out */
+		EFSYS_KMEM_ALLOC(enp->en_esip, vpd_length, svpd);
+		if (svpd == NULL) {
+			rc = ENOMEM;
+			goto fail8;
+		}
+		memcpy(svpd, (caddr_t)scfg + vpd_offset, vpd_length);
+	}
+
+	EFSYS_KMEM_FREE(enp->en_esip, size, scfg);
+
+	*svpdp = svpd;
+	*sizep = vpd_length;
+
+	return (0);
+
+fail8:
+	EFSYS_PROBE(fail8);
+fail7:
+	EFSYS_PROBE(fail7);
+fail6:
+	EFSYS_PROBE(fail6);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+
+	EFSYS_KMEM_FREE(enp->en_esip, size, scfg);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_init(
+	__in			efx_nic_t *enp)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	caddr_t svpd = NULL;
+	unsigned partn;
+	size_t size = 0;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	partn = (emip->emi_port == 1)
+		? MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT0
+		: MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT1;
+
+	/*
+	 * We need the static VPD sector to present a unified static+dynamic
+	 * VPD, that is, basically on every read, write, verify cycle. Since
+	 * it should *never* change we can just cache it here.
+	 */
+	if ((rc = siena_vpd_get_static(enp, partn, &svpd, &size)) != 0)
+		goto fail1;
+
+	if (svpd != NULL && size > 0) {
+		if ((rc = efx_vpd_hunk_verify(svpd, size, NULL)) != 0)
+			goto fail2;
+	}
+
+	enp->en_u.siena.enu_svpd = svpd;
+	enp->en_u.siena.enu_svpd_length = size;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+
+	EFSYS_KMEM_FREE(enp->en_esip, size, svpd);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_size(
+	__in			efx_nic_t *enp,
+	__out			size_t *sizep)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	unsigned int partn;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	/*
+	 * This function returns the total size the user should allocate
+	 * for all VPD operations. We've already cached the static vpd,
+	 * so we just need to return an upper bound on the dynamic vpd.
+	 * Since the dynamic_config structure can change under our feet,
+	 * (as version numbers are inserted), just be safe and return the
+	 * total size of the dynamic_config *sector*
+	 */
+	partn = (emip->emi_port == 1)
+		? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0
+		: MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1;
+
+	if ((rc = siena_nvram_partn_size(enp, partn, sizep)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_read(
+	__in			efx_nic_t *enp,
+	__out_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	siena_mc_dynamic_config_hdr_t *dcfg;
+	unsigned int vpd_length;
+	unsigned int vpd_offset;
+	unsigned int dcfg_partn;
+	size_t dcfg_size;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	dcfg_partn = (emip->emi_port == 1)
+		? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0
+		: MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1;
+
+	if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn,
+	    B_TRUE, &dcfg, &dcfg_size)) != 0)
+		goto fail1;
+
+	vpd_length = EFX_DWORD_FIELD(dcfg->dynamic_vpd_length, EFX_DWORD_0);
+	vpd_offset = EFX_DWORD_FIELD(dcfg->dynamic_vpd_offset, EFX_DWORD_0);
+
+	if (vpd_length > size) {
+		rc = EFAULT;	/* Invalid dcfg: header bigger than sector */
+		goto fail2;
+	}
+
+	EFSYS_ASSERT3U(vpd_length, <=, size);
+	memcpy(data, (caddr_t)dcfg + vpd_offset, vpd_length);
+
+	/* Pad data with all-1s, consistent with update operations */
+	memset(data + vpd_length, 0xff, size - vpd_length);
+
+	EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+
+	EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_verify(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_vpd_tag_t stag;
+	efx_vpd_tag_t dtag;
+	efx_vpd_keyword_t skey;
+	efx_vpd_keyword_t dkey;
+	unsigned int scont;
+	unsigned int dcont;
+
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	/*
+	 * Strictly you could take the view that dynamic vpd is optional.
+	 * Instead, to conform more closely to the read/verify/reinit()
+	 * paradigm, we require dynamic vpd. siena_vpd_reinit() will
+	 * reinitialize it as required.
+	 */
+	if ((rc = efx_vpd_hunk_verify(data, size, NULL)) != 0)
+		goto fail1;
+
+	/*
+	 * Verify that there is no duplication between the static and
+	 * dynamic cfg sectors.
+	 */
+	if (enp->en_u.siena.enu_svpd_length == 0)
+		goto done;
+
+	dcont = 0;
+	_NOTE(CONSTANTCONDITION)
+	while (1) {
+		if ((rc = efx_vpd_hunk_next(data, size, &dtag,
+		    &dkey, NULL, NULL, &dcont)) != 0)
+			goto fail2;
+		if (dcont == 0)
+			break;
+
+		scont = 0;
+		_NOTE(CONSTANTCONDITION)
+		while (1) {
+			if ((rc = efx_vpd_hunk_next(
+			    enp->en_u.siena.enu_svpd,
+			    enp->en_u.siena.enu_svpd_length, &stag, &skey,
+			    NULL, NULL, &scont)) != 0)
+				goto fail3;
+			if (scont == 0)
+				break;
+
+			if (stag == dtag && skey == dkey) {
+				rc = EEXIST;
+				goto fail4;
+			}
+		}
+	}
+
+done:
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_reinit(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	boolean_t wantpid;
+	int rc;
+
+	/*
+	 * Only create a PID if the dynamic cfg doesn't have one
+	 */
+	if (enp->en_u.siena.enu_svpd_length == 0)
+		wantpid = B_TRUE;
+	else {
+		unsigned int offset;
+		uint8_t length;
+
+		rc = efx_vpd_hunk_get(enp->en_u.siena.enu_svpd,
+				    enp->en_u.siena.enu_svpd_length,
+				    EFX_VPD_ID, 0, &offset, &length);
+		if (rc == 0)
+			wantpid = B_FALSE;
+		else if (rc == ENOENT)
+			wantpid = B_TRUE;
+		else
+			goto fail1;
+	}
+
+	if ((rc = efx_vpd_hunk_reinit(data, size, wantpid)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_get(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__inout			efx_vpd_value_t *evvp)
+{
+	unsigned int offset;
+	uint8_t length;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	/* Attempt to satisfy the request from svpd first */
+	if (enp->en_u.siena.enu_svpd_length > 0) {
+		if ((rc = efx_vpd_hunk_get(enp->en_u.siena.enu_svpd,
+		    enp->en_u.siena.enu_svpd_length, evvp->evv_tag,
+		    evvp->evv_keyword, &offset, &length)) == 0) {
+			evvp->evv_length = length;
+			memcpy(evvp->evv_value,
+			    enp->en_u.siena.enu_svpd + offset, length);
+			return (0);
+		} else if (rc != ENOENT)
+			goto fail1;
+	}
+
+	/* And then from the provided data buffer */
+	if ((rc = efx_vpd_hunk_get(data, size, evvp->evv_tag,
+	    evvp->evv_keyword, &offset, &length)) != 0)
+		goto fail2;
+
+	evvp->evv_length = length;
+	memcpy(evvp->evv_value, data + offset, length);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_set(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__in			efx_vpd_value_t *evvp)
+{
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	/* If the provided (tag,keyword) exists in svpd, then it is readonly */
+	if (enp->en_u.siena.enu_svpd_length > 0) {
+		unsigned int offset;
+		uint8_t length;
+
+		if ((rc = efx_vpd_hunk_get(enp->en_u.siena.enu_svpd,
+		    enp->en_u.siena.enu_svpd_length, evvp->evv_tag,
+		    evvp->evv_keyword, &offset, &length)) == 0) {
+			rc = EACCES;
+			goto fail1;
+		}
+	}
+
+	if ((rc = efx_vpd_hunk_set(data, size, evvp)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+	__checkReturn		int
+siena_vpd_next(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size,
+	__out			efx_vpd_value_t *evvp,
+	__inout			unsigned int *contp)
+{
+	_NOTE(ARGUNUSED(enp, data, size, evvp, contp))
+
+	return (ENOTSUP);
+}
+
+	__checkReturn		int
+siena_vpd_write(
+	__in			efx_nic_t *enp,
+	__in_bcount(size)	caddr_t data,
+	__in			size_t size)
+{
+	efx_mcdi_iface_t *emip = &(enp->en_u.siena.enu_mip);
+	siena_mc_dynamic_config_hdr_t *dcfg;
+	unsigned int vpd_offset;
+	unsigned int dcfg_partn;
+	unsigned int hdr_length;
+	unsigned int pos;
+	uint8_t cksum;
+	size_t partn_size, dcfg_size;
+	size_t vpd_length;
+	int rc;
+
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	/* Determine total length of all tags */
+	if ((rc = efx_vpd_hunk_length(data, size, &vpd_length)) != 0)
+		goto fail1;
+
+	/* Lock dynamic config sector for write, and read structure only */
+	dcfg_partn = (emip->emi_port == 1)
+		? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0
+		: MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1;
+
+	if ((rc = siena_nvram_partn_size(enp, dcfg_partn, &partn_size)) != 0)
+		goto fail2;
+
+	if ((rc = siena_nvram_partn_lock(enp, dcfg_partn)) != 0)
+		goto fail2;
+
+	if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn,
+	    B_FALSE, &dcfg, &dcfg_size)) != 0)
+		goto fail3;
+
+	hdr_length = EFX_WORD_FIELD(dcfg->length, EFX_WORD_0);
+
+	/* Allocated memory should have room for the new VPD */
+	if (hdr_length + vpd_length > dcfg_size) {
+		rc = ENOSPC;
+		goto fail3;
+	}
+
+	/* Copy in new vpd and update header */
+	vpd_offset = dcfg_size - vpd_length;
+	EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_offset,
+			     EFX_DWORD_0, vpd_offset);
+	memcpy((caddr_t)dcfg + vpd_offset, data, vpd_length);
+	EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_length,
+			    EFX_DWORD_0, vpd_length);
+
+	/* Update the checksum */
+	cksum = 0;
+	for (pos = 0; pos < hdr_length; pos++)
+		cksum += ((uint8_t *)dcfg)[pos];
+	dcfg->csum.eb_u8[0] -= cksum;
+
+	/* Erase and write the new sector */
+	if ((rc = siena_nvram_partn_erase(enp, dcfg_partn, 0, partn_size)) != 0)
+		goto fail4;
+
+	/* Write out the new structure to nvram */
+	if ((rc = siena_nvram_partn_write(enp, dcfg_partn, 0, (caddr_t)dcfg,
+	    vpd_offset + vpd_length)) != 0)
+		goto fail5;
+
+	EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg);
+
+	siena_nvram_partn_unlock(enp, dcfg_partn);
+
+	return (0);
+
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+
+	EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg);
+fail2:
+	EFSYS_PROBE(fail2);
+
+	siena_nvram_partn_unlock(enp, dcfg_partn);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	return (rc);
+}
+
+				void
+siena_vpd_fini(
+	__in			efx_nic_t *enp)
+{
+	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
+
+	if (enp->en_u.siena.enu_svpd_length > 0) {
+		EFSYS_KMEM_FREE(enp->en_esip, enp->en_u.siena.enu_svpd_length,
+				enp->en_u.siena.enu_svpd);
+
+		enp->en_u.siena.enu_svpd = NULL;
+		enp->en_u.siena.enu_svpd_length = 0;
+	}
+}
+
+#endif	/* EFSYS_OPT_SIENA */
+
+#endif	/* EFSYS_OPT_VPD */
diff --git a/sys/dev/sfxge/sfxge.c b/sys/dev/sfxge/sfxge.c
new file mode 100644
index 0000000..380215a
--- /dev/null
+++ b/sys/dev/sfxge/sfxge.c
@@ -0,0 +1,775 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/taskqueue.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+
+#include "common/efx.h"
+
+#include "sfxge.h"
+#include "sfxge_rx.h"
+
+#define SFXGE_CAP (IFCAP_VLAN_MTU | \
+		   IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO |	\
+		   IFCAP_JUMBO_MTU | IFCAP_LRO |			\
+		   IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE)
+#define SFXGE_CAP_ENABLE SFXGE_CAP
+#define SFXGE_CAP_FIXED (IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | \
+			 IFCAP_JUMBO_MTU | IFCAP_LINKSTATE)
+
+MALLOC_DEFINE(M_SFXGE, "sfxge", "Solarflare 10GigE driver");
+
+static void
+sfxge_reset(void *arg, int npending);
+
+static int
+sfxge_start(struct sfxge_softc *sc)
+{
+	int rc;
+
+	sx_assert(&sc->softc_lock, LA_XLOCKED);
+
+	if (sc->init_state == SFXGE_STARTED)
+		return 0;
+
+	if (sc->init_state != SFXGE_REGISTERED) {
+		rc = EINVAL;
+		goto fail;
+	}
+
+	if ((rc = efx_nic_init(sc->enp)) != 0)
+		goto fail;
+
+	/* Start processing interrupts. */
+	if ((rc = sfxge_intr_start(sc)) != 0)
+		goto fail2;
+
+	/* Start processing events. */
+	if ((rc = sfxge_ev_start(sc)) != 0)
+		goto fail3;
+
+	/* Start the receiver side. */
+	if ((rc = sfxge_rx_start(sc)) != 0)
+		goto fail4;
+
+	/* Start the transmitter side. */
+	if ((rc = sfxge_tx_start(sc)) != 0)
+		goto fail5;
+
+	/* Fire up the port. */
+	if ((rc = sfxge_port_start(sc)) != 0)
+		goto fail6;
+
+	sc->init_state = SFXGE_STARTED;
+
+	/* Tell the stack we're running. */
+	sc->ifnet->if_drv_flags |= IFF_DRV_RUNNING;
+	sc->ifnet->if_drv_flags &= ~IFF_DRV_OACTIVE;
+
+	return (0);
+
+fail6:
+	sfxge_tx_stop(sc);
+
+fail5:
+	sfxge_rx_stop(sc);
+
+fail4:
+	sfxge_ev_stop(sc);
+
+fail3:
+	sfxge_intr_stop(sc);
+
+fail2:
+	efx_nic_fini(sc->enp);
+
+fail:
+	device_printf(sc->dev, "sfxge_start: %d\n", rc);
+
+	return (rc);
+}
+
+static void
+sfxge_if_init(void *arg)
+{
+	struct sfxge_softc *sc;
+
+	sc = (struct sfxge_softc *)arg;
+
+	sx_xlock(&sc->softc_lock);
+	(void)sfxge_start(sc);
+	sx_xunlock(&sc->softc_lock);
+}
+
+static void
+sfxge_stop(struct sfxge_softc *sc)
+{
+	sx_assert(&sc->softc_lock, LA_XLOCKED);
+
+	if (sc->init_state != SFXGE_STARTED)
+		return;
+
+	sc->init_state = SFXGE_REGISTERED;
+
+	/* Stop the port. */
+	sfxge_port_stop(sc);
+
+	/* Stop the transmitter. */
+	sfxge_tx_stop(sc);
+
+	/* Stop the receiver. */
+	sfxge_rx_stop(sc);
+
+	/* Stop processing events. */
+	sfxge_ev_stop(sc);
+
+	/* Stop processing interrupts. */
+	sfxge_intr_stop(sc);
+
+	efx_nic_fini(sc->enp);
+
+	sc->ifnet->if_drv_flags &= ~IFF_DRV_RUNNING;
+}
+
+static int
+sfxge_if_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
+{
+	struct sfxge_softc *sc;
+	struct ifreq *ifr;
+	int error;
+
+	ifr = (struct ifreq *)data;
+	sc = ifp->if_softc;
+	error = 0;
+
+	switch (command) {
+	case SIOCSIFFLAGS:
+		sx_xlock(&sc->softc_lock);
+		if (ifp->if_flags & IFF_UP) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+				if ((ifp->if_flags ^ sc->if_flags) &
+				    (IFF_PROMISC | IFF_ALLMULTI)) {
+					sfxge_mac_filter_set(sc);
+				}
+			} else
+				sfxge_start(sc);
+		} else
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+				sfxge_stop(sc);
+		sc->if_flags = ifp->if_flags;
+		sx_xunlock(&sc->softc_lock);
+		break;
+	case SIOCSIFMTU:
+		if (ifr->ifr_mtu == ifp->if_mtu) {
+			/* Nothing to do */
+			error = 0;
+		} else if (ifr->ifr_mtu > SFXGE_MAX_MTU) {
+			error = EINVAL;
+		} else if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+			ifp->if_mtu = ifr->ifr_mtu;
+			error = 0;
+		} else {
+			/* Restart required */
+			sx_xlock(&sc->softc_lock);
+			sfxge_stop(sc);
+			ifp->if_mtu = ifr->ifr_mtu;
+			error = sfxge_start(sc);
+			sx_xunlock(&sc->softc_lock);
+			if (error) {
+				ifp->if_flags &= ~IFF_UP;
+				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+				if_down(ifp);
+			}
+		}
+		break;
+	case SIOCADDMULTI:
+	case SIOCDELMULTI:
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			sfxge_mac_filter_set(sc);
+		break;
+	case SIOCSIFCAP:
+		sx_xlock(&sc->softc_lock);
+
+		/*
+		 * The networking core already rejects attempts to
+		 * enable capabilities we don't have.  We still have
+		 * to reject attempts to disable capabilities that we
+		 * can't (yet) disable.
+		 */
+		if (~ifr->ifr_reqcap & SFXGE_CAP_FIXED) {
+			error = EINVAL;
+			sx_xunlock(&sc->softc_lock);
+			break;
+		}
+
+		ifp->if_capenable = ifr->ifr_reqcap;
+		if (ifp->if_capenable & IFCAP_TXCSUM)
+			ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
+		else
+			ifp->if_hwassist &= ~(CSUM_IP | CSUM_TCP | CSUM_UDP);
+		if (ifp->if_capenable & IFCAP_TSO)
+			ifp->if_hwassist |= CSUM_TSO;
+		else
+			ifp->if_hwassist &= ~CSUM_TSO;
+
+		sx_xunlock(&sc->softc_lock);
+		break;
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
+		break;
+	default:
+		error = ether_ioctl(ifp, command, data);
+	}
+
+	return (error);
+}
+
+static void
+sfxge_ifnet_fini(struct ifnet *ifp)
+{
+	struct sfxge_softc *sc = ifp->if_softc;
+
+	sx_xlock(&sc->softc_lock);
+	sfxge_stop(sc);
+	sx_xunlock(&sc->softc_lock);
+
+	ifmedia_removeall(&sc->media);
+	ether_ifdetach(ifp);
+	if_free(ifp);
+}
+
+static int 
+sfxge_ifnet_init(struct ifnet *ifp, struct sfxge_softc *sc)
+{
+	const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp);
+	device_t dev;
+	int rc;
+
+	dev = sc->dev;
+	sc->ifnet = ifp;
+
+	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
+	ifp->if_init = sfxge_if_init;
+	ifp->if_softc = sc;
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_ioctl = sfxge_if_ioctl;
+
+	ifp->if_capabilities = SFXGE_CAP;
+	ifp->if_capenable = SFXGE_CAP_ENABLE;
+	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
+
+	ether_ifattach(ifp, encp->enc_mac_addr);
+
+#ifdef SFXGE_HAVE_MQ
+	ifp->if_transmit = sfxge_if_transmit;
+	ifp->if_qflush = sfxge_if_qflush;
+#else
+	ifp->if_start = sfxge_if_start;
+	IFQ_SET_MAXLEN(&ifp->if_snd, SFXGE_NDESCS - 1);
+	ifp->if_snd.ifq_drv_maxlen = SFXGE_NDESCS - 1;
+	IFQ_SET_READY(&ifp->if_snd);
+
+	mtx_init(&sc->tx_lock, "txq", NULL, MTX_DEF);
+#endif
+
+	if ((rc = sfxge_port_ifmedia_init(sc)) != 0)
+		goto fail;
+
+	return 0;
+
+fail:
+	ether_ifdetach(sc->ifnet);
+	return rc;
+}
+
+void
+sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp)
+{
+	KASSERT(sc->buffer_table_next + n <=
+		efx_nic_cfg_get(sc->enp)->enc_buftbl_limit,
+		("buffer table full"));
+
+	*idp = sc->buffer_table_next;
+	sc->buffer_table_next += n;
+}
+
+static int
+sfxge_bar_init(struct sfxge_softc *sc)
+{
+	efsys_bar_t *esbp = &sc->bar;
+
+	esbp->esb_rid = PCIR_BAR(EFX_MEM_BAR);	
+	if ((esbp->esb_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
+	    &esbp->esb_rid, RF_ACTIVE)) == NULL) {
+		device_printf(sc->dev, "Cannot allocate BAR region %d\n",
+		    EFX_MEM_BAR);
+		return (ENXIO);
+	}
+	esbp->esb_tag = rman_get_bustag(esbp->esb_res);
+	esbp->esb_handle = rman_get_bushandle(esbp->esb_res);
+	mtx_init(&esbp->esb_lock, "sfxge_efsys_bar", NULL, MTX_DEF);
+
+	return (0);
+}
+
+static void
+sfxge_bar_fini(struct sfxge_softc *sc)
+{
+	efsys_bar_t *esbp = &sc->bar;
+
+	bus_release_resource(sc->dev, SYS_RES_MEMORY, esbp->esb_rid,
+	    esbp->esb_res);
+	mtx_destroy(&esbp->esb_lock);
+}
+
+static int
+sfxge_create(struct sfxge_softc *sc)
+{
+	device_t dev;
+	efx_nic_t *enp;
+	int error;
+
+	dev = sc->dev;
+
+	sx_init(&sc->softc_lock, "sfxge_softc");
+
+	sc->stats_node = SYSCTL_ADD_NODE(
+		device_get_sysctl_ctx(dev),
+		SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+		OID_AUTO, "stats", CTLFLAG_RD, NULL, "Statistics");
+	if (!sc->stats_node) {
+		error = ENOMEM;
+		goto fail;
+	}
+
+	TASK_INIT(&sc->task_reset, 0, sfxge_reset, sc);
+
+	(void) pci_enable_busmaster(dev);
+
+	/* Initialize DMA mappings. */
+	if ((error = sfxge_dma_init(sc)) != 0)
+		goto fail;
+
+	/* Map the device registers. */
+	if ((error = sfxge_bar_init(sc)) != 0)
+		goto fail;
+
+	error = efx_family(pci_get_vendor(dev), pci_get_device(dev),
+	    &sc->family);
+	KASSERT(error == 0, ("Family should be filtered by sfxge_probe()"));
+
+	/* Create the common code nic object. */
+	mtx_init(&sc->enp_lock, "sfxge_nic", NULL, MTX_DEF);
+	if ((error = efx_nic_create(sc->family, (efsys_identifier_t *)sc,
+	    &sc->bar, &sc->enp_lock, &enp)) != 0)
+		goto fail3;
+	sc->enp = enp;
+
+	/* Initialize MCDI to talk to the microcontroller. */
+	if ((error = sfxge_mcdi_init(sc)) != 0)
+		goto fail4;
+
+	/* Probe the NIC and build the configuration data area. */
+	if ((error = efx_nic_probe(enp)) != 0)
+		goto fail5;
+
+	/* Initialize the NVRAM. */
+	if ((error = efx_nvram_init(enp)) != 0)
+		goto fail6;
+
+	/* Initialize the VPD. */
+	if ((error = efx_vpd_init(enp)) != 0)
+		goto fail7;
+
+	/* Reset the NIC. */
+	if ((error = efx_nic_reset(enp)) != 0)
+		goto fail8;
+
+	/* Initialize buffer table allocation. */
+	sc->buffer_table_next = 0;
+
+	/* Set up interrupts. */
+	if ((error = sfxge_intr_init(sc)) != 0)
+		goto fail8;
+
+	/* Initialize event processing state. */
+	if ((error = sfxge_ev_init(sc)) != 0)
+		goto fail11;
+
+	/* Initialize receive state. */
+	if ((error = sfxge_rx_init(sc)) != 0)
+		goto fail12;
+
+	/* Initialize transmit state. */
+	if ((error = sfxge_tx_init(sc)) != 0)
+		goto fail13;
+
+	/* Initialize port state. */
+	if ((error = sfxge_port_init(sc)) != 0)
+		goto fail14;
+
+	sc->init_state = SFXGE_INITIALIZED;
+
+	return (0);
+
+fail14:
+	sfxge_tx_fini(sc);
+
+fail13:
+	sfxge_rx_fini(sc);
+
+fail12:
+	sfxge_ev_fini(sc);
+
+fail11:
+	sfxge_intr_fini(sc);
+
+fail8:
+	efx_vpd_fini(enp);
+
+fail7:
+	efx_nvram_fini(enp);
+
+fail6:
+	efx_nic_unprobe(enp);
+
+fail5:
+	sfxge_mcdi_fini(sc);
+
+fail4:
+	sc->enp = NULL;
+	efx_nic_destroy(enp);
+	mtx_destroy(&sc->enp_lock);
+
+fail3:
+	sfxge_bar_fini(sc);
+	(void) pci_disable_busmaster(sc->dev);
+
+fail:
+	sc->dev = NULL;
+	sx_destroy(&sc->softc_lock);
+	return (error);
+}
+
+static void
+sfxge_destroy(struct sfxge_softc *sc)
+{
+	efx_nic_t *enp;
+
+	/* Clean up port state. */
+	sfxge_port_fini(sc);
+
+	/* Clean up transmit state. */
+	sfxge_tx_fini(sc);
+
+	/* Clean up receive state. */
+	sfxge_rx_fini(sc);
+
+	/* Clean up event processing state. */
+	sfxge_ev_fini(sc);
+
+	/* Clean up interrupts. */
+	sfxge_intr_fini(sc);
+
+	/* Tear down common code subsystems. */
+	efx_nic_reset(sc->enp);
+	efx_vpd_fini(sc->enp);
+	efx_nvram_fini(sc->enp);
+	efx_nic_unprobe(sc->enp);
+
+	/* Tear down MCDI. */
+	sfxge_mcdi_fini(sc);
+
+	/* Destroy common code context. */
+	enp = sc->enp;
+	sc->enp = NULL;
+	efx_nic_destroy(enp);
+
+	/* Free DMA memory. */
+	sfxge_dma_fini(sc);
+
+	/* Free mapped BARs. */
+	sfxge_bar_fini(sc);
+
+	(void) pci_disable_busmaster(sc->dev);
+
+	taskqueue_drain(taskqueue_thread, &sc->task_reset);
+
+	/* Destroy the softc lock. */
+	sx_destroy(&sc->softc_lock);
+}
+
+static int
+sfxge_vpd_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	efx_vpd_value_t value;
+	int rc;
+	
+	value.evv_tag = arg2 >> 16;
+	value.evv_keyword = arg2 & 0xffff;
+	if ((rc = efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value))
+	    != 0)
+		return rc;
+
+	return SYSCTL_OUT(req, value.evv_value, value.evv_length);
+}
+
+static void
+sfxge_vpd_try_add(struct sfxge_softc *sc, struct sysctl_oid_list *list,
+		  efx_vpd_tag_t tag, const char *keyword)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	efx_vpd_value_t value;
+
+	/* Check whether VPD tag/keyword is present */
+	value.evv_tag = tag;
+	value.evv_keyword = EFX_VPD_KEYWORD(keyword[0], keyword[1]);
+	if (efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value) != 0)
+		return;
+
+	SYSCTL_ADD_PROC(
+		ctx, list, OID_AUTO, keyword, CTLTYPE_STRING|CTLFLAG_RD,
+		sc, tag << 16 | EFX_VPD_KEYWORD(keyword[0], keyword[1]),
+		sfxge_vpd_handler, "A", "");
+}
+
+static int
+sfxge_vpd_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid *vpd_node;
+	struct sysctl_oid_list *vpd_list;
+	char keyword[3];
+	efx_vpd_value_t value;
+	int rc;
+
+	if ((rc = efx_vpd_size(sc->enp, &sc->vpd_size)) != 0)
+		goto fail;
+	sc->vpd_data = malloc(sc->vpd_size, M_SFXGE, M_WAITOK);
+	if ((rc = efx_vpd_read(sc->enp, sc->vpd_data, sc->vpd_size)) != 0)
+		goto fail2;
+
+	/* Copy ID (product name) into device description, and log it. */
+	value.evv_tag = EFX_VPD_ID;
+	if (efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value) == 0) {
+		value.evv_value[value.evv_length] = 0;
+		device_set_desc_copy(sc->dev, value.evv_value);
+		device_printf(sc->dev, "%s\n", value.evv_value);
+	}
+
+	vpd_node = SYSCTL_ADD_NODE(
+		ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)),
+		OID_AUTO, "vpd", CTLFLAG_RD, NULL, "Vital Product Data");
+	vpd_list = SYSCTL_CHILDREN(vpd_node);
+
+	/* Add sysctls for all expected and any vendor-defined keywords. */
+	sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "PN");
+	sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "EC");
+	sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "SN");
+	keyword[0] = 'V';
+	keyword[2] = 0;
+	for (keyword[1] = '0'; keyword[1] <= '9'; keyword[1]++)
+		sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, keyword);
+	for (keyword[1] = 'A'; keyword[1] <= 'Z'; keyword[1]++)
+		sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, keyword);
+
+	return 0;
+	
+fail2:
+	free(sc->vpd_data, M_SFXGE);
+fail:
+	return rc;
+}
+
+static void
+sfxge_vpd_fini(struct sfxge_softc *sc)
+{
+	free(sc->vpd_data, M_SFXGE);
+}
+
+static void
+sfxge_reset(void *arg, int npending)
+{
+	struct sfxge_softc *sc;
+	int rc;
+
+	(void)npending;
+
+	sc = (struct sfxge_softc *)arg;
+
+	sx_xlock(&sc->softc_lock);
+
+	if (sc->init_state != SFXGE_STARTED)
+		goto done;
+
+	sfxge_stop(sc);
+	efx_nic_reset(sc->enp);
+	if ((rc = sfxge_start(sc)) != 0)
+		device_printf(sc->dev,
+			      "reset failed (%d); interface is now stopped\n",
+			      rc);
+
+done:
+	sx_xunlock(&sc->softc_lock);
+}
+
+void
+sfxge_schedule_reset(struct sfxge_softc *sc)
+{
+	taskqueue_enqueue(taskqueue_thread, &sc->task_reset);
+}
+
+static int
+sfxge_attach(device_t dev)
+{
+	struct sfxge_softc *sc;
+	struct ifnet *ifp;
+	int error;
+
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+
+	/* Allocate ifnet. */
+	ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		device_printf(dev, "Couldn't allocate ifnet\n");
+		error = ENOMEM;
+		goto fail;
+	}
+	sc->ifnet = ifp;
+
+	/* Initialize hardware. */
+	if ((error = sfxge_create(sc)) != 0)
+		goto fail2;
+
+	/* Create the ifnet for the port. */
+	if ((error = sfxge_ifnet_init(ifp, sc)) != 0)
+		goto fail3;
+
+	if ((error = sfxge_vpd_init(sc)) != 0)
+		goto fail4;
+
+	sc->init_state = SFXGE_REGISTERED;
+
+	return (0);
+
+fail4:
+	sfxge_ifnet_fini(ifp);
+fail3:
+	sfxge_destroy(sc);
+
+fail2:
+	if_free(sc->ifnet);
+
+fail:
+	return (error);
+}
+
+static int
+sfxge_detach(device_t dev)
+{
+	struct sfxge_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	sfxge_vpd_fini(sc);
+
+	/* Destroy the ifnet. */
+	sfxge_ifnet_fini(sc->ifnet);
+
+	/* Tear down hardware. */
+	sfxge_destroy(sc);
+
+	return (0);
+}
+
+static int
+sfxge_probe(device_t dev)
+{
+	uint16_t pci_vendor_id;
+	uint16_t pci_device_id;
+	efx_family_t family;
+	int rc;
+
+	pci_vendor_id = pci_get_vendor(dev);
+	pci_device_id = pci_get_device(dev);
+
+	rc = efx_family(pci_vendor_id, pci_device_id, &family);
+	if (rc)
+		return ENXIO;
+
+	KASSERT(family == EFX_FAMILY_SIENA, ("impossible controller family"));
+	device_set_desc(dev, "Solarflare SFC9000 family");
+	return 0;
+}
+
+static device_method_t sfxge_methods[] = {
+	DEVMETHOD(device_probe,		sfxge_probe),
+	DEVMETHOD(device_attach,	sfxge_attach),
+	DEVMETHOD(device_detach,	sfxge_detach),
+
+	/* Bus interface. */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
+
+	{ 0, 0 }
+};
+
+static devclass_t sfxge_devclass;
+
+static driver_t sfxge_driver = {
+	"sfxge",
+	sfxge_methods,
+	sizeof(struct sfxge_softc)
+};
+
+DRIVER_MODULE(sfxge, pci, sfxge_driver, sfxge_devclass, 0, 0);
diff --git a/sys/dev/sfxge/sfxge.h b/sys/dev/sfxge/sfxge.h
new file mode 100644
index 0000000..2d3e042
--- /dev/null
+++ b/sys/dev/sfxge/sfxge.h
@@ -0,0 +1,303 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SFXGE_H
+#define _SFXGE_H
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/condvar.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/sx.h>
+#include <vm/uma.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+
+/*
+ * Backward-compatibility
+ */
+#ifndef CACHE_LINE_SIZE
+/* This should be right on most machines the driver will be used on, and
+ * we needn't care too much about wasting a few KB per interface.
+ */
+#define CACHE_LINE_SIZE 128
+#endif
+#ifndef IFCAP_LINKSTATE
+#define IFCAP_LINKSTATE 0
+#endif
+#ifndef IFCAP_VLAN_HWTSO
+#define IFCAP_VLAN_HWTSO 0
+#endif
+#ifndef IFM_10G_T
+#define IFM_10G_T IFM_UNKNOWN
+#endif
+#ifndef IFM_10G_KX4
+#define IFM_10G_KX4 IFM_10G_CX4
+#endif
+#if __FreeBSD_version >= 800054
+/* Networking core is multiqueue aware. We can manage our own TX
+ * queues and use m_pkthdr.flowid.
+ */
+#define SFXGE_HAVE_MQ
+#endif
+#if (__FreeBSD_version >= 800501 && __FreeBSD_version < 900000) || \
+	__FreeBSD_version >= 900003
+#define SFXGE_HAVE_DESCRIBE_INTR
+#endif
+#ifdef IFM_ETH_RXPAUSE
+#define SFXGE_HAVE_PAUSE_MEDIAOPTS
+#endif
+#ifndef CTLTYPE_U64
+#define CTLTYPE_U64 CTLTYPE_QUAD
+#endif
+
+#include "sfxge_rx.h"
+#include "sfxge_tx.h"
+
+#define SFXGE_IP_ALIGN 2
+
+#define SFXGE_ETHERTYPE_LOOPBACK        0x9000  /* Xerox loopback */
+
+enum sfxge_evq_state {
+	SFXGE_EVQ_UNINITIALIZED = 0,
+	SFXGE_EVQ_INITIALIZED,
+	SFXGE_EVQ_STARTING,
+	SFXGE_EVQ_STARTED
+};
+
+#define	SFXGE_EV_BATCH	16384
+
+struct sfxge_evq {
+	struct sfxge_softc	*sc  __aligned(CACHE_LINE_SIZE);
+	struct mtx		lock __aligned(CACHE_LINE_SIZE);
+
+	enum sfxge_evq_state	init_state;
+	unsigned int		index;
+	efsys_mem_t		mem;
+	unsigned int		buf_base_id;
+
+	boolean_t		exception;
+
+	efx_evq_t		*common;
+	unsigned int		read_ptr;
+	unsigned int		rx_done;
+	unsigned int		tx_done;
+
+	/* Linked list of TX queues with completions to process */
+	struct sfxge_txq	*txq;
+	struct sfxge_txq	**txqs;
+};
+
+#define	SFXGE_NEVS	4096
+#define	SFXGE_NDESCS	1024
+#define	SFXGE_MODERATION	30
+
+enum sfxge_intr_state {
+	SFXGE_INTR_UNINITIALIZED = 0,
+	SFXGE_INTR_INITIALIZED,
+	SFXGE_INTR_TESTING,
+	SFXGE_INTR_STARTED
+};
+
+struct sfxge_intr_hdl {
+	int                eih_rid;
+	void               *eih_tag;
+	struct resource    *eih_res;
+};
+
+struct sfxge_intr {
+	enum sfxge_intr_state	state;
+	struct resource		*msix_res;
+	struct sfxge_intr_hdl	*table;
+	int			n_alloc;
+	int			type;
+	efsys_mem_t		status;
+	uint32_t		zero_count;
+};
+
+enum sfxge_mcdi_state {
+	SFXGE_MCDI_UNINITIALIZED = 0,
+	SFXGE_MCDI_INITIALIZED,
+	SFXGE_MCDI_BUSY,
+	SFXGE_MCDI_COMPLETED
+};
+
+struct sfxge_mcdi {
+	struct mtx		lock;
+	struct cv		cv;
+	enum sfxge_mcdi_state	state;
+	efx_mcdi_transport_t	transport;
+};
+
+struct sfxge_hw_stats {
+	clock_t			update_time;
+	efsys_mem_t		dma_buf;
+	void			*decode_buf;
+};
+
+enum sfxge_port_state {
+	SFXGE_PORT_UNINITIALIZED = 0,
+	SFXGE_PORT_INITIALIZED,
+	SFXGE_PORT_STARTED
+};
+
+struct sfxge_port {
+	struct sfxge_softc	*sc;
+	struct mtx		lock;
+	enum sfxge_port_state	init_state;
+#ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS
+	unsigned int		wanted_fc;
+#endif
+	struct sfxge_hw_stats	phy_stats;
+	struct sfxge_hw_stats	mac_stats;
+	efx_link_mode_t		link_mode;
+};
+
+enum sfxge_softc_state {
+	SFXGE_UNINITIALIZED = 0,
+	SFXGE_INITIALIZED,
+	SFXGE_REGISTERED,
+	SFXGE_STARTED
+};
+
+struct sfxge_softc {
+	device_t			dev;
+	struct sx			softc_lock;
+	enum sfxge_softc_state		init_state;
+	struct ifnet                    *ifnet;
+	unsigned int			if_flags;
+	struct sysctl_oid		*stats_node;
+
+	struct task			task_reset;
+
+	efx_family_t			family;
+	caddr_t				vpd_data;
+	size_t				vpd_size;
+	efx_nic_t			*enp;
+	struct mtx			enp_lock;
+
+	bus_dma_tag_t                   parent_dma_tag;
+	efsys_bar_t			bar;
+
+	struct sfxge_intr		intr;
+	struct sfxge_mcdi		mcdi;
+	struct sfxge_port		port;
+	uint32_t			buffer_table_next;
+
+	struct sfxge_evq		*evq[SFXGE_RX_SCALE_MAX];
+	unsigned int			ev_moderation;
+	clock_t				ev_stats_update_time;
+	uint64_t			ev_stats[EV_NQSTATS];
+
+	uma_zone_t			rxq_cache;
+	struct sfxge_rxq		*rxq[SFXGE_RX_SCALE_MAX];
+	unsigned int			rx_indir_table[SFXGE_RX_SCALE_MAX];
+
+#ifdef SFXGE_HAVE_MQ
+	struct sfxge_txq		*txq[SFXGE_TXQ_NTYPES + SFXGE_RX_SCALE_MAX];
+#else
+	struct sfxge_txq		*txq[SFXGE_TXQ_NTYPES];
+#endif
+
+	struct ifmedia			media;
+
+	size_t				rx_prefix_size;
+	size_t				rx_buffer_size;
+	uma_zone_t			rx_buffer_zone;
+
+#ifndef SFXGE_HAVE_MQ
+	struct mtx			tx_lock __aligned(CACHE_LINE_SIZE);
+#endif
+};
+
+#define SFXGE_LINK_UP(sc) ((sc)->port.link_mode != EFX_LINK_DOWN)
+#define SFXGE_RUNNING(sc) ((sc)->ifnet->if_drv_flags & IFF_DRV_RUNNING)
+
+/*
+ * From sfxge.c.
+ */
+extern void sfxge_schedule_reset(struct sfxge_softc *sc);
+extern void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n,
+				     uint32_t *idp);
+
+/*
+ * From sfxge_dma.c.
+ */
+extern int sfxge_dma_init(struct sfxge_softc *sc);
+extern void sfxge_dma_fini(struct sfxge_softc *sc);
+extern int sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len,
+    efsys_mem_t *esmp);
+extern void sfxge_dma_free(efsys_mem_t *esmp);
+extern int sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map,
+    struct mbuf **mp, bus_dma_segment_t *segs, int *nsegs, int maxsegs);
+
+/*
+ * From sfxge_ev.c.
+ */
+extern int sfxge_ev_init(struct sfxge_softc *sc);
+extern void sfxge_ev_fini(struct sfxge_softc *sc);
+extern int sfxge_ev_start(struct sfxge_softc *sc);
+extern void sfxge_ev_stop(struct sfxge_softc *sc);
+extern int sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index);
+
+/*
+ * From sfxge_intr.c.
+ */
+extern int sfxge_intr_init(struct sfxge_softc *sc);
+extern void sfxge_intr_fini(struct sfxge_softc *sc);
+extern int sfxge_intr_start(struct sfxge_softc *sc);
+extern void sfxge_intr_stop(struct sfxge_softc *sc);
+
+/*
+ * From sfxge_mcdi.c.
+ */
+extern int sfxge_mcdi_init(struct sfxge_softc *sc);
+extern void sfxge_mcdi_fini(struct sfxge_softc *sc);
+
+/*
+ * From sfxge_port.c.
+ */
+extern int sfxge_port_init(struct sfxge_softc *sc);
+extern void sfxge_port_fini(struct sfxge_softc *sc);
+extern int sfxge_port_start(struct sfxge_softc *sc);
+extern void sfxge_port_stop(struct sfxge_softc *sc);
+extern void sfxge_mac_link_update(struct sfxge_softc *sc,
+    efx_link_mode_t mode);
+extern int sfxge_mac_filter_set(struct sfxge_softc *sc);
+extern int sfxge_port_ifmedia_init(struct sfxge_softc *sc);
+
+#define SFXGE_MAX_MTU (9 * 1024)
+
+#endif /* _SFXGE_H */
diff --git a/sys/dev/sfxge/sfxge_dma.c b/sys/dev/sfxge/sfxge_dma.c
new file mode 100644
index 0000000..076c7c6
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_dma.c
@@ -0,0 +1,202 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+
+#include <machine/bus.h>
+
+#include "common/efx.h"
+
+#include "sfxge.h"
+
+static void
+sfxge_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	bus_addr_t *addr;
+
+	addr = arg;
+
+	if (error) {
+		*addr = 0;
+		return;
+	}
+
+	*addr = segs[0].ds_addr;
+}
+
+int
+sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map,
+    struct mbuf **mp, bus_dma_segment_t *segs, int *nsegs, int maxsegs)
+{
+	bus_dma_segment_t *psegs;
+	struct mbuf *m;
+	int seg_count;
+	int defragged;
+	int err;
+
+	m = *mp;
+	defragged = err = seg_count = 0;
+
+	KASSERT(m->m_pkthdr.len, ("packet has zero header length"));
+
+retry:
+	psegs = segs;
+	seg_count = 0;
+	if (m->m_next == NULL) {
+		sfxge_map_mbuf_fast(tag, map, m, segs);
+		*nsegs = 1;
+		return (0);
+	}
+#if defined(__i386__) || defined(__amd64__)
+	while (m && seg_count < maxsegs) {
+		/*
+		 * firmware doesn't like empty segments
+		 */
+		if (m->m_len != 0) {
+			seg_count++;
+			sfxge_map_mbuf_fast(tag, map, m, psegs);
+			psegs++;
+		}
+		m = m->m_next;
+	}
+#else
+	err = bus_dmamap_load_mbuf_sg(tag, map, *mp, segs, &seg_count, 0);
+#endif
+	if (seg_count == 0) {
+		err = EFBIG;
+		goto err_out;
+	} else if (err == EFBIG || seg_count >= maxsegs) {
+		if (!defragged) {
+			m = m_defrag(*mp, M_DONTWAIT);
+			if (m == NULL) {
+				err = ENOBUFS;
+				goto err_out;
+			}
+			*mp = m;
+			defragged = 1;
+			goto retry;
+		}
+		err = EFBIG;
+		goto err_out;
+	}
+	*nsegs = seg_count;
+
+err_out:
+	return (err);
+}
+
+void
+sfxge_dma_free(efsys_mem_t *esmp)
+{
+
+	bus_dmamap_unload(esmp->esm_tag, esmp->esm_map);
+	bus_dmamem_free(esmp->esm_tag, esmp->esm_base, esmp->esm_map);
+	bus_dma_tag_destroy(esmp->esm_tag);
+
+	esmp->esm_addr = 0;
+	esmp->esm_base = NULL;
+}
+
+int
+sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len, efsys_mem_t *esmp)
+{
+	void *vaddr;
+
+	/* Create the child DMA tag. */
+	if (bus_dma_tag_create(sc->parent_dma_tag, PAGE_SIZE, 0,
+	    MIN(0x3FFFFFFFFFFFUL, BUS_SPACE_MAXADDR), BUS_SPACE_MAXADDR, NULL,
+	    NULL, len, 1, len, 0, NULL, NULL, &esmp->esm_tag) != 0) {
+		device_printf(sc->dev, "Couldn't allocate txq DMA tag\n");
+		return (ENOMEM);
+	}
+
+	/* Allocate kernel memory. */
+	if (bus_dmamem_alloc(esmp->esm_tag, (void **)&vaddr,
+	    BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
+	    &esmp->esm_map) != 0) {
+		device_printf(sc->dev, "Couldn't allocate DMA memory\n");
+		bus_dma_tag_destroy(esmp->esm_tag);
+		return (ENOMEM);
+	}
+
+	/* Load map into device memory. */
+	if (bus_dmamap_load(esmp->esm_tag, esmp->esm_map, vaddr, len,
+	    sfxge_dma_cb, &esmp->esm_addr, 0) != 0) {
+		device_printf(sc->dev, "Couldn't load DMA mapping\n");
+		bus_dmamem_free(esmp->esm_tag, esmp->esm_base, esmp->esm_map);
+		bus_dma_tag_destroy(esmp->esm_tag);
+		return (ENOMEM);
+	}
+
+	/*
+	 * The callback gets error information about the mapping
+	 * and will have set our vaddr to NULL if something went
+	 * wrong.
+	 */
+	if (vaddr == NULL)
+		return (ENOMEM);
+
+	esmp->esm_base = vaddr;
+
+	return (0);
+}
+
+void
+sfxge_dma_fini(struct sfxge_softc *sc)
+{
+
+	bus_dma_tag_destroy(sc->parent_dma_tag);
+}
+
+int
+sfxge_dma_init(struct sfxge_softc *sc)
+{
+
+	/* Create the parent dma tag. */
+	if (bus_dma_tag_create(bus_get_dma_tag(sc->dev),	/* parent */
+	    1, 0,			/* algnmnt, boundary */
+	    BUS_SPACE_MAXADDR,		/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    BUS_SPACE_MAXSIZE_32BIT,	/* maxsize */
+	    BUS_SPACE_UNRESTRICTED,	/* nsegments */
+	    BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lock, lockarg */
+	    &sc->parent_dma_tag)) {
+		device_printf(sc->dev, "Cannot allocate parent DMA tag\n");
+		return (ENOMEM);
+	}
+
+	return (0);
+}
diff --git a/sys/dev/sfxge/sfxge_ev.c b/sys/dev/sfxge/sfxge_ev.c
new file mode 100644
index 0000000..b506b27
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_ev.c
@@ -0,0 +1,862 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include "common/efx.h"
+
+#include "sfxge.h"
+
+static void
+sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
+{
+	struct sfxge_softc *sc;
+	unsigned int index;
+	struct sfxge_rxq *rxq;
+	struct sfxge_txq *txq;
+
+	sc = evq->sc;
+	index = evq->index;
+	rxq = sc->rxq[index];
+
+	if ((txq = evq->txq) != NULL) {
+		evq->txq = NULL;
+		evq->txqs = &(evq->txq);
+
+		do {
+			struct sfxge_txq *next;
+
+			next = txq->next;
+			txq->next = NULL;
+
+			KASSERT(txq->evq_index == index,
+			    ("txq->evq_index != index"));
+
+			if (txq->pending != txq->completed)
+				sfxge_tx_qcomplete(txq);
+
+			txq = next;
+		} while (txq != NULL);
+	}
+
+	if (rxq->pending != rxq->completed)
+		sfxge_rx_qcomplete(rxq, eop);
+}
+
+static boolean_t
+sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
+    uint16_t flags)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	struct sfxge_rxq *rxq;
+	unsigned int expected;
+	struct sfxge_rx_sw_desc *rx_desc;
+
+	evq = arg;
+	sc = evq->sc;
+
+	if (evq->exception)
+		goto done;
+
+	rxq = sc->rxq[label];
+	KASSERT(rxq != NULL, ("rxq == NULL"));
+	KASSERT(evq->index == rxq->index,
+	    ("evq->index != rxq->index"));
+
+	if (rxq->init_state != SFXGE_RXQ_STARTED)
+		goto done;
+
+	expected = rxq->pending++ & (SFXGE_NDESCS - 1);
+	if (id != expected) {
+		evq->exception = B_TRUE;
+
+		device_printf(sc->dev, "RX completion out of order"
+			      " (id=%#x expected=%#x flags=%#x); resetting\n",
+			      id, expected, flags);
+		sfxge_schedule_reset(sc);
+
+		goto done;
+	}
+
+	rx_desc = &rxq->queue[id];
+
+	KASSERT(rx_desc->flags == EFX_DISCARD,
+	    ("rx_desc->flags != EFX_DISCARD"));
+	rx_desc->flags = flags;
+
+	KASSERT(size < (1 << 16), ("size > (1 << 16)"));
+	rx_desc->size = (uint16_t)size;
+	prefetch_read_many(rx_desc->mbuf);
+
+	evq->rx_done++;
+
+	if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
+		sfxge_ev_qcomplete(evq, B_FALSE);
+
+done:
+	return (evq->rx_done >= SFXGE_EV_BATCH);
+}
+
+static boolean_t
+sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+
+	evq->exception = B_TRUE;
+
+	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
+		device_printf(sc->dev,
+			      "hardware exception (code=%u); resetting\n",
+			      code);
+		sfxge_schedule_reset(sc);
+	}
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	struct sfxge_rxq *rxq;
+	unsigned int index;
+	uint16_t magic;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+	rxq = sc->rxq[label];
+
+	KASSERT(rxq != NULL, ("rxq == NULL"));
+
+	/* Resend a software event on the correct queue */
+	index = rxq->index;
+	evq = sc->evq[index];
+
+	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
+	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
+	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
+
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq not started"));
+	efx_ev_qpost(evq->common, magic);
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	struct sfxge_rxq *rxq;
+	unsigned int index;
+	uint16_t magic;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+	rxq = sc->rxq[label];
+
+	KASSERT(rxq != NULL, ("rxq == NULL"));
+
+	/* Resend a software event on the correct queue */
+	index = rxq->index;
+	evq = sc->evq[index];
+
+	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
+	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
+	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
+
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq not started"));
+	efx_ev_qpost(evq->common, magic);
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	struct sfxge_txq *txq;
+	unsigned int stop;
+	unsigned int delta;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+	txq = sc->txq[label];
+
+	KASSERT(txq != NULL, ("txq == NULL"));
+	KASSERT(evq->index == txq->evq_index,
+	    ("evq->index != txq->evq_index"));
+
+	if (txq->init_state != SFXGE_TXQ_STARTED)
+		goto done;
+
+	stop = (id + 1) & (SFXGE_NDESCS - 1);
+	id = txq->pending & (SFXGE_NDESCS - 1);
+
+	delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
+	txq->pending += delta;
+
+	evq->tx_done++;
+
+	if (txq->next == NULL &&
+	    evq->txqs != &(txq->next)) {
+		*(evq->txqs) = txq;
+		evq->txqs = &(txq->next);
+	}
+
+	if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
+		sfxge_tx_qcomplete(txq);
+
+done:
+	return (evq->tx_done >= SFXGE_EV_BATCH);
+}
+
+static boolean_t
+sfxge_ev_txq_flush_done(void *arg, uint32_t label)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	struct sfxge_txq *txq;
+	uint16_t magic;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+	txq = sc->txq[label];
+
+	KASSERT(txq != NULL, ("txq == NULL"));
+	KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
+	    ("txq not initialized"));
+
+	/* Resend a software event on the correct queue */
+	evq = sc->evq[txq->evq_index];
+
+	KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
+	    ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
+	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
+
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq not started"));
+	efx_ev_qpost(evq->common, magic);
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_software(void *arg, uint16_t magic)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	unsigned int label;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+
+	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
+	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
+
+	switch (magic) {
+	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
+		struct sfxge_rxq *rxq = sc->rxq[label];
+
+		KASSERT(rxq != NULL, ("rxq == NULL"));
+		KASSERT(evq->index == rxq->index,
+		    ("evq->index != rxq->index"));
+
+		sfxge_rx_qflush_done(rxq);
+		break;
+	}
+	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
+		struct sfxge_rxq *rxq = sc->rxq[label];
+
+		KASSERT(rxq != NULL, ("rxq == NULL"));
+		KASSERT(evq->index == rxq->index,
+		    ("evq->index != rxq->index"));
+
+		sfxge_rx_qflush_failed(rxq);
+		break;
+	}
+	case SFXGE_MAGIC_RX_QREFILL: {
+		struct sfxge_rxq *rxq = sc->rxq[label];
+
+		KASSERT(rxq != NULL, ("rxq == NULL"));
+		KASSERT(evq->index == rxq->index,
+		    ("evq->index != rxq->index"));
+
+		sfxge_rx_qrefill(rxq);
+		break;
+	}
+	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
+		struct sfxge_txq *txq = sc->txq[label];
+
+		KASSERT(txq != NULL, ("txq == NULL"));
+		KASSERT(evq->index == txq->evq_index,
+		    ("evq->index != txq->evq_index"));
+
+		sfxge_tx_qflush_done(txq);
+		break;
+	}
+	default:
+		break;
+	}
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_sram(void *arg, uint32_t code)
+{
+	(void)arg;
+	(void)code;
+
+	switch (code) {
+	case EFX_SRAM_UPDATE:
+		EFSYS_PROBE(sram_update);
+		break;
+
+	case EFX_SRAM_CLEAR:
+		EFSYS_PROBE(sram_clear);
+		break;
+
+	case EFX_SRAM_ILLEGAL_CLEAR:
+		EFSYS_PROBE(sram_illegal_clear);
+		break;
+
+	default:
+		KASSERT(B_FALSE, ("Impossible SRAM event"));
+		break;
+	}
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_timer(void *arg, uint32_t index)
+{
+	(void)arg;
+	(void)index;
+
+	return (B_FALSE);
+}
+
+static boolean_t
+sfxge_ev_wake_up(void *arg, uint32_t index)
+{
+	(void)arg;
+	(void)index;
+
+	return (B_FALSE);
+}
+
+static void
+sfxge_ev_stat_update(struct sfxge_softc *sc)
+{
+	struct sfxge_evq *evq;
+	unsigned int index;
+	clock_t now;
+
+	sx_xlock(&sc->softc_lock);
+
+	if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
+		goto out;
+
+	now = ticks;
+	if (now - sc->ev_stats_update_time < hz)
+		goto out;
+
+	sc->ev_stats_update_time = now;
+
+	/* Add event counts from each event queue in turn */
+	for (index = 0; index < sc->intr.n_alloc; index++) {
+		evq = sc->evq[index];
+		mtx_lock(&evq->lock);
+		efx_ev_qstats_update(evq->common, sc->ev_stats);
+		mtx_unlock(&evq->lock);
+	}
+out:
+	sx_xunlock(&sc->softc_lock);
+}
+
+static int
+sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	unsigned int id = arg2;
+
+	sfxge_ev_stat_update(sc);
+
+	return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]));
+}
+
+static void
+sfxge_ev_stat_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid_list *stat_list;
+	unsigned int id;
+	char name[40];
+
+	stat_list = SYSCTL_CHILDREN(sc->stats_node);
+
+	for (id = 0; id < EV_NQSTATS; id++) {
+		snprintf(name, sizeof(name), "ev_%s",
+			 efx_ev_qstat_name(sc->enp, id));
+		SYSCTL_ADD_PROC(
+			ctx, stat_list,
+			OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
+			sc, id, sfxge_ev_stat_handler, "Q",
+			"");
+	}
+}
+
+static void
+sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
+{
+	struct sfxge_evq *evq;
+	efx_evq_t *eep;
+
+	evq = sc->evq[idx];
+	eep = evq->common;
+
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq->init_state != SFXGE_EVQ_STARTED"));
+
+	(void)efx_ev_qmoderate(eep, us);
+}
+
+static int
+sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	struct sfxge_intr *intr = &sc->intr;
+	unsigned int moderation;
+	int error;
+	int index;
+
+	sx_xlock(&sc->softc_lock);
+
+	if (req->newptr) {
+		if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
+		    != 0)
+			goto out;
+
+		/* We may not be calling efx_ev_qmoderate() now,
+		 * so we have to range-check the value ourselves.
+		 */
+		if (moderation >
+		    efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
+			error = EINVAL;
+			goto out;
+		}
+
+		sc->ev_moderation = moderation;
+		if (intr->state == SFXGE_INTR_STARTED) {
+			for (index = 0; index < intr->n_alloc; index++)
+				sfxge_ev_qmoderate(sc, index, moderation);
+		}
+	} else {
+		error = SYSCTL_OUT(req, &sc->ev_moderation,
+				   sizeof(sc->ev_moderation));
+	}
+
+out:
+	sx_xunlock(&sc->softc_lock);
+
+	return error;
+}
+
+static boolean_t
+sfxge_ev_initialized(void *arg)
+{
+	struct sfxge_evq *evq;
+	
+	evq = (struct sfxge_evq *)arg;
+
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
+	    ("evq not starting"));
+
+	evq->init_state = SFXGE_EVQ_STARTED;
+
+	return (0);
+}
+
+static boolean_t
+sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+
+	sfxge_mac_link_update(sc, link_mode);
+
+	return (0);
+}
+
+static const efx_ev_callbacks_t sfxge_ev_callbacks = {
+	.eec_initialized	= sfxge_ev_initialized,
+	.eec_rx			= sfxge_ev_rx,
+	.eec_tx			= sfxge_ev_tx,
+	.eec_exception		= sfxge_ev_exception,
+	.eec_rxq_flush_done	= sfxge_ev_rxq_flush_done,
+	.eec_rxq_flush_failed	= sfxge_ev_rxq_flush_failed,
+	.eec_txq_flush_done	= sfxge_ev_txq_flush_done,
+	.eec_software		= sfxge_ev_software,
+	.eec_sram		= sfxge_ev_sram,
+	.eec_wake_up		= sfxge_ev_wake_up,
+	.eec_timer		= sfxge_ev_timer,
+	.eec_link_change	= sfxge_ev_link_change,
+};
+
+
+int
+sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_evq *evq;
+	int rc;
+
+	evq = sc->evq[index];
+
+	mtx_lock(&evq->lock);
+
+	if (evq->init_state != SFXGE_EVQ_STARTING &&
+	    evq->init_state != SFXGE_EVQ_STARTED) {
+		rc = EINVAL;
+		goto fail;
+	}
+
+	/* Synchronize the DMA memory for reading */
+	bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
+	    BUS_DMASYNC_POSTREAD);
+
+	KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
+	KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
+	KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
+	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
+
+	/* Poll the queue */
+	efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
+
+	evq->rx_done = 0;
+	evq->tx_done = 0;
+
+	/* Perform any pending completion processing */
+	sfxge_ev_qcomplete(evq, B_TRUE);
+
+	/* Re-prime the event queue for interrupts */
+	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
+		goto fail;
+
+	mtx_unlock(&evq->lock);
+
+	return (0);
+
+fail:
+	mtx_unlock(&(evq->lock));
+	return (rc);
+}
+
+static void
+sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_evq *evq;
+
+	evq = sc->evq[index];
+
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq->init_state != SFXGE_EVQ_STARTED"));
+
+	mtx_lock(&evq->lock);
+	evq->init_state = SFXGE_EVQ_INITIALIZED;
+	evq->read_ptr = 0;
+	evq->exception = B_FALSE;
+
+	/* Add event counts before discarding the common evq state */
+	efx_ev_qstats_update(evq->common, sc->ev_stats);
+
+	efx_ev_qdestroy(evq->common);
+	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
+	    EFX_EVQ_NBUFS(SFXGE_NEVS));
+	mtx_unlock(&evq->lock);
+}
+
+static int
+sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_evq *evq;
+	efsys_mem_t *esmp;
+	int count;
+	int rc;
+
+	evq = sc->evq[index];
+	esmp = &evq->mem;
+
+	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
+	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
+
+	/* Clear all events. */
+	(void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS));
+
+	/* Program the buffer table. */
+	if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
+	    EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0)
+		return rc;
+
+	/* Create the common code event queue. */
+	if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS,
+	    evq->buf_base_id, &evq->common)) != 0)
+		goto fail;
+
+	mtx_lock(&evq->lock);
+
+	/* Set the default moderation */
+	(void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
+
+	/* Prime the event queue for interrupts */
+	if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
+		goto fail2;
+
+	evq->init_state = SFXGE_EVQ_STARTING;
+
+	mtx_unlock(&evq->lock);
+
+	/* Wait for the initialization event */
+	count = 0;
+	do {
+		/* Pause for 100 ms */
+		pause("sfxge evq init", hz / 10);
+
+		/* Check to see if the test event has been processed */
+		if (evq->init_state == SFXGE_EVQ_STARTED)
+			goto done;
+
+	} while (++count < 20);
+
+	rc = ETIMEDOUT;
+	goto fail3;
+
+done:
+	return (0);
+
+fail3:
+	mtx_lock(&evq->lock);
+	evq->init_state = SFXGE_EVQ_INITIALIZED;
+fail2:
+	mtx_unlock(&evq->lock);
+	efx_ev_qdestroy(evq->common);
+fail:
+	efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
+	    EFX_EVQ_NBUFS(SFXGE_NEVS));
+
+	return (rc);
+}
+
+void
+sfxge_ev_stop(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	efx_nic_t *enp;
+	int index;
+
+	intr = &sc->intr;
+	enp = sc->enp;
+
+	KASSERT(intr->state == SFXGE_INTR_STARTED,
+	    ("Interrupts not started"));
+
+	/* Stop the event queue(s) */
+	index = intr->n_alloc;
+	while (--index >= 0)
+		sfxge_ev_qstop(sc, index);
+
+	/* Tear down the event module */
+	efx_ev_fini(enp);
+}
+
+int
+sfxge_ev_start(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+	int rc;
+
+	intr = &sc->intr;
+
+	KASSERT(intr->state == SFXGE_INTR_STARTED,
+	    ("intr->state != SFXGE_INTR_STARTED"));
+
+	/* Initialize the event module */
+	if ((rc = efx_ev_init(sc->enp)) != 0)
+		return rc;
+
+	/* Start the event queues */
+	for (index = 0; index < intr->n_alloc; index++) {
+		if ((rc = sfxge_ev_qstart(sc, index)) != 0)
+			goto fail;
+	}
+
+	return (0);
+
+fail:
+	/* Stop the event queue(s) */
+	while (--index >= 0)
+		sfxge_ev_qstop(sc, index);
+
+	/* Tear down the event module */
+	efx_ev_fini(sc->enp);
+
+	return (rc);
+}
+
+static void
+sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_evq *evq;
+
+	evq = sc->evq[index];
+
+	KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
+	    ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
+	KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
+
+	sfxge_dma_free(&evq->mem);
+
+	sc->evq[index] = NULL;
+
+	mtx_destroy(&evq->lock);
+
+	free(evq, M_SFXGE);
+}
+
+static int
+sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_evq *evq;
+	efsys_mem_t *esmp;
+	int rc;
+
+	KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
+
+	evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
+	evq->sc = sc;
+	evq->index = index;
+	sc->evq[index] = evq;
+	esmp = &evq->mem;
+
+	/* Initialise TX completion list */
+	evq->txqs = &evq->txq;
+
+	/* Allocate DMA space. */
+	if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0)
+		return (rc);
+
+	/* Allocate buffer table entries. */
+	sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS),
+				 &evq->buf_base_id);
+
+	mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
+
+	evq->init_state = SFXGE_EVQ_INITIALIZED;
+
+	return (0);
+}
+
+void
+sfxge_ev_fini(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+
+	intr = &sc->intr;
+
+	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
+	    ("intr->state != SFXGE_INTR_INITIALIZED"));
+
+	sc->ev_moderation = 0;
+
+	/* Tear down the event queue(s). */
+	index = intr->n_alloc;
+	while (--index >= 0)
+		sfxge_ev_qfini(sc, index);
+}
+
+int
+sfxge_ev_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
+	struct sfxge_intr *intr;
+	int index;
+	int rc;
+
+	intr = &sc->intr;
+
+	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
+	    ("intr->state != SFXGE_INTR_INITIALIZED"));
+
+	/* Set default interrupt moderation; add a sysctl to
+	 * read and change it.
+	 */
+	sc->ev_moderation = 30;
+	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
+			OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
+			sc, 0, sfxge_int_mod_handler, "IU",
+			"sfxge interrupt moderation (us)");
+
+	/*
+	 * Initialize the event queue(s) - one per interrupt.
+	 */
+	for (index = 0; index < intr->n_alloc; index++) {
+		if ((rc = sfxge_ev_qinit(sc, index)) != 0)
+			goto fail;
+	}
+
+	sfxge_ev_stat_init(sc);
+
+	return (0);
+
+fail:
+	while (--index >= 0)
+		sfxge_ev_qfini(sc, index);
+
+	return (rc);
+}
diff --git a/sys/dev/sfxge/sfxge_intr.c b/sys/dev/sfxge/sfxge_intr.c
new file mode 100644
index 0000000..c8f4c3d
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_intr.c
@@ -0,0 +1,556 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+#include <sys/syslog.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "common/efx.h"
+
+#include "sfxge.h"
+
+static int
+sfxge_intr_line_filter(void *arg)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	efx_nic_t *enp;
+	struct sfxge_intr *intr;
+	boolean_t fatal;
+	uint32_t qmask;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+	enp = sc->enp;
+	intr = &sc->intr;
+
+	KASSERT(intr != NULL, ("intr == NULL"));
+	KASSERT(intr->type == EFX_INTR_LINE,
+	    ("intr->type != EFX_INTR_LINE"));
+
+	if (intr->state != SFXGE_INTR_STARTED)
+		return FILTER_STRAY;
+
+	(void)efx_intr_status_line(enp, &fatal, &qmask);
+
+	if (fatal) {
+		(void) efx_intr_disable(enp);
+		(void) efx_intr_fatal(enp);
+		return FILTER_HANDLED;
+	}
+
+	if (qmask != 0) {
+		intr->zero_count = 0;
+		return FILTER_SCHEDULE_THREAD;
+	}
+
+	/* SF bug 15783: If the function is not asserting its IRQ and
+	 * we read the queue mask on the cycle before a flag is added
+	 * to the mask, this inhibits the function from asserting the
+	 * IRQ even though we don't see the flag set.  To work around
+	 * this, we must re-prime all event queues and report the IRQ
+	 * as handled when we see a mask of zero.  To allow for shared
+	 * IRQs, we don't repeat this if we see a mask of zero twice
+	 * or more in a row.
+	 */
+	if (intr->zero_count++ == 0) {
+		if (evq->init_state == SFXGE_EVQ_STARTED) {
+			if (efx_ev_qpending(evq->common, evq->read_ptr))
+				return FILTER_SCHEDULE_THREAD;
+			efx_ev_qprime(evq->common, evq->read_ptr);
+			return FILTER_HANDLED;
+		}
+	}
+
+	return FILTER_STRAY;
+}
+
+static void
+sfxge_intr_line(void *arg)
+{
+	struct sfxge_evq *evq = arg;
+	struct sfxge_softc *sc = evq->sc;
+
+	(void)sfxge_ev_qpoll(sc, 0);
+}
+
+static void
+sfxge_intr_message(void *arg)
+{
+	struct sfxge_evq *evq;
+	struct sfxge_softc *sc;
+	efx_nic_t *enp;
+	struct sfxge_intr *intr;
+	unsigned int index;
+	boolean_t fatal;
+
+	evq = (struct sfxge_evq *)arg;
+	sc = evq->sc;
+	enp = sc->enp;
+	intr = &sc->intr;
+	index = evq->index;
+
+	KASSERT(intr != NULL, ("intr == NULL"));
+	KASSERT(intr->type == EFX_INTR_MESSAGE,
+	    ("intr->type != EFX_INTR_MESSAGE"));
+
+	if (intr->state != SFXGE_INTR_STARTED)
+		return;
+
+	(void)efx_intr_status_message(enp, index, &fatal);
+
+	if (fatal) {
+		(void)efx_intr_disable(enp);
+		(void)efx_intr_fatal(enp);
+		return;
+	}
+
+	(void)sfxge_ev_qpoll(sc, index);
+}
+
+static int
+sfxge_intr_bus_enable(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	struct sfxge_intr_hdl *table;
+	driver_filter_t *filter;
+	driver_intr_t *handler;
+	int index;
+	int err;
+
+	intr = &sc->intr;
+	table = intr->table;
+
+	switch (intr->type) {
+	case EFX_INTR_MESSAGE:
+		filter = NULL; /* not shared */
+		handler = sfxge_intr_message;
+		break;
+
+	case EFX_INTR_LINE:
+		filter = sfxge_intr_line_filter;
+		handler = sfxge_intr_line;
+		break;
+
+	default:
+		KASSERT(0, ("Invalid interrupt type"));
+		return EINVAL;
+	}
+
+	/* Try to add the handlers */
+	for (index = 0; index < intr->n_alloc; index++) {
+		if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
+			    INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
+			    sc->evq[index], &table[index].eih_tag)) != 0) {
+			goto fail;
+		}
+#ifdef SFXGE_HAVE_DESCRIBE_INTR
+		if (intr->n_alloc > 1)
+			bus_describe_intr(sc->dev, table[index].eih_res,
+			    table[index].eih_tag, "%d", index);
+#endif
+		bus_bind_intr(sc->dev, table[index].eih_res, index);
+
+	}
+
+	return (0);
+
+fail:
+	/* Remove remaining handlers */
+	while (--index >= 0)
+		bus_teardown_intr(sc->dev, table[index].eih_res,
+		    table[index].eih_tag);
+
+	return (err);
+}
+
+static void
+sfxge_intr_bus_disable(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	struct sfxge_intr_hdl *table;
+	int i;
+
+	intr = &sc->intr;
+	table = intr->table;
+
+	/* Remove all handlers */
+	for (i = 0; i < intr->n_alloc; i++)
+		bus_teardown_intr(sc->dev, table[i].eih_res,
+		    table[i].eih_tag);
+}
+
+static int
+sfxge_intr_alloc(struct sfxge_softc *sc, int count)
+{
+	device_t dev;
+	struct sfxge_intr_hdl *table;
+	struct sfxge_intr *intr;
+	struct resource *res;
+	int rid;
+	int error;
+	int i;
+
+	dev = sc->dev;
+	intr = &sc->intr;
+	error = 0;
+
+	table = malloc(count * sizeof(struct sfxge_intr_hdl),
+	    M_SFXGE, M_WAITOK);
+	intr->table = table;
+
+	for (i = 0; i < count; i++) {
+		rid = i + 1;
+		res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+		    RF_SHAREABLE | RF_ACTIVE);
+		if (res == NULL) {
+			device_printf(dev, "Couldn't allocate interrupts for "
+			    "message %d\n", rid);
+			error = ENOMEM;
+			break;
+		}
+		table[i].eih_rid = rid;
+		table[i].eih_res = res;
+	}
+
+	if (error) {
+		count = i - 1;
+		for (i = 0; i < count; i++)
+			bus_release_resource(dev, SYS_RES_IRQ,
+			    table[i].eih_rid, table[i].eih_res);
+	}
+
+	return (error);
+}
+
+static void
+sfxge_intr_teardown_msix(struct sfxge_softc *sc)
+{
+	device_t dev;
+	struct resource *resp;
+	int rid;
+
+	dev = sc->dev;
+	resp = sc->intr.msix_res;
+
+	rid = rman_get_rid(resp);
+	bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
+}
+
+static int
+sfxge_intr_setup_msix(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	struct resource *resp;
+	device_t dev;
+	int count;
+	int rid;
+
+	dev = sc->dev;
+	intr = &sc->intr;
+
+	/* Check if MSI-X is available. */
+	count = pci_msix_count(dev);
+	if (count == 0)
+		return (EINVAL);
+
+	/* Limit the number of interrupts to the number of CPUs. */
+	if (count > mp_ncpus)
+		count = mp_ncpus;
+
+	/* Not very likely these days... */
+	if (count > EFX_MAXRSS)
+		count = EFX_MAXRSS;
+
+	rid = PCIR_BAR(4);
+	resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (resp == NULL)
+		return (ENOMEM);
+
+	if (pci_alloc_msix(dev, &count) != 0) {
+		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
+		return (ENOMEM);
+	}
+
+	/* Allocate interrupt handlers. */
+	if (sfxge_intr_alloc(sc, count) != 0) {
+		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
+		pci_release_msi(dev);
+		return (ENOMEM);
+	}
+
+	intr->type = EFX_INTR_MESSAGE;
+	intr->n_alloc = count;
+	intr->msix_res = resp;
+
+	return (0);
+}
+
+static int
+sfxge_intr_setup_msi(struct sfxge_softc *sc)
+{
+	struct sfxge_intr_hdl *table;
+	struct sfxge_intr *intr;
+	device_t dev;
+	int count;
+	int error;
+
+	dev = sc->dev;
+	intr = &sc->intr;
+	table = intr->table;
+
+	/*
+	 * Check if MSI is available.  All messages must be written to
+	 * the same address and on x86 this means the IRQs have the
+	 * same CPU affinity.  So we only ever allocate 1.
+	 */
+	count = pci_msi_count(dev) ? 1 : 0;
+	if (count == 0)
+		return (EINVAL);
+
+	if ((error = pci_alloc_msi(dev, &count)) != 0) 
+		return (ENOMEM);
+
+	/* Allocate interrupt handler. */
+	if (sfxge_intr_alloc(sc, count) != 0) {
+		pci_release_msi(dev);
+		return (ENOMEM);
+	}
+
+	intr->type = EFX_INTR_MESSAGE;
+	intr->n_alloc = count;
+
+	return (0);
+}
+
+static int
+sfxge_intr_setup_fixed(struct sfxge_softc *sc)
+{
+	struct sfxge_intr_hdl *table;
+	struct sfxge_intr *intr;
+	struct resource *res;
+	device_t dev;
+	int rid;
+
+	dev = sc->dev;
+	intr = &sc->intr;
+
+	rid = 0;
+	res =  bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+	    RF_SHAREABLE | RF_ACTIVE);
+	if (res == NULL)
+		return (ENOMEM);
+
+	table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
+	table[0].eih_rid = rid;
+	table[0].eih_res = res;
+
+	intr->type = EFX_INTR_LINE;
+	intr->n_alloc = 1;
+	intr->table = table;
+
+	return (0);
+}
+
+static const char *const __sfxge_err[] = {
+	"",
+	"SRAM out-of-bounds",
+	"Buffer ID out-of-bounds",
+	"Internal memory parity",
+	"Receive buffer ownership",
+	"Transmit buffer ownership",
+	"Receive descriptor ownership",
+	"Transmit descriptor ownership",
+	"Event queue ownership",
+	"Event queue FIFO overflow",
+	"Illegal address",
+	"SRAM parity"
+};
+
+void
+sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
+    uint32_t dword1)
+{
+	struct sfxge_softc *sc = (struct sfxge_softc *)arg;
+	device_t dev = sc->dev;
+
+	log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
+	    device_get_name(dev), device_get_unit(dev),
+		__sfxge_err[code], dword1, dword0);
+}
+
+void
+sfxge_intr_stop(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	
+	intr = &sc->intr;
+
+	KASSERT(intr->state == SFXGE_INTR_STARTED,
+	    ("Interrupts not started"));
+
+	intr->state = SFXGE_INTR_INITIALIZED;
+
+	/* Disable interrupts at the NIC */
+	efx_intr_disable(sc->enp);
+
+	/* Disable interrupts at the bus */
+	sfxge_intr_bus_disable(sc);
+
+	/* Tear down common code interrupt bits. */
+	efx_intr_fini(sc->enp);
+}
+
+int
+sfxge_intr_start(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	efsys_mem_t *esmp;
+	int rc;
+
+	intr = &sc->intr;
+	esmp = &intr->status;
+
+	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
+	    ("Interrupts not initialized"));
+
+	/* Zero the memory. */
+	(void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
+
+	/* Initialize common code interrupt bits. */
+	(void)efx_intr_init(sc->enp, intr->type, esmp);
+
+	/* Enable interrupts at the bus */
+	if ((rc = sfxge_intr_bus_enable(sc)) != 0)
+		goto fail;
+
+	intr->state = SFXGE_INTR_STARTED;
+
+	/* Enable interrupts at the NIC */
+	efx_intr_enable(sc->enp);
+
+	return (0);
+
+fail:
+	/* Tear down common code interrupt bits. */
+	efx_intr_fini(sc->enp);
+
+	intr->state = SFXGE_INTR_INITIALIZED;
+
+	return (rc);
+}
+
+void
+sfxge_intr_fini(struct sfxge_softc *sc)
+{
+	struct sfxge_intr_hdl *table;
+	struct sfxge_intr *intr;
+	efsys_mem_t *esmp;
+	device_t dev;
+	int i;
+
+	dev = sc->dev;
+	intr = &sc->intr;
+	esmp = &intr->status;
+	table = intr->table;
+
+	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
+	    ("intr->state != SFXGE_INTR_INITIALIZED"));
+
+	/* Free DMA memory. */
+	sfxge_dma_free(esmp);
+
+	/* Free interrupt handles. */
+	for (i = 0; i < intr->n_alloc; i++)
+		bus_release_resource(dev, SYS_RES_IRQ,
+		    table[i].eih_rid, table[i].eih_res);
+
+	if (table[0].eih_rid != 0)
+		pci_release_msi(dev);
+
+	if (intr->msix_res != NULL)
+		sfxge_intr_teardown_msix(sc);
+
+	/* Free the handle table */
+	free(table, M_SFXGE);
+	intr->table = NULL;
+	intr->n_alloc = 0;
+
+	/* Clear the interrupt type */
+	intr->type = EFX_INTR_INVALID;
+
+	intr->state = SFXGE_INTR_UNINITIALIZED;
+}
+
+int
+sfxge_intr_init(struct sfxge_softc *sc)
+{
+	device_t dev;
+	struct sfxge_intr *intr;
+	efsys_mem_t *esmp;
+	int rc;
+
+	dev = sc->dev;
+	intr = &sc->intr;
+	esmp = &intr->status;
+
+	KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
+	    ("Interrupts already initialized"));
+
+	/* Try to setup MSI-X or MSI interrupts if available. */
+	if ((rc = sfxge_intr_setup_msix(sc)) == 0)
+		device_printf(dev, "Using MSI-X interrupts\n");
+	else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
+		device_printf(dev, "Using MSI interrupts\n");
+	else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
+		device_printf(dev, "Using fixed interrupts\n");
+	} else {
+		device_printf(dev, "Couldn't setup interrupts\n");
+		return (ENOMEM);
+	}
+
+	/* Set up DMA for interrupts. */
+	if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
+		return (ENOMEM);
+
+	intr->state = SFXGE_INTR_INITIALIZED;
+
+	return (0);
+}
diff --git a/sys/dev/sfxge/sfxge_mcdi.c b/sys/dev/sfxge/sfxge_mcdi.c
new file mode 100644
index 0000000..6368ab4
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_mcdi.c
@@ -0,0 +1,250 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/condvar.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/syslog.h>
+#include <sys/taskqueue.h>
+
+#include "common/efx.h"
+#include "common/efx_mcdi.h"
+#include "common/efx_regs_mcdi.h"
+
+#include "sfxge.h"
+
+#define	SFXGE_MCDI_POLL_INTERVAL_MIN 10		/* 10us in 1us units */
+#define	SFXGE_MCDI_POLL_INTERVAL_MAX 100000	/* 100ms in 1us units */
+#define	SFXGE_MCDI_WATCHDOG_INTERVAL 10000000	/* 10s in 1us units */
+
+/* Acquire exclusive access to MCDI for the duration of a request. */
+static void
+sfxge_mcdi_acquire(struct sfxge_mcdi *mcdi)
+{
+
+	mtx_lock(&mcdi->lock);
+	KASSERT(mcdi->state != SFXGE_MCDI_UNINITIALIZED,
+	    ("MCDI not initialized"));
+
+	while (mcdi->state != SFXGE_MCDI_INITIALIZED)
+		(void)cv_wait_sig(&mcdi->cv, &mcdi->lock);
+	mcdi->state = SFXGE_MCDI_BUSY;
+
+	mtx_unlock(&mcdi->lock);
+}
+
+/* Release ownership of MCDI on request completion. */
+static void
+sfxge_mcdi_release(struct sfxge_mcdi *mcdi)
+{
+
+	mtx_lock(&mcdi->lock);
+	KASSERT((mcdi->state == SFXGE_MCDI_BUSY ||
+	    mcdi->state == SFXGE_MCDI_COMPLETED),
+	    ("MCDI not busy or task not completed"));
+
+	mcdi->state = SFXGE_MCDI_INITIALIZED;
+	cv_broadcast(&mcdi->cv);
+
+	mtx_unlock(&mcdi->lock);
+}
+
+static void
+sfxge_mcdi_timeout(struct sfxge_softc *sc)
+{
+	device_t dev = sc->dev;
+
+	log(LOG_WARNING, "[%s%d] MC_TIMEOUT", device_get_name(dev),
+		device_get_unit(dev));
+
+	EFSYS_PROBE(mcdi_timeout);
+	sfxge_schedule_reset(sc);
+}
+
+static void
+sfxge_mcdi_poll(struct sfxge_softc *sc)
+{
+	efx_nic_t *enp;
+	clock_t delay_total;
+	clock_t delay_us;
+	boolean_t aborted;
+
+	delay_total = 0;
+	delay_us = SFXGE_MCDI_POLL_INTERVAL_MIN;
+	enp = sc->enp;
+
+	do {
+		if (efx_mcdi_request_poll(enp)) {
+			EFSYS_PROBE1(mcdi_delay, clock_t, delay_total);
+			return;
+		}
+
+		if (delay_total > SFXGE_MCDI_WATCHDOG_INTERVAL) {
+			aborted = efx_mcdi_request_abort(enp);
+			KASSERT(aborted, ("abort failed"));
+			sfxge_mcdi_timeout(sc);
+			return;
+		}
+
+		/* Spin or block depending on delay interval. */
+		if (delay_us < 1000000)
+			DELAY(delay_us);
+		else
+			pause("mcdi wait", delay_us * hz / 1000000);
+
+		delay_total += delay_us;
+
+		/* Exponentially back off the poll frequency. */
+		delay_us = delay_us * 2;
+		if (delay_us > SFXGE_MCDI_POLL_INTERVAL_MAX)
+			delay_us = SFXGE_MCDI_POLL_INTERVAL_MAX;
+
+	} while (1);
+}
+
+static void
+sfxge_mcdi_execute(void *arg, efx_mcdi_req_t *emrp)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_mcdi *mcdi;
+
+	sc = (struct sfxge_softc *)arg;
+	mcdi = &sc->mcdi;
+
+	sfxge_mcdi_acquire(mcdi);
+
+	/* Issue request and poll for completion. */
+	efx_mcdi_request_start(sc->enp, emrp, B_FALSE);
+	sfxge_mcdi_poll(sc);
+
+	sfxge_mcdi_release(mcdi);
+}
+
+static void
+sfxge_mcdi_ev_cpl(void *arg)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_mcdi *mcdi;
+
+	sc = (struct sfxge_softc *)arg;
+	mcdi = &sc->mcdi;
+
+	mtx_lock(&mcdi->lock);
+	KASSERT(mcdi->state == SFXGE_MCDI_BUSY, ("MCDI not busy"));
+	mcdi->state = SFXGE_MCDI_COMPLETED;
+	cv_broadcast(&mcdi->cv);
+	mtx_unlock(&mcdi->lock);
+}
+
+static void
+sfxge_mcdi_exception(void *arg, efx_mcdi_exception_t eme)
+{
+	struct sfxge_softc *sc;
+	device_t dev;
+
+	sc = (struct sfxge_softc *)arg;
+	dev = sc->dev;
+
+	log(LOG_WARNING, "[%s%d] MC_%s", device_get_name(dev),
+	    device_get_unit(dev),
+	    (eme == EFX_MCDI_EXCEPTION_MC_REBOOT)
+	    ? "REBOOT"
+	    : (eme == EFX_MCDI_EXCEPTION_MC_BADASSERT)
+	    ? "BADASSERT" : "UNKNOWN");
+
+	EFSYS_PROBE(mcdi_exception);
+
+	sfxge_schedule_reset(sc);
+}
+
+int
+sfxge_mcdi_init(struct sfxge_softc *sc)
+{
+	efx_nic_t *enp;
+	struct sfxge_mcdi *mcdi;
+	efx_mcdi_transport_t *emtp;
+	int rc;
+
+	enp = sc->enp;
+	mcdi = &sc->mcdi;
+	emtp = &mcdi->transport;
+
+	KASSERT(mcdi->state == SFXGE_MCDI_UNINITIALIZED,
+	    ("MCDI already initialized"));
+
+	mtx_init(&mcdi->lock, "sfxge_mcdi", NULL, MTX_DEF);
+
+	mcdi->state = SFXGE_MCDI_INITIALIZED;
+
+	emtp->emt_context = sc;
+	emtp->emt_execute = sfxge_mcdi_execute;
+	emtp->emt_ev_cpl = sfxge_mcdi_ev_cpl;
+	emtp->emt_exception = sfxge_mcdi_exception;
+
+	cv_init(&mcdi->cv, "sfxge_mcdi");
+
+	if ((rc = efx_mcdi_init(enp, emtp)) != 0)
+		goto fail;
+
+	return (0);
+
+fail:
+	mtx_destroy(&mcdi->lock);
+	mcdi->state = SFXGE_MCDI_UNINITIALIZED;
+	return (rc);
+}
+
+void
+sfxge_mcdi_fini(struct sfxge_softc *sc)
+{
+	struct sfxge_mcdi *mcdi;
+	efx_nic_t *enp;
+	efx_mcdi_transport_t *emtp;
+
+	enp = sc->enp;
+	mcdi = &sc->mcdi;
+	emtp = &mcdi->transport;
+
+	mtx_lock(&mcdi->lock);
+	KASSERT(mcdi->state == SFXGE_MCDI_INITIALIZED,
+	    ("MCDI not initialized"));
+
+	efx_mcdi_fini(enp);
+	bzero(emtp, sizeof(*emtp));
+
+	cv_destroy(&mcdi->cv);
+	mtx_unlock(&mcdi->lock);
+
+	mtx_destroy(&mcdi->lock);
+}
diff --git a/sys/dev/sfxge/sfxge_port.c b/sys/dev/sfxge/sfxge_port.c
new file mode 100644
index 0000000..a38f40a
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_port.c
@@ -0,0 +1,789 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/limits.h>
+#include <net/ethernet.h>
+#include <net/if_dl.h>
+
+#include "common/efx.h"
+
+#include "sfxge.h"
+
+static int
+sfxge_mac_stat_update(struct sfxge_softc *sc)
+{
+	struct sfxge_port *port = &sc->port;
+	efsys_mem_t *esmp = &(port->mac_stats.dma_buf);
+	clock_t now;
+	unsigned int count;
+	int rc;
+
+	mtx_lock(&port->lock);
+
+	if (port->init_state != SFXGE_PORT_STARTED) {
+		rc = 0;
+		goto out;
+	}
+
+	now = ticks;
+	if (now - port->mac_stats.update_time < hz) {
+		rc = 0;
+		goto out;
+	}
+
+	port->mac_stats.update_time = now;
+
+	/* If we're unlucky enough to read statistics wduring the DMA, wait
+	 * up to 10ms for it to finish (typically takes <500us) */
+	for (count = 0; count < 100; ++count) {
+		EFSYS_PROBE1(wait, unsigned int, count);
+
+		/* Synchronize the DMA memory for reading */
+		bus_dmamap_sync(esmp->esm_tag, esmp->esm_map,
+		    BUS_DMASYNC_POSTREAD);
+
+		/* Try to update the cached counters */
+		if ((rc = efx_mac_stats_update(sc->enp, esmp,
+                    port->mac_stats.decode_buf, NULL)) != EAGAIN)
+			goto out;
+
+		DELAY(100);
+	}
+
+	rc = ETIMEDOUT;
+out:
+	mtx_unlock(&port->lock);
+	return rc;
+}
+
+static int
+sfxge_mac_stat_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	unsigned int id = arg2;
+	int rc;
+
+	if ((rc = sfxge_mac_stat_update(sc)) != 0)
+		return rc;
+
+	return SYSCTL_OUT(req,
+			  (uint64_t *)sc->port.mac_stats.decode_buf + id,
+			  sizeof(uint64_t));
+}
+
+static void
+sfxge_mac_stat_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid_list *stat_list;
+	unsigned int id;
+	const char *name;
+
+	stat_list = SYSCTL_CHILDREN(sc->stats_node);
+
+	/* Initialise the named stats */
+	for (id = 0; id < EFX_MAC_NSTATS; id++) {
+		name = efx_mac_stat_name(sc->enp, id);
+		SYSCTL_ADD_PROC(
+			ctx, stat_list,
+			OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
+			sc, id, sfxge_mac_stat_handler, "Q",
+			"");
+	}
+}
+
+#ifdef SFXGE_HAVE_PAUSE_MEDIAOPTS
+
+static unsigned int
+sfxge_port_wanted_fc(struct sfxge_softc *sc)
+{
+	struct ifmedia_entry *ifm = sc->media.ifm_cur;
+
+	if (ifm->ifm_media == (IFM_ETHER | IFM_AUTO))
+		return EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
+	return ((ifm->ifm_media & IFM_ETH_RXPAUSE) ? EFX_FCNTL_RESPOND : 0) |
+		((ifm->ifm_media & IFM_ETH_TXPAUSE) ? EFX_FCNTL_GENERATE : 0);
+}
+
+static unsigned int
+sfxge_port_link_fc_ifm(struct sfxge_softc *sc)
+{
+	unsigned int wanted_fc, link_fc;
+
+	efx_mac_fcntl_get(sc->enp, &wanted_fc, &link_fc);
+	return ((link_fc & EFX_FCNTL_RESPOND) ? IFM_ETH_RXPAUSE : 0) |
+		((link_fc & EFX_FCNTL_GENERATE) ? IFM_ETH_TXPAUSE : 0);
+}
+
+#else /* !SFXGE_HAVE_PAUSE_MEDIAOPTS */
+
+static unsigned int
+sfxge_port_wanted_fc(struct sfxge_softc *sc)
+{
+	return sc->port.wanted_fc;
+}
+
+static unsigned int
+sfxge_port_link_fc_ifm(struct sfxge_softc *sc)
+{
+	return 0;
+}
+
+static int
+sfxge_port_wanted_fc_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_port *port;
+	unsigned int fcntl;
+	int error;
+
+	sc = arg1;
+	port = &sc->port;
+
+	mtx_lock(&port->lock);
+
+	if (req->newptr) {
+		if ((error = SYSCTL_IN(req, &fcntl, sizeof(fcntl))) != 0)
+			goto out;
+
+		if (port->wanted_fc == fcntl)
+			goto out;
+
+		port->wanted_fc = fcntl;
+
+		if (port->init_state != SFXGE_PORT_STARTED)
+			goto out;
+
+		error = efx_mac_fcntl_set(sc->enp, port->wanted_fc, B_TRUE);
+	} else {
+		error = SYSCTL_OUT(req, &port->wanted_fc,
+				   sizeof(port->wanted_fc));
+	}
+
+out:
+	mtx_unlock(&port->lock);
+
+	return (error);
+}
+
+static int
+sfxge_port_link_fc_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_port *port;
+	unsigned int wanted_fc, link_fc;
+	int error;
+
+	sc = arg1;
+	port = &sc->port;
+
+	mtx_lock(&port->lock);
+	if (port->init_state == SFXGE_PORT_STARTED && SFXGE_LINK_UP(sc))
+		efx_mac_fcntl_get(sc->enp, &wanted_fc, &link_fc);
+	else
+		link_fc = 0;
+	error = SYSCTL_OUT(req, &link_fc, sizeof(link_fc));
+	mtx_unlock(&port->lock);
+
+	return (error);
+}
+
+#endif /* SFXGE_HAVE_PAUSE_MEDIAOPTS */
+
+static const u_long sfxge_link_baudrate[EFX_LINK_NMODES] = {
+	[EFX_LINK_10HDX]	= IF_Mbps(10),
+	[EFX_LINK_10FDX]	= IF_Mbps(10),
+	[EFX_LINK_100HDX]	= IF_Mbps(100),
+	[EFX_LINK_100FDX]	= IF_Mbps(100),
+	[EFX_LINK_1000HDX]	= IF_Gbps(1),
+	[EFX_LINK_1000FDX]	= IF_Gbps(1),
+	[EFX_LINK_10000FDX]	= MIN(IF_Gbps(10ULL), ULONG_MAX),
+};
+
+void
+sfxge_mac_link_update(struct sfxge_softc *sc, efx_link_mode_t mode)
+{
+	struct sfxge_port *port;
+	int link_state;
+	
+	port = &sc->port;
+
+	if (port->link_mode == mode)
+		return;
+
+	port->link_mode = mode;
+
+	/* Push link state update to the OS */
+	link_state = (port->link_mode != EFX_LINK_DOWN ?
+		      LINK_STATE_UP : LINK_STATE_DOWN);
+	sc->ifnet->if_baudrate = sfxge_link_baudrate[port->link_mode];
+	if_link_state_change(sc->ifnet, link_state);
+}
+
+static void
+sfxge_mac_poll_work(void *arg, int npending)
+{
+	struct sfxge_softc *sc;
+	efx_nic_t *enp;
+	struct sfxge_port *port;
+	efx_link_mode_t mode;
+
+	sc = (struct sfxge_softc *)arg;
+	enp = sc->enp;
+	port = &sc->port;
+
+	mtx_lock(&port->lock);
+
+	if (port->init_state != SFXGE_PORT_STARTED)
+		goto done;
+
+	/* This may sleep waiting for MCDI completion */
+	(void)efx_port_poll(enp, &mode);
+	sfxge_mac_link_update(sc, mode);
+
+done:
+	mtx_unlock(&port->lock);
+}
+
+static int
+sfxge_mac_filter_set_locked(struct sfxge_softc *sc)
+{
+	unsigned int bucket[EFX_MAC_HASH_BITS];
+	struct ifnet *ifp = sc->ifnet;
+	struct ifmultiaddr *ifma;
+	struct sockaddr_dl *sa;
+	efx_nic_t *enp = sc->enp;
+	unsigned int index;
+	int rc;
+
+	/* Set promisc-unicast and broadcast filter bits */
+	if ((rc = efx_mac_filter_set(enp, !!(ifp->if_flags & IFF_PROMISC),
+				     B_TRUE)) != 0)
+		return rc;
+
+	/* Set multicast hash filter */
+	if (ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) {
+		for (index = 0; index < EFX_MAC_HASH_BITS; index++)
+			bucket[index] = 1;
+	} else {
+		/* Broadcast frames also go through the multicast
+		 * filter, and the broadcast address hashes to
+		 * 0xff. */
+		bucket[0xff] = 1;
+
+		IF_ADDR_LOCK(ifp);
+		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+			if (ifma->ifma_addr->sa_family == AF_LINK) {
+				sa = (struct sockaddr_dl *)ifma->ifma_addr;
+				index = ether_crc32_le(LLADDR(sa), 6) & 0xff;
+				bucket[index] = 1;
+			}
+		}
+		IF_ADDR_UNLOCK(ifp);
+	}
+	return efx_mac_hash_set(enp, bucket);
+}
+
+int
+sfxge_mac_filter_set(struct sfxge_softc *sc)
+{
+	struct sfxge_port *port = &sc->port;
+	int rc;
+
+	KASSERT(port->init_state == SFXGE_PORT_STARTED, ("port not started"));
+
+	mtx_lock(&port->lock);
+	rc = sfxge_mac_filter_set_locked(sc);
+	mtx_unlock(&port->lock);
+	return rc;
+}
+
+void
+sfxge_port_stop(struct sfxge_softc *sc)
+{
+	struct sfxge_port *port;
+	efx_nic_t *enp;
+
+	port = &sc->port;
+	enp = sc->enp;
+
+	mtx_lock(&port->lock);
+
+	KASSERT(port->init_state == SFXGE_PORT_STARTED,
+	    ("port not started"));
+
+	port->init_state = SFXGE_PORT_INITIALIZED;
+
+	port->mac_stats.update_time = 0;
+
+	/* This may call MCDI */
+	(void)efx_mac_drain(enp, B_TRUE);
+
+	(void)efx_mac_stats_periodic(enp, &port->mac_stats.dma_buf, 0, B_FALSE);
+
+	port->link_mode = EFX_LINK_UNKNOWN;
+
+	/* Destroy the common code port object. */
+	efx_port_fini(sc->enp);
+
+	mtx_unlock(&port->lock);
+}
+
+int
+sfxge_port_start(struct sfxge_softc *sc)
+{
+	uint8_t mac_addr[ETHER_ADDR_LEN];
+	struct ifnet *ifp = sc->ifnet;
+	struct sfxge_port *port;
+	efx_nic_t *enp;
+	size_t pdu;
+	int rc;
+
+	port = &sc->port;
+	enp = sc->enp;
+
+	mtx_lock(&port->lock);
+
+	KASSERT(port->init_state == SFXGE_PORT_INITIALIZED,
+	    ("port not initialized"));
+
+	/* Initialize the port object in the common code. */
+	if ((rc = efx_port_init(sc->enp)) != 0)
+		goto fail;
+
+	/* Set the SDU */
+	pdu = EFX_MAC_PDU(ifp->if_mtu);
+	if ((rc = efx_mac_pdu_set(enp, pdu)) != 0)
+		goto fail2;
+
+	if ((rc = efx_mac_fcntl_set(enp, sfxge_port_wanted_fc(sc), B_TRUE))
+	    != 0)
+		goto fail2;
+
+	/* Set the unicast address */
+	IF_ADDR_LOCK(ifp);
+	bcopy(LLADDR((struct sockaddr_dl *)ifp->if_addr->ifa_addr),
+	      mac_addr, sizeof(mac_addr));
+	IF_ADDR_UNLOCK(ifp);
+	if ((rc = efx_mac_addr_set(enp, mac_addr)) != 0)
+		goto fail;
+
+	sfxge_mac_filter_set_locked(sc);
+
+	/* Update MAC stats by DMA every second */
+	if ((rc = efx_mac_stats_periodic(enp, &port->mac_stats.dma_buf,
+            1000, B_FALSE)) != 0)
+		goto fail2;
+
+	if ((rc = efx_mac_drain(enp, B_FALSE)) != 0)
+		goto fail3;
+
+	if ((rc = efx_phy_adv_cap_set(sc->enp, sc->media.ifm_cur->ifm_data))
+	    != 0)
+		goto fail4;
+
+	port->init_state = SFXGE_PORT_STARTED;
+
+	/* Single poll in case there were missing initial events */
+	mtx_unlock(&port->lock);
+	sfxge_mac_poll_work(sc, 0);
+
+	return (0);
+
+fail4:
+	(void)efx_mac_drain(enp, B_TRUE);
+fail3:
+	(void)efx_mac_stats_periodic(enp, &port->mac_stats.dma_buf,
+            0, B_FALSE);
+fail2:
+	efx_port_fini(sc->enp);
+fail:
+	mtx_unlock(&port->lock);
+
+	return (rc);
+}
+
+static int
+sfxge_phy_stat_update(struct sfxge_softc *sc)
+{
+	struct sfxge_port *port = &sc->port;
+	efsys_mem_t *esmp = &port->phy_stats.dma_buf;
+	clock_t now;
+	unsigned int count;
+	int rc;
+
+	mtx_lock(&port->lock);
+
+	if (port->init_state != SFXGE_PORT_STARTED) {
+		rc = 0;
+		goto out;
+	}
+
+	now = ticks;
+	if (now - port->phy_stats.update_time < hz) {
+		rc = 0;
+		goto out;
+	}
+
+	port->phy_stats.update_time = now;
+
+	/* If we're unlucky enough to read statistics wduring the DMA, wait
+	 * up to 10ms for it to finish (typically takes <500us) */
+	for (count = 0; count < 100; ++count) {
+		EFSYS_PROBE1(wait, unsigned int, count);
+
+		/* Synchronize the DMA memory for reading */
+		bus_dmamap_sync(esmp->esm_tag, esmp->esm_map,
+		    BUS_DMASYNC_POSTREAD);
+
+		/* Try to update the cached counters */
+		if ((rc = efx_phy_stats_update(sc->enp, esmp,
+		    port->phy_stats.decode_buf)) != EAGAIN)
+			goto out;
+
+		DELAY(100);
+	}
+
+	rc = ETIMEDOUT;
+out:
+	mtx_unlock(&port->lock);
+	return rc;
+}
+
+static int
+sfxge_phy_stat_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	unsigned int id = arg2;
+	int rc;
+
+	if ((rc = sfxge_phy_stat_update(sc)) != 0)
+		return rc;
+
+	return SYSCTL_OUT(req,
+			  (uint32_t *)sc->port.phy_stats.decode_buf + id,
+			  sizeof(uint32_t));
+}
+
+static void
+sfxge_phy_stat_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid_list *stat_list;
+	unsigned int id;
+	const char *name;
+	uint64_t stat_mask = efx_nic_cfg_get(sc->enp)->enc_phy_stat_mask;
+
+	stat_list = SYSCTL_CHILDREN(sc->stats_node);
+
+	/* Initialise the named stats */
+	for (id = 0; id < EFX_PHY_NSTATS; id++) {
+		if (!(stat_mask & ((uint64_t)1 << id)))
+			continue;
+		name = efx_phy_stat_name(sc->enp, id);
+		SYSCTL_ADD_PROC(
+			ctx, stat_list,
+			OID_AUTO, name, CTLTYPE_UINT|CTLFLAG_RD,
+			sc, id, sfxge_phy_stat_handler,
+			id == EFX_PHY_STAT_OUI ? "IX" : "IU",
+			"");
+	}
+}
+
+void
+sfxge_port_fini(struct sfxge_softc *sc)
+{
+	struct sfxge_port *port;
+	efsys_mem_t *esmp;
+
+	port = &sc->port;
+	esmp = &port->mac_stats.dma_buf;
+
+	KASSERT(port->init_state == SFXGE_PORT_INITIALIZED,
+	    ("Port not initialized"));
+
+	port->init_state = SFXGE_PORT_UNINITIALIZED;
+
+	port->link_mode = EFX_LINK_UNKNOWN;
+
+	/* Finish with PHY DMA memory */
+	sfxge_dma_free(&port->phy_stats.dma_buf);
+	free(port->phy_stats.decode_buf, M_SFXGE);
+
+	sfxge_dma_free(esmp);
+	free(port->mac_stats.decode_buf, M_SFXGE);
+
+	mtx_destroy(&port->lock);
+
+	port->sc = NULL;
+}
+
+int
+sfxge_port_init(struct sfxge_softc *sc)
+{
+	struct sfxge_port *port;
+	struct sysctl_ctx_list *sysctl_ctx;
+	struct sysctl_oid *sysctl_tree;
+	efsys_mem_t *mac_stats_buf, *phy_stats_buf;
+	int rc;
+
+	port = &sc->port;
+	mac_stats_buf = &port->mac_stats.dma_buf;
+	phy_stats_buf = &port->phy_stats.dma_buf;
+
+	KASSERT(port->init_state == SFXGE_PORT_UNINITIALIZED,
+	    ("Port already initialized"));
+
+	port->sc = sc;
+
+	mtx_init(&port->lock, "sfxge_port", NULL, MTX_DEF);
+
+	port->phy_stats.decode_buf = malloc(EFX_PHY_NSTATS * sizeof(uint32_t),
+					    M_SFXGE, M_WAITOK | M_ZERO);
+	if ((rc = sfxge_dma_alloc(sc, EFX_PHY_STATS_SIZE, phy_stats_buf)) != 0)
+		goto fail;
+	bzero(phy_stats_buf->esm_base, phy_stats_buf->esm_size);
+	sfxge_phy_stat_init(sc);
+
+	sysctl_ctx = device_get_sysctl_ctx(sc->dev);
+	sysctl_tree = device_get_sysctl_tree(sc->dev);
+
+#ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS
+	/* If flow control cannot be configured or reported through
+	 * ifmedia, provide sysctls for it. */
+	port->wanted_fc = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
+	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
+	    "wanted_fc", CTLTYPE_UINT|CTLFLAG_RW, sc, 0,
+	    sfxge_port_wanted_fc_handler, "IU", "wanted flow control mode");
+	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
+	    "link_fc", CTLTYPE_UINT|CTLFLAG_RD, sc, 0,
+	    sfxge_port_link_fc_handler, "IU", "link flow control mode");
+#endif
+
+	port->mac_stats.decode_buf = malloc(EFX_MAC_NSTATS * sizeof(uint64_t),
+					    M_SFXGE, M_WAITOK | M_ZERO);
+	if ((rc = sfxge_dma_alloc(sc, EFX_MAC_STATS_SIZE, mac_stats_buf)) != 0)
+		goto fail2;
+	bzero(mac_stats_buf->esm_base, mac_stats_buf->esm_size);
+	sfxge_mac_stat_init(sc);
+
+	port->init_state = SFXGE_PORT_INITIALIZED;
+
+	return (0);
+
+fail2:
+	free(port->mac_stats.decode_buf, M_SFXGE);
+	sfxge_dma_free(phy_stats_buf);
+fail:
+	free(port->phy_stats.decode_buf, M_SFXGE);
+	(void)mtx_destroy(&port->lock);
+	port->sc = NULL;
+	return rc;
+}
+
+static int sfxge_link_mode[EFX_PHY_MEDIA_NTYPES][EFX_LINK_NMODES] = {
+	[EFX_PHY_MEDIA_CX4] = {
+		[EFX_LINK_10000FDX]	= IFM_ETHER | IFM_FDX | IFM_10G_CX4,
+	},
+	[EFX_PHY_MEDIA_KX4] = {
+		[EFX_LINK_10000FDX]	= IFM_ETHER | IFM_FDX | IFM_10G_KX4,
+	},
+	[EFX_PHY_MEDIA_XFP] = {
+		/* Don't know the module type, but assume SR for now. */
+		[EFX_LINK_10000FDX]	= IFM_ETHER | IFM_FDX | IFM_10G_SR,
+	},
+	[EFX_PHY_MEDIA_SFP_PLUS] = {
+		/* Don't know the module type, but assume SX/SR for now. */
+		[EFX_LINK_1000FDX]	= IFM_ETHER | IFM_FDX | IFM_1000_SX,
+		[EFX_LINK_10000FDX]	= IFM_ETHER | IFM_FDX | IFM_10G_SR,
+	},
+	[EFX_PHY_MEDIA_BASE_T] = {
+		[EFX_LINK_10HDX]	= IFM_ETHER | IFM_HDX | IFM_10_T,
+		[EFX_LINK_10FDX]	= IFM_ETHER | IFM_FDX | IFM_10_T,
+		[EFX_LINK_100HDX]	= IFM_ETHER | IFM_HDX | IFM_100_TX,
+		[EFX_LINK_100FDX]	= IFM_ETHER | IFM_FDX | IFM_100_TX,
+		[EFX_LINK_1000HDX]	= IFM_ETHER | IFM_HDX | IFM_1000_T,
+		[EFX_LINK_1000FDX]	= IFM_ETHER | IFM_FDX | IFM_1000_T,
+		[EFX_LINK_10000FDX]	= IFM_ETHER | IFM_FDX | IFM_10G_T,
+	},
+};
+
+static void
+sfxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	struct sfxge_softc *sc;
+	efx_phy_media_type_t medium_type;
+	efx_link_mode_t mode;
+
+	sc = ifp->if_softc;
+	sx_xlock(&sc->softc_lock);
+
+	ifmr->ifm_status = IFM_AVALID;
+	ifmr->ifm_active = IFM_ETHER;
+
+	if (SFXGE_RUNNING(sc) && SFXGE_LINK_UP(sc)) {
+		ifmr->ifm_status |= IFM_ACTIVE;
+
+		efx_phy_media_type_get(sc->enp, &medium_type);
+		mode = sc->port.link_mode;
+		ifmr->ifm_active |= sfxge_link_mode[medium_type][mode];
+		ifmr->ifm_active |= sfxge_port_link_fc_ifm(sc);
+	}
+
+	sx_xunlock(&sc->softc_lock);
+}
+
+static int
+sfxge_media_change(struct ifnet *ifp)
+{
+	struct sfxge_softc *sc;
+	struct ifmedia_entry *ifm;
+	int rc;
+
+	sc = ifp->if_softc;
+	ifm = sc->media.ifm_cur;
+
+	sx_xlock(&sc->softc_lock);
+
+	if (!SFXGE_RUNNING(sc)) {
+		rc = 0;
+		goto out;
+	}
+
+	rc = efx_mac_fcntl_set(sc->enp, sfxge_port_wanted_fc(sc), B_TRUE);
+	if (rc != 0)
+		goto out;
+
+	rc = efx_phy_adv_cap_set(sc->enp, ifm->ifm_data);
+out:
+	sx_xunlock(&sc->softc_lock);	
+
+	return rc;
+}
+
+int sfxge_port_ifmedia_init(struct sfxge_softc *sc)
+{
+	efx_phy_media_type_t medium_type;
+	uint32_t cap_mask, mode_cap_mask;
+	efx_link_mode_t mode;
+	int mode_ifm, best_mode_ifm = 0;
+	int rc;
+
+	/* We need port state to initialise the ifmedia list. */
+	if ((rc = efx_nic_init(sc->enp)) != 0)
+		goto out;
+	if ((rc = efx_port_init(sc->enp)) != 0)
+		goto out2;
+
+	/*
+	 * Register ifconfig callbacks for querying and setting the
+	 * link mode and link status.
+	 */
+	ifmedia_init(&sc->media, IFM_IMASK, sfxge_media_change,
+	    sfxge_media_status);
+
+	/*
+	 * Map firmware medium type and capabilities to ifmedia types.
+	 * ifmedia does not distinguish between forcing the link mode
+	 * and disabling auto-negotiation.  1000BASE-T and 10GBASE-T
+	 * require AN even if only one link mode is enabled, and for
+	 * 100BASE-TX it is useful even if the link mode is forced.
+	 * Therefore we never disable auto-negotiation.
+	 *
+	 * Also enable and advertise flow control by default.
+	 */
+
+	efx_phy_media_type_get(sc->enp, &medium_type);
+	efx_phy_adv_cap_get(sc->enp, EFX_PHY_CAP_PERM, &cap_mask);
+
+	EFX_STATIC_ASSERT(EFX_LINK_10HDX == EFX_PHY_CAP_10HDX + 1);
+	EFX_STATIC_ASSERT(EFX_LINK_10FDX == EFX_PHY_CAP_10FDX + 1);
+	EFX_STATIC_ASSERT(EFX_LINK_100HDX == EFX_PHY_CAP_100HDX + 1);
+	EFX_STATIC_ASSERT(EFX_LINK_100FDX == EFX_PHY_CAP_100FDX + 1);
+	EFX_STATIC_ASSERT(EFX_LINK_1000HDX == EFX_PHY_CAP_1000HDX + 1);
+	EFX_STATIC_ASSERT(EFX_LINK_1000FDX == EFX_PHY_CAP_1000FDX + 1);
+	EFX_STATIC_ASSERT(EFX_LINK_10000FDX == EFX_PHY_CAP_10000FDX + 1);
+
+	for (mode = EFX_LINK_10HDX; mode <= EFX_LINK_10000FDX; mode++) {
+		mode_cap_mask = 1 << (mode - 1);
+		mode_ifm = sfxge_link_mode[medium_type][mode];
+
+		if ((cap_mask & mode_cap_mask) && mode_ifm) {
+			mode_cap_mask |= cap_mask & (1 << EFX_PHY_CAP_AN);
+
+#ifdef SFXGE_HAVE_PAUSE_MEDIAOPTS
+			/* No flow-control */
+			ifmedia_add(&sc->media, mode_ifm, mode_cap_mask, NULL);
+
+			/* Respond-only.  If using AN, we implicitly
+			 * offer symmetric as well, but that doesn't
+			 * mean we *have* to generate pause frames.
+			 */
+			mode_cap_mask |= cap_mask & ((1 << EFX_PHY_CAP_PAUSE) |
+						     (1 << EFX_PHY_CAP_ASYM));
+			mode_ifm |= IFM_ETH_RXPAUSE;
+			ifmedia_add(&sc->media, mode_ifm, mode_cap_mask, NULL);
+
+			/* Symmetric */
+			mode_cap_mask &= ~(1 << EFX_PHY_CAP_ASYM);
+			mode_ifm |= IFM_ETH_TXPAUSE;
+#else /* !SFXGE_HAVE_PAUSE_MEDIAOPTS */
+			mode_cap_mask |= cap_mask & (1 << EFX_PHY_CAP_PAUSE);
+#endif
+			ifmedia_add(&sc->media, mode_ifm, mode_cap_mask, NULL);
+
+			/* Link modes are numbered in order of speed,
+			 * so assume the last one available is the best.
+			 */
+			best_mode_ifm = mode_ifm;
+		}
+	}
+
+	if (cap_mask & (1 << EFX_PHY_CAP_AN)) {
+		/* Add autoselect mode. */
+		mode_ifm = IFM_ETHER | IFM_AUTO;
+		ifmedia_add(&sc->media, mode_ifm,
+			    cap_mask & ~(1 << EFX_PHY_CAP_ASYM), NULL);
+		best_mode_ifm = mode_ifm;
+	}
+
+	if (best_mode_ifm)
+		ifmedia_set(&sc->media, best_mode_ifm);
+
+	/* Now discard port state until interface is started. */
+	efx_port_fini(sc->enp);
+out2:
+	efx_nic_fini(sc->enp);
+out:
+	return rc;
+}
diff --git a/sys/dev/sfxge/sfxge_rx.c b/sys/dev/sfxge/sfxge_rx.c
new file mode 100644
index 0000000..7dd5160
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_rx.c
@@ -0,0 +1,1233 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/mbuf.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/limits.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_vlan_var.h>
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+
+#include <machine/in_cksum.h>
+
+#include "common/efx.h"
+
+
+#include "sfxge.h"
+#include "sfxge_rx.h"
+
+#define RX_REFILL_THRESHOLD (EFX_RXQ_LIMIT(SFXGE_NDESCS) * 9 / 10)
+#define RX_REFILL_THRESHOLD_2 (RX_REFILL_THRESHOLD / 2)
+
+/* Size of the LRO hash table.  Must be a power of 2.  A larger table
+ * means we can accelerate a larger number of streams.
+ */
+static unsigned lro_table_size = 128;
+
+/* Maximum length of a hash chain.  If chains get too long then the lookup
+ * time increases and may exceed the benefit of LRO.
+ */
+static unsigned lro_chain_max = 20;
+
+/* Maximum time (in ticks) that a connection can be idle before it's LRO
+ * state is discarded.
+ */
+static unsigned lro_idle_ticks; /* initialised in sfxge_rx_init() */
+
+/* Number of packets with payload that must arrive in-order before a
+ * connection is eligible for LRO.  The idea is we should avoid coalescing
+ * segments when the sender is in slow-start because reducing the ACK rate
+ * can damage performance.
+ */
+static int lro_slow_start_packets = 2000;
+
+/* Number of packets with payload that must arrive in-order following loss
+ * before a connection is eligible for LRO.  The idea is we should avoid
+ * coalescing segments when the sender is recovering from loss, because
+ * reducing the ACK rate can damage performance.
+ */
+static int lro_loss_packets = 20;
+
+/* Flags for sfxge_lro_conn::l2_id; must not collide with EVL_VLID_MASK */
+#define SFXGE_LRO_L2_ID_VLAN 0x4000
+#define SFXGE_LRO_L2_ID_IPV6 0x8000
+#define SFXGE_LRO_CONN_IS_VLAN_ENCAP(c) ((c)->l2_id & SFXGE_LRO_L2_ID_VLAN)
+#define SFXGE_LRO_CONN_IS_TCPIPV4(c) (!((c)->l2_id & SFXGE_LRO_L2_ID_IPV6))
+
+/* Compare IPv6 addresses, avoiding conditional branches */
+static __inline unsigned long ipv6_addr_cmp(const struct in6_addr *left,
+					    const struct in6_addr *right)
+{
+#if LONG_BIT == 64
+	const uint64_t *left64 = (const uint64_t *)left;
+	const uint64_t *right64 = (const uint64_t *)right;
+	return (left64[0] - right64[0]) | (left64[1] - right64[1]);
+#else
+	return (left->s6_addr32[0] - right->s6_addr32[0]) |
+	       (left->s6_addr32[1] - right->s6_addr32[1]) |
+	       (left->s6_addr32[2] - right->s6_addr32[2]) |
+	       (left->s6_addr32[3] - right->s6_addr32[3]);
+#endif
+}
+
+void
+sfxge_rx_qflush_done(struct sfxge_rxq *rxq)
+{
+
+	rxq->flush_state = SFXGE_FLUSH_DONE;
+}
+
+void
+sfxge_rx_qflush_failed(struct sfxge_rxq *rxq)
+{
+
+	rxq->flush_state = SFXGE_FLUSH_FAILED;
+}
+
+static uint8_t toep_key[] = {
+	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
+	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
+	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
+	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
+	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa
+};
+
+static void
+sfxge_rx_post_refill(void *arg)
+{
+	struct sfxge_rxq *rxq = arg;
+	struct sfxge_softc *sc;
+	unsigned int index;
+	struct sfxge_evq *evq;
+	uint16_t magic;
+
+	sc = rxq->sc;
+	index = rxq->index;
+	evq = sc->evq[index];
+
+	magic = SFXGE_MAGIC_RX_QREFILL | index;
+
+	/* This is guaranteed due to the start/stop order of rx and ev */
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq not started"));
+	KASSERT(rxq->init_state == SFXGE_RXQ_STARTED,
+	    ("rxq not started"));
+	efx_ev_qpost(evq->common, magic);
+}
+
+static void
+sfxge_rx_schedule_refill(struct sfxge_rxq *rxq, boolean_t retrying)
+{
+	/* Initially retry after 100 ms, but back off in case of
+	 * repeated failures as we probably have to wait for the
+	 * administrator to raise the pool limit. */
+	if (retrying)
+		rxq->refill_delay = min(rxq->refill_delay * 2, 10 * hz);
+	else
+		rxq->refill_delay = hz / 10;
+
+	callout_reset_curcpu(&rxq->refill_callout, rxq->refill_delay,
+			     sfxge_rx_post_refill, rxq);
+}
+
+static inline struct mbuf *sfxge_rx_alloc_mbuf(struct sfxge_softc *sc)
+{
+	struct mb_args args;
+	struct mbuf *m;
+
+	/* Allocate mbuf structure */
+	args.flags = M_PKTHDR;
+	args.type = MT_DATA;
+	m = (struct mbuf *)uma_zalloc_arg(zone_mbuf, &args, M_DONTWAIT);
+
+	/* Allocate (and attach) packet buffer */
+	if (m && !uma_zalloc_arg(sc->rx_buffer_zone, m, M_DONTWAIT)) {
+		uma_zfree(zone_mbuf, m);
+		m = NULL;
+	}
+
+	return m;
+}
+
+#define	SFXGE_REFILL_BATCH  64
+
+static void
+sfxge_rx_qfill(struct sfxge_rxq *rxq, unsigned int target, boolean_t retrying)
+{
+	struct sfxge_softc *sc;
+	unsigned int index;
+	struct sfxge_evq *evq;
+	unsigned int batch;
+	unsigned int rxfill;
+	unsigned int mblksize;
+	int ntodo;
+	efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
+
+	sc = rxq->sc;
+	index = rxq->index;
+	evq = sc->evq[index];
+
+	prefetch_read_many(sc->enp);
+	prefetch_read_many(rxq->common);
+
+	mtx_assert(&evq->lock, MA_OWNED);
+
+	if (rxq->init_state != SFXGE_RXQ_STARTED)
+		return;
+
+	rxfill = rxq->added - rxq->completed;
+	KASSERT(rxfill <= EFX_RXQ_LIMIT(SFXGE_NDESCS),
+	    ("rxfill > EFX_RXQ_LIMIT(SFXGE_NDESCS)"));
+	ntodo = min(EFX_RXQ_LIMIT(SFXGE_NDESCS) - rxfill, target);
+	KASSERT(ntodo <= EFX_RXQ_LIMIT(SFXGE_NDESCS),
+	    ("ntodo > EFX_RQX_LIMIT(SFXGE_NDESCS)"));
+
+	if (ntodo == 0)
+		return;
+
+	batch = 0;
+	mblksize = sc->rx_buffer_size;
+	while (ntodo-- > 0) {
+		unsigned int id;
+		struct sfxge_rx_sw_desc *rx_desc;
+		bus_dma_segment_t seg;
+		struct mbuf *m;
+
+		id = (rxq->added + batch) & (SFXGE_NDESCS - 1);
+		rx_desc = &rxq->queue[id];
+		KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL"));
+
+		rx_desc->flags = EFX_DISCARD;
+		m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc);
+		if (m == NULL)
+			break;
+		sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg);
+		addr[batch++] = seg.ds_addr;
+
+		if (batch == SFXGE_REFILL_BATCH) {
+			efx_rx_qpost(rxq->common, addr, mblksize, batch,
+			    rxq->completed, rxq->added);
+			rxq->added += batch;
+			batch = 0;
+		}
+	}
+
+	if (ntodo != 0)
+		sfxge_rx_schedule_refill(rxq, retrying);
+
+	if (batch != 0) {
+		efx_rx_qpost(rxq->common, addr, mblksize, batch,
+		    rxq->completed, rxq->added);
+		rxq->added += batch;
+	}
+
+	/* Make the descriptors visible to the hardware */
+	bus_dmamap_sync(rxq->mem.esm_tag, rxq->mem.esm_map,
+			BUS_DMASYNC_PREWRITE);
+
+	efx_rx_qpush(rxq->common, rxq->added);
+}
+
+void
+sfxge_rx_qrefill(struct sfxge_rxq *rxq)
+{
+
+	if (rxq->init_state != SFXGE_RXQ_STARTED)
+		return;
+
+	/* Make sure the queue is full */
+	sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(SFXGE_NDESCS), B_TRUE);
+}
+
+static void __sfxge_rx_deliver(struct sfxge_softc *sc, struct mbuf *m)
+{
+	struct ifnet *ifp = sc->ifnet;
+
+	m->m_pkthdr.rcvif = ifp;
+	m->m_pkthdr.header = m->m_data;
+	m->m_pkthdr.csum_data = 0xffff;
+	ifp->if_input(ifp, m);
+}
+
+static void
+sfxge_rx_deliver(struct sfxge_softc *sc, struct sfxge_rx_sw_desc *rx_desc)
+{
+	struct mbuf *m = rx_desc->mbuf;
+	int csum_flags;
+
+	/* Convert checksum flags */
+	csum_flags = (rx_desc->flags & EFX_CKSUM_IPV4) ?
+		(CSUM_IP_CHECKED | CSUM_IP_VALID) : 0;
+	if (rx_desc->flags & EFX_CKSUM_TCPUDP)
+		csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+
+#ifdef SFXGE_HAVE_MQ
+	/* The hash covers a 4-tuple for TCP only */
+	if (rx_desc->flags & EFX_PKT_TCP) {
+		m->m_pkthdr.flowid = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ,
+						       mtod(m, uint8_t *));
+		m->m_flags |= M_FLOWID;
+	}
+#endif
+	m->m_data += sc->rx_prefix_size;
+	m->m_len = rx_desc->size - sc->rx_prefix_size;
+	m->m_pkthdr.len = m->m_len;
+	m->m_pkthdr.csum_flags = csum_flags;
+	__sfxge_rx_deliver(sc, rx_desc->mbuf);
+
+	rx_desc->flags = EFX_DISCARD;
+	rx_desc->mbuf = NULL;
+}
+
+static void
+sfxge_lro_deliver(struct sfxge_lro_state *st, struct sfxge_lro_conn *c)
+{
+	struct sfxge_softc *sc = st->sc;
+	struct mbuf *m = c->mbuf;
+	struct tcphdr *c_th;
+	int csum_flags;
+
+	KASSERT(m, ("no mbuf to deliver"));
+
+	++st->n_bursts;
+
+	/* Finish off packet munging and recalculate IP header checksum. */
+	if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
+		struct ip *iph = c->nh;
+		iph->ip_len = htons(iph->ip_len);
+		iph->ip_sum = 0;
+		iph->ip_sum = in_cksum_hdr(iph);
+		c_th = (struct tcphdr *)(iph + 1);
+		csum_flags = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
+			      CSUM_IP_CHECKED | CSUM_IP_VALID);
+	} else {
+		struct ip6_hdr *iph = c->nh;
+		iph->ip6_plen = htons(iph->ip6_plen);
+		c_th = (struct tcphdr *)(iph + 1);
+		csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+	}
+
+	c_th->th_win = c->th_last->th_win;
+	c_th->th_ack = c->th_last->th_ack;
+	if (c_th->th_off == c->th_last->th_off) {
+		/* Copy TCP options (take care to avoid going negative). */
+		int optlen = ((c_th->th_off - 5) & 0xf) << 2u;
+		memcpy(c_th + 1, c->th_last + 1, optlen);
+	}
+
+#ifdef SFXGE_HAVE_MQ
+	m->m_pkthdr.flowid = c->conn_hash;
+	m->m_flags |= M_FLOWID;
+#endif
+	m->m_pkthdr.csum_flags = csum_flags;
+	__sfxge_rx_deliver(sc, m);
+
+	c->mbuf = NULL;
+	c->delivered = 1;
+}
+
+/* Drop the given connection, and add it to the free list. */
+static void sfxge_lro_drop(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c)
+{
+	unsigned bucket;
+
+	KASSERT(!c->mbuf, ("found orphaned mbuf"));
+
+	if (c->next_buf.mbuf) {
+		sfxge_rx_deliver(rxq->sc, &c->next_buf);
+		LIST_REMOVE(c, active_link);
+	}
+
+	bucket = c->conn_hash & rxq->lro.conns_mask;
+	KASSERT(rxq->lro.conns_n[bucket] > 0, ("LRO: bucket fill level wrong"));
+	--rxq->lro.conns_n[bucket];
+	TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link);
+	TAILQ_INSERT_HEAD(&rxq->lro.free_conns, c, link);
+}
+
+/* Stop tracking connections that have gone idle in order to keep hash
+ * chains short.
+ */
+static void sfxge_lro_purge_idle(struct sfxge_rxq *rxq, unsigned now)
+{
+	struct sfxge_lro_conn *c;
+	unsigned i;
+
+	KASSERT(LIST_EMPTY(&rxq->lro.active_conns),
+		("found active connections"));
+
+	rxq->lro.last_purge_ticks = now;
+	for (i = 0; i <= rxq->lro.conns_mask; ++i) {
+		if (TAILQ_EMPTY(&rxq->lro.conns[i]))
+			continue;
+
+		c = TAILQ_LAST(&rxq->lro.conns[i], sfxge_lro_tailq);
+		if (now - c->last_pkt_ticks > lro_idle_ticks) {
+			++rxq->lro.n_drop_idle;
+			sfxge_lro_drop(rxq, c);
+		}
+	}
+}
+
+static void
+sfxge_lro_merge(struct sfxge_lro_state *st, struct sfxge_lro_conn *c,
+		struct mbuf *mbuf, struct tcphdr *th)
+{
+	struct tcphdr *c_th;
+
+	/* Tack the new mbuf onto the chain. */
+	KASSERT(!mbuf->m_next, ("mbuf already chained"));
+	c->mbuf_tail->m_next = mbuf;
+	c->mbuf_tail = mbuf;
+
+	/* Increase length appropriately */
+	c->mbuf->m_pkthdr.len += mbuf->m_len;
+
+	/* Update the connection state flags */
+	if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
+		struct ip *iph = c->nh;
+		iph->ip_len += mbuf->m_len;
+		c_th = (struct tcphdr *)(iph + 1);
+	} else {
+		struct ip6_hdr *iph = c->nh;
+		iph->ip6_plen += mbuf->m_len;
+		c_th = (struct tcphdr *)(iph + 1);
+	}
+	c_th->th_flags |= (th->th_flags & TH_PUSH);
+	c->th_last = th;
+	++st->n_merges;
+
+	/* Pass packet up now if another segment could overflow the IP
+	 * length.
+	 */
+	if (c->mbuf->m_pkthdr.len > 65536 - 9200)
+		sfxge_lro_deliver(st, c);
+}
+
+static void
+sfxge_lro_start(struct sfxge_lro_state *st, struct sfxge_lro_conn *c,
+		struct mbuf *mbuf, void *nh, struct tcphdr *th)
+{
+	/* Start the chain */
+	c->mbuf = mbuf;
+	c->mbuf_tail = c->mbuf;
+	c->nh = nh;
+	c->th_last = th;
+
+	mbuf->m_pkthdr.len = mbuf->m_len;
+
+	/* Mangle header fields for later processing */
+	if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
+		struct ip *iph = nh;
+		iph->ip_len = ntohs(iph->ip_len);
+	} else {
+		struct ip6_hdr *iph = nh;
+		iph->ip6_plen = ntohs(iph->ip6_plen);
+	}
+}
+
+/* Try to merge or otherwise hold or deliver (as appropriate) the
+ * packet buffered for this connection (c->next_buf).  Return a flag
+ * indicating whether the connection is still active for LRO purposes.
+ */
+static int
+sfxge_lro_try_merge(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c)
+{
+	struct sfxge_rx_sw_desc *rx_buf = &c->next_buf;
+	char *eh = c->next_eh;
+	int data_length, hdr_length, dont_merge;
+	unsigned th_seq, pkt_length;
+	struct tcphdr *th;
+	unsigned now;
+
+	if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
+		struct ip *iph = c->next_nh;
+		th = (struct tcphdr *)(iph + 1);
+		pkt_length = ntohs(iph->ip_len) + (char *) iph - eh;
+	} else {
+		struct ip6_hdr *iph = c->next_nh;
+		th = (struct tcphdr *)(iph + 1);
+		pkt_length = ntohs(iph->ip6_plen) + (char *) th - eh;
+	}
+
+	hdr_length = (char *) th + th->th_off * 4 - eh;
+	data_length = (min(pkt_length, rx_buf->size - rxq->sc->rx_prefix_size) -
+		       hdr_length);
+	th_seq = ntohl(th->th_seq);
+	dont_merge = ((data_length <= 0)
+		      | (th->th_flags & (TH_URG | TH_SYN | TH_RST | TH_FIN)));
+
+	/* Check for options other than aligned timestamp. */
+	if (th->th_off != 5) {
+		const uint32_t *opt_ptr = (const uint32_t *) (th + 1);
+		if (th->th_off == 8 &&
+		    opt_ptr[0] == ntohl((TCPOPT_NOP << 24) |
+					(TCPOPT_NOP << 16) |
+					(TCPOPT_TIMESTAMP << 8) |
+					TCPOLEN_TIMESTAMP)) {
+			/* timestamp option -- okay */
+		} else {
+			dont_merge = 1;
+		}
+	}
+
+	if (__predict_false(th_seq != c->next_seq)) {
+		/* Out-of-order, so start counting again. */
+		if (c->mbuf)
+			sfxge_lro_deliver(&rxq->lro, c);
+		c->n_in_order_pkts -= lro_loss_packets;
+		c->next_seq = th_seq + data_length;
+		++rxq->lro.n_misorder;
+		goto deliver_buf_out;
+	}
+	c->next_seq = th_seq + data_length;
+
+	now = ticks;
+	if (now - c->last_pkt_ticks > lro_idle_ticks) {
+		++rxq->lro.n_drop_idle;
+		if (c->mbuf)
+			sfxge_lro_deliver(&rxq->lro, c);
+		sfxge_lro_drop(rxq, c);
+		return 0;
+	}
+	c->last_pkt_ticks = ticks;
+
+	if (c->n_in_order_pkts < lro_slow_start_packets) {
+		/* May be in slow-start, so don't merge. */
+		++rxq->lro.n_slow_start;
+		++c->n_in_order_pkts;
+		goto deliver_buf_out;
+	}
+
+	if (__predict_false(dont_merge)) {
+		if (c->mbuf)
+			sfxge_lro_deliver(&rxq->lro, c);
+		if (th->th_flags & (TH_FIN | TH_RST)) {
+			++rxq->lro.n_drop_closed;
+			sfxge_lro_drop(rxq, c);
+			return 0;
+		}
+		goto deliver_buf_out;
+	}
+
+	rx_buf->mbuf->m_data += rxq->sc->rx_prefix_size;
+
+	if (__predict_true(c->mbuf != NULL)) {
+		/* Remove headers and any padding */
+		rx_buf->mbuf->m_data += hdr_length;
+		rx_buf->mbuf->m_len = data_length;
+
+		sfxge_lro_merge(&rxq->lro, c, rx_buf->mbuf, th);
+	} else {
+		/* Remove any padding */
+		rx_buf->mbuf->m_len = pkt_length;
+
+		sfxge_lro_start(&rxq->lro, c, rx_buf->mbuf, c->next_nh, th);
+	}
+
+	rx_buf->mbuf = NULL;
+	return 1;
+
+ deliver_buf_out:
+	sfxge_rx_deliver(rxq->sc, rx_buf);
+	return 1;
+}
+
+static void sfxge_lro_new_conn(struct sfxge_lro_state *st, uint32_t conn_hash,
+			       uint16_t l2_id, void *nh, struct tcphdr *th)
+{
+	unsigned bucket = conn_hash & st->conns_mask;
+	struct sfxge_lro_conn *c;
+
+	if (st->conns_n[bucket] >= lro_chain_max) {
+		++st->n_too_many;
+		return;
+	}
+
+	if (!TAILQ_EMPTY(&st->free_conns)) {
+		c = TAILQ_FIRST(&st->free_conns);
+		TAILQ_REMOVE(&st->free_conns, c, link);
+	} else {
+		c = malloc(sizeof(*c), M_SFXGE, M_DONTWAIT);
+		if (c == NULL)
+			return;
+		c->mbuf = NULL;
+		c->next_buf.mbuf = NULL;
+	}
+
+	/* Create the connection tracking data */
+	++st->conns_n[bucket];
+	TAILQ_INSERT_HEAD(&st->conns[bucket], c, link);
+	c->l2_id = l2_id;
+	c->conn_hash = conn_hash;
+	c->source = th->th_sport;
+	c->dest = th->th_dport;
+	c->n_in_order_pkts = 0;
+	c->last_pkt_ticks = *(volatile int *)&ticks;
+	c->delivered = 0;
+	++st->n_new_stream;
+	/* NB. We don't initialise c->next_seq, and it doesn't matter what
+	 * value it has.  Most likely the next packet received for this
+	 * connection will not match -- no harm done.
+	 */
+}
+
+/* Process mbuf and decide whether to dispatch it to the stack now or
+ * later.
+ */
+static void
+sfxge_lro(struct sfxge_rxq *rxq, struct sfxge_rx_sw_desc *rx_buf)
+{
+	struct sfxge_softc *sc = rxq->sc;
+	struct mbuf *m = rx_buf->mbuf;
+	struct ether_header *eh;
+	struct sfxge_lro_conn *c;
+	uint16_t l2_id;
+	uint16_t l3_proto;
+        void *nh;
+	struct tcphdr *th;
+	uint32_t conn_hash;
+	unsigned bucket;
+
+	/* Get the hardware hash */
+	conn_hash = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ,
+				      mtod(m, uint8_t *));
+
+	eh = (struct ether_header *)(m->m_data + sc->rx_prefix_size);
+	if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
+		struct ether_vlan_header *veh = (struct ether_vlan_header *)eh;
+		l2_id = EVL_VLANOFTAG(ntohs(veh->evl_tag)) |
+			SFXGE_LRO_L2_ID_VLAN;
+		l3_proto = veh->evl_proto;
+		nh = veh + 1;
+	} else {
+		l2_id = 0;
+		l3_proto = eh->ether_type;
+		nh = eh + 1;
+	}
+
+	/* Check whether this is a suitable packet (unfragmented
+	 * TCP/IPv4 or TCP/IPv6).  If so, find the TCP header and
+	 * length, and compute a hash if necessary.  If not, return.
+	 */
+	if (l3_proto == htons(ETHERTYPE_IP)) {
+		struct ip *iph = nh;
+		if ((iph->ip_p - IPPROTO_TCP) |
+		    (iph->ip_hl - (sizeof(*iph) >> 2u)) |
+		    (iph->ip_off & htons(IP_MF | IP_OFFMASK)))
+			goto deliver_now;
+		th = (struct tcphdr *)(iph + 1);
+	} else if (l3_proto == htons(ETHERTYPE_IPV6)) {
+		struct ip6_hdr *iph = nh;
+		if (iph->ip6_nxt != IPPROTO_TCP)
+			goto deliver_now;
+		l2_id |= SFXGE_LRO_L2_ID_IPV6;
+		th = (struct tcphdr *)(iph + 1);
+	} else {
+		goto deliver_now;
+	}
+
+	bucket = conn_hash & rxq->lro.conns_mask;
+
+	TAILQ_FOREACH(c, &rxq->lro.conns[bucket], link) {
+		if ((c->l2_id - l2_id) | (c->conn_hash - conn_hash))
+			continue;
+		if ((c->source - th->th_sport) | (c->dest - th->th_dport))
+			continue;
+		if (c->mbuf) {
+			if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
+				struct ip *c_iph, *iph = nh;
+				c_iph = c->nh;
+				if ((c_iph->ip_src.s_addr - iph->ip_src.s_addr) |
+				    (c_iph->ip_dst.s_addr - iph->ip_dst.s_addr))
+					continue;
+			} else {
+				struct ip6_hdr *c_iph, *iph = nh;
+				c_iph = c->nh;
+				if (ipv6_addr_cmp(&c_iph->ip6_src, &iph->ip6_src) |
+				    ipv6_addr_cmp(&c_iph->ip6_dst, &iph->ip6_dst))
+					continue;
+			}
+		}
+
+		/* Re-insert at head of list to reduce lookup time. */
+		TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link);
+		TAILQ_INSERT_HEAD(&rxq->lro.conns[bucket], c, link);
+
+		if (c->next_buf.mbuf) {
+			if (!sfxge_lro_try_merge(rxq, c))
+				goto deliver_now;
+		} else {
+			LIST_INSERT_HEAD(&rxq->lro.active_conns, c,
+			    active_link);
+		}
+		c->next_buf = *rx_buf;
+		c->next_eh = eh;
+		c->next_nh = nh;
+
+		rx_buf->mbuf = NULL;
+		rx_buf->flags = EFX_DISCARD;
+		return;
+	}
+
+	sfxge_lro_new_conn(&rxq->lro, conn_hash, l2_id, nh, th);
+ deliver_now:
+	sfxge_rx_deliver(sc, rx_buf);
+}
+
+static void sfxge_lro_end_of_burst(struct sfxge_rxq *rxq)
+{
+	struct sfxge_lro_state *st = &rxq->lro;
+	struct sfxge_lro_conn *c;
+	unsigned t;
+
+	while (!LIST_EMPTY(&st->active_conns)) {
+		c = LIST_FIRST(&st->active_conns);
+		if (!c->delivered && c->mbuf)
+			sfxge_lro_deliver(st, c);
+		if (sfxge_lro_try_merge(rxq, c)) {
+			if (c->mbuf)
+				sfxge_lro_deliver(st, c);
+			LIST_REMOVE(c, active_link);
+		}
+		c->delivered = 0;
+	}
+
+	t = *(volatile int *)&ticks;
+	if (__predict_false(t != st->last_purge_ticks))
+		sfxge_lro_purge_idle(rxq, t);
+}
+
+void
+sfxge_rx_qcomplete(struct sfxge_rxq *rxq, boolean_t eop)
+{
+	struct sfxge_softc *sc = rxq->sc;
+	int lro_enabled = sc->ifnet->if_capenable & IFCAP_LRO;
+	unsigned int index;
+	struct sfxge_evq *evq;
+	unsigned int completed;
+	unsigned int level;
+	struct mbuf *m;
+	struct sfxge_rx_sw_desc *prev = NULL;
+
+	index = rxq->index;
+	evq = sc->evq[index];
+
+	mtx_assert(&evq->lock, MA_OWNED);
+
+	completed = rxq->completed;
+	while (completed != rxq->pending) {
+		unsigned int id;
+		struct sfxge_rx_sw_desc *rx_desc;
+
+		id = completed++ & (SFXGE_NDESCS - 1);
+		rx_desc = &rxq->queue[id];
+		m = rx_desc->mbuf;
+
+		if (rxq->init_state != SFXGE_RXQ_STARTED)
+			goto discard;
+
+		if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
+			goto discard;
+
+		prefetch_read_many(mtod(m, caddr_t));
+
+		/* Check for loopback packets */
+		if (!(rx_desc->flags & EFX_PKT_IPV4) &&
+		    !(rx_desc->flags & EFX_PKT_IPV6)) {
+			struct ether_header *etherhp;
+
+			/*LINTED*/
+			etherhp = mtod(m, struct ether_header *);
+
+			if (etherhp->ether_type ==
+			    htons(SFXGE_ETHERTYPE_LOOPBACK)) {
+				EFSYS_PROBE(loopback);
+
+				rxq->loopback++;
+				goto discard;
+			}
+		}
+
+		/* Pass packet up the stack or into LRO (pipelined) */
+		if (prev != NULL) {
+			if (lro_enabled)
+				sfxge_lro(rxq, prev);
+			else
+				sfxge_rx_deliver(sc, prev);
+		}
+		prev = rx_desc;
+		continue;
+
+discard:
+		/* Return the packet to the pool */
+		m_free(m);
+		rx_desc->mbuf = NULL;
+	}
+	rxq->completed = completed;
+
+	level = rxq->added - rxq->completed;
+
+	/* Pass last packet up the stack or into LRO */
+	if (prev != NULL) {
+		if (lro_enabled)
+			sfxge_lro(rxq, prev);
+		else
+			sfxge_rx_deliver(sc, prev);
+	}
+
+	/*
+	 * If there are any pending flows and this is the end of the
+	 * poll then they must be completed.
+	 */
+	if (eop)
+		sfxge_lro_end_of_burst(rxq);
+
+	/* Top up the queue if necessary */
+	if (level < RX_REFILL_THRESHOLD)
+		sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(SFXGE_NDESCS), B_FALSE);
+}
+
+static void
+sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_rxq *rxq;
+	struct sfxge_evq *evq;
+	unsigned int count;
+
+	rxq = sc->rxq[index];
+	evq = sc->evq[index];
+
+	mtx_lock(&evq->lock);
+	
+	KASSERT(rxq->init_state == SFXGE_RXQ_STARTED,
+	    ("rxq not started"));
+
+	rxq->init_state = SFXGE_RXQ_INITIALIZED;
+
+	callout_stop(&rxq->refill_callout);
+
+again:
+	rxq->flush_state = SFXGE_FLUSH_PENDING;
+
+	/* Flush the receive queue */
+	efx_rx_qflush(rxq->common);
+
+	mtx_unlock(&evq->lock);
+
+	count = 0;
+	do {
+		/* Spin for 100 ms */
+		DELAY(100000);
+
+		if (rxq->flush_state != SFXGE_FLUSH_PENDING)
+			break;
+
+	} while (++count < 20);
+
+	mtx_lock(&evq->lock);
+
+	if (rxq->flush_state == SFXGE_FLUSH_FAILED)
+		goto again;
+
+	rxq->flush_state = SFXGE_FLUSH_DONE;
+
+	rxq->pending = rxq->added;
+	sfxge_rx_qcomplete(rxq, B_TRUE);
+
+	KASSERT(rxq->completed == rxq->pending,
+	    ("rxq->completed != rxq->pending"));
+
+	rxq->added = 0;
+	rxq->pending = 0;
+	rxq->completed = 0;
+	rxq->loopback = 0;
+
+	/* Destroy the common code receive queue. */
+	efx_rx_qdestroy(rxq->common);	
+
+	efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
+	    EFX_RXQ_NBUFS(SFXGE_NDESCS));
+
+	mtx_unlock(&evq->lock);
+}
+
+static int
+sfxge_rx_qstart(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_rxq *rxq;
+	efsys_mem_t *esmp;
+	struct sfxge_evq *evq;
+	int rc;
+
+	rxq = sc->rxq[index];
+	esmp = &rxq->mem;
+	evq = sc->evq[index];
+
+	KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
+	    ("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq->init_state != SFXGE_EVQ_STARTED"));
+
+	/* Program the buffer table. */
+	if ((rc = efx_sram_buf_tbl_set(sc->enp, rxq->buf_base_id, esmp,
+	    EFX_RXQ_NBUFS(SFXGE_NDESCS))) != 0)
+		return rc;
+
+	/* Create the common code receive queue. */
+	if ((rc = efx_rx_qcreate(sc->enp, index, index, EFX_RXQ_TYPE_DEFAULT,
+	    esmp, SFXGE_NDESCS, rxq->buf_base_id, evq->common,
+	    &rxq->common)) != 0)
+		goto fail;
+
+	mtx_lock(&evq->lock);
+
+	/* Enable the receive queue. */
+	efx_rx_qenable(rxq->common);
+
+	rxq->init_state = SFXGE_RXQ_STARTED;
+
+	/* Try to fill the queue from the pool. */
+	sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(SFXGE_NDESCS), B_FALSE);
+
+	mtx_unlock(&evq->lock);
+
+	return (0);
+
+fail:
+	efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
+	    EFX_RXQ_NBUFS(SFXGE_NDESCS));
+	return rc;
+}
+
+void
+sfxge_rx_stop(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+
+	intr = &sc->intr;
+
+	/* Stop the receive queue(s) */
+	index = intr->n_alloc;
+	while (--index >= 0)
+		sfxge_rx_qstop(sc, index);
+
+	sc->rx_prefix_size = 0;
+	sc->rx_buffer_size = 0;
+
+	efx_rx_fini(sc->enp);
+}
+
+int
+sfxge_rx_start(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+	int rc;
+
+	intr = &sc->intr;
+
+	/* Initialize the common code receive module. */
+	if ((rc = efx_rx_init(sc->enp)) != 0)
+		return (rc);
+
+	/* Calculate the receive packet buffer size. */
+	sc->rx_prefix_size = EFX_RX_PREFIX_SIZE;
+	sc->rx_buffer_size = (EFX_MAC_PDU(sc->ifnet->if_mtu) +
+			      sc->rx_prefix_size);
+
+	/* Select zone for packet buffers */
+	if (sc->rx_buffer_size <= MCLBYTES)
+		sc->rx_buffer_zone = zone_clust;
+	else if (sc->rx_buffer_size <= MJUMPAGESIZE)
+		sc->rx_buffer_zone = zone_jumbop;
+	else if (sc->rx_buffer_size <= MJUM9BYTES)
+		sc->rx_buffer_zone = zone_jumbo9;
+	else
+		sc->rx_buffer_zone = zone_jumbo16;
+
+	/*
+	 * Set up the scale table.  Enable all hash types and hash insertion.
+	 */
+	for (index = 0; index < SFXGE_RX_SCALE_MAX; index++)
+		sc->rx_indir_table[index] = index % sc->intr.n_alloc;
+	if ((rc = efx_rx_scale_tbl_set(sc->enp, sc->rx_indir_table,
+				       SFXGE_RX_SCALE_MAX)) != 0)
+		goto fail;
+	(void)efx_rx_scale_mode_set(sc->enp, EFX_RX_HASHALG_TOEPLITZ,
+	    (1 << EFX_RX_HASH_IPV4) | (1 << EFX_RX_HASH_TCPIPV4) |
+	    (1 << EFX_RX_HASH_IPV6) | (1 << EFX_RX_HASH_TCPIPV6), B_TRUE);
+
+	if ((rc = efx_rx_scale_toeplitz_ipv4_key_set(sc->enp, toep_key,
+	    sizeof(toep_key))) != 0)
+		goto fail;
+
+	/* Start the receive queue(s). */
+	for (index = 0; index < intr->n_alloc; index++) {
+		if ((rc = sfxge_rx_qstart(sc, index)) != 0)
+			goto fail2;
+	}
+
+	return (0);
+
+fail2:
+	while (--index >= 0)
+		sfxge_rx_qstop(sc, index);
+
+fail:
+	efx_rx_fini(sc->enp);
+
+	return (rc);
+}
+
+static void sfxge_lro_init(struct sfxge_rxq *rxq)
+{
+	struct sfxge_lro_state *st = &rxq->lro;
+	unsigned i;
+
+	st->conns_mask = lro_table_size - 1;
+	KASSERT(!((st->conns_mask + 1) & st->conns_mask),
+		("lro_table_size must be a power of 2"));
+	st->sc = rxq->sc;
+	st->conns = malloc((st->conns_mask + 1) * sizeof(st->conns[0]),
+			   M_SFXGE, M_WAITOK);
+	st->conns_n = malloc((st->conns_mask + 1) * sizeof(st->conns_n[0]),
+			     M_SFXGE, M_WAITOK);
+	for (i = 0; i <= st->conns_mask; ++i) {
+		TAILQ_INIT(&st->conns[i]);
+		st->conns_n[i] = 0;
+	}
+	LIST_INIT(&st->active_conns);
+	TAILQ_INIT(&st->free_conns);
+}
+
+static void sfxge_lro_fini(struct sfxge_rxq *rxq)
+{
+	struct sfxge_lro_state *st = &rxq->lro;
+	struct sfxge_lro_conn *c;
+	unsigned i;
+
+	/* Return cleanly if sfxge_lro_init() has not been called. */
+	if (st->conns == NULL)
+		return;
+
+	KASSERT(LIST_EMPTY(&st->active_conns), ("found active connections"));
+
+	for (i = 0; i <= st->conns_mask; ++i) {
+		while (!TAILQ_EMPTY(&st->conns[i])) {
+			c = TAILQ_LAST(&st->conns[i], sfxge_lro_tailq);
+			sfxge_lro_drop(rxq, c);
+		}
+	}
+
+	while (!TAILQ_EMPTY(&st->free_conns)) {
+		c = TAILQ_FIRST(&st->free_conns);
+		TAILQ_REMOVE(&st->free_conns, c, link);
+		KASSERT(!c->mbuf, ("found orphaned mbuf"));
+		free(c, M_SFXGE);
+	}
+
+	free(st->conns_n, M_SFXGE);
+	free(st->conns, M_SFXGE);
+	st->conns = NULL;
+}
+
+static void
+sfxge_rx_qfini(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_rxq *rxq;
+
+	rxq = sc->rxq[index];
+
+	KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
+	    ("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
+
+	/* Free the context array and the flow table. */
+	free(rxq->queue, M_SFXGE);
+	sfxge_lro_fini(rxq);
+
+	/* Release DMA memory. */
+	sfxge_dma_free(&rxq->mem);
+
+	sc->rxq[index] = NULL;
+
+	free(rxq, M_SFXGE);
+}
+
+static int
+sfxge_rx_qinit(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_rxq *rxq;
+	struct sfxge_evq *evq;
+	efsys_mem_t *esmp;
+	int rc;
+
+	KASSERT(index < sc->intr.n_alloc, ("index >= %d", sc->intr.n_alloc));
+
+	rxq = malloc(sizeof(struct sfxge_rxq), M_SFXGE, M_ZERO | M_WAITOK);
+	rxq->sc = sc;
+	rxq->index = index;
+
+	sc->rxq[index] = rxq;
+	esmp = &rxq->mem;
+
+	evq = sc->evq[index];
+
+	/* Allocate and zero DMA space. */
+	if ((rc = sfxge_dma_alloc(sc, EFX_RXQ_SIZE(SFXGE_NDESCS), esmp)) != 0)
+		return (rc);
+	(void)memset(esmp->esm_base, 0, EFX_RXQ_SIZE(SFXGE_NDESCS));
+
+	/* Allocate buffer table entries. */
+	sfxge_sram_buf_tbl_alloc(sc, EFX_RXQ_NBUFS(SFXGE_NDESCS),
+				 &rxq->buf_base_id);
+
+	/* Allocate the context array and the flow table. */
+	rxq->queue = malloc(sizeof(struct sfxge_rx_sw_desc) * SFXGE_NDESCS,
+	    M_SFXGE, M_WAITOK | M_ZERO);
+	sfxge_lro_init(rxq);
+
+	callout_init(&rxq->refill_callout, B_TRUE);
+
+	rxq->init_state = SFXGE_RXQ_INITIALIZED;
+
+	return (0);
+}
+
+static const struct {
+	const char *name;
+	size_t offset;
+} sfxge_rx_stats[] = {
+#define SFXGE_RX_STAT(name, member) \
+	{ #name, offsetof(struct sfxge_rxq, member) }
+	SFXGE_RX_STAT(lro_merges, lro.n_merges),
+	SFXGE_RX_STAT(lro_bursts, lro.n_bursts),
+	SFXGE_RX_STAT(lro_slow_start, lro.n_slow_start),
+	SFXGE_RX_STAT(lro_misorder, lro.n_misorder),
+	SFXGE_RX_STAT(lro_too_many, lro.n_too_many),
+	SFXGE_RX_STAT(lro_new_stream, lro.n_new_stream),
+	SFXGE_RX_STAT(lro_drop_idle, lro.n_drop_idle),
+	SFXGE_RX_STAT(lro_drop_closed, lro.n_drop_closed)
+};
+
+static int
+sfxge_rx_stat_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	unsigned int id = arg2;
+	unsigned int sum, index;
+
+	/* Sum across all RX queues */
+	sum = 0;
+	for (index = 0; index < sc->intr.n_alloc; index++)
+		sum += *(unsigned int *)((caddr_t)sc->rxq[index] +
+					 sfxge_rx_stats[id].offset);
+
+	return SYSCTL_OUT(req, &sum, sizeof(sum));
+}
+
+static void
+sfxge_rx_stat_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid_list *stat_list;
+	unsigned int id;
+
+	stat_list = SYSCTL_CHILDREN(sc->stats_node);
+
+	for (id = 0;
+	     id < sizeof(sfxge_rx_stats) / sizeof(sfxge_rx_stats[0]);
+	     id++) {
+		SYSCTL_ADD_PROC(
+			ctx, stat_list,
+			OID_AUTO, sfxge_rx_stats[id].name,
+			CTLTYPE_UINT|CTLFLAG_RD,
+			sc, id, sfxge_rx_stat_handler, "IU",
+			"");
+	}
+}
+
+void
+sfxge_rx_fini(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+
+	intr = &sc->intr;
+
+	index = intr->n_alloc;
+	while (--index >= 0)
+		sfxge_rx_qfini(sc, index);
+}
+
+int
+sfxge_rx_init(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+	int rc;
+
+	if (lro_idle_ticks == 0)
+		lro_idle_ticks = hz / 10 + 1; /* 100 ms */
+
+	intr = &sc->intr;
+
+	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
+	    ("intr->state != SFXGE_INTR_INITIALIZED"));
+
+	/* Initialize the receive queue(s) - one per interrupt. */
+	for (index = 0; index < intr->n_alloc; index++) {
+		if ((rc = sfxge_rx_qinit(sc, index)) != 0)
+			goto fail;
+	}
+
+	sfxge_rx_stat_init(sc);
+
+	return (0);
+
+fail:
+	/* Tear down the receive queue(s). */
+	while (--index >= 0)
+		sfxge_rx_qfini(sc, index);
+
+	return (rc);
+}
diff --git a/sys/dev/sfxge/sfxge_rx.h b/sys/dev/sfxge/sfxge_rx.h
new file mode 100644
index 0000000..5a80fdb
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_rx.h
@@ -0,0 +1,189 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SFXGE_RX_H
+#define _SFXGE_RX_H
+
+#define SFXGE_MAGIC_RESERVED    0x8000
+
+#define SFXGE_MAGIC_DMAQ_LABEL_WIDTH  6
+#define SFXGE_MAGIC_DMAQ_LABEL_MASK   \
+        ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)
+
+#define SFXGE_MAGIC_RX_QFLUSH_DONE                                      \
+        (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
+
+#define SFXGE_MAGIC_RX_QFLUSH_FAILED                                    \
+        (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
+
+#define SFXGE_MAGIC_RX_QREFILL                                          \
+        (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
+
+#define SFXGE_MAGIC_TX_QFLUSH_DONE                                      \
+        (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
+
+#define	SFXGE_RX_SCALE_MAX	EFX_MAXRSS
+
+struct sfxge_rx_sw_desc {
+	struct mbuf	*mbuf;
+	bus_dmamap_t	map;
+	int		flags;
+	int		size;
+};
+
+/**
+ * struct sfxge_lro_conn - Connection state for software LRO
+ * @link: Link for hash table and free list.
+ * @active_link: Link for active_conns list
+ * @l2_id: Identifying information from layer 2
+ * @conn_hash: Hash of connection 4-tuple
+ * @nh: IP (v4 or v6) header of super-packet
+ * @source: Source TCP port number
+ * @dest: Destination TCP port number
+ * @n_in_order_pkts: Number of in-order packets with payload.
+ * @next_seq: Next in-order sequence number.
+ * @last_pkt_ticks: Time we last saw a packet on this connection.
+ * @mbuf: The mbuf we are currently holding.
+ *	If %NULL, then all following fields are undefined.
+ * @mbuf_tail: The tail of the frag_list of mbufs we're holding.
+ *	Only valid after at least one merge.
+ * @th_last: The TCP header of the last packet merged.
+ * @next_buf: The next RX buffer to process.
+ * @next_eh: Ethernet header of the next buffer.
+ * @next_nh: IP header of the next buffer.
+ * @delivered: True if we've delivered a payload packet up this interrupt.
+ */
+struct sfxge_lro_conn {
+	TAILQ_ENTRY(sfxge_lro_conn) link;
+	LIST_ENTRY(sfxge_lro_conn) active_link;
+	uint16_t l2_id;
+	uint32_t conn_hash;
+	void *nh;
+	uint16_t source, dest;
+	int n_in_order_pkts;
+	unsigned next_seq;
+	unsigned last_pkt_ticks;
+	struct mbuf *mbuf;
+	struct mbuf *mbuf_tail;
+	struct tcphdr *th_last;
+	struct sfxge_rx_sw_desc next_buf;
+	void *next_eh;
+	void *next_nh;
+	int delivered;
+};
+
+/**
+ * struct sfxge_lro_state - Port state for software LRO
+ * @sc: The associated NIC.
+ * @conns_mask: Number of hash buckets - 1.
+ * @conns: Hash buckets for tracked connections.
+ * @conns_n: Length of linked list for each hash bucket.
+ * @active_conns: Connections that are holding a packet.
+ *	Connections are self-linked when not in this list.
+ * @free_conns: Free sfxge_lro_conn instances.
+ * @last_purge_ticks: The value of ticks last time we purged idle
+ *	connections.
+ * @n_merges: Number of packets absorbed by LRO.
+ * @n_bursts: Number of bursts spotted by LRO.
+ * @n_slow_start: Number of packets not merged because connection may be in
+ *	slow-start.
+ * @n_misorder: Number of out-of-order packets seen in tracked streams.
+ * @n_too_many: Incremented when we're trying to track too many streams.
+ * @n_new_stream: Number of distinct streams we've tracked.
+ * @n_drop_idle: Number of streams discarded because they went idle.
+ * @n_drop_closed: Number of streams that have seen a FIN or RST.
+ */
+struct sfxge_lro_state {
+	struct sfxge_softc *sc;
+	unsigned conns_mask;
+	TAILQ_HEAD(sfxge_lro_tailq, sfxge_lro_conn) *conns;
+	unsigned *conns_n;
+	LIST_HEAD(, sfxge_lro_conn) active_conns;
+	TAILQ_HEAD(, sfxge_lro_conn) free_conns;
+	unsigned last_purge_ticks;
+	unsigned n_merges;
+	unsigned n_bursts;
+	unsigned n_slow_start;
+	unsigned n_misorder;
+	unsigned n_too_many;
+	unsigned n_new_stream;
+	unsigned n_drop_idle;
+	unsigned n_drop_closed;
+};
+
+enum sfxge_flush_state {
+	SFXGE_FLUSH_DONE = 0,
+	SFXGE_FLUSH_PENDING,
+	SFXGE_FLUSH_FAILED
+};
+
+enum sfxge_rxq_state {
+	SFXGE_RXQ_UNINITIALIZED = 0,
+	SFXGE_RXQ_INITIALIZED,
+	SFXGE_RXQ_STARTED
+};
+
+#define	SFXGE_RX_BATCH	128
+
+struct sfxge_rxq {
+	struct sfxge_softc		*sc __aligned(CACHE_LINE_SIZE);
+	unsigned int			index;
+	efsys_mem_t			mem;
+	unsigned int			buf_base_id;
+	enum sfxge_rxq_state		init_state;
+
+	struct sfxge_rx_sw_desc		*queue __aligned(CACHE_LINE_SIZE);
+	unsigned int			added;
+	unsigned int			pending;
+	unsigned int			completed;
+	unsigned int			loopback;
+	struct sfxge_lro_state		lro;
+	struct callout			refill_callout;
+	unsigned int			refill_delay;
+
+	efx_rxq_t			*common __aligned(CACHE_LINE_SIZE);
+	volatile enum sfxge_flush_state	flush_state;
+};
+
+/*
+ * From sfxge_rx.c.
+ */
+extern int sfxge_rx_init(struct sfxge_softc *sc);
+extern void sfxge_rx_fini(struct sfxge_softc *sc);
+extern int sfxge_rx_start(struct sfxge_softc *sc);
+extern void sfxge_rx_stop(struct sfxge_softc *sc);
+extern void sfxge_rx_qcomplete(struct sfxge_rxq *rxq, boolean_t eop);
+extern void sfxge_rx_qrefill(struct sfxge_rxq *rxq);
+extern void sfxge_rx_qflush_done(struct sfxge_rxq *rxq);
+extern void sfxge_rx_qflush_failed(struct sfxge_rxq *rxq);
+extern void sfxge_rx_scale_update(void *arg, int npending);
+
+#endif
diff --git a/sys/dev/sfxge/sfxge_tx.c b/sys/dev/sfxge/sfxge_tx.c
new file mode 100644
index 0000000..801787a
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_tx.c
@@ -0,0 +1,1491 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/mbuf.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+
+#include <net/bpf.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_vlan_var.h>
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+
+#include "common/efx.h"
+
+#include "sfxge.h"
+#include "sfxge_tx.h"
+
+/* Set the block level to ensure there is space to generate a
+ * large number of descriptors for TSO.  With minimum MSS and
+ * maximum mbuf length we might need more than a ring-ful of
+ * descriptors, but this should not happen in practice except
+ * due to deliberate attack.  In that case we will truncate
+ * the output at a packet boundary.  Allow for a reasonable
+ * minimum MSS of 512.
+ */
+#define SFXGE_TSO_MAX_DESC ((65535 / 512) * 2 + SFXGE_TX_MAPPING_MAX_SEG - 1)
+#define SFXGE_TXQ_BLOCK_LEVEL (SFXGE_NDESCS - SFXGE_TSO_MAX_DESC)
+
+/* Forward declarations. */
+static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq);
+static void sfxge_tx_qlist_post(struct sfxge_txq *txq);
+static void sfxge_tx_qunblock(struct sfxge_txq *txq);
+static int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf,
+			      const bus_dma_segment_t *dma_seg, int n_dma_seg);
+
+void
+sfxge_tx_qcomplete(struct sfxge_txq *txq)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_evq *evq;
+	unsigned int completed;
+
+	sc = txq->sc;
+	evq = sc->evq[txq->evq_index];
+
+	mtx_assert(&evq->lock, MA_OWNED);
+
+	completed = txq->completed;
+	while (completed != txq->pending) {
+		struct sfxge_tx_mapping *stmp;
+		unsigned int id;
+
+		id = completed++ & (SFXGE_NDESCS - 1);
+
+		stmp = &txq->stmp[id];
+		if (stmp->flags & TX_BUF_UNMAP) {
+			bus_dmamap_unload(txq->packet_dma_tag, stmp->map);
+			if (stmp->flags & TX_BUF_MBUF) {
+				struct mbuf *m = stmp->u.mbuf;
+				do
+					m = m_free(m);
+				while (m != NULL);
+			} else {
+				free(stmp->u.heap_buf, M_SFXGE);
+			}
+			stmp->flags = 0;
+		}
+	}
+	txq->completed = completed;
+
+	/* Check whether we need to unblock the queue. */
+	mb();
+	if (txq->blocked) {
+		unsigned int level;
+
+		level = txq->added - txq->completed;
+		if (level <= SFXGE_TXQ_UNBLOCK_LEVEL)
+			sfxge_tx_qunblock(txq);
+	}
+}
+
+#ifdef SFXGE_HAVE_MQ
+
+/*
+ * Reorder the put list and append it to the get list.
+ */
+static void
+sfxge_tx_qdpl_swizzle(struct sfxge_txq *txq)
+{
+	struct sfxge_tx_dpl *stdp;
+	struct mbuf *mbuf, *get_next, **get_tailp;
+	volatile uintptr_t *putp;
+	uintptr_t put;
+	unsigned int count;
+
+	mtx_assert(&txq->lock, MA_OWNED);
+
+	stdp = &txq->dpl;
+
+	/* Acquire the put list. */
+	putp = &stdp->std_put;
+	put = atomic_readandclear_ptr(putp);
+	mbuf = (void *)put;
+
+	if (mbuf == NULL)
+		return;
+
+	/* Reverse the put list. */
+	get_tailp = &mbuf->m_nextpkt;
+	get_next = NULL;
+
+	count = 0;
+	do {
+		struct mbuf *put_next;
+
+		put_next = mbuf->m_nextpkt;
+		mbuf->m_nextpkt = get_next;
+		get_next = mbuf;
+		mbuf = put_next;
+
+		count++;
+	} while (mbuf != NULL);
+
+	/* Append the reversed put list to the get list. */
+	KASSERT(*get_tailp == NULL, ("*get_tailp != NULL"));
+	*stdp->std_getp = get_next;
+	stdp->std_getp = get_tailp;
+	stdp->std_count += count;
+}
+
+#endif /* SFXGE_HAVE_MQ */
+
+static void
+sfxge_tx_qreap(struct sfxge_txq *txq)
+{
+	mtx_assert(SFXGE_TXQ_LOCK(txq), MA_OWNED);
+
+	txq->reaped = txq->completed;
+}
+
+static void
+sfxge_tx_qlist_post(struct sfxge_txq *txq)
+{
+	unsigned int old_added;
+	unsigned int level;
+	int rc;
+
+	mtx_assert(SFXGE_TXQ_LOCK(txq), MA_OWNED);
+
+	KASSERT(txq->n_pend_desc != 0, ("txq->n_pend_desc == 0"));
+	KASSERT(txq->n_pend_desc <= SFXGE_TSO_MAX_DESC,
+		("txq->n_pend_desc too large"));
+	KASSERT(!txq->blocked, ("txq->blocked"));
+
+	old_added = txq->added;
+
+	/* Post the fragment list. */
+	rc = efx_tx_qpost(txq->common, txq->pend_desc, txq->n_pend_desc,
+			  txq->reaped, &txq->added);
+	KASSERT(rc == 0, ("efx_tx_qpost() failed"));
+
+	/* If efx_tx_qpost() had to refragment, our information about
+	 * buffers to free may be associated with the wrong
+	 * descriptors.
+	 */
+	KASSERT(txq->added - old_added == txq->n_pend_desc,
+		("efx_tx_qpost() refragmented descriptors"));
+
+	level = txq->added - txq->reaped;
+	KASSERT(level <= SFXGE_NDESCS, ("overfilled TX queue"));
+
+	/* Clear the fragment list. */
+	txq->n_pend_desc = 0;
+
+	/* Have we reached the block level? */
+	if (level < SFXGE_TXQ_BLOCK_LEVEL)
+		return;
+
+	/* Reap, and check again */
+	sfxge_tx_qreap(txq);
+	level = txq->added - txq->reaped;
+	if (level < SFXGE_TXQ_BLOCK_LEVEL)
+		return;
+
+	txq->blocked = 1;
+
+	/*
+	 * Avoid a race with completion interrupt handling that could leave
+	 * the queue blocked.
+	 */
+	mb();
+	sfxge_tx_qreap(txq);
+	level = txq->added - txq->reaped;
+	if (level < SFXGE_TXQ_BLOCK_LEVEL) {
+		mb();
+		txq->blocked = 0;
+	}
+}
+
+static int sfxge_tx_queue_mbuf(struct sfxge_txq *txq, struct mbuf *mbuf)
+{
+	bus_dmamap_t *used_map;
+	bus_dmamap_t map;
+	bus_dma_segment_t dma_seg[SFXGE_TX_MAPPING_MAX_SEG];
+	unsigned int id;
+	struct sfxge_tx_mapping *stmp;
+	efx_buffer_t *desc;
+	int n_dma_seg;
+	int rc;
+	int i;
+
+	KASSERT(!txq->blocked, ("txq->blocked"));
+
+	if (mbuf->m_pkthdr.csum_flags & CSUM_TSO)
+		prefetch_read_many(mbuf->m_data);
+
+	if (txq->init_state != SFXGE_TXQ_STARTED) {
+		rc = EINTR;
+		goto reject;
+	}
+
+	/* Load the packet for DMA. */
+	id = txq->added & (SFXGE_NDESCS - 1);
+	stmp = &txq->stmp[id];
+	rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag, stmp->map,
+				     mbuf, dma_seg, &n_dma_seg, 0);
+	if (rc == EFBIG) {
+		/* Try again. */
+		struct mbuf *new_mbuf = m_collapse(mbuf, M_DONTWAIT,
+						   SFXGE_TX_MAPPING_MAX_SEG);
+		if (new_mbuf == NULL)
+			goto reject;
+		++txq->collapses;
+		mbuf = new_mbuf;
+		rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag,
+					     stmp->map, mbuf,
+					     dma_seg, &n_dma_seg, 0);
+	}
+	if (rc != 0)
+		goto reject;
+
+	/* Make the packet visible to the hardware. */
+	bus_dmamap_sync(txq->packet_dma_tag, stmp->map, BUS_DMASYNC_PREWRITE);
+
+	used_map = &stmp->map;
+
+	if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) {
+		rc = sfxge_tx_queue_tso(txq, mbuf, dma_seg, n_dma_seg);
+		if (rc < 0)
+			goto reject_mapped;
+		stmp = &txq->stmp[rc];
+	} else {
+		/* Add the mapping to the fragment list, and set flags
+		 * for the buffer.
+		 */
+		i = 0;
+		for (;;) {
+			desc = &txq->pend_desc[i];
+			desc->eb_addr = dma_seg[i].ds_addr;
+			desc->eb_size = dma_seg[i].ds_len;
+			if (i == n_dma_seg - 1) {
+				desc->eb_eop = 1;
+				break;
+			}
+			desc->eb_eop = 0;
+			i++;
+
+			stmp->flags = 0;
+			if (__predict_false(stmp ==
+					    &txq->stmp[SFXGE_NDESCS - 1]))
+				stmp = &txq->stmp[0];
+			else
+				stmp++;
+		}
+		txq->n_pend_desc = n_dma_seg;
+	}
+
+	/*
+	 * If the mapping required more than one descriptor
+	 * then we need to associate the DMA map with the last
+	 * descriptor, not the first.
+	 */
+	if (used_map != &stmp->map) {
+		map = stmp->map;
+		stmp->map = *used_map;
+		*used_map = map;
+	}
+
+	stmp->u.mbuf = mbuf;
+	stmp->flags = TX_BUF_UNMAP | TX_BUF_MBUF;
+
+	/* Post the fragment list. */
+	sfxge_tx_qlist_post(txq);
+
+	return 0;
+
+reject_mapped:
+	bus_dmamap_unload(txq->packet_dma_tag, *used_map);
+reject:
+	/* Drop the packet on the floor. */
+	m_freem(mbuf);
+	++txq->drops;
+
+	return rc;
+}
+
+#ifdef SFXGE_HAVE_MQ
+
+/*
+ * Drain the deferred packet list into the transmit queue.
+ */
+static void
+sfxge_tx_qdpl_drain(struct sfxge_txq *txq)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_tx_dpl *stdp;
+	struct mbuf *mbuf, *next;
+	unsigned int count;
+	unsigned int pushed;
+	int rc;
+
+	mtx_assert(&txq->lock, MA_OWNED);
+
+	sc = txq->sc;
+	stdp = &txq->dpl;
+	pushed = txq->added;
+
+	prefetch_read_many(sc->enp);
+	prefetch_read_many(txq->common);
+
+	mbuf = stdp->std_get;
+	count = stdp->std_count;
+
+	while (count != 0) {
+		KASSERT(mbuf != NULL, ("mbuf == NULL"));
+
+		next = mbuf->m_nextpkt;
+		mbuf->m_nextpkt = NULL;
+
+		ETHER_BPF_MTAP(sc->ifnet, mbuf); /* packet capture */
+
+		if (next != NULL)
+			prefetch_read_many(next);
+
+		rc = sfxge_tx_queue_mbuf(txq, mbuf);
+		--count;
+		mbuf = next;
+		if (rc != 0)
+			continue;
+
+		if (txq->blocked)
+			break;
+
+		/* Push the fragments to the hardware in batches. */
+		if (txq->added - pushed >= SFXGE_TX_BATCH) {
+			efx_tx_qpush(txq->common, txq->added);
+			pushed = txq->added;
+		}
+	}
+
+	if (count == 0) {
+		KASSERT(mbuf == NULL, ("mbuf != NULL"));
+		stdp->std_get = NULL;
+		stdp->std_count = 0;
+		stdp->std_getp = &stdp->std_get;
+	} else {
+		stdp->std_get = mbuf;
+		stdp->std_count = count;
+	}
+
+	if (txq->added != pushed)
+		efx_tx_qpush(txq->common, txq->added);
+
+	KASSERT(txq->blocked || stdp->std_count == 0,
+		("queue unblocked but count is non-zero"));
+}
+
+#define SFXGE_TX_QDPL_PENDING(_txq)					\
+    ((_txq)->dpl.std_put != 0)
+
+/*
+ * Service the deferred packet list.
+ *
+ * NOTE: drops the txq mutex!
+ */
+static inline void
+sfxge_tx_qdpl_service(struct sfxge_txq *txq)
+{
+	mtx_assert(&txq->lock, MA_OWNED);
+
+	do {
+		if (SFXGE_TX_QDPL_PENDING(txq))
+			sfxge_tx_qdpl_swizzle(txq);
+
+		if (!txq->blocked)
+			sfxge_tx_qdpl_drain(txq);
+
+		mtx_unlock(&txq->lock);
+	} while (SFXGE_TX_QDPL_PENDING(txq) &&
+		 mtx_trylock(&txq->lock));
+}
+
+/*
+ * Put a packet on the deferred packet list.
+ *
+ * If we are called with the txq lock held, we put the packet on the "get
+ * list", otherwise we atomically push it on the "put list".  The swizzle
+ * function takes care of ordering.
+ *
+ * The length of the put list is bounded by SFXGE_TX_MAX_DEFFERED.  We
+ * overload the csum_data field in the mbuf to keep track of this length
+ * because there is no cheap alternative to avoid races.
+ */
+static inline int
+sfxge_tx_qdpl_put(struct sfxge_txq *txq, struct mbuf *mbuf, int locked)
+{
+	struct sfxge_tx_dpl *stdp;
+
+	stdp = &txq->dpl;
+
+	KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL"));
+
+	if (locked) {
+		mtx_assert(&txq->lock, MA_OWNED);
+
+		sfxge_tx_qdpl_swizzle(txq);
+
+		*(stdp->std_getp) = mbuf;
+		stdp->std_getp = &mbuf->m_nextpkt;
+		stdp->std_count++;
+	} else {
+		volatile uintptr_t *putp;
+		uintptr_t old;
+		uintptr_t new;
+		unsigned old_len;
+
+		putp = &stdp->std_put;
+		new = (uintptr_t)mbuf;
+
+		do {
+			old = *putp;
+			if (old) {
+				struct mbuf *mp = (struct mbuf *)old;
+				old_len = mp->m_pkthdr.csum_data;
+			} else
+				old_len = 0;
+			if (old_len >= SFXGE_TX_MAX_DEFERRED)
+				return ENOBUFS;
+			mbuf->m_pkthdr.csum_data = old_len + 1;
+			mbuf->m_nextpkt = (void *)old;
+		} while (atomic_cmpset_ptr(putp, old, new) == 0);
+	}
+
+	return (0);
+}
+
+/*
+ * Called from if_transmit - will try to grab the txq lock and enqueue to the
+ * put list if it succeeds, otherwise will push onto the defer list.
+ */
+int
+sfxge_tx_packet_add(struct sfxge_txq *txq, struct mbuf *m)
+{
+	int locked;
+	int rc;
+
+	/*
+	 * Try to grab the txq lock.  If we are able to get the lock,
+	 * the packet will be appended to the "get list" of the deferred
+	 * packet list.  Otherwise, it will be pushed on the "put list".
+	 */
+	locked = mtx_trylock(&txq->lock);
+
+	/*
+	 * Can only fail if we weren't able to get the lock.
+	 */
+	if (sfxge_tx_qdpl_put(txq, m, locked) != 0) {
+		KASSERT(!locked,
+		    ("sfxge_tx_qdpl_put() failed locked"));
+		rc = ENOBUFS;
+		goto fail;
+	}
+
+	/*
+	 * Try to grab the lock again.
+	 *
+	 * If we are able to get the lock, we need to process the deferred
+	 * packet list.  If we are not able to get the lock, another thread
+	 * is processing the list.
+	 */
+	if (!locked)
+		locked = mtx_trylock(&txq->lock);
+
+	if (locked) {
+		/* Try to service the list. */
+		sfxge_tx_qdpl_service(txq);
+		/* Lock has been dropped. */
+	}
+
+	return (0);
+
+fail:
+	return (rc);
+	
+}
+
+static void
+sfxge_tx_qdpl_flush(struct sfxge_txq *txq)
+{
+	struct sfxge_tx_dpl *stdp = &txq->dpl;
+	struct mbuf *mbuf, *next;
+
+	mtx_lock(&txq->lock);
+
+	sfxge_tx_qdpl_swizzle(txq);
+	for (mbuf = stdp->std_get; mbuf != NULL; mbuf = next) {
+		next = mbuf->m_nextpkt;
+		m_freem(mbuf);
+	}
+	stdp->std_get = NULL;
+	stdp->std_count = 0;
+	stdp->std_getp = &stdp->std_get;	
+
+	mtx_unlock(&txq->lock);
+}
+
+void
+sfxge_if_qflush(struct ifnet *ifp)
+{
+	struct sfxge_softc *sc;
+	int i;
+
+	sc = ifp->if_softc;
+
+	for (i = 0; i < SFXGE_TX_SCALE(sc); i++)
+		sfxge_tx_qdpl_flush(sc->txq[i]);
+}
+
+/*
+ * TX start -- called by the stack.
+ */
+int
+sfxge_if_transmit(struct ifnet *ifp, struct mbuf *m)
+{	
+	struct sfxge_softc *sc;
+	struct sfxge_txq *txq;
+	int rc;
+
+	sc = (struct sfxge_softc *)ifp->if_softc;
+
+	KASSERT(ifp->if_flags & IFF_UP, ("interface not up"));
+
+	if (!SFXGE_LINK_UP(sc)) {
+		m_freem(m);
+		return (0);
+	}
+
+	/* Pick the desired transmit queue. */
+	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO)) {
+		int index = 0;
+
+		if (m->m_flags & M_FLOWID) {
+			uint32_t hash = m->m_pkthdr.flowid;
+
+			index = sc->rx_indir_table[hash % SFXGE_RX_SCALE_MAX];
+		}
+		txq = sc->txq[SFXGE_TXQ_IP_TCP_UDP_CKSUM + index];
+	} else if (m->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
+		txq = sc->txq[SFXGE_TXQ_IP_CKSUM];
+	} else {
+		txq = sc->txq[SFXGE_TXQ_NON_CKSUM];
+	}
+
+	rc = sfxge_tx_packet_add(txq, m);
+
+	return (rc);
+}
+
+#else /* !SFXGE_HAVE_MQ */
+
+static void sfxge_if_start_locked(struct ifnet *ifp)
+{
+	struct sfxge_softc *sc = ifp->if_softc;
+	struct sfxge_txq *txq;
+	struct mbuf *mbuf;
+	unsigned int pushed[SFXGE_TXQ_NTYPES];
+	unsigned int q_index;
+
+	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
+	    IFF_DRV_RUNNING)
+		return;
+
+	if (!sc->port.link_up)
+		return;
+
+	for (q_index = 0; q_index < SFXGE_TXQ_NTYPES; q_index++) {
+		txq = sc->txq[q_index];
+		pushed[q_index] = txq->added;
+	}
+
+	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
+                IFQ_DRV_DEQUEUE(&ifp->if_snd, mbuf);
+		if (mbuf == NULL)
+			break;
+
+		ETHER_BPF_MTAP(ifp, mbuf); /* packet capture */
+
+		/* Pick the desired transmit queue. */
+		if (mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO))
+			q_index = SFXGE_TXQ_IP_TCP_UDP_CKSUM;
+		else if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP)
+			q_index = SFXGE_TXQ_IP_CKSUM;
+		else
+			q_index = SFXGE_TXQ_NON_CKSUM;
+		txq = sc->txq[q_index];
+
+		if (sfxge_tx_queue_mbuf(txq, mbuf) != 0)
+			continue;
+
+		if (txq->blocked) {
+			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+			break;
+		}
+
+		/* Push the fragments to the hardware in batches. */
+		if (txq->added - pushed[q_index] >= SFXGE_TX_BATCH) {
+			efx_tx_qpush(txq->common, txq->added);
+			pushed[q_index] = txq->added;
+		}
+	}
+
+	for (q_index = 0; q_index < SFXGE_TXQ_NTYPES; q_index++) {
+		txq = sc->txq[q_index];
+		if (txq->added != pushed[q_index])
+			efx_tx_qpush(txq->common, txq->added);
+	}
+}
+
+void sfxge_if_start(struct ifnet *ifp)
+{
+	struct sfxge_softc *sc = ifp->if_softc;
+
+	mtx_lock(&sc->tx_lock);
+	sfxge_if_start_locked(ifp);
+	mtx_unlock(&sc->tx_lock);
+}
+
+static inline void
+sfxge_tx_qdpl_service(struct sfxge_txq *txq)
+{
+	struct sfxge_softc *sc = txq->sc;
+	struct ifnet *ifp = sc->ifnet;
+
+	mtx_assert(&sc->tx_lock, MA_OWNED);
+	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+	sfxge_if_start_locked(ifp);
+	mtx_unlock(&sc->tx_lock);
+}
+
+#endif /* SFXGE_HAVE_MQ */
+
+/*
+ * Software "TSO".  Not quite as good as doing it in hardware, but
+ * still faster than segmenting in the stack.
+ */
+
+struct sfxge_tso_state {
+	/* Output position */
+	unsigned out_len;	/* Remaining length in current segment */
+	unsigned seqnum;	/* Current sequence number */
+	unsigned packet_space;	/* Remaining space in current packet */
+
+	/* Input position */
+	unsigned dma_seg_i;	/* Current DMA segment number */
+	uint64_t dma_addr;	/* DMA address of current position */
+	unsigned in_len;	/* Remaining length in current mbuf */
+
+	const struct mbuf *mbuf; /* Input mbuf (head of chain) */
+	u_short protocol;	/* Network protocol (after VLAN decap) */
+	ssize_t nh_off;		/* Offset of network header */
+	ssize_t tcph_off;	/* Offset of TCP header */
+	unsigned header_len;	/* Number of bytes of header */
+	int full_packet_size;	/* Number of bytes to put in each outgoing
+				 * segment */
+};
+
+static inline const struct ip *tso_iph(const struct sfxge_tso_state *tso)
+{
+	KASSERT(tso->protocol == htons(ETHERTYPE_IP),
+		("tso_iph() in non-IPv4 state"));
+	return (const struct ip *)(tso->mbuf->m_data + tso->nh_off);
+}
+static inline const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso)
+{
+	KASSERT(tso->protocol == htons(ETHERTYPE_IPV6),
+		("tso_ip6h() in non-IPv6 state"));
+	return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off);
+}
+static inline const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso)
+{
+	return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off);
+}
+
+/* Size of preallocated TSO header buffers.  Larger blocks must be
+ * allocated from the heap.
+ */
+#define TSOH_STD_SIZE	128
+
+/* At most half the descriptors in the queue at any time will refer to
+ * a TSO header buffer, since they must always be followed by a
+ * payload descriptor referring to an mbuf.
+ */
+#define TSOH_COUNT	(SFXGE_NDESCS / 2u)
+#define TSOH_PER_PAGE	(PAGE_SIZE / TSOH_STD_SIZE)
+#define TSOH_PAGE_COUNT	((TSOH_COUNT + TSOH_PER_PAGE - 1) / TSOH_PER_PAGE)
+
+static int tso_init(struct sfxge_txq *txq)
+{
+	struct sfxge_softc *sc = txq->sc;
+	int i, rc;
+
+	/* Allocate TSO header buffers */
+	txq->tsoh_buffer = malloc(TSOH_PAGE_COUNT * sizeof(txq->tsoh_buffer[0]),
+				  M_SFXGE, M_WAITOK);
+
+	for (i = 0; i < TSOH_PAGE_COUNT; i++) {
+		rc = sfxge_dma_alloc(sc, PAGE_SIZE, &txq->tsoh_buffer[i]);
+		if (rc)
+			goto fail;
+	}
+
+	return 0;
+
+fail:
+	while (i-- > 0)
+		sfxge_dma_free(&txq->tsoh_buffer[i]);
+	free(txq->tsoh_buffer, M_SFXGE);
+	txq->tsoh_buffer = NULL;
+	return rc;
+}
+
+static void tso_fini(struct sfxge_txq *txq)
+{
+	int i;
+
+	if (txq->tsoh_buffer) {
+		for (i = 0; i < TSOH_PAGE_COUNT; i++)
+			sfxge_dma_free(&txq->tsoh_buffer[i]);
+		free(txq->tsoh_buffer, M_SFXGE);
+	}
+}
+
+static void tso_start(struct sfxge_tso_state *tso, struct mbuf *mbuf)
+{
+	struct ether_header *eh = mtod(mbuf, struct ether_header *);
+
+	tso->mbuf = mbuf;
+
+	/* Find network protocol and header */
+	tso->protocol = eh->ether_type;
+	if (tso->protocol == htons(ETHERTYPE_VLAN)) {
+		struct ether_vlan_header *veh =
+			mtod(mbuf, struct ether_vlan_header *);
+		tso->protocol = veh->evl_proto;
+		tso->nh_off = sizeof(*veh);
+	} else {
+		tso->nh_off = sizeof(*eh);
+	}
+
+	/* Find TCP header */
+	if (tso->protocol == htons(ETHERTYPE_IP)) {
+		KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP,
+			("TSO required on non-TCP packet"));
+		tso->tcph_off = tso->nh_off + 4 * tso_iph(tso)->ip_hl;
+	} else {
+		KASSERT(tso->protocol == htons(ETHERTYPE_IPV6),
+			("TSO required on non-IP packet"));
+		KASSERT(tso_ip6h(tso)->ip6_nxt == IPPROTO_TCP,
+			("TSO required on non-TCP packet"));
+		tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr);
+	}
+
+	/* We assume all headers are linear in the head mbuf */
+	tso->header_len = tso->tcph_off + 4 * tso_tcph(tso)->th_off;
+	KASSERT(tso->header_len <= mbuf->m_len, ("packet headers fragmented"));
+	tso->full_packet_size = tso->header_len + mbuf->m_pkthdr.tso_segsz;
+
+	tso->seqnum = ntohl(tso_tcph(tso)->th_seq);
+
+	/* These flags must not be duplicated */
+	KASSERT(!(tso_tcph(tso)->th_flags & (TH_URG | TH_SYN | TH_RST)),
+		("incompatible TCP flag on TSO packet"));
+
+	tso->out_len = mbuf->m_pkthdr.len - tso->header_len;
+}
+
+/*
+ * tso_fill_packet_with_fragment - form descriptors for the current fragment
+ *
+ * Form descriptors for the current fragment, until we reach the end
+ * of fragment or end-of-packet.  Return 0 on success, 1 if not enough
+ * space.
+ */
+static void tso_fill_packet_with_fragment(struct sfxge_txq *txq,
+					  struct sfxge_tso_state *tso)
+{
+	efx_buffer_t *desc;
+	int n;
+
+	if (tso->in_len == 0 || tso->packet_space == 0)
+		return;
+
+	KASSERT(tso->in_len > 0, ("TSO input length went negative"));
+	KASSERT(tso->packet_space > 0, ("TSO packet space went negative"));
+
+	n = min(tso->in_len, tso->packet_space);
+
+	tso->packet_space -= n;
+	tso->out_len -= n;
+	tso->in_len -= n;
+
+	desc = &txq->pend_desc[txq->n_pend_desc++];
+	desc->eb_addr = tso->dma_addr;
+	desc->eb_size = n;
+	desc->eb_eop = tso->out_len == 0 || tso->packet_space == 0;
+
+	tso->dma_addr += n;
+}
+
+/* Callback from bus_dmamap_load() for long TSO headers. */
+static void tso_map_long_header(void *dma_addr_ret,
+				bus_dma_segment_t *segs, int nseg,
+				int error)
+{
+	*(uint64_t *)dma_addr_ret = ((__predict_true(error == 0) &&
+				      __predict_true(nseg == 1)) ?
+				     segs->ds_addr : 0);
+}
+
+/*
+ * tso_start_new_packet - generate a new header and prepare for the new packet
+ *
+ * Generate a new header and prepare for the new packet.  Return 0 on
+ * success, or an error code if failed to alloc header.
+ */
+static int tso_start_new_packet(struct sfxge_txq *txq,
+				struct sfxge_tso_state *tso,
+				unsigned int id)
+{
+	struct sfxge_tx_mapping *stmp = &txq->stmp[id];
+	struct tcphdr *tsoh_th;
+	unsigned ip_length;
+	caddr_t header;
+	uint64_t dma_addr;
+	bus_dmamap_t map;
+	efx_buffer_t *desc;
+	int rc;
+
+	/* Allocate a DMA-mapped header buffer. */
+	if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) {
+		unsigned int page_index = (id / 2) / TSOH_PER_PAGE;
+		unsigned int buf_index = (id / 2) % TSOH_PER_PAGE;
+
+		header = (txq->tsoh_buffer[page_index].esm_base +
+			  buf_index * TSOH_STD_SIZE);
+		dma_addr = (txq->tsoh_buffer[page_index].esm_addr +
+			    buf_index * TSOH_STD_SIZE);
+		map = txq->tsoh_buffer[page_index].esm_map;
+
+		stmp->flags = 0;
+	} else {
+		/* We cannot use bus_dmamem_alloc() as that may sleep */
+		header = malloc(tso->header_len, M_SFXGE, M_NOWAIT);
+		if (__predict_false(!header))
+			return ENOMEM;     
+		rc = bus_dmamap_load(txq->packet_dma_tag, stmp->map,
+				     header, tso->header_len,
+				     tso_map_long_header, &dma_addr,
+				     BUS_DMA_NOWAIT);
+		if (__predict_false(dma_addr == 0)) {
+			if (rc == 0) {
+				/* Succeeded but got >1 segment */
+				bus_dmamap_unload(txq->packet_dma_tag,
+						  stmp->map);
+				rc = EINVAL;
+			}
+			free(header, M_SFXGE);
+			return rc;
+		}
+		map = stmp->map;
+
+		txq->tso_long_headers++;
+		stmp->u.heap_buf = header;
+		stmp->flags = TX_BUF_UNMAP;
+	}
+
+	tsoh_th = (struct tcphdr *)(header + tso->tcph_off);
+
+	/* Copy and update the headers. */
+	memcpy(header, tso->mbuf->m_data, tso->header_len);
+
+	tsoh_th->th_seq = htonl(tso->seqnum);
+	tso->seqnum += tso->mbuf->m_pkthdr.tso_segsz;
+	if (tso->out_len > tso->mbuf->m_pkthdr.tso_segsz) {
+		/* This packet will not finish the TSO burst. */
+		ip_length = tso->full_packet_size - tso->nh_off;
+		tsoh_th->th_flags &= ~(TH_FIN | TH_PUSH);
+	} else {
+		/* This packet will be the last in the TSO burst. */
+		ip_length = tso->header_len - tso->nh_off + tso->out_len;
+	}
+
+	if (tso->protocol == htons(ETHERTYPE_IP)) {
+		struct ip *tsoh_iph = (struct ip *)(header + tso->nh_off);
+		tsoh_iph->ip_len = htons(ip_length);
+		/* XXX We should increment ip_id, but FreeBSD doesn't
+		 * currently allocate extra IDs for multiple segments.
+		 */
+	} else {
+		struct ip6_hdr *tsoh_iph =
+			(struct ip6_hdr *)(header + tso->nh_off);
+		tsoh_iph->ip6_plen = htons(ip_length - sizeof(*tsoh_iph));
+	}
+
+	/* Make the header visible to the hardware. */
+	bus_dmamap_sync(txq->packet_dma_tag, map, BUS_DMASYNC_PREWRITE);
+
+	tso->packet_space = tso->mbuf->m_pkthdr.tso_segsz;
+	txq->tso_packets++;
+
+	/* Form a descriptor for this header. */
+	desc = &txq->pend_desc[txq->n_pend_desc++];
+	desc->eb_addr = dma_addr;
+	desc->eb_size = tso->header_len;
+	desc->eb_eop = 0;
+
+	return 0;
+}
+
+static int
+sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf,
+		   const bus_dma_segment_t *dma_seg, int n_dma_seg)
+{
+	struct sfxge_tso_state tso;
+	unsigned int id, next_id;
+
+	tso_start(&tso, mbuf);
+
+	/* Grab the first payload fragment. */
+	if (dma_seg->ds_len == tso.header_len) {
+		--n_dma_seg;
+		KASSERT(n_dma_seg, ("no payload found in TSO packet"));
+		++dma_seg;
+		tso.in_len = dma_seg->ds_len;
+		tso.dma_addr = dma_seg->ds_addr;
+	} else {
+		tso.in_len = dma_seg->ds_len - tso.header_len;
+		tso.dma_addr = dma_seg->ds_addr + tso.header_len;
+	}
+
+	id = txq->added & (SFXGE_NDESCS - 1);
+	if (__predict_false(tso_start_new_packet(txq, &tso, id)))
+		return -1;
+
+	while (1) {
+		id = (id + 1) & (SFXGE_NDESCS - 1);
+		tso_fill_packet_with_fragment(txq, &tso);
+
+		/* Move onto the next fragment? */
+		if (tso.in_len == 0) {
+			--n_dma_seg;
+			if (n_dma_seg == 0)
+				break;
+			++dma_seg;
+			tso.in_len = dma_seg->ds_len;
+			tso.dma_addr = dma_seg->ds_addr;
+		}
+
+		/* End of packet? */
+		if (tso.packet_space == 0) {
+			/* If the queue is now full due to tiny MSS,
+			 * or we can't create another header, discard
+			 * the remainder of the input mbuf but do not
+			 * roll back the work we have done.
+			 */
+			if (txq->n_pend_desc >
+			    SFXGE_TSO_MAX_DESC - (1 + SFXGE_TX_MAPPING_MAX_SEG))
+				break;
+			next_id = (id + 1) & (SFXGE_NDESCS - 1);
+			if (__predict_false(tso_start_new_packet(txq, &tso,
+								 next_id)))
+				break;
+			id = next_id;
+		}
+	}
+
+	txq->tso_bursts++;
+	return id;
+}
+
+static void
+sfxge_tx_qunblock(struct sfxge_txq *txq)
+{
+	struct sfxge_softc *sc;
+	struct sfxge_evq *evq;
+
+	sc = txq->sc;
+	evq = sc->evq[txq->evq_index];
+
+	mtx_assert(&evq->lock, MA_OWNED);
+
+	if (txq->init_state != SFXGE_TXQ_STARTED)
+		return;
+
+	mtx_lock(SFXGE_TXQ_LOCK(txq));
+
+	if (txq->blocked) {
+		unsigned int level;
+
+		level = txq->added - txq->completed;
+		if (level <= SFXGE_TXQ_UNBLOCK_LEVEL)
+			txq->blocked = 0;
+	}
+
+	sfxge_tx_qdpl_service(txq);
+	/* note: lock has been dropped */
+}
+
+void
+sfxge_tx_qflush_done(struct sfxge_txq *txq)
+{
+
+	txq->flush_state = SFXGE_FLUSH_DONE;
+}
+
+static void
+sfxge_tx_qstop(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_txq *txq;
+	struct sfxge_evq *evq;
+	unsigned int count;
+
+	txq = sc->txq[index];
+	evq = sc->evq[txq->evq_index];
+
+	mtx_lock(SFXGE_TXQ_LOCK(txq));
+
+	KASSERT(txq->init_state == SFXGE_TXQ_STARTED,
+	    ("txq->init_state != SFXGE_TXQ_STARTED"));
+
+	txq->init_state = SFXGE_TXQ_INITIALIZED;
+	txq->flush_state = SFXGE_FLUSH_PENDING;
+
+	/* Flush the transmit queue. */
+	efx_tx_qflush(txq->common);
+
+	mtx_unlock(SFXGE_TXQ_LOCK(txq));
+
+	count = 0;
+	do {
+		/* Spin for 100ms. */
+		DELAY(100000);
+
+		if (txq->flush_state != SFXGE_FLUSH_PENDING)
+			break;
+	} while (++count < 20);
+
+	mtx_lock(&evq->lock);
+	mtx_lock(SFXGE_TXQ_LOCK(txq));
+
+	KASSERT(txq->flush_state != SFXGE_FLUSH_FAILED,
+	    ("txq->flush_state == SFXGE_FLUSH_FAILED"));
+
+	txq->flush_state = SFXGE_FLUSH_DONE;
+
+	txq->blocked = 0;
+	txq->pending = txq->added;
+
+	sfxge_tx_qcomplete(txq);
+	KASSERT(txq->completed == txq->added,
+	    ("txq->completed != txq->added"));
+
+	sfxge_tx_qreap(txq);
+	KASSERT(txq->reaped == txq->completed,
+	    ("txq->reaped != txq->completed"));
+
+	txq->added = 0;
+	txq->pending = 0;
+	txq->completed = 0;
+	txq->reaped = 0;
+
+	/* Destroy the common code transmit queue. */
+	efx_tx_qdestroy(txq->common);
+	txq->common = NULL;
+
+	efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
+	    EFX_TXQ_NBUFS(SFXGE_NDESCS));
+
+	mtx_unlock(&evq->lock);
+	mtx_unlock(SFXGE_TXQ_LOCK(txq));
+}
+
+static int
+sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_txq *txq;
+	efsys_mem_t *esmp;
+	uint16_t flags;
+	struct sfxge_evq *evq;
+	int rc;
+
+	txq = sc->txq[index];
+	esmp = &txq->mem;
+	evq = sc->evq[txq->evq_index];
+
+	KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
+	    ("txq->init_state != SFXGE_TXQ_INITIALIZED"));
+	KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
+	    ("evq->init_state != SFXGE_EVQ_STARTED"));
+
+	/* Program the buffer table. */
+	if ((rc = efx_sram_buf_tbl_set(sc->enp, txq->buf_base_id, esmp,
+	    EFX_TXQ_NBUFS(SFXGE_NDESCS))) != 0)
+		return rc;
+
+	/* Determine the kind of queue we are creating. */
+	switch (txq->type) {
+	case SFXGE_TXQ_NON_CKSUM:
+		flags = 0;
+		break;
+	case SFXGE_TXQ_IP_CKSUM:
+		flags = EFX_CKSUM_IPV4;
+		break;
+	case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
+		flags = EFX_CKSUM_IPV4 | EFX_CKSUM_TCPUDP;
+		break;
+	default:
+		KASSERT(0, ("Impossible TX queue"));
+		flags = 0;
+		break;
+	}
+
+	/* Create the common code transmit queue. */
+	if ((rc = efx_tx_qcreate(sc->enp, index, index, esmp,
+	    SFXGE_NDESCS, txq->buf_base_id, flags, evq->common,
+	    &txq->common)) != 0)
+		goto fail;
+
+	mtx_lock(SFXGE_TXQ_LOCK(txq));
+	
+	/* Enable the transmit queue. */
+	efx_tx_qenable(txq->common);
+
+	txq->init_state = SFXGE_TXQ_STARTED;
+
+	mtx_unlock(SFXGE_TXQ_LOCK(txq));
+
+	return (0);
+
+fail:
+	efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
+	    EFX_TXQ_NBUFS(SFXGE_NDESCS));
+	return rc;
+}
+
+void
+sfxge_tx_stop(struct sfxge_softc *sc)
+{
+	const efx_nic_cfg_t *encp;
+	int index;
+
+	index = SFXGE_TX_SCALE(sc);
+	while (--index >= 0)
+		sfxge_tx_qstop(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index);
+
+	sfxge_tx_qstop(sc, SFXGE_TXQ_IP_CKSUM);
+
+	encp = efx_nic_cfg_get(sc->enp);
+        sfxge_tx_qstop(sc, SFXGE_TXQ_NON_CKSUM);
+
+	/* Tear down the transmit module */
+	efx_tx_fini(sc->enp);
+}
+
+int
+sfxge_tx_start(struct sfxge_softc *sc)
+{
+	int index;
+	int rc;
+
+	/* Initialize the common code transmit module. */
+	if ((rc = efx_tx_init(sc->enp)) != 0)
+		return (rc);
+
+	if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_NON_CKSUM)) != 0)
+		goto fail;
+
+	if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_IP_CKSUM)) != 0)
+		goto fail2;
+
+	for (index = 0; index < SFXGE_TX_SCALE(sc); index++) {
+		if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM +
+		    index)) != 0)
+			goto fail3;
+	}
+
+	return (0);
+
+fail3:
+	while (--index >= 0)
+		sfxge_tx_qstop(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index);
+
+	sfxge_tx_qstop(sc, SFXGE_TXQ_IP_CKSUM);
+
+fail2:
+        sfxge_tx_qstop(sc, SFXGE_TXQ_NON_CKSUM);
+
+fail:
+	efx_tx_fini(sc->enp);
+
+	return (rc);
+}
+
+/**
+ * Destroy a transmit queue.
+ */
+static void
+sfxge_tx_qfini(struct sfxge_softc *sc, unsigned int index)
+{
+	struct sfxge_txq *txq;
+	unsigned int nmaps = SFXGE_NDESCS;
+
+	txq = sc->txq[index];
+
+	KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
+	    ("txq->init_state != SFXGE_TXQ_INITIALIZED"));
+
+	if (txq->type == SFXGE_TXQ_IP_TCP_UDP_CKSUM)
+		tso_fini(txq);
+
+	/* Free the context arrays. */
+	free(txq->pend_desc, M_SFXGE);
+	while (nmaps--)
+		bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map);
+	free(txq->stmp, M_SFXGE);
+
+	/* Release DMA memory mapping. */
+	sfxge_dma_free(&txq->mem);
+
+	sc->txq[index] = NULL;
+
+#ifdef SFXGE_HAVE_MQ
+	mtx_destroy(&txq->lock);
+#endif
+
+	free(txq, M_SFXGE);
+}
+
+static int
+sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index,
+    enum sfxge_txq_type type, unsigned int evq_index)
+{
+	struct sfxge_txq *txq;
+	struct sfxge_evq *evq;
+#ifdef SFXGE_HAVE_MQ
+	struct sfxge_tx_dpl *stdp;
+#endif
+	efsys_mem_t *esmp;
+	unsigned int nmaps;
+	int rc;
+
+	txq = malloc(sizeof(struct sfxge_txq), M_SFXGE, M_ZERO | M_WAITOK);
+	txq->sc = sc;
+
+	sc->txq[txq_index] = txq;
+	esmp = &txq->mem;
+
+	evq = sc->evq[evq_index];
+
+	/* Allocate and zero DMA space for the descriptor ring. */
+	if ((rc = sfxge_dma_alloc(sc, EFX_TXQ_SIZE(SFXGE_NDESCS), esmp)) != 0)
+		return (rc);
+	(void)memset(esmp->esm_base, 0, EFX_TXQ_SIZE(SFXGE_NDESCS));
+
+	/* Allocate buffer table entries. */
+	sfxge_sram_buf_tbl_alloc(sc, EFX_TXQ_NBUFS(SFXGE_NDESCS),
+				 &txq->buf_base_id);
+
+	/* Create a DMA tag for packet mappings. */
+	if (bus_dma_tag_create(sc->parent_dma_tag, 1, 0x1000,
+	    MIN(0x3FFFFFFFFFFFUL, BUS_SPACE_MAXADDR), BUS_SPACE_MAXADDR, NULL,
+	    NULL, 0x11000, SFXGE_TX_MAPPING_MAX_SEG, 0x1000, 0, NULL, NULL,
+	    &txq->packet_dma_tag) != 0) {
+		device_printf(sc->dev, "Couldn't allocate txq DMA tag\n");
+		rc = ENOMEM;
+		goto fail;
+	}
+
+	/* Allocate pending descriptor array for batching writes. */
+	txq->pend_desc = malloc(sizeof(efx_buffer_t) * SFXGE_NDESCS,
+				M_SFXGE, M_ZERO | M_WAITOK);
+
+	/* Allocate and initialise mbuf DMA mapping array. */
+	txq->stmp = malloc(sizeof(struct sfxge_tx_mapping) * SFXGE_NDESCS,
+	    M_SFXGE, M_ZERO | M_WAITOK);
+	for (nmaps = 0; nmaps < SFXGE_NDESCS; nmaps++) {
+		rc = bus_dmamap_create(txq->packet_dma_tag, 0,
+				       &txq->stmp[nmaps].map);
+		if (rc != 0)
+			goto fail2;
+	}
+
+	if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM &&
+	    (rc = tso_init(txq)) != 0)
+		goto fail3;
+
+#ifdef SFXGE_HAVE_MQ
+	/* Initialize the deferred packet list. */
+	stdp = &txq->dpl;
+	stdp->std_getp = &stdp->std_get;
+
+	mtx_init(&txq->lock, "txq", NULL, MTX_DEF);
+#endif
+
+	txq->type = type;
+	txq->evq_index = evq_index;
+	txq->txq_index = txq_index;
+	txq->init_state = SFXGE_TXQ_INITIALIZED;
+
+	return (0);
+
+fail3:
+	free(txq->pend_desc, M_SFXGE);
+fail2:
+	while (nmaps--)
+		bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map);
+	free(txq->stmp, M_SFXGE);
+	bus_dma_tag_destroy(txq->packet_dma_tag);
+
+fail:
+	sfxge_dma_free(esmp);
+
+	return (rc);
+}
+
+static const struct {
+	const char *name;
+	size_t offset;
+} sfxge_tx_stats[] = {
+#define SFXGE_TX_STAT(name, member) \
+	{ #name, offsetof(struct sfxge_txq, member) }
+	SFXGE_TX_STAT(tso_bursts, tso_bursts),
+	SFXGE_TX_STAT(tso_packets, tso_packets),
+	SFXGE_TX_STAT(tso_long_headers, tso_long_headers),
+	SFXGE_TX_STAT(tx_collapses, collapses),
+	SFXGE_TX_STAT(tx_drops, drops),
+};
+
+static int
+sfxge_tx_stat_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct sfxge_softc *sc = arg1;
+	unsigned int id = arg2;
+	unsigned long sum;
+	unsigned int index;
+
+	/* Sum across all TX queues */
+	sum = 0;
+	for (index = 0;
+	     index < SFXGE_TXQ_IP_TCP_UDP_CKSUM + SFXGE_TX_SCALE(sc);
+	     index++)
+		sum += *(unsigned long *)((caddr_t)sc->txq[index] +
+					  sfxge_tx_stats[id].offset);
+
+	return SYSCTL_OUT(req, &sum, sizeof(sum));
+}
+
+static void
+sfxge_tx_stat_init(struct sfxge_softc *sc)
+{
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
+	struct sysctl_oid_list *stat_list;
+	unsigned int id;
+
+	stat_list = SYSCTL_CHILDREN(sc->stats_node);
+
+	for (id = 0;
+	     id < sizeof(sfxge_tx_stats) / sizeof(sfxge_tx_stats[0]);
+	     id++) {
+		SYSCTL_ADD_PROC(
+			ctx, stat_list,
+			OID_AUTO, sfxge_tx_stats[id].name,
+			CTLTYPE_ULONG|CTLFLAG_RD,
+			sc, id, sfxge_tx_stat_handler, "LU",
+			"");
+	}
+}
+
+void
+sfxge_tx_fini(struct sfxge_softc *sc)
+{
+	int index;
+
+	index = SFXGE_TX_SCALE(sc);
+	while (--index >= 0)
+		sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index);
+
+	sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM);
+        sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM);
+}
+
+
+int
+sfxge_tx_init(struct sfxge_softc *sc)
+{
+	struct sfxge_intr *intr;
+	int index;
+	int rc;
+
+	intr = &sc->intr;
+
+	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
+	    ("intr->state != SFXGE_INTR_INITIALIZED"));
+
+	/* Initialize the transmit queues */
+	if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_NON_CKSUM,
+	    SFXGE_TXQ_NON_CKSUM, 0)) != 0)
+		goto fail;
+
+	if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_CKSUM,
+	    SFXGE_TXQ_IP_CKSUM, 0)) != 0)
+		goto fail2;
+
+	for (index = 0; index < SFXGE_TX_SCALE(sc); index++) {
+		if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index,
+		    SFXGE_TXQ_IP_TCP_UDP_CKSUM, index)) != 0)
+			goto fail3;
+	}
+
+	sfxge_tx_stat_init(sc);
+
+	return (0);
+
+fail3:
+	sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM);
+
+	while (--index >= 0)
+		sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index);
+
+fail2:
+	sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM);
+
+fail:
+	return (rc);
+}
diff --git a/sys/dev/sfxge/sfxge_tx.h b/sys/dev/sfxge/sfxge_tx.h
new file mode 100644
index 0000000..483a16a
--- /dev/null
+++ b/sys/dev/sfxge/sfxge_tx.h
@@ -0,0 +1,185 @@
+/*-
+ * Copyright (c) 2010-2011 Solarflare Communications, Inc.
+ * All rights reserved.
+ *
+ * This software was developed in part by Philip Paeps under contract for
+ * Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SFXGE_TX_H
+#define _SFXGE_TX_H
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+
+/* Maximum number of DMA segments needed to map an mbuf chain.  With
+ * TSO, the mbuf length may be just over 64K, divided into 2K mbuf
+ * clusters.  (The chain could be longer than this initially, but can
+ * be shortened with m_collapse().)
+ */
+#define	SFXGE_TX_MAPPING_MAX_SEG (64 / 2 + 1)
+
+/* Maximum number of DMA segments needed to map an output packet.  It
+ * could overlap all mbufs in the chain and also require an extra
+ * segment for a TSO header.
+ */
+#define SFXGE_TX_PACKET_MAX_SEG (SFXGE_TX_MAPPING_MAX_SEG + 1)
+
+/*
+ * Buffer mapping flags.
+ *
+ * Buffers and DMA mappings must be freed when the last descriptor
+ * referring to them is completed.  Set the TX_BUF_UNMAP and
+ * TX_BUF_MBUF flags on the last descriptor generated for an mbuf
+ * chain.  Set only the TX_BUF_UNMAP flag on a descriptor referring to
+ * a heap buffer.
+ */
+enum sfxge_tx_buf_flags {
+	TX_BUF_UNMAP = 1,
+	TX_BUF_MBUF = 2,
+};
+
+/*
+ * Buffer mapping information for descriptors in flight.
+ */
+struct sfxge_tx_mapping {
+	union {
+		struct mbuf	*mbuf;
+		caddr_t		heap_buf;
+	}			u;
+	bus_dmamap_t		map;
+	enum sfxge_tx_buf_flags	flags;
+};
+
+#define SFXGE_TX_MAX_DEFERRED 64
+
+/*
+ * Deferred packet list.
+ */
+struct sfxge_tx_dpl {
+	uintptr_t		std_put;    /* Head of put list. */
+	struct mbuf		*std_get;   /* Head of get list. */
+	struct mbuf		**std_getp; /* Tail of get list. */
+	unsigned int		std_count;  /* Count of packets. */
+};
+
+
+#define	SFXGE_TX_BUFFER_SIZE	0x400
+#define	SFXGE_TX_HEADER_SIZE	0x100
+#define	SFXGE_TX_COPY_THRESHOLD	0x200
+
+enum sfxge_txq_state {
+	SFXGE_TXQ_UNINITIALIZED = 0,
+	SFXGE_TXQ_INITIALIZED,
+	SFXGE_TXQ_STARTED
+};
+
+enum sfxge_txq_type {
+	SFXGE_TXQ_NON_CKSUM = 0,
+	SFXGE_TXQ_IP_CKSUM,
+	SFXGE_TXQ_IP_TCP_UDP_CKSUM,
+	SFXGE_TXQ_NTYPES
+};
+
+#define	SFXGE_TXQ_UNBLOCK_LEVEL		(EFX_TXQ_LIMIT(SFXGE_NDESCS) / 4)
+
+#define	SFXGE_TX_BATCH	64
+
+#ifdef SFXGE_HAVE_MQ
+#define SFXGE_TXQ_LOCK(txq)		(&(txq)->lock)
+#define SFXGE_TX_SCALE(sc)		((sc)->intr.n_alloc)
+#else
+#define SFXGE_TXQ_LOCK(txq)		(&(txq)->sc->tx_lock)
+#define SFXGE_TX_SCALE(sc)		1
+#endif
+
+struct sfxge_txq {
+	/* The following fields should be written very rarely */
+	struct sfxge_softc		*sc;
+	enum sfxge_txq_state		init_state;
+	enum sfxge_flush_state		flush_state;
+	enum sfxge_txq_type		type;
+	unsigned int			txq_index;
+	unsigned int			evq_index;
+	efsys_mem_t			mem;
+	unsigned int			buf_base_id;
+
+	struct sfxge_tx_mapping		*stmp;	/* Packets in flight. */
+	bus_dma_tag_t			packet_dma_tag;
+	efx_buffer_t			*pend_desc;
+	efx_txq_t			*common;
+	struct sfxge_txq		*next;
+
+	efsys_mem_t			*tsoh_buffer;
+
+	/* This field changes more often and is read regularly on both
+	 * the initiation and completion paths
+	 */
+	int				blocked __aligned(CACHE_LINE_SIZE);
+
+	/* The following fields change more often, and are used mostly
+	 * on the initiation path
+	 */
+#ifdef SFXGE_HAVE_MQ
+	struct mtx			lock __aligned(CACHE_LINE_SIZE);
+	struct sfxge_tx_dpl		dpl;	/* Deferred packet list. */
+	unsigned int			n_pend_desc;
+#else
+	unsigned int			n_pend_desc __aligned(CACHE_LINE_SIZE);
+#endif
+	unsigned int			added;
+	unsigned int			reaped;
+	/* Statistics */
+	unsigned long			tso_bursts;
+	unsigned long			tso_packets;
+	unsigned long			tso_long_headers;
+	unsigned long			collapses;
+	unsigned long			drops;
+
+	/* The following fields change more often, and are used mostly
+	 * on the completion path
+	 */
+	unsigned int			pending __aligned(CACHE_LINE_SIZE);
+	unsigned int			completed;
+};
+
+extern int sfxge_tx_packet_add(struct sfxge_txq *, struct mbuf *);
+
+extern int sfxge_tx_init(struct sfxge_softc *sc);
+extern void sfxge_tx_fini(struct sfxge_softc *sc);
+extern int sfxge_tx_start(struct sfxge_softc *sc);
+extern void sfxge_tx_stop(struct sfxge_softc *sc);
+extern void sfxge_tx_qcomplete(struct sfxge_txq *txq);
+extern void sfxge_tx_qflush_done(struct sfxge_txq *txq);
+#ifdef SFXGE_HAVE_MQ
+extern void sfxge_if_qflush(struct ifnet *ifp);
+extern int sfxge_if_transmit(struct ifnet *ifp, struct mbuf *m);
+#else
+extern void sfxge_if_start(struct ifnet *ifp);
+#endif
+
+#endif
diff --git a/sys/dev/siba/siba_core.c b/sys/dev/siba/siba_core.c
index 2b1a84f..61652ad 100644
--- a/sys/dev/siba/siba_core.c
+++ b/sys/dev/siba/siba_core.c
@@ -214,16 +214,8 @@ siba_core_attach(struct siba_softc *siba)
 int
 siba_core_detach(struct siba_softc *siba)
 {
-	device_t *devlistp;
-	int devcnt, error = 0, i;
-
-	error = device_get_children(siba->siba_dev, &devlistp, &devcnt);
-	if (error != 0)
-		return (0);
-
-	for ( i = 0 ; i < devcnt ; i++)
-		device_delete_child(siba->siba_dev, devlistp[i]);
-	free(devlistp, M_TEMP);
+	/* detach & delete all children */
+	device_delete_all_children(siba->siba_dev);
 	return (0);
 }
 
diff --git a/sys/dev/siis/siis.c b/sys/dev/siis/siis.c
index a7b018a..40c17cb 100644
--- a/sys/dev/siis/siis.c
+++ b/sys/dev/siis/siis.c
@@ -91,7 +91,7 @@ static void siis_process_request_sense(device_t dev, union ccb *ccb);
 static void siisaction(struct cam_sim *sim, union ccb *ccb);
 static void siispoll(struct cam_sim *sim);
 
-MALLOC_DEFINE(M_SIIS, "SIIS driver", "SIIS driver data buffers");
+static MALLOC_DEFINE(M_SIIS, "SIIS driver", "SIIS driver data buffers");
 
 static struct {
 	uint32_t	id;
@@ -205,15 +205,10 @@ static int
 siis_detach(device_t dev)
 {
 	struct siis_controller *ctlr = device_get_softc(dev);
-	device_t *children;
-	int nchildren, i;
 
 	/* Detach & delete all children */
-	if (!device_get_children(dev, &children, &nchildren)) {
-		for (i = 0; i < nchildren; i++)
-			device_delete_child(dev, children[i]);
-		free(children, M_TEMP);
-	}
+	device_delete_all_children(dev);
+
 	/* Free interrupts. */
 	if (ctlr->irq.r_irq) {
 		bus_teardown_intr(dev, ctlr->irq.r_irq,
diff --git a/sys/dev/sio/sio_pci.c b/sys/dev/sio/sio_pci.c
index 3564194..a693536 100644
--- a/sys/dev/sio/sio_pci.c
+++ b/sys/dev/sio/sio_pci.c
@@ -76,7 +76,8 @@ static struct pci_ids pci_ids[] = {
 	{ 0x7101135e, "SeaLevel Ultra 530.PCI Single Port Serial", 0x18 },
 	{ 0x0000151f, "SmartLink 5634PCV SurfRider", 0x10 },
 	{ 0x0103115d, "Xircom Cardbus modem", 0x10 },
-	{ 0x432214e4, "Broadcom 802.11g/GPRS CardBus (Serial)", 0x10 },
+	{ 0x432214e4, "Broadcom 802.11b/GPRS CardBus (Serial)", 0x10 },
+	{ 0x434414e4, "Broadcom 802.11bg/EDGE/GPRS CardBus (Serial)", 0x10 },
 	{ 0x01c0135c, "Quatech SSCLP-200/300", 0x18 
 		/* 
 		 * NB: You must mount the "SPAD" jumper to correctly detect
diff --git a/sys/dev/sis/if_sis.c b/sys/dev/sis/if_sis.c
index d0be783..630281d 100644
--- a/sys/dev/sis/if_sis.c
+++ b/sys/dev/sis/if_sis.c
@@ -90,6 +90,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -119,10 +120,13 @@ MODULE_DEPEND(sis, miibus, 1, 1, 1);
 
 #define CSR_READ_2(sc, reg)		bus_read_2(sc->sis_res[0], reg)
 
+#define	CSR_BARRIER(sc, reg, length, flags)				\
+	bus_barrier(sc->sis_res[0], reg, length, flags)
+
 /*
  * Various supported device vendors/types and their names.
  */
-static struct sis_type sis_devs[] = {
+static const struct sis_type const sis_devs[] = {
 	{ SIS_VENDORID, SIS_DEVICEID_900, "SiS 900 10/100BaseTX" },
 	{ SIS_VENDORID, SIS_DEVICEID_7016, "SiS 7016 10/100BaseTX" },
 	{ NS_VENDORID, NS_DEVICEID_DP83815, "NatSemi DP8381[56] 10/100BaseTX" },
@@ -145,6 +149,8 @@ static void sis_init(void *);
 static void sis_initl(struct sis_softc *);
 static void sis_intr(void *);
 static int sis_ioctl(struct ifnet *, u_long, caddr_t);
+static uint32_t sis_mii_bitbang_read(device_t);
+static void sis_mii_bitbang_write(device_t, uint32_t);
 static int sis_newbuf(struct sis_softc *, struct sis_rxdesc *);
 static int sis_resume(device_t);
 static int sis_rxeof(struct sis_softc *);
@@ -159,6 +165,20 @@ static void sis_add_sysctls(struct sis_softc *);
 static void sis_watchdog(struct sis_softc *);
 static void sis_wol(struct sis_softc *);
 
+/*
+ * MII bit-bang glue
+ */
+static const struct mii_bitbang_ops sis_mii_bitbang_ops = {
+	sis_mii_bitbang_read,
+	sis_mii_bitbang_write,
+	{
+		SIS_MII_DATA,		/* MII_BIT_MDO */
+		SIS_MII_DATA,		/* MII_BIT_MDI */
+		SIS_MII_CLK,		/* MII_BIT_MDC */
+		SIS_MII_DIR,		/* MII_BIT_DIR_HOST_PHY */
+		0,			/* MII_BIT_DIR_PHY_HOST */
+	}
+};
 
 static struct resource_spec sis_res_spec[] = {
 #ifdef SIS_USEIOSPACE
@@ -412,179 +432,41 @@ sis_read_mac(struct sis_softc *sc, device_t dev, caddr_t dest)
 #endif
 
 /*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static void
-sis_mii_sync(struct sis_softc *sc)
+static uint32_t
+sis_mii_bitbang_read(device_t dev)
 {
-	int		i;
+	struct sis_softc	*sc;
+	uint32_t		val;
 
- 	SIO_SET(SIS_MII_DIR|SIS_MII_DATA);
+	sc = device_get_softc(dev);
 
- 	for (i = 0; i < 32; i++) {
- 		SIO_SET(SIS_MII_CLK);
- 		DELAY(1);
- 		SIO_CLR(SIS_MII_CLK);
- 		DELAY(1);
- 	}
+	val = CSR_READ_4(sc, SIS_EECTL);
+	CSR_BARRIER(sc, SIS_EECTL, 4,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
+	return (val);
 }
 
 /*
- * Clock a series of bits through the MII.
+ * Write the MII serial port for the MII bit-bang module.
  */
 static void
-sis_mii_send(struct sis_softc *sc, uint32_t bits, int cnt)
-{
-	int			i;
-
-	SIO_CLR(SIS_MII_CLK);
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		if (bits & i) {
-			SIO_SET(SIS_MII_DATA);
-		} else {
-			SIO_CLR(SIS_MII_DATA);
-		}
-		DELAY(1);
-		SIO_CLR(SIS_MII_CLK);
-		DELAY(1);
-		SIO_SET(SIS_MII_CLK);
-	}
-}
-
-/*
- * Read an PHY register through the MII.
- */
-static int
-sis_mii_readreg(struct sis_softc *sc, struct sis_mii_frame *frame)
+sis_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	int			i, ack;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = SIS_MII_STARTDELIM;
-	frame->mii_opcode = SIS_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	/*
- 	 * Turn on data xmit.
-	 */
-	SIO_SET(SIS_MII_DIR);
-
-	sis_mii_sync(sc);
-
-	/*
-	 * Send command/address info.
-	 */
-	sis_mii_send(sc, frame->mii_stdelim, 2);
-	sis_mii_send(sc, frame->mii_opcode, 2);
-	sis_mii_send(sc, frame->mii_phyaddr, 5);
-	sis_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Idle bit */
-	SIO_CLR((SIS_MII_CLK|SIS_MII_DATA));
-	DELAY(1);
-	SIO_SET(SIS_MII_CLK);
-	DELAY(1);
-
-	/* Turn off xmit. */
-	SIO_CLR(SIS_MII_DIR);
-
-	/* Check for ack */
-	SIO_CLR(SIS_MII_CLK);
-	DELAY(1);
-	ack = CSR_READ_4(sc, SIS_EECTL) & SIS_MII_DATA;
-	SIO_SET(SIS_MII_CLK);
-	DELAY(1);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for (i = 0; i < 16; i++) {
-			SIO_CLR(SIS_MII_CLK);
-			DELAY(1);
-			SIO_SET(SIS_MII_CLK);
-			DELAY(1);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		SIO_CLR(SIS_MII_CLK);
-		DELAY(1);
-		if (!ack) {
-			if (CSR_READ_4(sc, SIS_EECTL) & SIS_MII_DATA)
-				frame->mii_data |= i;
-			DELAY(1);
-		}
-		SIO_SET(SIS_MII_CLK);
-		DELAY(1);
-	}
-
-fail:
-
-	SIO_CLR(SIS_MII_CLK);
-	DELAY(1);
-	SIO_SET(SIS_MII_CLK);
-	DELAY(1);
-
-	if (ack)
-		return (1);
-	return (0);
-}
-
-/*
- * Write to a PHY register through the MII.
- */
-static int
-sis_mii_writereg(struct sis_softc *sc, struct sis_mii_frame *frame)
-{
-
- 	/*
- 	 * Set up frame for TX.
- 	 */
-
- 	frame->mii_stdelim = SIS_MII_STARTDELIM;
- 	frame->mii_opcode = SIS_MII_WRITEOP;
- 	frame->mii_turnaround = SIS_MII_TURNAROUND;
-
- 	/*
-  	 * Turn on data output.
- 	 */
- 	SIO_SET(SIS_MII_DIR);
-
- 	sis_mii_sync(sc);
-
- 	sis_mii_send(sc, frame->mii_stdelim, 2);
- 	sis_mii_send(sc, frame->mii_opcode, 2);
- 	sis_mii_send(sc, frame->mii_phyaddr, 5);
- 	sis_mii_send(sc, frame->mii_regaddr, 5);
- 	sis_mii_send(sc, frame->mii_turnaround, 2);
- 	sis_mii_send(sc, frame->mii_data, 16);
-
- 	/* Idle bit. */
- 	SIO_SET(SIS_MII_CLK);
- 	DELAY(1);
- 	SIO_CLR(SIS_MII_CLK);
- 	DELAY(1);
+	struct sis_softc	*sc;
 
- 	/*
- 	 * Turn off xmit.
- 	 */
- 	SIO_CLR(SIS_MII_DIR);
+	sc = device_get_softc(dev);
 
- 	return (0);
+	CSR_WRITE_4(sc, SIS_EECTL, val);
+	CSR_BARRIER(sc, SIS_EECTL, 4,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
 sis_miibus_readreg(device_t dev, int phy, int reg)
 {
 	struct sis_softc	*sc;
-	struct sis_mii_frame    frame;
 
 	sc = device_get_softc(dev);
 
@@ -628,7 +510,8 @@ sis_miibus_readreg(device_t dev, int phy, int reg)
 		}
 
 		if (i == SIS_TIMEOUT) {
-			device_printf(sc->sis_dev, "PHY failed to come ready\n");
+			device_printf(sc->sis_dev,
+			    "PHY failed to come ready\n");
 			return (0);
 		}
 
@@ -638,22 +521,15 @@ sis_miibus_readreg(device_t dev, int phy, int reg)
 			return (0);
 
 		return (val);
-	} else {
-		bzero((char *)&frame, sizeof(frame));
-
-		frame.mii_phyaddr = phy;
-		frame.mii_regaddr = reg;
-		sis_mii_readreg(sc, &frame);
-
-		return (frame.mii_data);
-	}
+	} else
+		return (mii_bitbang_readreg(dev, &sis_mii_bitbang_ops, phy,
+		    reg));
 }
 
 static int
 sis_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	struct sis_softc	*sc;
-	struct sis_mii_frame	frame;
 
 	sc = device_get_softc(dev);
 
@@ -686,15 +562,11 @@ sis_miibus_writereg(device_t dev, int phy, int reg, int data)
 		}
 
 		if (i == SIS_TIMEOUT)
-			device_printf(sc->sis_dev, "PHY failed to come ready\n");
-	} else {
-		bzero((char *)&frame, sizeof(frame));
-
-		frame.mii_phyaddr = phy;
-		frame.mii_regaddr = reg;
-		frame.mii_data = data;
-		sis_mii_writereg(sc, &frame);
-	}
+			device_printf(sc->sis_dev,
+			    "PHY failed to come ready\n");
+	} else
+		mii_bitbang_writereg(dev, &sis_mii_bitbang_ops, phy, reg,
+		    data);
 	return (0);
 }
 
@@ -989,7 +861,7 @@ sis_reset(struct sis_softc *sc)
 static int
 sis_probe(device_t dev)
 {
-	struct sis_type		*t;
+	const struct sis_type	*t;
 
 	t = sis_devs;
 
diff --git a/sys/dev/sis/if_sisreg.h b/sys/dev/sis/if_sisreg.h
index c86a13d..b400806 100644
--- a/sys/dev/sis/if_sisreg.h
+++ b/sys/dev/sis/if_sisreg.h
@@ -433,26 +433,9 @@ struct sis_desc {
 struct sis_type {
 	uint16_t		sis_vid;
 	uint16_t		sis_did;
-	char			*sis_name;
+	const char		*sis_name;
 };
 
-struct sis_mii_frame {
-	uint8_t			mii_stdelim;
-	uint8_t			mii_opcode;
-	uint8_t			mii_phyaddr;
-	uint8_t			mii_regaddr;
-	uint8_t			mii_turnaround;
-	uint16_t		mii_data;
-};
-
-/*
- * MII constants
- */
-#define	SIS_MII_STARTDELIM	0x01
-#define	SIS_MII_READOP		0x02
-#define	SIS_MII_WRITEOP		0x01
-#define	SIS_MII_TURNAROUND	0x02
-
 #define SIS_TYPE_900	1
 #define SIS_TYPE_7016	2
 #define SIS_TYPE_83815	3
diff --git a/sys/dev/smc/if_smc.c b/sys/dev/smc/if_smc.c
index 4aa0396..9404f56 100644
--- a/sys/dev/smc/if_smc.c
+++ b/sys/dev/smc/if_smc.c
@@ -74,6 +74,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/smc/if_smcvar.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #define	SMC_LOCK(sc)		mtx_lock(&(sc)->smc_mtx)
@@ -123,11 +124,33 @@ static timeout_t	smc_watchdog;
 static poll_handler_t	smc_poll;
 #endif
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t smc_mii_bitbang_read(device_t);
+static void smc_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops smc_mii_bitbang_ops = {
+	smc_mii_bitbang_read,
+	smc_mii_bitbang_write,
+	{
+		MGMT_MDO,	/* MII_BIT_MDO */
+		MGMT_MDI,	/* MII_BIT_MDI */
+		MGMT_MCLK,	/* MII_BIT_MDC */
+		MGMT_MDOE,	/* MII_BIT_DIR_HOST_PHY */
+		0,		/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static __inline void
 smc_select_bank(struct smc_softc *sc, uint16_t bank)
 {
 
+	bus_barrier(sc->smc_reg, BSR, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	bus_write_2(sc->smc_reg, BSR, bank & BSR_BANK_MASK);
+	bus_barrier(sc->smc_reg, BSR, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 /* Never call this when not in bank 2. */
@@ -143,35 +166,35 @@ smc_mmu_wait(struct smc_softc *sc)
 }
 
 static __inline uint8_t
-smc_read_1(struct smc_softc *sc, bus_addr_t offset)
+smc_read_1(struct smc_softc *sc, bus_size_t offset)
 {
 
 	return (bus_read_1(sc->smc_reg, offset));
 }
 
 static __inline void
-smc_write_1(struct smc_softc *sc, bus_addr_t offset, uint8_t val)
+smc_write_1(struct smc_softc *sc, bus_size_t offset, uint8_t val)
 {
 
 	bus_write_1(sc->smc_reg, offset, val);
 }
 
 static __inline uint16_t
-smc_read_2(struct smc_softc *sc, bus_addr_t offset)
+smc_read_2(struct smc_softc *sc, bus_size_t offset)
 {
 
 	return (bus_read_2(sc->smc_reg, offset));
 }
 
 static __inline void
-smc_write_2(struct smc_softc *sc, bus_addr_t offset, uint16_t val)
+smc_write_2(struct smc_softc *sc, bus_size_t offset, uint16_t val)
 {
 
 	bus_write_2(sc->smc_reg, offset, val);
 }
 
 static __inline void
-smc_read_multi_2(struct smc_softc *sc, bus_addr_t offset, uint16_t *datap,
+smc_read_multi_2(struct smc_softc *sc, bus_size_t offset, uint16_t *datap,
     bus_size_t count)
 {
 
@@ -179,13 +202,21 @@ smc_read_multi_2(struct smc_softc *sc, bus_addr_t offset, uint16_t *datap,
 }
 
 static __inline void
-smc_write_multi_2(struct smc_softc *sc, bus_addr_t offset, uint16_t *datap,
+smc_write_multi_2(struct smc_softc *sc, bus_size_t offset, uint16_t *datap,
     bus_size_t count)
 {
 
 	bus_write_multi_2(sc->smc_reg, offset, datap, count);
 }
 
+static __inline void
+smc_barrier(struct smc_softc *sc, bus_size_t offset, bus_size_t length,
+    int flags)
+{
+
+	bus_barrier(sc->smc_reg, offset, length, flags);
+}
+
 int
 smc_probe(device_t dev)
 {
@@ -900,70 +931,43 @@ smc_task_intr(void *context, int pending)
 	SMC_UNLOCK(sc);
 }
 
-static u_int
-smc_mii_readbits(struct smc_softc *sc, int nbits)
+static uint32_t
+smc_mii_bitbang_read(device_t dev)
 {
-	u_int	mgmt, mask, val;
+	struct smc_softc	*sc;
+	uint32_t		val;
+
+	sc = device_get_softc(dev);
 
 	SMC_ASSERT_LOCKED(sc);
 	KASSERT((smc_read_2(sc, BSR) & BSR_BANK_MASK) == 3,
-	    ("%s: smc_mii_readbits called with bank %d (!= 3)",
+	    ("%s: smc_mii_bitbang_read called with bank %d (!= 3)",
 	    device_get_nameunit(sc->smc_dev),
 	    smc_read_2(sc, BSR) & BSR_BANK_MASK));
 
-	/*
-	 * Set up the MGMT (aka MII) register.
-	 */
-	mgmt = smc_read_2(sc, MGMT) & ~(MGMT_MCLK | MGMT_MDOE | MGMT_MDO);
-	smc_write_2(sc, MGMT, mgmt);
-
-	/*
-	 * Read the bits in.
-	 */
-	for (mask = 1 << (nbits - 1), val = 0; mask; mask >>= 1) {
-		if (smc_read_2(sc, MGMT) & MGMT_MDI)
-			val |= mask;
-
-		smc_write_2(sc, MGMT, mgmt);
-		DELAY(1);
-		smc_write_2(sc, MGMT, mgmt | MGMT_MCLK);
-		DELAY(1);
-	}
+	val = smc_read_2(sc, MGMT);
+	smc_barrier(sc, MGMT, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
 	return (val);
 }
 
 static void
-smc_mii_writebits(struct smc_softc *sc, u_int val, int nbits)
+smc_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	u_int	mgmt, mask;
+	struct smc_softc	*sc;
+
+	sc = device_get_softc(dev);
 
 	SMC_ASSERT_LOCKED(sc);
 	KASSERT((smc_read_2(sc, BSR) & BSR_BANK_MASK) == 3,
-	    ("%s: smc_mii_writebits called with bank %d (!= 3)",
+	    ("%s: smc_mii_bitbang_write called with bank %d (!= 3)",
 	    device_get_nameunit(sc->smc_dev),
 	    smc_read_2(sc, BSR) & BSR_BANK_MASK));
 
-	/*
-	 * Set up the MGMT (aka MII) register).
-	 */
-	mgmt = smc_read_2(sc, MGMT) & ~(MGMT_MCLK | MGMT_MDOE | MGMT_MDO);
-	mgmt |= MGMT_MDOE;
-
-	/*
-	 * Push the bits out.
-	 */
-	for (mask = 1 << (nbits - 1); mask; mask >>= 1) {
-		if (val & mask)
-			mgmt |= MGMT_MDO;
-		else
-			mgmt &= ~MGMT_MDO;
-
-		smc_write_2(sc, MGMT, mgmt);
-		DELAY(1);
-		smc_write_2(sc, MGMT, mgmt | MGMT_MCLK);
-		DELAY(1);
-	}
+	smc_write_2(sc, MGMT, val);
+	smc_barrier(sc, MGMT, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 int
@@ -978,26 +982,7 @@ smc_miibus_readreg(device_t dev, int phy, int reg)
 
 	smc_select_bank(sc, 3);
 
-	/*
-	 * Send out the idle pattern.
-	 */
-	smc_mii_writebits(sc, 0xffffffff, 32);
-
-	/*
-	 * Start code + read opcode + phy address + phy register
-	 */
-	smc_mii_writebits(sc, 6 << 10 | phy << 5 | reg, 14);
-
-	/*
-	 * Turnaround + data
-	 */
-	val = smc_mii_readbits(sc, 18);
-
-	/*
-	 * Reset the MDIO interface.
-	 */
-	smc_write_2(sc, MGMT,
-	    smc_read_2(sc, MGMT) & ~(MGMT_MCLK | MGMT_MDOE | MGMT_MDO));
+	val = mii_bitbang_readreg(dev, &smc_mii_bitbang_ops, phy, reg);
 
 	SMC_UNLOCK(sc);
 	return (val);
@@ -1014,23 +999,7 @@ smc_miibus_writereg(device_t dev, int phy, int reg, int data)
 
 	smc_select_bank(sc, 3);
 
-	/*
-	 * Send idle pattern.
-	 */
-	smc_mii_writebits(sc, 0xffffffff, 32);
-
-	/*
-	 * Start code + write opcode + phy address + phy register + turnaround
-	 * + data.
-	 */
-	smc_mii_writebits(sc, 5 << 28 | phy << 23 | reg << 18 | 2 << 16 | data,
-	    32);
-
-	/*
-	 * Reset MDIO interface.
-	 */
-	smc_write_2(sc, MGMT,
-	    smc_read_2(sc, MGMT) & ~(MGMT_MCLK | MGMT_MDOE | MGMT_MDO));
+	mii_bitbang_writereg(dev, &smc_mii_bitbang_ops, phy, reg, data);
 
 	SMC_UNLOCK(sc);
 	return (0);
diff --git a/sys/dev/sound/midi/midi.c b/sys/dev/sound/midi/midi.c
index 97492a7..964e1bd 100644
--- a/sys/dev/sound/midi/midi.c
+++ b/sys/dev/sound/midi/midi.c
@@ -239,7 +239,7 @@ static int      midi_unload(void);
  * Misc declr.
  */
 SYSCTL_NODE(_hw, OID_AUTO, midi, CTLFLAG_RD, 0, "Midi driver");
-SYSCTL_NODE(_hw_midi, OID_AUTO, stat, CTLFLAG_RD, 0, "Status device");
+static SYSCTL_NODE(_hw_midi, OID_AUTO, stat, CTLFLAG_RD, 0, "Status device");
 
 int             midi_debug;
 /* XXX: should this be moved into debug.midi? */
diff --git a/sys/dev/sound/pci/envy24.c b/sys/dev/sound/pci/envy24.c
index 57a8ed6..1c59765 100644
--- a/sys/dev/sound/pci/envy24.c
+++ b/sys/dev/sound/pci/envy24.c
@@ -41,7 +41,7 @@
 
 SND_DECLARE_FILE("$FreeBSD$");
 
-MALLOC_DEFINE(M_ENVY24, "envy24", "envy24 audio");
+static MALLOC_DEFINE(M_ENVY24, "envy24", "envy24 audio");
 
 /* -------------------------------------------------------------------- */
 
diff --git a/sys/dev/sound/pci/envy24ht.c b/sys/dev/sound/pci/envy24ht.c
index 0358625..25d2e3f 100644
--- a/sys/dev/sound/pci/envy24ht.c
+++ b/sys/dev/sound/pci/envy24ht.c
@@ -53,7 +53,7 @@
 
 SND_DECLARE_FILE("$FreeBSD$");
 
-MALLOC_DEFINE(M_ENVY24HT, "envy24ht", "envy24ht audio");
+static MALLOC_DEFINE(M_ENVY24HT, "envy24ht", "envy24ht audio");
 
 /* -------------------------------------------------------------------- */
 
diff --git a/sys/dev/sound/pci/maestro.c b/sys/dev/sound/pci/maestro.c
index 7c977ad..66567fc 100644
--- a/sys/dev/sound/pci/maestro.c
+++ b/sys/dev/sound/pci/maestro.c
@@ -191,7 +191,7 @@ static unsigned int powerstate_init   = PCI_POWERSTATE_D2;
 /* XXX: this should move to a device specific sysctl dev.pcm.X.debug.Y via
    device_get_sysctl_*() as discussed on multimedia@ in msg-id
    <861wujij2q.fsf@xps.des.no> */
-SYSCTL_NODE(_debug, OID_AUTO, maestro, CTLFLAG_RD, 0, "");
+static SYSCTL_NODE(_debug, OID_AUTO, maestro, CTLFLAG_RD, 0, "");
 SYSCTL_UINT(_debug_maestro, OID_AUTO, powerstate_active, CTLFLAG_RW,
 	    &powerstate_active, 0, "The Dx power state when active (0-1)");
 SYSCTL_UINT(_debug_maestro, OID_AUTO, powerstate_idle, CTLFLAG_RW,
diff --git a/sys/dev/sound/pci/spicds.c b/sys/dev/sound/pci/spicds.c
index 3a4002c..3e1df4c 100644
--- a/sys/dev/sound/pci/spicds.c
+++ b/sys/dev/sound/pci/spicds.c
@@ -35,7 +35,7 @@
 
 #include <dev/sound/pci/spicds.h>
 
-MALLOC_DEFINE(M_SPICDS, "spicds", "SPI codec");
+static MALLOC_DEFINE(M_SPICDS, "spicds", "SPI codec");
 
 #define SPICDS_NAMELEN	16
 struct spicds_info {
diff --git a/sys/dev/sound/pcm/ac97.c b/sys/dev/sound/pcm/ac97.c
index ad164d6..bc36948 100644
--- a/sys/dev/sound/pcm/ac97.c
+++ b/sys/dev/sound/pcm/ac97.c
@@ -38,7 +38,7 @@
 
 SND_DECLARE_FILE("$FreeBSD$");
 
-MALLOC_DEFINE(M_AC97, "ac97", "ac97 codec");
+static MALLOC_DEFINE(M_AC97, "ac97", "ac97 codec");
 
 struct ac97mixtable_entry {
 	int reg;		/* register index		*/
diff --git a/sys/dev/sound/pcm/feeder.c b/sys/dev/sound/pcm/feeder.c
index 916b2d9..720f676 100644
--- a/sys/dev/sound/pcm/feeder.c
+++ b/sys/dev/sound/pcm/feeder.c
@@ -35,7 +35,7 @@
 
 SND_DECLARE_FILE("$FreeBSD$");
 
-MALLOC_DEFINE(M_FEEDER, "feeder", "pcm feeder");
+static MALLOC_DEFINE(M_FEEDER, "feeder", "pcm feeder");
 
 #define MAXFEEDERS 	256
 #undef FEEDER_DEBUG
diff --git a/sys/dev/sound/pcm/mixer.c b/sys/dev/sound/pcm/mixer.c
index 4c60c50..58b9326 100644
--- a/sys/dev/sound/pcm/mixer.c
+++ b/sys/dev/sound/pcm/mixer.c
@@ -37,7 +37,7 @@
 
 SND_DECLARE_FILE("$FreeBSD$");
 
-MALLOC_DEFINE(M_MIXER, "mixer", "mixer");
+static MALLOC_DEFINE(M_MIXER, "mixer", "mixer");
 
 static int mixer_bypass = 1;
 TUNABLE_INT("hw.snd.vpc_mixer_bypass", &mixer_bypass);
diff --git a/sys/dev/sound/usb/uaudio.c b/sys/dev/sound/usb/uaudio.c
index 955b4c5..0445572 100644
--- a/sys/dev/sound/usb/uaudio.c
+++ b/sys/dev/sound/usb/uaudio.c
@@ -93,7 +93,7 @@ static int uaudio_default_channels = 0;		/* use default */
 #ifdef USB_DEBUG
 static int uaudio_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uaudio, CTLFLAG_RW, 0, "USB uaudio");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uaudio, CTLFLAG_RW, 0, "USB uaudio");
 
 SYSCTL_INT(_hw_usb_uaudio, OID_AUTO, debug, CTLFLAG_RW,
     &uaudio_debug, 0, "uaudio debug level");
diff --git a/sys/dev/ste/if_ste.c b/sys/dev/ste/if_ste.c
index 846761c..a6fc00b 100644
--- a/sys/dev/ste/if_ste.c
+++ b/sys/dev/ste/if_ste.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/resource.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -84,7 +85,7 @@ MODULE_DEPEND(ste, miibus, 1, 1, 1);
 /*
  * Various supported device vendors/types and their names.
  */
-static struct ste_type ste_devs[] = {
+static const struct ste_type const ste_devs[] = {
 	{ ST_VENDORID, ST_DEVICEID_ST201_1, "Sundance ST201 10/100BaseTX" },
 	{ ST_VENDORID, ST_DEVICEID_ST201_2, "Sundance ST201 10/100BaseTX" },
 	{ DL_VENDORID, DL_DEVICEID_DL10050, "D-Link DL10050 10/100BaseTX" },
@@ -112,10 +113,8 @@ static int	ste_init_rx_list(struct ste_softc *);
 static void	ste_init_tx_list(struct ste_softc *);
 static void	ste_intr(void *);
 static int	ste_ioctl(struct ifnet *, u_long, caddr_t);
-static int	ste_mii_readreg(struct ste_softc *, struct ste_mii_frame *);
-static void	ste_mii_send(struct ste_softc *, uint32_t, int);
-static void	ste_mii_sync(struct ste_softc *);
-static int	ste_mii_writereg(struct ste_softc *, struct ste_mii_frame *);
+static uint32_t ste_mii_bitbang_read(device_t);
+static void	ste_mii_bitbang_write(device_t, uint32_t);
 static int	ste_miibus_readreg(device_t, int, int);
 static void	ste_miibus_statchg(device_t);
 static int	ste_miibus_writereg(device_t, int, int, int);
@@ -138,6 +137,21 @@ static void	ste_txeof(struct ste_softc *);
 static void	ste_wait(struct ste_softc *);
 static void	ste_watchdog(struct ste_softc *);
 
+/*
+ * MII bit-bang glue
+ */
+static const struct mii_bitbang_ops ste_mii_bitbang_ops = {
+	ste_mii_bitbang_read,
+	ste_mii_bitbang_write,
+	{
+		STE_PHYCTL_MDATA,	/* MII_BIT_MDO */
+		STE_PHYCTL_MDATA,	/* MII_BIT_MDI */
+		STE_PHYCTL_MCLK,	/* MII_BIT_MDC */
+		STE_PHYCTL_MDIR,	/* MII_BIT_DIR_HOST_PHY */
+		0,			/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static device_method_t ste_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		ste_probe),
@@ -188,210 +202,51 @@ DRIVER_MODULE(miibus, ste, miibus_driver, miibus_devclass, 0, 0);
 #define STE_CLRBIT1(sc, reg, x)				\
 	CSR_WRITE_1(sc, reg, CSR_READ_1(sc, reg) & ~(x))
 
-
-#define MII_SET(x)		STE_SETBIT1(sc, STE_PHYCTL, x)
-#define MII_CLR(x)		STE_CLRBIT1(sc, STE_PHYCTL, x)
-
-/*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
- */
-static void
-ste_mii_sync(struct ste_softc *sc)
-{
-	int i;
-
-	MII_SET(STE_PHYCTL_MDIR|STE_PHYCTL_MDATA);
-
-	for (i = 0; i < 32; i++) {
-		MII_SET(STE_PHYCTL_MCLK);
-		DELAY(1);
-		MII_CLR(STE_PHYCTL_MCLK);
-		DELAY(1);
-	}
-}
-
-/*
- * Clock a series of bits through the MII.
- */
-static void
-ste_mii_send(struct ste_softc *sc, uint32_t bits, int cnt)
-{
-	int i;
-
-	MII_CLR(STE_PHYCTL_MCLK);
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		if (bits & i) {
-			MII_SET(STE_PHYCTL_MDATA);
-                } else {
-			MII_CLR(STE_PHYCTL_MDATA);
-                }
-		DELAY(1);
-		MII_CLR(STE_PHYCTL_MCLK);
-		DELAY(1);
-		MII_SET(STE_PHYCTL_MCLK);
-	}
-}
-
 /*
- * Read an PHY register through the MII.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static int
-ste_mii_readreg(struct ste_softc *sc, struct ste_mii_frame *frame)
+static uint32_t
+ste_mii_bitbang_read(device_t dev)
 {
-	int i, ack;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = STE_MII_STARTDELIM;
-	frame->mii_opcode = STE_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	CSR_WRITE_2(sc, STE_PHYCTL, 0);
-	/*
- 	 * Turn on data xmit.
-	 */
-	MII_SET(STE_PHYCTL_MDIR);
-
-	ste_mii_sync(sc);
-
-	/*
-	 * Send command/address info.
-	 */
-	ste_mii_send(sc, frame->mii_stdelim, 2);
-	ste_mii_send(sc, frame->mii_opcode, 2);
-	ste_mii_send(sc, frame->mii_phyaddr, 5);
-	ste_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Turn off xmit. */
-	MII_CLR(STE_PHYCTL_MDIR);
-
-	/* Idle bit */
-	MII_CLR((STE_PHYCTL_MCLK|STE_PHYCTL_MDATA));
-	DELAY(1);
-	MII_SET(STE_PHYCTL_MCLK);
-	DELAY(1);
-
-	/* Check for ack */
-	MII_CLR(STE_PHYCTL_MCLK);
-	DELAY(1);
-	ack = CSR_READ_2(sc, STE_PHYCTL) & STE_PHYCTL_MDATA;
-	MII_SET(STE_PHYCTL_MCLK);
-	DELAY(1);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for (i = 0; i < 16; i++) {
-			MII_CLR(STE_PHYCTL_MCLK);
-			DELAY(1);
-			MII_SET(STE_PHYCTL_MCLK);
-			DELAY(1);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		MII_CLR(STE_PHYCTL_MCLK);
-		DELAY(1);
-		if (!ack) {
-			if (CSR_READ_2(sc, STE_PHYCTL) & STE_PHYCTL_MDATA)
-				frame->mii_data |= i;
-			DELAY(1);
-		}
-		MII_SET(STE_PHYCTL_MCLK);
-		DELAY(1);
-	}
+	struct ste_softc *sc;
+	uint32_t val;
 
-fail:
+	sc = device_get_softc(dev);
 
-	MII_CLR(STE_PHYCTL_MCLK);
-	DELAY(1);
-	MII_SET(STE_PHYCTL_MCLK);
-	DELAY(1);
+	val = CSR_READ_1(sc, STE_PHYCTL);
+	CSR_BARRIER(sc, STE_PHYCTL, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	if (ack)
-		return (1);
-	return (0);
+	return (val);
 }
 
 /*
- * Write to a PHY register through the MII.
+ * Write the MII serial port for the MII bit-bang module.
  */
-static int
-ste_mii_writereg(struct ste_softc *sc, struct ste_mii_frame *frame)
+static void
+ste_mii_bitbang_write(device_t dev, uint32_t val)
 {
+	struct ste_softc *sc;
 
-	/*
-	 * Set up frame for TX.
-	 */
-
-	frame->mii_stdelim = STE_MII_STARTDELIM;
-	frame->mii_opcode = STE_MII_WRITEOP;
-	frame->mii_turnaround = STE_MII_TURNAROUND;
-
-	/*
- 	 * Turn on data output.
-	 */
-	MII_SET(STE_PHYCTL_MDIR);
-
-	ste_mii_sync(sc);
-
-	ste_mii_send(sc, frame->mii_stdelim, 2);
-	ste_mii_send(sc, frame->mii_opcode, 2);
-	ste_mii_send(sc, frame->mii_phyaddr, 5);
-	ste_mii_send(sc, frame->mii_regaddr, 5);
-	ste_mii_send(sc, frame->mii_turnaround, 2);
-	ste_mii_send(sc, frame->mii_data, 16);
-
-	/* Idle bit. */
-	MII_SET(STE_PHYCTL_MCLK);
-	DELAY(1);
-	MII_CLR(STE_PHYCTL_MCLK);
-	DELAY(1);
-
-	/*
-	 * Turn off xmit.
-	 */
-	MII_CLR(STE_PHYCTL_MDIR);
+	sc = device_get_softc(dev);
 
-	return (0);
+	CSR_WRITE_1(sc, STE_PHYCTL, val);
+	CSR_BARRIER(sc, STE_PHYCTL, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
 ste_miibus_readreg(device_t dev, int phy, int reg)
 {
-	struct ste_softc *sc;
-	struct ste_mii_frame frame;
-
-	sc = device_get_softc(dev);
-	bzero((char *)&frame, sizeof(frame));
 
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	ste_mii_readreg(sc, &frame);
-
-	return (frame.mii_data);
+	return (mii_bitbang_readreg(dev, &ste_mii_bitbang_ops, phy, reg));
 }
 
 static int
 ste_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
-	struct ste_softc *sc;
-	struct ste_mii_frame frame;
-
-	sc = device_get_softc(dev);
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
 
-	ste_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &ste_mii_bitbang_ops, phy, reg, data);
 
 	return (0);
 }
@@ -1027,7 +882,7 @@ ste_stats_update(struct ste_softc *sc)
 static int
 ste_probe(device_t dev)
 {
-	struct ste_type *t;
+	const struct ste_type *t;
 
 	t = ste_devs;
 
diff --git a/sys/dev/ste/if_stereg.h b/sys/dev/ste/if_stereg.h
index 840e0bf..e3aa51b 100644
--- a/sys/dev/ste/if_stereg.h
+++ b/sys/dev/ste/if_stereg.h
@@ -492,6 +492,9 @@ struct ste_desc_onefrag {
 #define CSR_READ_1(sc, reg)		\
 	bus_read_1((sc)->ste_res, reg)
 
+#define	CSR_BARRIER(sc, reg, length, flags)				\
+	bus_barrier((sc)->ste_res, reg, length, flags)
+
 #define	STE_DESC_ALIGN		8
 #define STE_RX_LIST_CNT		128
 #define STE_TX_LIST_CNT		128
@@ -519,7 +522,7 @@ struct ste_desc_onefrag {
 struct ste_type {
 	uint16_t		ste_vid;
 	uint16_t		ste_did;
-	char			*ste_name;
+	const char		*ste_name;
 };
 
 struct ste_list_data {
@@ -590,20 +593,3 @@ struct ste_softc {
 #define	STE_LOCK(_sc)		mtx_lock(&(_sc)->ste_mtx)
 #define	STE_UNLOCK(_sc)		mtx_unlock(&(_sc)->ste_mtx)
 #define	STE_LOCK_ASSERT(_sc)	mtx_assert(&(_sc)->ste_mtx, MA_OWNED)
-
-struct ste_mii_frame {
-	uint8_t			mii_stdelim;
-	uint8_t			mii_opcode;
-	uint8_t			mii_phyaddr;
-	uint8_t			mii_regaddr;
-	uint8_t			mii_turnaround;
-	uint16_t		mii_data;
-};
-
-/*
- * MII constants
- */
-#define STE_MII_STARTDELIM	0x01
-#define STE_MII_READOP		0x02
-#define STE_MII_WRITEOP		0x01
-#define STE_MII_TURNAROUND	0x02
diff --git a/sys/dev/stge/if_stge.c b/sys/dev/stge/if_stge.c
index 5058a4d..b18da27 100644
--- a/sys/dev/stge/if_stge.c
+++ b/sys/dev/stge/if_stge.c
@@ -67,6 +67,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -86,11 +87,11 @@ MODULE_DEPEND(stge, miibus, 1, 1, 1);
 /*
  * Devices supported by this driver.
  */
-static struct stge_product {
+static const struct stge_product {
 	uint16_t	stge_vendorid;
 	uint16_t	stge_deviceid;
 	const char	*stge_name;
-} stge_products[] = {
+} const stge_products[] = {
 	{ VENDOR_SUNDANCETI,	DEVICEID_SUNDANCETI_ST1023,
 	  "Sundance ST-1023 Gigabit Ethernet" },
 
@@ -160,10 +161,6 @@ static int	stge_newbuf(struct stge_softc *, int);
 static __inline struct mbuf *stge_fixup_rx(struct stge_softc *, struct mbuf *);
 #endif
 
-static void	stge_mii_sync(struct stge_softc *);
-static void	stge_mii_send(struct stge_softc *, uint32_t, int);
-static int	stge_mii_readreg(struct stge_softc *, struct stge_mii_frame *);
-static int	stge_mii_writereg(struct stge_softc *, struct stge_mii_frame *);
 static int	stge_miibus_readreg(device_t, int, int);
 static int	stge_miibus_writereg(device_t, int, int, int);
 static void	stge_miibus_statchg(device_t);
@@ -185,6 +182,24 @@ static int	sysctl_int_range(SYSCTL_HANDLER_ARGS, int, int);
 static int	sysctl_hw_stge_rxint_nframe(SYSCTL_HANDLER_ARGS);
 static int	sysctl_hw_stge_rxint_dmawait(SYSCTL_HANDLER_ARGS);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t stge_mii_bitbang_read(device_t);
+static void	stge_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops stge_mii_bitbang_ops = {
+	stge_mii_bitbang_read,
+	stge_mii_bitbang_write,
+	{
+		PC_MgmtData,		/* MII_BIT_MDO */
+		PC_MgmtData,		/* MII_BIT_MDI */
+		PC_MgmtClk,		/* MII_BIT_MDC */
+		PC_MgmtDir,		/* MII_BIT_DIR_HOST_PHY */
+		0,			/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static device_method_t stge_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		stge_probe),
@@ -225,176 +240,40 @@ static struct resource_spec stge_res_spec_mem[] = {
 	{ -1,			0,		0 }
 };
 
-#define	MII_SET(x)	\
-	CSR_WRITE_1(sc, STGE_PhyCtrl, CSR_READ_1(sc, STGE_PhyCtrl) | (x))
-#define	MII_CLR(x)	\
-	CSR_WRITE_1(sc, STGE_PhyCtrl, CSR_READ_1(sc, STGE_PhyCtrl) & ~(x))
-
 /*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
+ * stge_mii_bitbang_read: [mii bit-bang interface function]
+ *
+ *	Read the MII serial port for the MII bit-bang module.
  */
-static void
-stge_mii_sync(struct stge_softc	*sc)
+static uint32_t
+stge_mii_bitbang_read(device_t dev)
 {
-	int i;
+	struct stge_softc *sc;
+	uint32_t val;
 
-	MII_SET(PC_MgmtDir | PC_MgmtData);
+	sc = device_get_softc(dev);
 
-	for (i = 0; i < 32; i++) {
-		MII_SET(PC_MgmtClk);
-		DELAY(1);
-		MII_CLR(PC_MgmtClk);
-		DELAY(1);
-	}
+	val = CSR_READ_1(sc, STGE_PhyCtrl);
+	CSR_BARRIER(sc, STGE_PhyCtrl, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
+	return (val);
 }
 
 /*
- * Clock a series of bits through the MII.
+ * stge_mii_bitbang_write: [mii big-bang interface function]
+ *
+ *	Write the MII serial port for the MII bit-bang module.
  */
 static void
-stge_mii_send(struct stge_softc *sc, uint32_t bits, int cnt)
-{
-	int i;
-
-	MII_CLR(PC_MgmtClk);
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		if (bits & i)
-			MII_SET(PC_MgmtData);
-                else
-			MII_CLR(PC_MgmtData);
-		DELAY(1);
-		MII_CLR(PC_MgmtClk);
-		DELAY(1);
-		MII_SET(PC_MgmtClk);
-	}
-}
-
-/*
- * Read an PHY register through the MII.
- */
-static int
-stge_mii_readreg(struct stge_softc *sc, struct stge_mii_frame *frame)
+stge_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	int i, ack;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = STGE_MII_STARTDELIM;
-	frame->mii_opcode = STGE_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	CSR_WRITE_1(sc, STGE_PhyCtrl, 0 | sc->sc_PhyCtrl);
-	/*
- 	 * Turn on data xmit.
-	 */
-	MII_SET(PC_MgmtDir);
-
-	stge_mii_sync(sc);
-
-	/*
-	 * Send command/address info.
-	 */
-	stge_mii_send(sc, frame->mii_stdelim, 2);
-	stge_mii_send(sc, frame->mii_opcode, 2);
-	stge_mii_send(sc, frame->mii_phyaddr, 5);
-	stge_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Turn off xmit. */
-	MII_CLR(PC_MgmtDir);
-
-	/* Idle bit */
-	MII_CLR((PC_MgmtClk | PC_MgmtData));
-	DELAY(1);
-	MII_SET(PC_MgmtClk);
-	DELAY(1);
-
-	/* Check for ack */
-	MII_CLR(PC_MgmtClk);
-	DELAY(1);
-	ack = CSR_READ_1(sc, STGE_PhyCtrl) & PC_MgmtData;
-	MII_SET(PC_MgmtClk);
-	DELAY(1);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for(i = 0; i < 16; i++) {
-			MII_CLR(PC_MgmtClk);
-			DELAY(1);
-			MII_SET(PC_MgmtClk);
-			DELAY(1);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		MII_CLR(PC_MgmtClk);
-		DELAY(1);
-		if (!ack) {
-			if (CSR_READ_1(sc, STGE_PhyCtrl) & PC_MgmtData)
-				frame->mii_data |= i;
-			DELAY(1);
-		}
-		MII_SET(PC_MgmtClk);
-		DELAY(1);
-	}
-
-fail:
-	MII_CLR(PC_MgmtClk);
-	DELAY(1);
-	MII_SET(PC_MgmtClk);
-	DELAY(1);
-
-	if (ack)
-		return(1);
-	return(0);
-}
-
-/*
- * Write to a PHY register through the MII.
- */
-static int
-stge_mii_writereg(struct stge_softc *sc, struct stge_mii_frame *frame)
-{
-
-	/*
-	 * Set up frame for TX.
-	 */
-	frame->mii_stdelim = STGE_MII_STARTDELIM;
-	frame->mii_opcode = STGE_MII_WRITEOP;
-	frame->mii_turnaround = STGE_MII_TURNAROUND;
-
-	/*
- 	 * Turn on data output.
-	 */
-	MII_SET(PC_MgmtDir);
-
-	stge_mii_sync(sc);
-
-	stge_mii_send(sc, frame->mii_stdelim, 2);
-	stge_mii_send(sc, frame->mii_opcode, 2);
-	stge_mii_send(sc, frame->mii_phyaddr, 5);
-	stge_mii_send(sc, frame->mii_regaddr, 5);
-	stge_mii_send(sc, frame->mii_turnaround, 2);
-	stge_mii_send(sc, frame->mii_data, 16);
-
-	/* Idle bit. */
-	MII_SET(PC_MgmtClk);
-	DELAY(1);
-	MII_CLR(PC_MgmtClk);
-	DELAY(1);
+	struct stge_softc *sc;
 
-	/*
-	 * Turn off xmit.
-	 */
-	MII_CLR(PC_MgmtDir);
+	sc = device_get_softc(dev);
 
-	return(0);
+	CSR_WRITE_1(sc, STGE_PhyCtrl, val);
+	CSR_BARRIER(sc, STGE_PhyCtrl, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 /*
@@ -406,8 +285,7 @@ static int
 stge_miibus_readreg(device_t dev, int phy, int reg)
 {
 	struct stge_softc *sc;
-	struct stge_mii_frame frame;
-	int error;
+	int error, val;
 
 	sc = device_get_softc(dev);
 
@@ -418,21 +296,11 @@ stge_miibus_readreg(device_t dev, int phy, int reg)
 		STGE_MII_UNLOCK(sc);
 		return (error);
 	}
-	bzero(&frame, sizeof(frame));
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
 
 	STGE_MII_LOCK(sc);
-	error = stge_mii_readreg(sc, &frame);
+	val = mii_bitbang_readreg(dev, &stge_mii_bitbang_ops, phy, reg);
 	STGE_MII_UNLOCK(sc);
-
-	if (error != 0) {
-		/* Don't show errors for PHY probe request */
-		if (reg != 1)
-			device_printf(sc->sc_dev, "phy read fail\n");
-		return (0);
-	}
-	return (frame.mii_data);
+	return (val);
 }
 
 /*
@@ -444,22 +312,12 @@ static int
 stge_miibus_writereg(device_t dev, int phy, int reg, int val)
 {
 	struct stge_softc *sc;
-	struct stge_mii_frame frame;
-	int error;
 
 	sc = device_get_softc(dev);
 
-	bzero(&frame, sizeof(frame));
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = val;
-
 	STGE_MII_LOCK(sc);
-	error = stge_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &stge_mii_bitbang_ops, phy, reg, val);
 	STGE_MII_UNLOCK(sc);
-
-	if (error != 0)
-		device_printf(sc->sc_dev, "phy write fail\n");
 	return (0);
 }
 
@@ -550,7 +408,7 @@ stge_read_eeprom(struct stge_softc *sc, int offset, uint16_t *data)
 static int
 stge_probe(device_t dev)
 {
-	struct stge_product *sp;
+	const struct stge_product *sp;
 	int i;
 	uint16_t vendor, devid;
 
diff --git a/sys/dev/stge/if_stgereg.h b/sys/dev/stge/if_stgereg.h
index 6ecbdf9..f14142a 100644
--- a/sys/dev/stge/if_stgereg.h
+++ b/sys/dev/stge/if_stgereg.h
@@ -99,6 +99,9 @@
 #define CSR_READ_1(_sc, reg)		\
 	bus_read_1((_sc)->sc_res[0], (reg))
 
+#define	CSR_BARRIER(_sc, reg, length, flags)				\
+	bus_barrier((_sc)->sc_res[0], reg, length, flags)
+
 /*
  * TC9021 buffer fragment descriptor.
  */
@@ -677,23 +680,6 @@ do {									\
 
 #define STGE_TIMEOUT 1000
 
-struct stge_mii_frame {
-	uint8_t	mii_stdelim;
-	uint8_t	mii_opcode;
-	uint8_t	mii_phyaddr;
-	uint8_t	mii_regaddr;
-	uint8_t	mii_turnaround;
-	uint16_t mii_data;
-};
-
-/*
- * MII constants
- */
-#define STGE_MII_STARTDELIM	0x01
-#define STGE_MII_READOP		0x02
-#define STGE_MII_WRITEOP	0x01
-#define STGE_MII_TURNAROUND	0x02
-
 #define	STGE_RESET_NONE	0x00
 #define	STGE_RESET_TX	0x01
 #define	STGE_RESET_RX	0x02
diff --git a/sys/dev/syscons/scterm-teken.c b/sys/dev/syscons/scterm-teken.c
index 725f9f5..6762388 100644
--- a/sys/dev/syscons/scterm-teken.c
+++ b/sys/dev/syscons/scterm-teken.c
@@ -424,10 +424,18 @@ static const struct unicp437 cp437table[] = {
 	{ 0x00b6, 0x14, 0x00 }, { 0x00b7, 0xfa, 0x00 },
 	{ 0x00ba, 0xa7, 0x00 }, { 0x00bb, 0xaf, 0x00 },
 	{ 0x00bc, 0xac, 0x00 }, { 0x00bd, 0xab, 0x00 },
-	{ 0x00bf, 0xa8, 0x00 }, { 0x00c4, 0x8e, 0x01 },
-	{ 0x00c6, 0x92, 0x00 }, { 0x00c7, 0x80, 0x00 },
-	{ 0x00c9, 0x90, 0x00 }, { 0x00d1, 0xa5, 0x00 },
-	{ 0x00d6, 0x99, 0x00 }, { 0x00dc, 0x9a, 0x00 },
+	{ 0x00bf, 0xa8, 0x00 }, { 0x00c0, 0x41, 0x00 },
+	{ 0x00c1, 0x41, 0x00 }, { 0x00c2, 0x41, 0x00 },
+	{ 0x00c4, 0x8e, 0x01 }, { 0x00c6, 0x92, 0x00 }, 
+	{ 0x00c7, 0x80, 0x00 }, { 0x00c8, 0x45, 0x00 },
+	{ 0x00c9, 0x90, 0x00 }, { 0x00ca, 0x45, 0x00 },
+	{ 0x00cb, 0x45, 0x00 }, { 0x00cc, 0x49, 0x00 },
+	{ 0x00cd, 0x49, 0x00 }, { 0x00ce, 0x49, 0x00 },
+	{ 0x00cf, 0x49, 0x00 }, { 0x00d1, 0xa5, 0x00 },
+	{ 0x00d2, 0x4f, 0x00 }, { 0x00d3, 0x4f, 0x00 },
+	{ 0x00d4, 0x4f, 0x00 }, { 0x00d6, 0x99, 0x00 }, 
+	{ 0x00d9, 0x55, 0x00 }, { 0x00da, 0x55, 0x00 },
+	{ 0x00db, 0x55, 0x00 }, { 0x00dc, 0x9a, 0x00 },
 	{ 0x00df, 0xe1, 0x00 }, { 0x00e0, 0x85, 0x00 },
 	{ 0x00e1, 0xa0, 0x00 }, { 0x00e2, 0x83, 0x00 },
 	{ 0x00e4, 0x84, 0x00 }, { 0x00e5, 0x86, 0x00 },
@@ -442,6 +450,7 @@ static const struct unicp437 cp437table[] = {
 	{ 0x00f8, 0xed, 0x00 }, { 0x00f9, 0x97, 0x00 },
 	{ 0x00fa, 0xa3, 0x00 }, { 0x00fb, 0x96, 0x00 },
 	{ 0x00fc, 0x81, 0x00 }, { 0x00ff, 0x98, 0x00 },
+	{ 0x013f, 0x4c, 0x00 }, { 0x0140, 0x6c, 0x00 },
 	{ 0x0192, 0x9f, 0x00 }, { 0x0393, 0xe2, 0x00 },
 	{ 0x0398, 0xe9, 0x00 }, { 0x03a3, 0xe4, 0x00 },
 	{ 0x03a6, 0xe8, 0x00 }, { 0x03a9, 0xea, 0x00 },
@@ -490,7 +499,8 @@ static const struct unicp437 cp437table[] = {
 	{ 0x2584, 0xdc, 0x00 }, { 0x2588, 0xdb, 0x00 },
 	{ 0x258c, 0xdd, 0x00 }, { 0x2590, 0xde, 0x00 },
 	{ 0x2591, 0xb0, 0x02 }, { 0x25a0, 0xfe, 0x00 },
-	{ 0x25ac, 0x16, 0x00 }, { 0x25b2, 0x1e, 0x00 },
+	{ 0x25ac, 0x16, 0x00 }, 
+	{ 0x25ae, 0xdb, 0x00 }, { 0x25b2, 0x1e, 0x00 },
 	{ 0x25ba, 0x10, 0x00 }, { 0x25bc, 0x1f, 0x00 },
 	{ 0x25c4, 0x11, 0x00 }, { 0x25cb, 0x09, 0x00 },
 	{ 0x25d8, 0x08, 0x00 }, { 0x25d9, 0x0a, 0x00 },
diff --git a/sys/dev/syscons/syscons.c b/sys/dev/syscons/syscons.c
index 227350e..3928d91 100644
--- a/sys/dev/syscons/syscons.c
+++ b/sys/dev/syscons/syscons.c
@@ -136,8 +136,8 @@ static	int		sc_no_suspend_vtswitch = 0;
 #endif
 static	int		sc_susp_scr;
 
-SYSCTL_NODE(_hw, OID_AUTO, syscons, CTLFLAG_RD, 0, "syscons");
-SYSCTL_NODE(_hw_syscons, OID_AUTO, saver, CTLFLAG_RD, 0, "saver");
+static SYSCTL_NODE(_hw, OID_AUTO, syscons, CTLFLAG_RD, 0, "syscons");
+static SYSCTL_NODE(_hw_syscons, OID_AUTO, saver, CTLFLAG_RD, 0, "saver");
 SYSCTL_INT(_hw_syscons_saver, OID_AUTO, keybonly, CTLFLAG_RW,
     &sc_saver_keyb_only, 0, "screen saver interrupted by input only");
 SYSCTL_INT(_hw_syscons, OID_AUTO, bell, CTLFLAG_RW, &enable_bell, 
diff --git a/sys/dev/tdfx/tdfx_pci.c b/sys/dev/tdfx/tdfx_pci.c
index 27308a9..0992e5f 100644
--- a/sys/dev/tdfx/tdfx_pci.c
+++ b/sys/dev/tdfx/tdfx_pci.c
@@ -89,7 +89,7 @@ static device_method_t tdfx_methods[] = {
 	{ 0, 0 }
 };
 
-MALLOC_DEFINE(M_TDFX,"tdfx_driver","3DFX Graphics[/2D]/3D Accelerator(s)");
+static MALLOC_DEFINE(M_TDFX,"tdfx_driver","3DFX Graphics[/2D]/3D Accelerators");
 
 /* Char. Dev. file operations structure */
 static struct cdevsw tdfx_cdev = {
diff --git a/sys/dev/ti/if_ti.c b/sys/dev/ti/if_ti.c
index 8468565..d57be53 100644
--- a/sys/dev/ti/if_ti.c
+++ b/sys/dev/ti/if_ti.c
@@ -112,8 +112,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/rman.h>
 
-/* #define TI_PRIVATE_JUMBOS */
-#ifndef TI_PRIVATE_JUMBOS
+#ifdef TI_SF_BUF_JUMBO
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 #endif
@@ -126,21 +125,22 @@ __FBSDID("$FreeBSD$");
 #include <dev/ti/ti_fw.h>
 #include <dev/ti/ti_fw2.h>
 
+#include <sys/sysctl.h>
+
 #define TI_CSUM_FEATURES	(CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_IP_FRAGS)
 /*
  * We can only turn on header splitting if we're using extended receive
  * BDs.
  */
-#if defined(TI_JUMBO_HDRSPLIT) && defined(TI_PRIVATE_JUMBOS)
-#error "options TI_JUMBO_HDRSPLIT and TI_PRIVATE_JUMBOS are mutually exclusive"
-#endif /* TI_JUMBO_HDRSPLIT && TI_JUMBO_HDRSPLIT */
+#if defined(TI_JUMBO_HDRSPLIT) && !defined(TI_SF_BUF_JUMBO)
+#error "options TI_JUMBO_HDRSPLIT requires TI_SF_BUF_JUMBO"
+#endif /* TI_JUMBO_HDRSPLIT && !TI_SF_BUF_JUMBO */
 
 typedef enum {
 	TI_SWAP_HTON,
 	TI_SWAP_NTOH
 } ti_swap_type;
 
-
 /*
  * Various supported device vendors/types and their names.
  */
@@ -197,36 +197,38 @@ static void ti_stop(struct ti_softc *);
 static void ti_watchdog(void *);
 static int ti_shutdown(device_t);
 static int ti_ifmedia_upd(struct ifnet *);
+static int ti_ifmedia_upd_locked(struct ti_softc *);
 static void ti_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 
-static u_int32_t ti_eeprom_putbyte(struct ti_softc *, int);
-static u_int8_t	ti_eeprom_getbyte(struct ti_softc *, int, u_int8_t *);
+static uint32_t ti_eeprom_putbyte(struct ti_softc *, int);
+static uint8_t	ti_eeprom_getbyte(struct ti_softc *, int, uint8_t *);
 static int ti_read_eeprom(struct ti_softc *, caddr_t, int, int);
 
 static void ti_add_mcast(struct ti_softc *, struct ether_addr *);
 static void ti_del_mcast(struct ti_softc *, struct ether_addr *);
 static void ti_setmulti(struct ti_softc *);
 
-static void ti_mem_read(struct ti_softc *, u_int32_t, u_int32_t, void *);
-static void ti_mem_write(struct ti_softc *, u_int32_t, u_int32_t, void *);
-static void ti_mem_zero(struct ti_softc *, u_int32_t, u_int32_t);
-static int ti_copy_mem(struct ti_softc *, u_int32_t, u_int32_t, caddr_t, int, int);
-static int ti_copy_scratch(struct ti_softc *, u_int32_t, u_int32_t, caddr_t,
-		int, int, int);
+static void ti_mem_read(struct ti_softc *, uint32_t, uint32_t, void *);
+static void ti_mem_write(struct ti_softc *, uint32_t, uint32_t, void *);
+static void ti_mem_zero(struct ti_softc *, uint32_t, uint32_t);
+static int ti_copy_mem(struct ti_softc *, uint32_t, uint32_t, caddr_t, int,
+    int);
+static int ti_copy_scratch(struct ti_softc *, uint32_t, uint32_t, caddr_t,
+    int, int, int);
 static int ti_bcopy_swap(const void *, void *, size_t, ti_swap_type);
 static void ti_loadfw(struct ti_softc *);
 static void ti_cmd(struct ti_softc *, struct ti_cmd_desc *);
 static void ti_cmd_ext(struct ti_softc *, struct ti_cmd_desc *, caddr_t, int);
 static void ti_handle_events(struct ti_softc *);
-static int ti_alloc_dmamaps(struct ti_softc *);
-static void ti_free_dmamaps(struct ti_softc *);
-static int ti_alloc_jumbo_mem(struct ti_softc *);
-#ifdef TI_PRIVATE_JUMBOS
-static void *ti_jalloc(struct ti_softc *);
-static void ti_jfree(void *, void *);
-#endif /* TI_PRIVATE_JUMBOS */
-static int ti_newbuf_std(struct ti_softc *, int, struct mbuf *);
-static int ti_newbuf_mini(struct ti_softc *, int, struct mbuf *);
+static void ti_dma_map_addr(void *, bus_dma_segment_t *, int, int);
+static int ti_dma_alloc(struct ti_softc *);
+static void ti_dma_free(struct ti_softc *);
+static int ti_dma_ring_alloc(struct ti_softc *, bus_size_t, bus_size_t,
+    bus_dma_tag_t *, uint8_t **, bus_dmamap_t *, bus_addr_t *, const char *);
+static void ti_dma_ring_free(struct ti_softc *, bus_dma_tag_t *, uint8_t **,
+    bus_dmamap_t *);
+static int ti_newbuf_std(struct ti_softc *, int);
+static int ti_newbuf_mini(struct ti_softc *, int);
 static int ti_newbuf_jumbo(struct ti_softc *, int, struct mbuf *);
 static int ti_init_rx_ring_std(struct ti_softc *);
 static void ti_free_rx_ring_std(struct ti_softc *);
@@ -236,16 +238,23 @@ static int ti_init_rx_ring_mini(struct ti_softc *);
 static void ti_free_rx_ring_mini(struct ti_softc *);
 static void ti_free_tx_ring(struct ti_softc *);
 static int ti_init_tx_ring(struct ti_softc *);
+static void ti_discard_std(struct ti_softc *, int);
+#ifndef TI_SF_BUF_JUMBO
+static void ti_discard_jumbo(struct ti_softc *, int);
+#endif
+static void ti_discard_mini(struct ti_softc *, int);
 
 static int ti_64bitslot_war(struct ti_softc *);
 static int ti_chipinit(struct ti_softc *);
 static int ti_gibinit(struct ti_softc *);
 
 #ifdef TI_JUMBO_HDRSPLIT
-static __inline void ti_hdr_split	(struct mbuf *top, int hdr_len,
-					     int pkt_len, int idx);
+static __inline void ti_hdr_split(struct mbuf *top, int hdr_len, int pkt_len,
+    int idx);
 #endif /* TI_JUMBO_HDRSPLIT */
 
+static void ti_sysctl_node(struct ti_softc *);
+
 static device_method_t ti_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		ti_probe),
@@ -270,11 +279,10 @@ MODULE_DEPEND(ti, ether, 1, 1, 1);
 /*
  * Send an instruction or address to the EEPROM, check for ACK.
  */
-static u_int32_t ti_eeprom_putbyte(sc, byte)
-	struct ti_softc		*sc;
-	int			byte;
+static uint32_t
+ti_eeprom_putbyte(struct ti_softc *sc, int byte)
 {
-	int			i, ack = 0;
+	int i, ack = 0;
 
 	/*
 	 * Make sure we're in TX mode.
@@ -316,13 +324,11 @@ static u_int32_t ti_eeprom_putbyte(sc, byte)
  * We have to send two address bytes since the EEPROM can hold
  * more than 256 bytes of data.
  */
-static u_int8_t ti_eeprom_getbyte(sc, addr, dest)
-	struct ti_softc		*sc;
-	int			addr;
-	u_int8_t		*dest;
+static uint8_t
+ti_eeprom_getbyte(struct ti_softc *sc, int addr, uint8_t *dest)
 {
-	int			i;
-	u_int8_t		byte = 0;
+	int i;
+	uint8_t byte = 0;
 
 	EEPROM_START;
 
@@ -393,14 +399,10 @@ static u_int8_t ti_eeprom_getbyte(sc, addr, dest)
  * Read a sequence of bytes from the EEPROM.
  */
 static int
-ti_read_eeprom(sc, dest, off, cnt)
-	struct ti_softc		*sc;
-	caddr_t			dest;
-	int			off;
-	int			cnt;
+ti_read_eeprom(struct ti_softc *sc, caddr_t dest, int off, int cnt)
 {
-	int			err = 0, i;
-	u_int8_t		byte = 0;
+	int err = 0, i;
+	uint8_t byte = 0;
 
 	for (i = 0; i < cnt; i++) {
 		err = ti_eeprom_getbyte(sc, off + i, &byte);
@@ -417,13 +419,10 @@ ti_read_eeprom(sc, dest, off, cnt)
  * Can be used to copy data from NIC local memory.
  */
 static void
-ti_mem_read(sc, addr, len, buf)
-	struct ti_softc		*sc;
-	u_int32_t		addr, len;
-	void			*buf;
+ti_mem_read(struct ti_softc *sc, uint32_t addr, uint32_t len, void *buf)
 {
-	int			segptr, segsize, cnt;
-	char			*ptr;
+	int segptr, segsize, cnt;
+	char *ptr;
 
 	segptr = addr;
 	cnt = len;
@@ -436,7 +435,7 @@ ti_mem_read(sc, addr, len, buf)
 			segsize = TI_WINLEN - (segptr % TI_WINLEN);
 		CSR_WRITE_4(sc, TI_WINBASE, (segptr & ~(TI_WINLEN - 1)));
 		bus_space_read_region_4(sc->ti_btag, sc->ti_bhandle,
-		    TI_WINDOW + (segptr & (TI_WINLEN - 1)), (u_int32_t *)ptr,
+		    TI_WINDOW + (segptr & (TI_WINLEN - 1)), (uint32_t *)ptr,
 		    segsize / 4);
 		ptr += segsize;
 		segptr += segsize;
@@ -450,13 +449,10 @@ ti_mem_read(sc, addr, len, buf)
  * Can be used to copy data into NIC local memory.
  */
 static void
-ti_mem_write(sc, addr, len, buf)
-	struct ti_softc		*sc;
-	u_int32_t		addr, len;
-	void			*buf;
+ti_mem_write(struct ti_softc *sc, uint32_t addr, uint32_t len, void *buf)
 {
-	int			segptr, segsize, cnt;
-	char			*ptr;
+	int segptr, segsize, cnt;
+	char *ptr;
 
 	segptr = addr;
 	cnt = len;
@@ -469,7 +465,7 @@ ti_mem_write(sc, addr, len, buf)
 			segsize = TI_WINLEN - (segptr % TI_WINLEN);
 		CSR_WRITE_4(sc, TI_WINBASE, (segptr & ~(TI_WINLEN - 1)));
 		bus_space_write_region_4(sc->ti_btag, sc->ti_bhandle,
-		    TI_WINDOW + (segptr & (TI_WINLEN - 1)), (u_int32_t *)ptr,
+		    TI_WINDOW + (segptr & (TI_WINLEN - 1)), (uint32_t *)ptr,
 		    segsize / 4);
 		ptr += segsize;
 		segptr += segsize;
@@ -482,11 +478,9 @@ ti_mem_write(sc, addr, len, buf)
  * Can be used to clear a section of NIC local memory.
  */
 static void
-ti_mem_zero(sc, addr, len)
-	struct ti_softc		*sc;
-	u_int32_t		addr, len;
+ti_mem_zero(struct ti_softc *sc, uint32_t addr, uint32_t len)
 {
-	int			segptr, segsize, cnt;
+	int segptr, segsize, cnt;
 
 	segptr = addr;
 	cnt = len;
@@ -505,18 +499,14 @@ ti_mem_zero(sc, addr, len)
 }
 
 static int
-ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
-	struct ti_softc		*sc;
-	u_int32_t		tigon_addr, len;
-	caddr_t			buf;
-	int			useraddr, readdata;
-{
-	int		segptr, segsize, cnt;
-	caddr_t		ptr;
-	u_int32_t	origwin;
-	u_int8_t	tmparray[TI_WINLEN], tmparray2[TI_WINLEN];
-	int		resid, segresid;
-	int		first_pass;
+ti_copy_mem(struct ti_softc *sc, uint32_t tigon_addr, uint32_t len,
+    caddr_t buf, int useraddr, int readdata)
+{
+	int segptr, segsize, cnt;
+	caddr_t ptr;
+	uint32_t origwin;
+	int resid, segresid;
+	int first_pass;
 
 	TI_LOCK_ASSERT(sc);
 
@@ -524,8 +514,7 @@ ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
 	 * At the moment, we don't handle non-aligned cases, we just bail.
 	 * If this proves to be a problem, it will be fixed.
 	 */
-	if ((readdata == 0)
-	 && (tigon_addr & 0x3)) {
+	if (readdata == 0 && (tigon_addr & 0x3) != 0) {
 		device_printf(sc->ti_dev, "%s: tigon address %#x isn't "
 		    "word-aligned\n", __func__, tigon_addr);
 		device_printf(sc->ti_dev, "%s: unaligned writes aren't "
@@ -574,58 +563,54 @@ ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
 		ti_offset = TI_WINDOW + (segptr & (TI_WINLEN -1));
 
 		if (readdata) {
-
-			bus_space_read_region_4(sc->ti_btag,
-						sc->ti_bhandle, ti_offset,
-						(u_int32_t *)tmparray,
-						segsize >> 2);
+			bus_space_read_region_4(sc->ti_btag, sc->ti_bhandle,
+			    ti_offset, (uint32_t *)sc->ti_membuf, segsize >> 2);
 			if (useraddr) {
 				/*
 				 * Yeah, this is a little on the kludgy
 				 * side, but at least this code is only
 				 * used for debugging.
 				 */
-				ti_bcopy_swap(tmparray, tmparray2, segsize,
-					      TI_SWAP_NTOH);
+				ti_bcopy_swap(sc->ti_membuf, sc->ti_membuf2,
+				    segsize, TI_SWAP_NTOH);
 
 				TI_UNLOCK(sc);
 				if (first_pass) {
-					copyout(&tmparray2[segresid], ptr,
-						segsize - segresid);
+					copyout(&sc->ti_membuf2[segresid], ptr,
+					    segsize - segresid);
 					first_pass = 0;
 				} else
-					copyout(tmparray2, ptr, segsize);
+					copyout(sc->ti_membuf2, ptr, segsize);
 				TI_LOCK(sc);
 			} else {
 				if (first_pass) {
 
-					ti_bcopy_swap(tmparray, tmparray2,
-						      segsize, TI_SWAP_NTOH);
+					ti_bcopy_swap(sc->ti_membuf,
+					    sc->ti_membuf2, segsize,
+					    TI_SWAP_NTOH);
 					TI_UNLOCK(sc);
-					bcopy(&tmparray2[segresid], ptr,
-					      segsize - segresid);
+					bcopy(&sc->ti_membuf2[segresid], ptr,
+					    segsize - segresid);
 					TI_LOCK(sc);
 					first_pass = 0;
 				} else
-					ti_bcopy_swap(tmparray, ptr, segsize,
-						      TI_SWAP_NTOH);
+					ti_bcopy_swap(sc->ti_membuf, ptr,
+					    segsize, TI_SWAP_NTOH);
 			}
 
 		} else {
 			if (useraddr) {
 				TI_UNLOCK(sc);
-				copyin(ptr, tmparray2, segsize);
+				copyin(ptr, sc->ti_membuf2, segsize);
 				TI_LOCK(sc);
-				ti_bcopy_swap(tmparray2, tmparray, segsize,
-					      TI_SWAP_HTON);
+				ti_bcopy_swap(sc->ti_membuf2, sc->ti_membuf,
+				    segsize, TI_SWAP_HTON);
 			} else
-				ti_bcopy_swap(ptr, tmparray, segsize,
-					      TI_SWAP_HTON);
+				ti_bcopy_swap(ptr, sc->ti_membuf, segsize,
+				    TI_SWAP_HTON);
 
-			bus_space_write_region_4(sc->ti_btag,
-						 sc->ti_bhandle, ti_offset,
-						 (u_int32_t *)tmparray,
-						 segsize >> 2);
+			bus_space_write_region_4(sc->ti_btag, sc->ti_bhandle,
+			    ti_offset, (uint32_t *)sc->ti_membuf, segsize >> 2);
 		}
 		segptr += segsize;
 		ptr += segsize;
@@ -636,8 +621,8 @@ ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
 	 * Handle leftover, non-word-aligned bytes.
 	 */
 	if (resid != 0) {
-		u_int32_t	tmpval, tmpval2;
-		bus_size_t	ti_offset;
+		uint32_t tmpval, tmpval2;
+		bus_size_t ti_offset;
 
 		/*
 		 * Set the segment pointer.
@@ -652,7 +637,7 @@ ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
 		 * writes, since we'll be doing read/modify/write.
 		 */
 		bus_space_read_region_4(sc->ti_btag, sc->ti_bhandle,
-					ti_offset, &tmpval, 1);
+		    ti_offset, &tmpval, 1);
 
 		/*
 		 * Next, translate this from little-endian to big-endian
@@ -694,7 +679,7 @@ ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
 			tmpval = htonl(tmpval2);
 
 			bus_space_write_region_4(sc->ti_btag, sc->ti_bhandle,
-						 ti_offset, &tmpval, 1);
+			    ti_offset, &tmpval, 1);
 		}
 	}
 
@@ -704,17 +689,13 @@ ti_copy_mem(sc, tigon_addr, len, buf, useraddr, readdata)
 }
 
 static int
-ti_copy_scratch(sc, tigon_addr, len, buf, useraddr, readdata, cpu)
-	struct ti_softc		*sc;
-	u_int32_t		tigon_addr, len;
-	caddr_t			buf;
-	int			useraddr, readdata;
-	int			cpu;
-{
-	u_int32_t	segptr;
-	int		cnt;
-	u_int32_t	tmpval, tmpval2;
-	caddr_t		ptr;
+ti_copy_scratch(struct ti_softc *sc, uint32_t tigon_addr, uint32_t len,
+    caddr_t buf, int useraddr, int readdata, int cpu)
+{
+	uint32_t segptr;
+	int cnt;
+	uint32_t tmpval, tmpval2;
+	caddr_t ptr;
 
 	TI_LOCK_ASSERT(sc);
 
@@ -802,19 +783,14 @@ ti_copy_scratch(sc, tigon_addr, len, buf, useraddr, readdata, cpu)
 }
 
 static int
-ti_bcopy_swap(src, dst, len, swap_type)
-	const void	*src;
-	void		*dst;
-	size_t		len;
-	ti_swap_type	swap_type;
-{
-	const u_int8_t *tmpsrc;
-	u_int8_t *tmpdst;
+ti_bcopy_swap(const void *src, void *dst, size_t len, ti_swap_type swap_type)
+{
+	const uint8_t *tmpsrc;
+	uint8_t *tmpdst;
 	size_t tmplen;
 
 	if (len & 0x3) {
-		printf("ti_bcopy_swap: length %zd isn't 32-bit aligned\n",
-		       len);
+		printf("ti_bcopy_swap: length %zd isn't 32-bit aligned\n", len);
 		return (-1);
 	}
 
@@ -824,12 +800,9 @@ ti_bcopy_swap(src, dst, len, swap_type)
 
 	while (tmplen) {
 		if (swap_type == TI_SWAP_NTOH)
-			*(u_int32_t *)tmpdst =
-				ntohl(*(const u_int32_t *)tmpsrc);
+			*(uint32_t *)tmpdst = ntohl(*(const uint32_t *)tmpsrc);
 		else
-			*(u_int32_t *)tmpdst =
-				htonl(*(const u_int32_t *)tmpsrc);
-
+			*(uint32_t *)tmpdst = htonl(*(const uint32_t *)tmpsrc);
 		tmpsrc += 4;
 		tmpdst += 4;
 		tmplen -= 4;
@@ -844,8 +817,7 @@ ti_bcopy_swap(src, dst, len, swap_type)
  * Tigon 2.
  */
 static void
-ti_loadfw(sc)
-	struct ti_softc		*sc;
+ti_loadfw(struct ti_softc *sc)
 {
 
 	TI_LOCK_ASSERT(sc);
@@ -902,14 +874,12 @@ ti_loadfw(sc)
  * Send the NIC a command via the command ring.
  */
 static void
-ti_cmd(sc, cmd)
-	struct ti_softc		*sc;
-	struct ti_cmd_desc	*cmd;
+ti_cmd(struct ti_softc *sc, struct ti_cmd_desc *cmd)
 {
-	int			index;
+	int index;
 
 	index = sc->ti_cmd_saved_prodidx;
-	CSR_WRITE_4(sc, TI_GCR_CMDRING + (index * 4), *(u_int32_t *)(cmd));
+	CSR_WRITE_4(sc, TI_GCR_CMDRING + (index * 4), *(uint32_t *)(cmd));
 	TI_INC(index, TI_CMD_RING_CNT);
 	CSR_WRITE_4(sc, TI_MB_CMDPROD_IDX, index);
 	sc->ti_cmd_saved_prodidx = index;
@@ -920,21 +890,17 @@ ti_cmd(sc, cmd)
  * number of command slots to include after the initial command.
  */
 static void
-ti_cmd_ext(sc, cmd, arg, len)
-	struct ti_softc		*sc;
-	struct ti_cmd_desc	*cmd;
-	caddr_t			arg;
-	int			len;
+ti_cmd_ext(struct ti_softc *sc, struct ti_cmd_desc *cmd, caddr_t arg, int len)
 {
-	int			index;
-	int			i;
+	int index;
+	int i;
 
 	index = sc->ti_cmd_saved_prodidx;
-	CSR_WRITE_4(sc, TI_GCR_CMDRING + (index * 4), *(u_int32_t *)(cmd));
+	CSR_WRITE_4(sc, TI_GCR_CMDRING + (index * 4), *(uint32_t *)(cmd));
 	TI_INC(index, TI_CMD_RING_CNT);
 	for (i = 0; i < len; i++) {
 		CSR_WRITE_4(sc, TI_GCR_CMDRING + (index * 4),
-		    *(u_int32_t *)(&arg[i * 4]));
+		    *(uint32_t *)(&arg[i * 4]));
 		TI_INC(index, TI_CMD_RING_CNT);
 	}
 	CSR_WRITE_4(sc, TI_MB_CMDPROD_IDX, index);
@@ -945,25 +911,40 @@ ti_cmd_ext(sc, cmd, arg, len)
  * Handle events that have triggered interrupts.
  */
 static void
-ti_handle_events(sc)
-	struct ti_softc		*sc;
+ti_handle_events(struct ti_softc *sc)
 {
-	struct ti_event_desc	*e;
+	struct ti_event_desc *e;
 
-	if (sc->ti_rdata->ti_event_ring == NULL)
+	if (sc->ti_rdata.ti_event_ring == NULL)
 		return;
 
+	bus_dmamap_sync(sc->ti_cdata.ti_event_ring_tag,
+	    sc->ti_cdata.ti_event_ring_map, BUS_DMASYNC_POSTREAD);
 	while (sc->ti_ev_saved_considx != sc->ti_ev_prodidx.ti_idx) {
-		e = &sc->ti_rdata->ti_event_ring[sc->ti_ev_saved_considx];
+		e = &sc->ti_rdata.ti_event_ring[sc->ti_ev_saved_considx];
 		switch (TI_EVENT_EVENT(e)) {
 		case TI_EV_LINKSTAT_CHANGED:
 			sc->ti_linkstat = TI_EVENT_CODE(e);
-			if (sc->ti_linkstat == TI_EV_CODE_LINK_UP)
-				device_printf(sc->ti_dev, "10/100 link up\n");
-			else if (sc->ti_linkstat == TI_EV_CODE_GIG_LINK_UP)
-				device_printf(sc->ti_dev, "gigabit link up\n");
-			else if (sc->ti_linkstat == TI_EV_CODE_LINK_DOWN)
-				device_printf(sc->ti_dev, "link down\n");
+			if (sc->ti_linkstat == TI_EV_CODE_LINK_UP) {
+				if_link_state_change(sc->ti_ifp, LINK_STATE_UP);
+				sc->ti_ifp->if_baudrate = IF_Mbps(100);
+				if (bootverbose)
+					device_printf(sc->ti_dev,
+					    "10/100 link up\n");
+			} else if (sc->ti_linkstat == TI_EV_CODE_GIG_LINK_UP) {
+				if_link_state_change(sc->ti_ifp, LINK_STATE_UP);
+				sc->ti_ifp->if_baudrate = IF_Gbps(1UL);
+				if (bootverbose)
+					device_printf(sc->ti_dev,
+					    "gigabit link up\n");
+			} else if (sc->ti_linkstat == TI_EV_CODE_LINK_DOWN) {
+				if_link_state_change(sc->ti_ifp,
+				    LINK_STATE_DOWN);
+				sc->ti_ifp->if_baudrate = 0;
+				if (bootverbose)
+					device_printf(sc->ti_dev,
+					    "link down\n");
+			}
 			break;
 		case TI_EV_ERROR:
 			if (TI_EVENT_CODE(e) == TI_EV_CODE_ERR_INVAL_CMD)
@@ -992,298 +973,453 @@ ti_handle_events(sc)
 		TI_INC(sc->ti_ev_saved_considx, TI_EVENT_RING_CNT);
 		CSR_WRITE_4(sc, TI_GCR_EVENTCONS_IDX, sc->ti_ev_saved_considx);
 	}
+	bus_dmamap_sync(sc->ti_cdata.ti_event_ring_tag,
+	    sc->ti_cdata.ti_event_ring_map, BUS_DMASYNC_PREREAD);
+}
+
+struct ti_dmamap_arg {
+	bus_addr_t	ti_busaddr;
+};
+
+static void
+ti_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	struct ti_dmamap_arg *ctx;
+
+	if (error)
+		return;
+
+	KASSERT(nseg == 1, ("%s: %d segments returned!", __func__, nseg));
+
+	ctx = arg;
+	ctx->ti_busaddr = segs->ds_addr;
 }
 
 static int
-ti_alloc_dmamaps(struct ti_softc *sc)
+ti_dma_ring_alloc(struct ti_softc *sc, bus_size_t alignment, bus_size_t maxsize,
+    bus_dma_tag_t *tag, uint8_t **ring, bus_dmamap_t *map, bus_addr_t *paddr,
+    const char *msg)
 {
-	int i;
+	struct ti_dmamap_arg ctx;
+	int error;
 
-	for (i = 0; i < TI_TX_RING_CNT; i++) {
-		sc->ti_cdata.ti_txdesc[i].tx_m = NULL;
-		sc->ti_cdata.ti_txdesc[i].tx_dmamap = 0;
-		if (bus_dmamap_create(sc->ti_mbuftx_dmat, 0,
-				      &sc->ti_cdata.ti_txdesc[i].tx_dmamap))
-			return (ENOBUFS);
-	}
-	for (i = 0; i < TI_STD_RX_RING_CNT; i++) {
-		if (bus_dmamap_create(sc->ti_mbufrx_dmat, 0,
-				      &sc->ti_cdata.ti_rx_std_maps[i]))
-			return (ENOBUFS);
+	error = bus_dma_tag_create(sc->ti_cdata.ti_parent_tag,
+	    alignment, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL,
+	    NULL, maxsize, 1, maxsize, 0, NULL, NULL, tag);
+	if (error != 0) {
+		device_printf(sc->ti_dev,
+		    "could not create %s dma tag\n", msg);
+		return (error);
 	}
-
-	for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++) {
-		if (bus_dmamap_create(sc->ti_jumbo_dmat, 0,
-				      &sc->ti_cdata.ti_rx_jumbo_maps[i]))
-			return (ENOBUFS);
+	/* Allocate DMA'able memory for ring. */
+	error = bus_dmamem_alloc(*tag, (void **)ring,
+	    BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, map);
+	if (error != 0) {
+		device_printf(sc->ti_dev,
+		    "could not allocate DMA'able memory for %s\n", msg);
+		return (error);
 	}
-	for (i = 0; i < TI_MINI_RX_RING_CNT; i++) {
-		if (bus_dmamap_create(sc->ti_mbufrx_dmat, 0,
-				      &sc->ti_cdata.ti_rx_mini_maps[i]))
-			return (ENOBUFS);
+	/* Load the address of the ring. */
+	ctx.ti_busaddr = 0;
+	error = bus_dmamap_load(*tag, *map, *ring, maxsize, ti_dma_map_addr,
+	    &ctx, BUS_DMA_NOWAIT);
+	if (error != 0) {
+		device_printf(sc->ti_dev,
+		    "could not load DMA'able memory for %s\n", msg);
+		return (error);
 	}
-
+	*paddr = ctx.ti_busaddr;
 	return (0);
 }
 
 static void
-ti_free_dmamaps(struct ti_softc *sc)
+ti_dma_ring_free(struct ti_softc *sc, bus_dma_tag_t *tag, uint8_t **ring,
+    bus_dmamap_t *map)
 {
-	int i;
 
-	if (sc->ti_mbuftx_dmat)
-		for (i = 0; i < TI_TX_RING_CNT; i++)
-			if (sc->ti_cdata.ti_txdesc[i].tx_dmamap) {
-				bus_dmamap_destroy(sc->ti_mbuftx_dmat,
-				    sc->ti_cdata.ti_txdesc[i].tx_dmamap);
-				sc->ti_cdata.ti_txdesc[i].tx_dmamap = 0;
-			}
+	if (*map != NULL)
+		bus_dmamap_unload(*tag, *map);
+	if (*map != NULL && *ring != NULL) {
+		bus_dmamem_free(*tag, *ring, *map);
+		*ring = NULL;
+		*map = NULL;
+	}
+	if (*tag) {
+		bus_dma_tag_destroy(*tag);
+		*tag = NULL;
+	}
+}
 
-	if (sc->ti_mbufrx_dmat)
-		for (i = 0; i < TI_STD_RX_RING_CNT; i++)
-			if (sc->ti_cdata.ti_rx_std_maps[i]) {
-				bus_dmamap_destroy(sc->ti_mbufrx_dmat,
-				    sc->ti_cdata.ti_rx_std_maps[i]);
-				sc->ti_cdata.ti_rx_std_maps[i] = 0;
-			}
+static int
+ti_dma_alloc(struct ti_softc *sc)
+{
+	bus_addr_t lowaddr;
+	int i, error;
+
+	lowaddr = BUS_SPACE_MAXADDR;
+	if (sc->ti_dac == 0)
+		lowaddr = BUS_SPACE_MAXADDR_32BIT;
+
+	error = bus_dma_tag_create(bus_get_dma_tag(sc->ti_dev), 1, 0, lowaddr,
+	    BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE_32BIT, 0,
+	    BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
+	    &sc->ti_cdata.ti_parent_tag);
+	if (error != 0) {
+		device_printf(sc->ti_dev,
+		    "could not allocate parent dma tag\n");
+		return (ENOMEM);
+	}
 
-	if (sc->ti_jumbo_dmat)
-		for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++)
-			if (sc->ti_cdata.ti_rx_jumbo_maps[i]) {
-				bus_dmamap_destroy(sc->ti_jumbo_dmat,
-				    sc->ti_cdata.ti_rx_jumbo_maps[i]);
-				sc->ti_cdata.ti_rx_jumbo_maps[i] = 0;
-			}
-	if (sc->ti_mbufrx_dmat)
-		for (i = 0; i < TI_MINI_RX_RING_CNT; i++)
-			if (sc->ti_cdata.ti_rx_mini_maps[i]) {
-				bus_dmamap_destroy(sc->ti_mbufrx_dmat,
-				    sc->ti_cdata.ti_rx_mini_maps[i]);
-				sc->ti_cdata.ti_rx_mini_maps[i] = 0;
-			}
-}
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, sizeof(struct ti_gib),
+	    &sc->ti_cdata.ti_gib_tag, (uint8_t **)&sc->ti_rdata.ti_info,
+	    &sc->ti_cdata.ti_gib_map, &sc->ti_rdata.ti_info_paddr, "GIB");
+	if (error)
+		return (error);
 
-#ifdef TI_PRIVATE_JUMBOS
+	/* Producer/consumer status */
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, sizeof(struct ti_status),
+	    &sc->ti_cdata.ti_status_tag, (uint8_t **)&sc->ti_rdata.ti_status,
+	    &sc->ti_cdata.ti_status_map, &sc->ti_rdata.ti_status_paddr,
+	    "event ring");
+	if (error)
+		return (error);
 
-/*
- * Memory management for the jumbo receive ring is a pain in the
- * butt. We need to allocate at least 9018 bytes of space per frame,
- * _and_ it has to be contiguous (unless you use the extended
- * jumbo descriptor format). Using malloc() all the time won't
- * work: malloc() allocates memory in powers of two, which means we
- * would end up wasting a considerable amount of space by allocating
- * 9K chunks. We don't have a jumbo mbuf cluster pool. Thus, we have
- * to do our own memory management.
- *
- * The driver needs to allocate a contiguous chunk of memory at boot
- * time. We then chop this up ourselves into 9K pieces and use them
- * as external mbuf storage.
- *
- * One issue here is how much memory to allocate. The jumbo ring has
- * 256 slots in it, but at 9K per slot than can consume over 2MB of
- * RAM. This is a bit much, especially considering we also need
- * RAM for the standard ring and mini ring (on the Tigon 2). To
- * save space, we only actually allocate enough memory for 64 slots
- * by default, which works out to between 500 and 600K. This can
- * be tuned by changing a #define in if_tireg.h.
- */
+	/* Event ring */
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, TI_EVENT_RING_SZ,
+	    &sc->ti_cdata.ti_event_ring_tag,
+	    (uint8_t **)&sc->ti_rdata.ti_event_ring,
+	    &sc->ti_cdata.ti_event_ring_map, &sc->ti_rdata.ti_event_ring_paddr,
+	    "event ring");
+	if (error)
+		return (error);
 
-static int
-ti_alloc_jumbo_mem(sc)
-	struct ti_softc		*sc;
-{
-	caddr_t			ptr;
-	int			i;
-	struct ti_jpool_entry   *entry;
+	/* Command ring lives in shared memory so no need to create DMA area. */
 
+	/* Standard RX ring */
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, TI_STD_RX_RING_SZ,
+	    &sc->ti_cdata.ti_rx_std_ring_tag,
+	    (uint8_t **)&sc->ti_rdata.ti_rx_std_ring,
+	    &sc->ti_cdata.ti_rx_std_ring_map,
+	    &sc->ti_rdata.ti_rx_std_ring_paddr, "RX ring");
+	if (error)
+		return (error);
+
+	/* Jumbo RX ring */
+	error = ti_dma_ring_alloc(sc, TI_JUMBO_RING_ALIGN, TI_JUMBO_RX_RING_SZ,
+	    &sc->ti_cdata.ti_rx_jumbo_ring_tag,
+	    (uint8_t **)&sc->ti_rdata.ti_rx_jumbo_ring,
+	    &sc->ti_cdata.ti_rx_jumbo_ring_map,
+	    &sc->ti_rdata.ti_rx_jumbo_ring_paddr, "jumbo RX ring");
+	if (error)
+		return (error);
+
+	/* RX return ring */
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, TI_RX_RETURN_RING_SZ,
+	    &sc->ti_cdata.ti_rx_return_ring_tag,
+	    (uint8_t **)&sc->ti_rdata.ti_rx_return_ring,
+	    &sc->ti_cdata.ti_rx_return_ring_map,
+	    &sc->ti_rdata.ti_rx_return_ring_paddr, "RX return ring");
+	if (error)
+		return (error);
+
+	/* Create DMA tag for standard RX mbufs. */
+	error = bus_dma_tag_create(sc->ti_cdata.ti_parent_tag, 1, 0,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1,
+	    MCLBYTES, 0, NULL, NULL, &sc->ti_cdata.ti_rx_std_tag);
+	if (error) {
+		device_printf(sc->ti_dev, "could not allocate RX dma tag\n");
+		return (error);
+	}
+
+	/* Create DMA tag for jumbo RX mbufs. */
+#ifdef TI_SF_BUF_JUMBO
 	/*
-	 * Grab a big chunk o' storage.  Since we are chopping this pool up
-	 * into ~9k chunks, there doesn't appear to be a need to use page
-	 * alignment.
+	 * The VM system will take care of providing aligned pages.  Alignment
+	 * is set to 1 here so that busdma resources won't be wasted.
 	 */
-	if (bus_dma_tag_create(sc->ti_parent_dmat,	/* parent */
-				1, 0,			/* algnmnt, boundary */
-				BUS_SPACE_MAXADDR,	/* lowaddr */
-				BUS_SPACE_MAXADDR,	/* highaddr */
-				NULL, NULL,		/* filter, filterarg */
-				TI_JMEM,		/* maxsize */
-				1,			/* nsegments */
-				TI_JMEM,		/* maxsegsize */
-				0,			/* flags */
-				NULL, NULL,		/* lockfunc, lockarg */
-				&sc->ti_jumbo_dmat) != 0) {
-		device_printf(sc->ti_dev, "Failed to allocate jumbo dmat\n");
-		return (ENOBUFS);
+	error = bus_dma_tag_create(sc->ti_cdata.ti_parent_tag, 1, 0,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, PAGE_SIZE * 4, 4,
+	    PAGE_SIZE, 0, NULL, NULL, &sc->ti_cdata.ti_rx_jumbo_tag);
+#else
+	error = bus_dma_tag_create(sc->ti_cdata.ti_parent_tag, 1, 0,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MJUM9BYTES, 1,
+	    MJUM9BYTES, 0, NULL, NULL, &sc->ti_cdata.ti_rx_jumbo_tag);
+#endif
+	if (error) {
+		device_printf(sc->ti_dev,
+		    "could not allocate jumbo RX dma tag\n");
+		return (error);
 	}
 
-	if (bus_dmamem_alloc(sc->ti_jumbo_dmat,
-			     (void**)&sc->ti_cdata.ti_jumbo_buf,
-			     BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
-			     &sc->ti_jumbo_dmamap) != 0) {
-		device_printf(sc->ti_dev, "Failed to allocate jumbo memory\n");
-		return (ENOBUFS);
+	/* Create DMA tag for TX mbufs. */
+	error = bus_dma_tag_create(sc->ti_cdata.ti_parent_tag, 1,
+	    0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
+	    MCLBYTES * TI_MAXTXSEGS, TI_MAXTXSEGS, MCLBYTES, 0, NULL, NULL,
+	    &sc->ti_cdata.ti_tx_tag);
+	if (error) {
+		device_printf(sc->ti_dev, "could not allocate TX dma tag\n");
+		return (ENOMEM);
 	}
 
-	SLIST_INIT(&sc->ti_jfree_listhead);
-	SLIST_INIT(&sc->ti_jinuse_listhead);
+	/* Create DMA maps for RX buffers. */
+	for (i = 0; i < TI_STD_RX_RING_CNT; i++) {
+		error = bus_dmamap_create(sc->ti_cdata.ti_rx_std_tag, 0,
+		    &sc->ti_cdata.ti_rx_std_maps[i]);
+		if (error) {
+			device_printf(sc->ti_dev,
+			    "could not create DMA map for RX\n");
+			return (error);
+		}
+	}
+	error = bus_dmamap_create(sc->ti_cdata.ti_rx_std_tag, 0,
+	    &sc->ti_cdata.ti_rx_std_sparemap);
+	if (error) {
+		device_printf(sc->ti_dev,
+		    "could not create spare DMA map for RX\n");
+		return (error);
+	}
 
-	/*
-	 * Now divide it up into 9K pieces and save the addresses
-	 * in an array.
-	 */
-	ptr = sc->ti_cdata.ti_jumbo_buf;
-	for (i = 0; i < TI_JSLOTS; i++) {
-		sc->ti_cdata.ti_jslots[i] = ptr;
-		ptr += TI_JLEN;
-		entry = malloc(sizeof(struct ti_jpool_entry),
-			       M_DEVBUF, M_NOWAIT);
-		if (entry == NULL) {
-			device_printf(sc->ti_dev, "no memory for jumbo "
-			    "buffer queue!\n");
-			return (ENOBUFS);
+	/* Create DMA maps for jumbo RX buffers. */
+	for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++) {
+		error = bus_dmamap_create(sc->ti_cdata.ti_rx_jumbo_tag, 0,
+		    &sc->ti_cdata.ti_rx_jumbo_maps[i]);
+		if (error) {
+			device_printf(sc->ti_dev,
+			    "could not create DMA map for jumbo RX\n");
+			return (error);
 		}
-		entry->slot = i;
-		SLIST_INSERT_HEAD(&sc->ti_jfree_listhead, entry, jpool_entries);
+	}
+	error = bus_dmamap_create(sc->ti_cdata.ti_rx_jumbo_tag, 0,
+	    &sc->ti_cdata.ti_rx_jumbo_sparemap);
+	if (error) {
+		device_printf(sc->ti_dev,
+		    "could not create spare DMA map for jumbo RX\n");
+		return (error);
 	}
 
-	return (0);
-}
+	/* Create DMA maps for TX buffers. */
+	for (i = 0; i < TI_TX_RING_CNT; i++) {
+		error = bus_dmamap_create(sc->ti_cdata.ti_tx_tag, 0,
+		    &sc->ti_cdata.ti_txdesc[i].tx_dmamap);
+		if (error) {
+			device_printf(sc->ti_dev,
+			    "could not create DMA map for TX\n");
+			return (ENOMEM);
+		}
+	}
 
-/*
- * Allocate a jumbo buffer.
- */
-static void *ti_jalloc(sc)
-	struct ti_softc		*sc;
-{
-	struct ti_jpool_entry	*entry;
+	/* Mini ring and TX ring is not available on Tigon 1. */
+	if (sc->ti_hwrev == TI_HWREV_TIGON)
+		return (0);
 
-	entry = SLIST_FIRST(&sc->ti_jfree_listhead);
+	/* TX ring */
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, TI_TX_RING_SZ,
+	    &sc->ti_cdata.ti_tx_ring_tag, (uint8_t **)&sc->ti_rdata.ti_tx_ring,
+	    &sc->ti_cdata.ti_tx_ring_map, &sc->ti_rdata.ti_tx_ring_paddr,
+	    "TX ring");
+	if (error)
+		return (error);
+
+	/* Mini RX ring */
+	error = ti_dma_ring_alloc(sc, TI_RING_ALIGN, TI_MINI_RX_RING_SZ,
+	    &sc->ti_cdata.ti_rx_mini_ring_tag,
+	    (uint8_t **)&sc->ti_rdata.ti_rx_mini_ring,
+	    &sc->ti_cdata.ti_rx_mini_ring_map,
+	    &sc->ti_rdata.ti_rx_mini_ring_paddr, "mini RX ring");
+	if (error)
+		return (error);
+
+	/* Create DMA tag for mini RX mbufs. */
+	error = bus_dma_tag_create(sc->ti_cdata.ti_parent_tag, 1, 0,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MHLEN, 1,
+	    MHLEN, 0, NULL, NULL, &sc->ti_cdata.ti_rx_mini_tag);
+	if (error) {
+		device_printf(sc->ti_dev,
+		    "could not allocate mini RX dma tag\n");
+		return (error);
+	}
 
-	if (entry == NULL) {
-		device_printf(sc->ti_dev, "no free jumbo buffers\n");
-		return (NULL);
+	/* Create DMA maps for mini RX buffers. */
+	for (i = 0; i < TI_MINI_RX_RING_CNT; i++) {
+		error = bus_dmamap_create(sc->ti_cdata.ti_rx_mini_tag, 0,
+		    &sc->ti_cdata.ti_rx_mini_maps[i]);
+		if (error) {
+			device_printf(sc->ti_dev,
+			    "could not create DMA map for mini RX\n");
+			return (error);
+		}
+	}
+	error = bus_dmamap_create(sc->ti_cdata.ti_rx_mini_tag, 0,
+	    &sc->ti_cdata.ti_rx_mini_sparemap);
+	if (error) {
+		device_printf(sc->ti_dev,
+		    "could not create spare DMA map for mini RX\n");
+		return (error);
 	}
 
-	SLIST_REMOVE_HEAD(&sc->ti_jfree_listhead, jpool_entries);
-	SLIST_INSERT_HEAD(&sc->ti_jinuse_listhead, entry, jpool_entries);
-	return (sc->ti_cdata.ti_jslots[entry->slot]);
+	return (0);
 }
 
-/*
- * Release a jumbo buffer.
- */
 static void
-ti_jfree(buf, args)
-	void			*buf;
-	void			*args;
+ti_dma_free(struct ti_softc *sc)
 {
-	struct ti_softc		*sc;
-	int			i;
-	struct ti_jpool_entry	*entry;
-
-	/* Extract the softc struct pointer. */
-	sc = (struct ti_softc *)args;
-
-	if (sc == NULL)
-		panic("ti_jfree: didn't get softc pointer!");
-
-	/* calculate the slot this buffer belongs to */
-	i = ((vm_offset_t)buf
-	     - (vm_offset_t)sc->ti_cdata.ti_jumbo_buf) / TI_JLEN;
-
-	if ((i < 0) || (i >= TI_JSLOTS))
-		panic("ti_jfree: asked to free buffer that we don't manage!");
-
-	entry = SLIST_FIRST(&sc->ti_jinuse_listhead);
-	if (entry == NULL)
-		panic("ti_jfree: buffer not in use!");
-	entry->slot = i;
-	SLIST_REMOVE_HEAD(&sc->ti_jinuse_listhead, jpool_entries);
-	SLIST_INSERT_HEAD(&sc->ti_jfree_listhead, entry, jpool_entries);
-}
+	int i;
 
-#else
+	/* Destroy DMA maps for RX buffers. */
+	for (i = 0; i < TI_STD_RX_RING_CNT; i++) {
+		if (sc->ti_cdata.ti_rx_std_maps[i]) {
+			bus_dmamap_destroy(sc->ti_cdata.ti_rx_std_tag,
+			    sc->ti_cdata.ti_rx_std_maps[i]);
+			sc->ti_cdata.ti_rx_std_maps[i] = NULL;
+		}
+	}
+	if (sc->ti_cdata.ti_rx_std_sparemap) {
+		bus_dmamap_destroy(sc->ti_cdata.ti_rx_std_tag,
+		    sc->ti_cdata.ti_rx_std_sparemap);
+		sc->ti_cdata.ti_rx_std_sparemap = NULL;
+	}
+	if (sc->ti_cdata.ti_rx_std_tag) {
+		bus_dma_tag_destroy(sc->ti_cdata.ti_rx_std_tag);
+		sc->ti_cdata.ti_rx_std_tag = NULL;
+	}
 
-static int
-ti_alloc_jumbo_mem(sc)
-	struct ti_softc		*sc;
-{
+	/* Destroy DMA maps for jumbo RX buffers. */
+	for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++) {
+		if (sc->ti_cdata.ti_rx_jumbo_maps[i]) {
+			bus_dmamap_destroy(sc->ti_cdata.ti_rx_jumbo_tag,
+			    sc->ti_cdata.ti_rx_jumbo_maps[i]);
+			sc->ti_cdata.ti_rx_jumbo_maps[i] = NULL;
+		}
+	}
+	if (sc->ti_cdata.ti_rx_jumbo_sparemap) {
+		bus_dmamap_destroy(sc->ti_cdata.ti_rx_jumbo_tag,
+		    sc->ti_cdata.ti_rx_jumbo_sparemap);
+		sc->ti_cdata.ti_rx_jumbo_sparemap = NULL;
+	}
+	if (sc->ti_cdata.ti_rx_jumbo_tag) {
+		bus_dma_tag_destroy(sc->ti_cdata.ti_rx_jumbo_tag);
+		sc->ti_cdata.ti_rx_jumbo_tag = NULL;
+	}
 
-	/*
-	 * The VM system will take care of providing aligned pages.  Alignment
-	 * is set to 1 here so that busdma resources won't be wasted.
-	 */
-	if (bus_dma_tag_create(sc->ti_parent_dmat,	/* parent */
-				1, 0,			/* algnmnt, boundary */
-				BUS_SPACE_MAXADDR,	/* lowaddr */
-				BUS_SPACE_MAXADDR,	/* highaddr */
-				NULL, NULL,		/* filter, filterarg */
-				PAGE_SIZE * 4 /*XXX*/,	/* maxsize */
-				4,			/* nsegments */
-				PAGE_SIZE,		/* maxsegsize */
-				0,			/* flags */
-				NULL, NULL,		/* lockfunc, lockarg */
-				&sc->ti_jumbo_dmat) != 0) {
-		device_printf(sc->ti_dev, "Failed to allocate jumbo dmat\n");
-		return (ENOBUFS);
+	/* Destroy DMA maps for mini RX buffers. */
+	for (i = 0; i < TI_MINI_RX_RING_CNT; i++) {
+		if (sc->ti_cdata.ti_rx_mini_maps[i]) {
+			bus_dmamap_destroy(sc->ti_cdata.ti_rx_mini_tag,
+			    sc->ti_cdata.ti_rx_mini_maps[i]);
+			sc->ti_cdata.ti_rx_mini_maps[i] = NULL;
+		}
+	}
+	if (sc->ti_cdata.ti_rx_mini_sparemap) {
+		bus_dmamap_destroy(sc->ti_cdata.ti_rx_mini_tag,
+		    sc->ti_cdata.ti_rx_mini_sparemap);
+		sc->ti_cdata.ti_rx_mini_sparemap = NULL;
+	}
+	if (sc->ti_cdata.ti_rx_mini_tag) {
+		bus_dma_tag_destroy(sc->ti_cdata.ti_rx_mini_tag);
+		sc->ti_cdata.ti_rx_mini_tag = NULL;
 	}
 
-	return (0);
+	/* Destroy DMA maps for TX buffers. */
+	for (i = 0; i < TI_TX_RING_CNT; i++) {
+		if (sc->ti_cdata.ti_txdesc[i].tx_dmamap) {
+			bus_dmamap_destroy(sc->ti_cdata.ti_tx_tag,
+			    sc->ti_cdata.ti_txdesc[i].tx_dmamap);
+			sc->ti_cdata.ti_txdesc[i].tx_dmamap = NULL;
+		}
+	}
+	if (sc->ti_cdata.ti_tx_tag) {
+		bus_dma_tag_destroy(sc->ti_cdata.ti_tx_tag);
+		sc->ti_cdata.ti_tx_tag = NULL;
+	}
+
+	/* Destroy standard RX ring. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_rx_std_ring_tag,
+	    (void *)&sc->ti_rdata.ti_rx_std_ring,
+	    &sc->ti_cdata.ti_rx_std_ring_map);
+	/* Destroy jumbo RX ring. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_rx_jumbo_ring_tag,
+	    (void *)&sc->ti_rdata.ti_rx_jumbo_ring,
+	    &sc->ti_cdata.ti_rx_jumbo_ring_map);
+	/* Destroy mini RX ring. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_rx_mini_ring_tag,
+	    (void *)&sc->ti_rdata.ti_rx_mini_ring,
+	    &sc->ti_cdata.ti_rx_mini_ring_map);
+	/* Destroy RX return ring. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_rx_return_ring_tag,
+	    (void *)&sc->ti_rdata.ti_rx_return_ring,
+	    &sc->ti_cdata.ti_rx_return_ring_map);
+	/* Destroy TX ring. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_tx_ring_tag,
+	    (void *)&sc->ti_rdata.ti_tx_ring, &sc->ti_cdata.ti_tx_ring_map);
+	/* Destroy status block. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_status_tag,
+	    (void *)&sc->ti_rdata.ti_status, &sc->ti_cdata.ti_status_map);
+	/* Destroy event ring. */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_event_ring_tag,
+	    (void *)&sc->ti_rdata.ti_event_ring,
+	    &sc->ti_cdata.ti_event_ring_map);
+	/* Destroy GIB */
+	ti_dma_ring_free(sc, &sc->ti_cdata.ti_gib_tag,
+	    (void *)&sc->ti_rdata.ti_info, &sc->ti_cdata.ti_gib_map);
+
+	/* Destroy the parent tag. */
+	if (sc->ti_cdata.ti_parent_tag) {
+		bus_dma_tag_destroy(sc->ti_cdata.ti_parent_tag);
+		sc->ti_cdata.ti_parent_tag = NULL;
+	}
 }
 
-#endif /* TI_PRIVATE_JUMBOS */
-
 /*
  * Intialize a standard receive ring descriptor.
  */
 static int
-ti_newbuf_std(sc, i, m)
-	struct ti_softc		*sc;
-	int			i;
-	struct mbuf		*m;
-{
-	bus_dmamap_t		map;
-	bus_dma_segment_t	segs;
-	struct mbuf		*m_new = NULL;
-	struct ti_rx_desc	*r;
-	int			nsegs;
-
-	nsegs = 0;
-	if (m == NULL) {
-		MGETHDR(m_new, M_DONTWAIT, MT_DATA);
-		if (m_new == NULL)
-			return (ENOBUFS);
+ti_newbuf_std(struct ti_softc *sc, int i)
+{
+	bus_dmamap_t map;
+	bus_dma_segment_t segs[1];
+	struct mbuf *m;
+	struct ti_rx_desc *r;
+	int error, nsegs;
+
+	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+	if (m == NULL)
+		return (ENOBUFS);
+	m->m_len = m->m_pkthdr.len = MCLBYTES;
+	m_adj(m, ETHER_ALIGN);
 
-		MCLGET(m_new, M_DONTWAIT);
-		if (!(m_new->m_flags & M_EXT)) {
-			m_freem(m_new);
-			return (ENOBUFS);
-		}
-		m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
-	} else {
-		m_new = m;
-		m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
-		m_new->m_data = m_new->m_ext.ext_buf;
+	error = bus_dmamap_load_mbuf_sg(sc->ti_cdata.ti_rx_std_tag,
+	    sc->ti_cdata.ti_rx_std_sparemap, m, segs, &nsegs, 0);
+	if (error != 0) {
+		m_freem(m);
+		return (error);
+        }
+	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+	if (sc->ti_cdata.ti_rx_std_chain[i] != NULL) {
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_std_tag,
+		    sc->ti_cdata.ti_rx_std_maps[i], BUS_DMASYNC_POSTREAD);
+		bus_dmamap_unload(sc->ti_cdata.ti_rx_std_tag,
+		    sc->ti_cdata.ti_rx_std_maps[i]);
 	}
 
-	m_adj(m_new, ETHER_ALIGN);
-	sc->ti_cdata.ti_rx_std_chain[i] = m_new;
-	r = &sc->ti_rdata->ti_rx_std_ring[i];
 	map = sc->ti_cdata.ti_rx_std_maps[i];
-	if (bus_dmamap_load_mbuf_sg(sc->ti_mbufrx_dmat, map, m_new, &segs,
-				    &nsegs, 0))
-		return (ENOBUFS);
-	if (nsegs != 1)
-		return (ENOBUFS);
-	ti_hostaddr64(&r->ti_addr, segs.ds_addr);
-	r->ti_len = segs.ds_len;
+	sc->ti_cdata.ti_rx_std_maps[i] = sc->ti_cdata.ti_rx_std_sparemap;
+	sc->ti_cdata.ti_rx_std_sparemap = map;
+	sc->ti_cdata.ti_rx_std_chain[i] = m;
+
+	r = &sc->ti_rdata.ti_rx_std_ring[i];
+	ti_hostaddr64(&r->ti_addr, segs[0].ds_addr);
+	r->ti_len = segs[0].ds_len;
 	r->ti_type = TI_BDTYPE_RECV_BD;
 	r->ti_flags = 0;
-	if (sc->ti_ifp->if_hwassist)
+	r->ti_vlan_tag = 0;
+	r->ti_tcp_udp_cksum = 0;
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM;
 	r->ti_idx = i;
 
-	bus_dmamap_sync(sc->ti_mbufrx_dmat, map, BUS_DMASYNC_PREREAD);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_std_tag,
+	    sc->ti_cdata.ti_rx_std_maps[i], BUS_DMASYNC_PREREAD);
 	return (0);
 }
 
@@ -1292,117 +1428,112 @@ ti_newbuf_std(sc, i, m)
  * the Tigon 2.
  */
 static int
-ti_newbuf_mini(sc, i, m)
-	struct ti_softc		*sc;
-	int			i;
-	struct mbuf		*m;
-{
-	bus_dma_segment_t	segs;
-	bus_dmamap_t		map;
-	struct mbuf		*m_new = NULL;
-	struct ti_rx_desc	*r;
-	int			nsegs;
-
-	nsegs = 0;
-	if (m == NULL) {
-		MGETHDR(m_new, M_DONTWAIT, MT_DATA);
-		if (m_new == NULL) {
-			return (ENOBUFS);
-		}
-		m_new->m_len = m_new->m_pkthdr.len = MHLEN;
-	} else {
-		m_new = m;
-		m_new->m_data = m_new->m_pktdat;
-		m_new->m_len = m_new->m_pkthdr.len = MHLEN;
+ti_newbuf_mini(struct ti_softc *sc, int i)
+{
+	bus_dmamap_t map;
+	bus_dma_segment_t segs[1];
+	struct mbuf *m;
+	struct ti_rx_desc *r;
+	int error, nsegs;
+
+	MGETHDR(m, M_DONTWAIT, MT_DATA);
+	if (m == NULL)
+		return (ENOBUFS);
+	m->m_len = m->m_pkthdr.len = MHLEN;
+	m_adj(m, ETHER_ALIGN);
+
+	error = bus_dmamap_load_mbuf_sg(sc->ti_cdata.ti_rx_mini_tag,
+	    sc->ti_cdata.ti_rx_mini_sparemap, m, segs, &nsegs, 0);
+	if (error != 0) {
+		m_freem(m);
+		return (error);
+        }
+	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+	if (sc->ti_cdata.ti_rx_mini_chain[i] != NULL) {
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_mini_tag,
+		    sc->ti_cdata.ti_rx_mini_maps[i], BUS_DMASYNC_POSTREAD);
+		bus_dmamap_unload(sc->ti_cdata.ti_rx_mini_tag,
+		    sc->ti_cdata.ti_rx_mini_maps[i]);
 	}
 
-	m_adj(m_new, ETHER_ALIGN);
-	r = &sc->ti_rdata->ti_rx_mini_ring[i];
-	sc->ti_cdata.ti_rx_mini_chain[i] = m_new;
 	map = sc->ti_cdata.ti_rx_mini_maps[i];
-	if (bus_dmamap_load_mbuf_sg(sc->ti_mbufrx_dmat, map, m_new, &segs,
-				    &nsegs, 0))
-		return (ENOBUFS);
-	if (nsegs != 1)
-		return (ENOBUFS);
-	ti_hostaddr64(&r->ti_addr, segs.ds_addr);
-	r->ti_len = segs.ds_len;
+	sc->ti_cdata.ti_rx_mini_maps[i] = sc->ti_cdata.ti_rx_mini_sparemap;
+	sc->ti_cdata.ti_rx_mini_sparemap = map;
+	sc->ti_cdata.ti_rx_mini_chain[i] = m;
+
+	r = &sc->ti_rdata.ti_rx_mini_ring[i];
+	ti_hostaddr64(&r->ti_addr, segs[0].ds_addr);
+	r->ti_len = segs[0].ds_len;
 	r->ti_type = TI_BDTYPE_RECV_BD;
 	r->ti_flags = TI_BDFLAG_MINI_RING;
-	if (sc->ti_ifp->if_hwassist)
+	r->ti_vlan_tag = 0;
+	r->ti_tcp_udp_cksum = 0;
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM;
 	r->ti_idx = i;
 
-	bus_dmamap_sync(sc->ti_mbufrx_dmat, map, BUS_DMASYNC_PREREAD);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_mini_tag,
+	    sc->ti_cdata.ti_rx_mini_maps[i], BUS_DMASYNC_PREREAD);
 	return (0);
 }
 
-#ifdef TI_PRIVATE_JUMBOS
+#ifndef TI_SF_BUF_JUMBO
 
 /*
  * Initialize a jumbo receive ring descriptor. This allocates
  * a jumbo buffer from the pool managed internally by the driver.
  */
 static int
-ti_newbuf_jumbo(sc, i, m)
-	struct ti_softc		*sc;
-	int			i;
-	struct mbuf		*m;
+ti_newbuf_jumbo(struct ti_softc *sc, int i, struct mbuf *dummy)
 {
-	bus_dmamap_t		map;
-	struct mbuf		*m_new = NULL;
-	struct ti_rx_desc	*r;
-	int			nsegs;
-	bus_dma_segment_t	segs;
+	bus_dmamap_t map;
+	bus_dma_segment_t segs[1];
+	struct mbuf *m;
+	struct ti_rx_desc *r;
+	int error, nsegs;
 
-	if (m == NULL) {
-		caddr_t			*buf = NULL;
+	(void)dummy;
 
-		/* Allocate the mbuf. */
-		MGETHDR(m_new, M_DONTWAIT, MT_DATA);
-		if (m_new == NULL) {
-			return (ENOBUFS);
-		}
+	m = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, MJUM9BYTES);
+	if (m == NULL)
+		return (ENOBUFS);
+	m->m_len = m->m_pkthdr.len = MJUM9BYTES;
+	m_adj(m, ETHER_ALIGN);
 
-		/* Allocate the jumbo buffer */
-		buf = ti_jalloc(sc);
-		if (buf == NULL) {
-			m_freem(m_new);
-			device_printf(sc->ti_dev, "jumbo allocation failed "
-			    "-- packet dropped!\n");
-			return (ENOBUFS);
-		}
+	error = bus_dmamap_load_mbuf_sg(sc->ti_cdata.ti_rx_jumbo_tag,
+	    sc->ti_cdata.ti_rx_jumbo_sparemap, m, segs, &nsegs, 0);
+	if (error != 0) {
+		m_freem(m);
+		return (error);
+        }
+	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
 
-		/* Attach the buffer to the mbuf. */
-		m_new->m_data = (void *) buf;
-		m_new->m_len = m_new->m_pkthdr.len = TI_JUMBO_FRAMELEN;
-		MEXTADD(m_new, buf, TI_JUMBO_FRAMELEN, ti_jfree,
-		    (struct ti_softc *)sc, 0, EXT_NET_DRV);
-	} else {
-		m_new = m;
-		m_new->m_data = m_new->m_ext.ext_buf;
-		m_new->m_ext.ext_size = TI_JUMBO_FRAMELEN;
+	if (sc->ti_cdata.ti_rx_jumbo_chain[i] != NULL) {
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_tag,
+		    sc->ti_cdata.ti_rx_jumbo_maps[i], BUS_DMASYNC_POSTREAD);
+		bus_dmamap_unload(sc->ti_cdata.ti_rx_jumbo_tag,
+		    sc->ti_cdata.ti_rx_jumbo_maps[i]);
 	}
 
-	m_adj(m_new, ETHER_ALIGN);
-	/* Set up the descriptor. */
-	r = &sc->ti_rdata->ti_rx_jumbo_ring[i];
-	sc->ti_cdata.ti_rx_jumbo_chain[i] = m_new;
 	map = sc->ti_cdata.ti_rx_jumbo_maps[i];
-	if (bus_dmamap_load_mbuf_sg(sc->ti_jumbo_dmat, map, m_new, &segs,
-				    &nsegs, 0))
-		return (ENOBUFS);
-	if (nsegs != 1)
-		return (ENOBUFS);
-	ti_hostaddr64(&r->ti_addr, segs.ds_addr);
-	r->ti_len = segs.ds_len;
+	sc->ti_cdata.ti_rx_jumbo_maps[i] = sc->ti_cdata.ti_rx_jumbo_sparemap;
+	sc->ti_cdata.ti_rx_jumbo_sparemap = map;
+	sc->ti_cdata.ti_rx_jumbo_chain[i] = m;
+
+	r = &sc->ti_rdata.ti_rx_jumbo_ring[i];
+	ti_hostaddr64(&r->ti_addr, segs[0].ds_addr);
+	r->ti_len = segs[0].ds_len;
 	r->ti_type = TI_BDTYPE_RECV_JUMBO_BD;
 	r->ti_flags = TI_BDFLAG_JUMBO_RING;
-	if (sc->ti_ifp->if_hwassist)
+	r->ti_vlan_tag = 0;
+	r->ti_tcp_udp_cksum = 0;
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM;
 	r->ti_idx = i;
 
-	bus_dmamap_sync(sc->ti_jumbo_dmat, map, BUS_DMASYNC_PREREAD);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_tag,
+	    sc->ti_cdata.ti_rx_jumbo_maps[i], BUS_DMASYNC_PREREAD);
 	return (0);
 }
 
@@ -1417,30 +1548,26 @@ ti_newbuf_jumbo(sc, i, m)
 #define TCP_HDR_LEN (52 + sizeof(struct ether_header))
 #define UDP_HDR_LEN (28 + sizeof(struct ether_header))
 #define NFS_HDR_LEN (UDP_HDR_LEN)
-static int HDR_LEN =  TCP_HDR_LEN;
-
+static int HDR_LEN = TCP_HDR_LEN;
 
 /*
  * Initialize a jumbo receive ring descriptor. This allocates
  * a jumbo buffer from the pool managed internally by the driver.
  */
 static int
-ti_newbuf_jumbo(sc, idx, m_old)
-	struct ti_softc		*sc;
-	int			idx;
-	struct mbuf		*m_old;
-{
-	bus_dmamap_t		map;
-	struct mbuf		*cur, *m_new = NULL;
-	struct mbuf		*m[3] = {NULL, NULL, NULL};
-	struct ti_rx_desc_ext	*r;
-	vm_page_t		frame;
-	static int		color;
-				/* 1 extra buf to make nobufs easy*/
-	struct sf_buf		*sf[3] = {NULL, NULL, NULL};
-	int			i;
-	bus_dma_segment_t	segs[4];
-	int			nsegs;
+ti_newbuf_jumbo(struct ti_softc *sc, int idx, struct mbuf *m_old)
+{
+	bus_dmamap_t map;
+	struct mbuf *cur, *m_new = NULL;
+	struct mbuf *m[3] = {NULL, NULL, NULL};
+	struct ti_rx_desc_ext *r;
+	vm_page_t frame;
+	static int color;
+	/* 1 extra buf to make nobufs easy*/
+	struct sf_buf *sf[3] = {NULL, NULL, NULL};
+	int i;
+	bus_dma_segment_t segs[4];
+	int nsegs;
 
 	if (m_old != NULL) {
 		m_new = m_old;
@@ -1517,11 +1644,11 @@ ti_newbuf_jumbo(sc, idx, m_old)
 	}
 
 	/* Set up the descriptor. */
-	r = &sc->ti_rdata->ti_rx_jumbo_ring[idx];
+	r = &sc->ti_rdata.ti_rx_jumbo_ring[idx];
 	sc->ti_cdata.ti_rx_jumbo_chain[idx] = m_new;
 	map = sc->ti_cdata.ti_rx_jumbo_maps[i];
-	if (bus_dmamap_load_mbuf_sg(sc->ti_jumbo_dmat, map, m_new, segs,
-				    &nsegs, 0))
+	if (bus_dmamap_load_mbuf_sg(sc->ti_cdata.ti_rx_jumbo_tag, map, m_new,
+	    segs, &nsegs, 0))
 		return (ENOBUFS);
 	if ((nsegs < 1) || (nsegs > 4))
 		return (ENOBUFS);
@@ -1544,12 +1671,12 @@ ti_newbuf_jumbo(sc, idx, m_old)
 
 	r->ti_flags = TI_BDFLAG_JUMBO_RING|TI_RCB_FLAG_USE_EXT_RX_BD;
 
-	if (sc->ti_ifp->if_hwassist)
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM|TI_BDFLAG_IP_CKSUM;
 
 	r->ti_idx = idx;
 
-	bus_dmamap_sync(sc->ti_jumbo_dmat, map, BUS_DMASYNC_PREREAD);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_tag, map, BUS_DMASYNC_PREREAD);
 	return (0);
 
 nobufs:
@@ -1573,8 +1700,6 @@ nobufs:
 }
 #endif
 
-
-
 /*
  * The standard receive ring has 512 entries in it. At 2K per mbuf cluster,
  * that's 1MB or memory, which is a lot. For now, we fill only the first
@@ -1582,151 +1707,151 @@ nobufs:
  * the NIC.
  */
 static int
-ti_init_rx_ring_std(sc)
-	struct ti_softc		*sc;
+ti_init_rx_ring_std(struct ti_softc *sc)
 {
-	int			i;
-	struct ti_cmd_desc	cmd;
+	int i;
+	struct ti_cmd_desc cmd;
 
-	for (i = 0; i < TI_SSLOTS; i++) {
-		if (ti_newbuf_std(sc, i, NULL) == ENOBUFS)
+	for (i = 0; i < TI_STD_RX_RING_CNT; i++) {
+		if (ti_newbuf_std(sc, i) != 0)
 			return (ENOBUFS);
 	};
 
-	TI_UPDATE_STDPROD(sc, i - 1);
-	sc->ti_std = i - 1;
+	sc->ti_std = TI_STD_RX_RING_CNT - 1;
+	TI_UPDATE_STDPROD(sc, TI_STD_RX_RING_CNT - 1);
 
 	return (0);
 }
 
 static void
-ti_free_rx_ring_std(sc)
-	struct ti_softc		*sc;
+ti_free_rx_ring_std(struct ti_softc *sc)
 {
-	bus_dmamap_t		map;
-	int			i;
+	bus_dmamap_t map;
+	int i;
 
 	for (i = 0; i < TI_STD_RX_RING_CNT; i++) {
 		if (sc->ti_cdata.ti_rx_std_chain[i] != NULL) {
 			map = sc->ti_cdata.ti_rx_std_maps[i];
-			bus_dmamap_sync(sc->ti_mbufrx_dmat, map,
+			bus_dmamap_sync(sc->ti_cdata.ti_rx_std_tag, map,
 			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(sc->ti_mbufrx_dmat, map);
+			bus_dmamap_unload(sc->ti_cdata.ti_rx_std_tag, map);
 			m_freem(sc->ti_cdata.ti_rx_std_chain[i]);
 			sc->ti_cdata.ti_rx_std_chain[i] = NULL;
 		}
-		bzero((char *)&sc->ti_rdata->ti_rx_std_ring[i],
-		    sizeof(struct ti_rx_desc));
 	}
+	bzero(sc->ti_rdata.ti_rx_std_ring, TI_STD_RX_RING_SZ);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_std_ring_tag,
+	    sc->ti_cdata.ti_rx_std_ring_map, BUS_DMASYNC_PREWRITE);
 }
 
 static int
-ti_init_rx_ring_jumbo(sc)
-	struct ti_softc		*sc;
+ti_init_rx_ring_jumbo(struct ti_softc *sc)
 {
-	int			i;
-	struct ti_cmd_desc	cmd;
+	struct ti_cmd_desc cmd;
+	int i;
 
 	for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++) {
-		if (ti_newbuf_jumbo(sc, i, NULL) == ENOBUFS)
+		if (ti_newbuf_jumbo(sc, i, NULL) != 0)
 			return (ENOBUFS);
 	};
 
-	TI_UPDATE_JUMBOPROD(sc, i - 1);
-	sc->ti_jumbo = i - 1;
+	sc->ti_jumbo = TI_JUMBO_RX_RING_CNT - 1;
+	TI_UPDATE_JUMBOPROD(sc, TI_JUMBO_RX_RING_CNT - 1);
 
 	return (0);
 }
 
 static void
-ti_free_rx_ring_jumbo(sc)
-	struct ti_softc		*sc;
+ti_free_rx_ring_jumbo(struct ti_softc *sc)
 {
-	bus_dmamap_t		map;
-	int			i;
+	bus_dmamap_t map;
+	int i;
 
 	for (i = 0; i < TI_JUMBO_RX_RING_CNT; i++) {
 		if (sc->ti_cdata.ti_rx_jumbo_chain[i] != NULL) {
 			map = sc->ti_cdata.ti_rx_jumbo_maps[i];
-			bus_dmamap_sync(sc->ti_jumbo_dmat, map,
+			bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_tag, map,
 			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(sc->ti_jumbo_dmat, map);
+			bus_dmamap_unload(sc->ti_cdata.ti_rx_jumbo_tag, map);
 			m_freem(sc->ti_cdata.ti_rx_jumbo_chain[i]);
 			sc->ti_cdata.ti_rx_jumbo_chain[i] = NULL;
 		}
-		bzero((char *)&sc->ti_rdata->ti_rx_jumbo_ring[i],
-		    sizeof(struct ti_rx_desc));
 	}
+	bzero(sc->ti_rdata.ti_rx_jumbo_ring, TI_JUMBO_RX_RING_SZ);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_ring_tag,
+	    sc->ti_cdata.ti_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE);
 }
 
 static int
-ti_init_rx_ring_mini(sc)
-	struct ti_softc		*sc;
+ti_init_rx_ring_mini(struct ti_softc *sc)
 {
-	int			i;
+	int i;
 
-	for (i = 0; i < TI_MSLOTS; i++) {
-		if (ti_newbuf_mini(sc, i, NULL) == ENOBUFS)
+	for (i = 0; i < TI_MINI_RX_RING_CNT; i++) {
+		if (ti_newbuf_mini(sc, i) != 0)
 			return (ENOBUFS);
 	};
 
-	TI_UPDATE_MINIPROD(sc, i - 1);
-	sc->ti_mini = i - 1;
+	sc->ti_mini = TI_MINI_RX_RING_CNT - 1;
+	TI_UPDATE_MINIPROD(sc, TI_MINI_RX_RING_CNT - 1);
 
 	return (0);
 }
 
 static void
-ti_free_rx_ring_mini(sc)
-	struct ti_softc		*sc;
+ti_free_rx_ring_mini(struct ti_softc *sc)
 {
-	bus_dmamap_t		map;
-	int			i;
+	bus_dmamap_t map;
+	int i;
+
+	if (sc->ti_rdata.ti_rx_mini_ring == NULL)
+		return;
 
 	for (i = 0; i < TI_MINI_RX_RING_CNT; i++) {
 		if (sc->ti_cdata.ti_rx_mini_chain[i] != NULL) {
 			map = sc->ti_cdata.ti_rx_mini_maps[i];
-			bus_dmamap_sync(sc->ti_mbufrx_dmat, map,
+			bus_dmamap_sync(sc->ti_cdata.ti_rx_mini_tag, map,
 			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(sc->ti_mbufrx_dmat, map);
+			bus_dmamap_unload(sc->ti_cdata.ti_rx_mini_tag, map);
 			m_freem(sc->ti_cdata.ti_rx_mini_chain[i]);
 			sc->ti_cdata.ti_rx_mini_chain[i] = NULL;
 		}
-		bzero((char *)&sc->ti_rdata->ti_rx_mini_ring[i],
-		    sizeof(struct ti_rx_desc));
 	}
+	bzero(sc->ti_rdata.ti_rx_mini_ring, TI_MINI_RX_RING_SZ);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_mini_ring_tag,
+	    sc->ti_cdata.ti_rx_mini_ring_map, BUS_DMASYNC_PREWRITE);
 }
 
 static void
-ti_free_tx_ring(sc)
-	struct ti_softc		*sc;
+ti_free_tx_ring(struct ti_softc *sc)
 {
-	struct ti_txdesc	*txd;
-	int			i;
+	struct ti_txdesc *txd;
+	int i;
 
-	if (sc->ti_rdata->ti_tx_ring == NULL)
+	if (sc->ti_rdata.ti_tx_ring == NULL)
 		return;
 
 	for (i = 0; i < TI_TX_RING_CNT; i++) {
 		txd = &sc->ti_cdata.ti_txdesc[i];
 		if (txd->tx_m != NULL) {
-			bus_dmamap_sync(sc->ti_mbuftx_dmat, txd->tx_dmamap,
+			bus_dmamap_sync(sc->ti_cdata.ti_tx_tag, txd->tx_dmamap,
 			    BUS_DMASYNC_POSTWRITE);
-			bus_dmamap_unload(sc->ti_mbuftx_dmat, txd->tx_dmamap);
+			bus_dmamap_unload(sc->ti_cdata.ti_tx_tag,
+			    txd->tx_dmamap);
 			m_freem(txd->tx_m);
 			txd->tx_m = NULL;
 		}
-		bzero((char *)&sc->ti_rdata->ti_tx_ring[i],
-		    sizeof(struct ti_tx_desc));
 	}
+	bzero(sc->ti_rdata.ti_tx_ring, TI_TX_RING_SZ);
+	bus_dmamap_sync(sc->ti_cdata.ti_tx_ring_tag,
+	    sc->ti_cdata.ti_tx_ring_map, BUS_DMASYNC_PREWRITE);
 }
 
 static int
-ti_init_tx_ring(sc)
-	struct ti_softc		*sc;
+ti_init_tx_ring(struct ti_softc *sc)
 {
-	struct ti_txdesc	*txd;
-	int			i;
+	struct ti_txdesc *txd;
+	int i;
 
 	STAILQ_INIT(&sc->ti_cdata.ti_txfreeq);
 	STAILQ_INIT(&sc->ti_cdata.ti_txbusyq);
@@ -1747,15 +1872,13 @@ ti_init_tx_ring(sc)
  * work.
  */
 static void
-ti_add_mcast(sc, addr)
-	struct ti_softc		*sc;
-	struct ether_addr	*addr;
+ti_add_mcast(struct ti_softc *sc, struct ether_addr *addr)
 {
-	struct ti_cmd_desc	cmd;
-	u_int16_t		*m;
-	u_int32_t		ext[2] = {0, 0};
+	struct ti_cmd_desc cmd;
+	uint16_t *m;
+	uint32_t ext[2] = {0, 0};
 
-	m = (u_int16_t *)&addr->octet[0];
+	m = (uint16_t *)&addr->octet[0];
 
 	switch (sc->ti_hwrev) {
 	case TI_HWREV_TIGON:
@@ -1775,15 +1898,13 @@ ti_add_mcast(sc, addr)
 }
 
 static void
-ti_del_mcast(sc, addr)
-	struct ti_softc		*sc;
-	struct ether_addr	*addr;
+ti_del_mcast(struct ti_softc *sc, struct ether_addr *addr)
 {
-	struct ti_cmd_desc	cmd;
-	u_int16_t		*m;
-	u_int32_t		ext[2] = {0, 0};
+	struct ti_cmd_desc cmd;
+	uint16_t *m;
+	uint32_t ext[2] = {0, 0};
 
-	m = (u_int16_t *)&addr->octet[0];
+	m = (uint16_t *)&addr->octet[0];
 
 	switch (sc->ti_hwrev) {
 	case TI_HWREV_TIGON:
@@ -1817,14 +1938,13 @@ ti_del_mcast(sc, addr)
  * any given time.
  */
 static void
-ti_setmulti(sc)
-	struct ti_softc		*sc;
+ti_setmulti(struct ti_softc *sc)
 {
-	struct ifnet		*ifp;
-	struct ifmultiaddr	*ifma;
-	struct ti_cmd_desc	cmd;
-	struct ti_mc_entry	*mc;
-	u_int32_t		intrs;
+	struct ifnet *ifp;
+	struct ifmultiaddr *ifma;
+	struct ti_cmd_desc cmd;
+	struct ti_mc_entry *mc;
+	uint32_t intrs;
 
 	TI_LOCK_ASSERT(sc);
 
@@ -1877,9 +1997,10 @@ ti_setmulti(sc)
  * around it on the Tigon 2 by setting a bit in the PCI state register,
  * but for the Tigon 1 we must give up and abort the interface attach.
  */
-static int ti_64bitslot_war(sc)
-	struct ti_softc		*sc;
+static int
+ti_64bitslot_war(struct ti_softc *sc)
 {
+
 	if (!(CSR_READ_4(sc, TI_PCI_STATE) & TI_PCISTATE_32BIT_BUS)) {
 		CSR_WRITE_4(sc, 0x600, 0);
 		CSR_WRITE_4(sc, 0x604, 0);
@@ -1903,21 +2024,15 @@ static int ti_64bitslot_war(sc)
  * self-test results.
  */
 static int
-ti_chipinit(sc)
-	struct ti_softc		*sc;
+ti_chipinit(struct ti_softc *sc)
 {
-	u_int32_t		cacheline;
-	u_int32_t		pci_writemax = 0;
-	u_int32_t		hdrsplit;
+	uint32_t cacheline;
+	uint32_t pci_writemax = 0;
+	uint32_t hdrsplit;
 
 	/* Initialize link to down state. */
 	sc->ti_linkstat = TI_EV_CODE_LINK_DOWN;
 
-	if (sc->ti_ifp->if_capenable & IFCAP_HWCSUM)
-		sc->ti_ifp->if_hwassist = TI_CSUM_FEATURES;
-	else
-		sc->ti_ifp->if_hwassist = 0;
-
 	/* Set endianness before we access any non-PCI registers. */
 #if 0 && BYTE_ORDER == BIG_ENDIAN
 	CSR_WRITE_4(sc, TI_MISC_HOST_CTL,
@@ -2036,7 +2151,7 @@ ti_chipinit(sc)
 	 * the firmware racks up lots of nicDmaReadRingFull
 	 * errors.  This is not compatible with hardware checksums.
 	 */
-	if (sc->ti_ifp->if_hwassist == 0)
+	if ((sc->ti_ifp->if_capenable & (IFCAP_TXCSUM | IFCAP_RXCSUM)) == 0)
 		TI_SETBIT(sc, TI_GCR_OPMODE, TI_OPMODE_1_DMA_ACTIVE);
 
 	/* Recommended settings from Tigon manual. */
@@ -2057,30 +2172,24 @@ ti_chipinit(sc)
  * start the CPU(s) running.
  */
 static int
-ti_gibinit(sc)
-	struct ti_softc		*sc;
+ti_gibinit(struct ti_softc *sc)
 {
-	struct ti_rcb		*rcb;
-	int			i;
-	struct ifnet		*ifp;
-	uint32_t		rdphys;
+	struct ifnet *ifp;
+	struct ti_rcb *rcb;
+	int i;
 
 	TI_LOCK_ASSERT(sc);
 
 	ifp = sc->ti_ifp;
-	rdphys = sc->ti_rdata_phys;
 
 	/* Disable interrupts for now. */
 	CSR_WRITE_4(sc, TI_MB_HOSTINTR, 1);
 
-	/*
-	 * Tell the chip where to find the general information block.
-	 * While this struct could go into >4GB memory, we allocate it in a
-	 * single slab with the other descriptors, and those don't seem to
-	 * support being located in a 64-bit region.
-	 */
-	CSR_WRITE_4(sc, TI_GCR_GENINFO_HI, 0);
-	CSR_WRITE_4(sc, TI_GCR_GENINFO_LO, rdphys + TI_RD_OFF(ti_info));
+	/* Tell the chip where to find the general information block. */
+	CSR_WRITE_4(sc, TI_GCR_GENINFO_HI,
+	    (uint64_t)sc->ti_rdata.ti_info_paddr >> 32);
+	CSR_WRITE_4(sc, TI_GCR_GENINFO_LO,
+	    sc->ti_rdata.ti_info_paddr & 0xFFFFFFFF);
 
 	/* Load the firmware into SRAM. */
 	ti_loadfw(sc);
@@ -2088,20 +2197,20 @@ ti_gibinit(sc)
 	/* Set up the contents of the general info and ring control blocks. */
 
 	/* Set up the event ring and producer pointer. */
-	rcb = &sc->ti_rdata->ti_info.ti_ev_rcb;
-
-	TI_HOSTADDR(rcb->ti_hostaddr) = rdphys + TI_RD_OFF(ti_event_ring);
+	bzero(sc->ti_rdata.ti_event_ring, TI_EVENT_RING_SZ);
+	rcb = &sc->ti_rdata.ti_info->ti_ev_rcb;
+	ti_hostaddr64(&rcb->ti_hostaddr, sc->ti_rdata.ti_event_ring_paddr);
 	rcb->ti_flags = 0;
-	TI_HOSTADDR(sc->ti_rdata->ti_info.ti_ev_prodidx_ptr) =
-	    rdphys + TI_RD_OFF(ti_ev_prodidx_r);
+	ti_hostaddr64(&sc->ti_rdata.ti_info->ti_ev_prodidx_ptr,
+	    sc->ti_rdata.ti_status_paddr +
+	    offsetof(struct ti_status, ti_ev_prodidx_r));
 	sc->ti_ev_prodidx.ti_idx = 0;
 	CSR_WRITE_4(sc, TI_GCR_EVENTCONS_IDX, 0);
 	sc->ti_ev_saved_considx = 0;
 
 	/* Set up the command ring and producer mailbox. */
-	rcb = &sc->ti_rdata->ti_info.ti_cmd_rcb;
-
-	TI_HOSTADDR(rcb->ti_hostaddr) = TI_GCR_NIC_ADDR(TI_GCR_CMDRING);
+	rcb = &sc->ti_rdata.ti_info->ti_cmd_rcb;
+	ti_hostaddr64(&rcb->ti_hostaddr, TI_GCR_NIC_ADDR(TI_GCR_CMDRING));
 	rcb->ti_flags = 0;
 	rcb->ti_max_len = 0;
 	for (i = 0; i < TI_CMD_RING_CNT; i++) {
@@ -2116,61 +2225,66 @@ ti_gibinit(sc)
 	 * We re-use the current stats buffer for this to
 	 * conserve memory.
 	 */
-	TI_HOSTADDR(sc->ti_rdata->ti_info.ti_refresh_stats_ptr) =
-	    rdphys + TI_RD_OFF(ti_info.ti_stats);
+	bzero(&sc->ti_rdata.ti_info->ti_stats, sizeof(struct ti_stats));
+	ti_hostaddr64(&sc->ti_rdata.ti_info->ti_refresh_stats_ptr,
+	    sc->ti_rdata.ti_info_paddr + offsetof(struct ti_gib, ti_stats));
 
 	/* Set up the standard receive ring. */
-	rcb = &sc->ti_rdata->ti_info.ti_std_rx_rcb;
-	TI_HOSTADDR(rcb->ti_hostaddr) = rdphys + TI_RD_OFF(ti_rx_std_ring);
+	rcb = &sc->ti_rdata.ti_info->ti_std_rx_rcb;
+	ti_hostaddr64(&rcb->ti_hostaddr, sc->ti_rdata.ti_rx_std_ring_paddr);
 	rcb->ti_max_len = TI_FRAMELEN;
 	rcb->ti_flags = 0;
-	if (sc->ti_ifp->if_hwassist)
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM |
 		     TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM;
-	rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
+	if (sc->ti_ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
+		rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
 
 	/* Set up the jumbo receive ring. */
-	rcb = &sc->ti_rdata->ti_info.ti_jumbo_rx_rcb;
-	TI_HOSTADDR(rcb->ti_hostaddr) = rdphys + TI_RD_OFF(ti_rx_jumbo_ring);
+	rcb = &sc->ti_rdata.ti_info->ti_jumbo_rx_rcb;
+	ti_hostaddr64(&rcb->ti_hostaddr, sc->ti_rdata.ti_rx_jumbo_ring_paddr);
 
-#ifdef TI_PRIVATE_JUMBOS
-	rcb->ti_max_len = TI_JUMBO_FRAMELEN;
+#ifndef TI_SF_BUF_JUMBO
+	rcb->ti_max_len = MJUM9BYTES - ETHER_ALIGN;
 	rcb->ti_flags = 0;
 #else
 	rcb->ti_max_len = PAGE_SIZE;
 	rcb->ti_flags = TI_RCB_FLAG_USE_EXT_RX_BD;
 #endif
-	if (sc->ti_ifp->if_hwassist)
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM |
 		     TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM;
-	rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
+	if (sc->ti_ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
+		rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
 
 	/*
 	 * Set up the mini ring. Only activated on the
 	 * Tigon 2 but the slot in the config block is
 	 * still there on the Tigon 1.
 	 */
-	rcb = &sc->ti_rdata->ti_info.ti_mini_rx_rcb;
-	TI_HOSTADDR(rcb->ti_hostaddr) = rdphys + TI_RD_OFF(ti_rx_mini_ring);
+	rcb = &sc->ti_rdata.ti_info->ti_mini_rx_rcb;
+	ti_hostaddr64(&rcb->ti_hostaddr, sc->ti_rdata.ti_rx_mini_ring_paddr);
 	rcb->ti_max_len = MHLEN - ETHER_ALIGN;
 	if (sc->ti_hwrev == TI_HWREV_TIGON)
 		rcb->ti_flags = TI_RCB_FLAG_RING_DISABLED;
 	else
 		rcb->ti_flags = 0;
-	if (sc->ti_ifp->if_hwassist)
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
 		rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM |
 		     TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM;
-	rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
+	if (sc->ti_ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
+		rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
 
 	/*
 	 * Set up the receive return ring.
 	 */
-	rcb = &sc->ti_rdata->ti_info.ti_return_rcb;
-	TI_HOSTADDR(rcb->ti_hostaddr) = rdphys + TI_RD_OFF(ti_rx_return_ring);
+	rcb = &sc->ti_rdata.ti_info->ti_return_rcb;
+	ti_hostaddr64(&rcb->ti_hostaddr, sc->ti_rdata.ti_rx_return_ring_paddr);
 	rcb->ti_flags = 0;
 	rcb->ti_max_len = TI_RETURN_RING_CNT;
-	TI_HOSTADDR(sc->ti_rdata->ti_info.ti_return_prodidx_ptr) =
-	    rdphys + TI_RD_OFF(ti_return_prodidx_r);
+	ti_hostaddr64(&sc->ti_rdata.ti_info->ti_return_prodidx_ptr,
+	    sc->ti_rdata.ti_status_paddr +
+	    offsetof(struct ti_status, ti_return_prodidx_r));
 
 	/*
 	 * Set up the tx ring. Note: for the Tigon 2, we have the option
@@ -2182,31 +2296,42 @@ ti_gibinit(sc)
 	 * a Tigon 1 chip.
 	 */
 	CSR_WRITE_4(sc, TI_WINBASE, TI_TX_RING_BASE);
-	bzero((char *)sc->ti_rdata->ti_tx_ring,
-	    TI_TX_RING_CNT * sizeof(struct ti_tx_desc));
-	rcb = &sc->ti_rdata->ti_info.ti_tx_rcb;
+	if (sc->ti_rdata.ti_tx_ring != NULL)
+		bzero(sc->ti_rdata.ti_tx_ring, TI_TX_RING_SZ);
+	rcb = &sc->ti_rdata.ti_info->ti_tx_rcb;
 	if (sc->ti_hwrev == TI_HWREV_TIGON)
 		rcb->ti_flags = 0;
 	else
 		rcb->ti_flags = TI_RCB_FLAG_HOST_RING;
-	rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
-	if (sc->ti_ifp->if_hwassist)
+	if (sc->ti_ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
+		rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST;
+	if (sc->ti_ifp->if_capenable & IFCAP_TXCSUM)
 		rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM |
 		     TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM;
 	rcb->ti_max_len = TI_TX_RING_CNT;
 	if (sc->ti_hwrev == TI_HWREV_TIGON)
-		TI_HOSTADDR(rcb->ti_hostaddr) = TI_TX_RING_BASE;
+		ti_hostaddr64(&rcb->ti_hostaddr, TI_TX_RING_BASE);
 	else
-		TI_HOSTADDR(rcb->ti_hostaddr) = rdphys + TI_RD_OFF(ti_tx_ring);
-	TI_HOSTADDR(sc->ti_rdata->ti_info.ti_tx_considx_ptr) =
-	    rdphys + TI_RD_OFF(ti_tx_considx_r);
-
-	bus_dmamap_sync(sc->ti_rdata_dmat, sc->ti_rdata_dmamap,
-	    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
-
-	/* Set up tuneables */
+		ti_hostaddr64(&rcb->ti_hostaddr,
+		    sc->ti_rdata.ti_tx_ring_paddr);
+	ti_hostaddr64(&sc->ti_rdata.ti_info->ti_tx_considx_ptr,
+	    sc->ti_rdata.ti_status_paddr +
+	    offsetof(struct ti_status, ti_tx_considx_r));
+
+	bus_dmamap_sync(sc->ti_cdata.ti_gib_tag, sc->ti_cdata.ti_gib_map,
+	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	bus_dmamap_sync(sc->ti_cdata.ti_status_tag, sc->ti_cdata.ti_status_map,
+	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	bus_dmamap_sync(sc->ti_cdata.ti_event_ring_tag,
+	    sc->ti_cdata.ti_event_ring_map,
+	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	if (sc->ti_rdata.ti_tx_ring != NULL)
+		bus_dmamap_sync(sc->ti_cdata.ti_tx_ring_tag,
+		    sc->ti_cdata.ti_tx_ring_map, BUS_DMASYNC_PREWRITE);
+
+	/* Set up tunables */
 #if 0
-	if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN))
+	if (ifp->if_mtu > ETHERMTU + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)
 		CSR_WRITE_4(sc, TI_GCR_RX_COAL_TICKS,
 		    (sc->ti_rx_coal_ticks / 10));
 	else
@@ -2228,32 +2353,14 @@ ti_gibinit(sc)
 	return (0);
 }
 
-static void
-ti_rdata_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
-{
-	struct ti_softc *sc;
-
-	sc = arg;
-	if (error || nseg != 1)
-		return;
-
-	/*
-	 * All of the Tigon data structures need to live at <4GB.  This
-	 * cast is fine since busdma was told about this constraint.
-	 */
-	sc->ti_rdata_phys = segs[0].ds_addr;
-	return;
-}
-	
 /*
  * Probe for a Tigon chip. Check the PCI vendor and device IDs
  * against our list and return its name if we find a match.
  */
 static int
-ti_probe(dev)
-	device_t		dev;
+ti_probe(device_t dev)
 {
-	const struct ti_type	*t;
+	const struct ti_type *t;
 
 	t = ti_devs;
 
@@ -2270,16 +2377,14 @@ ti_probe(dev)
 }
 
 static int
-ti_attach(dev)
-	device_t		dev;
+ti_attach(device_t dev)
 {
-	struct ifnet		*ifp;
-	struct ti_softc		*sc;
-	int			error = 0, rid;
-	u_char			eaddr[6];
+	struct ifnet *ifp;
+	struct ti_softc *sc;
+	int error = 0, rid;
+	u_char eaddr[6];
 
 	sc = device_get_softc(dev);
-	sc->ti_unit = device_get_unit(dev);
 	sc->ti_dev = dev;
 
 	mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
@@ -2292,8 +2397,8 @@ ti_attach(dev)
 		error = ENOSPC;
 		goto fail;
 	}
-	sc->ti_ifp->if_capabilities = IFCAP_HWCSUM |
-	    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
+	sc->ti_ifp->if_hwassist = TI_CSUM_FEATURES;
+	sc->ti_ifp->if_capabilities = IFCAP_TXCSUM | IFCAP_RXCSUM;
 	sc->ti_ifp->if_capenable = sc->ti_ifp->if_capabilities;
 
 	/*
@@ -2301,7 +2406,7 @@ ti_attach(dev)
 	 */
 	pci_enable_busmaster(dev);
 
-	rid = TI_PCI_LOMEM;
+	rid = PCIR_BAR(0);
 	sc->ti_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 	    RF_ACTIVE);
 
@@ -2349,108 +2454,23 @@ ti_attach(dev)
 	 * the NIC). This means the MAC address is actually preceded
 	 * by two zero bytes. We need to skip over those.
 	 */
-	if (ti_read_eeprom(sc, eaddr,
-				TI_EE_MAC_OFFSET + 2, ETHER_ADDR_LEN)) {
+	if (ti_read_eeprom(sc, eaddr, TI_EE_MAC_OFFSET + 2, ETHER_ADDR_LEN)) {
 		device_printf(dev, "failed to read station address\n");
 		error = ENXIO;
 		goto fail;
 	}
 
-	/* Allocate the general information block and ring buffers. */
-	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
-				1, 0,			/* algnmnt, boundary */
-				BUS_SPACE_MAXADDR,	/* lowaddr */
-				BUS_SPACE_MAXADDR,	/* highaddr */
-				NULL, NULL,		/* filter, filterarg */
-				BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
-				0,			/* nsegments */
-				BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
-				0,			/* flags */
-				NULL, NULL,		/* lockfunc, lockarg */
-				&sc->ti_parent_dmat) != 0) {
-		device_printf(dev, "Failed to allocate parent dmat\n");
-		error = ENOMEM;
-		goto fail;
-	}
-
-	if (bus_dma_tag_create(sc->ti_parent_dmat,	/* parent */
-				PAGE_SIZE, 0,		/* algnmnt, boundary */
-				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
-				BUS_SPACE_MAXADDR,	/* highaddr */
-				NULL, NULL,		/* filter, filterarg */
-				sizeof(struct ti_ring_data),	/* maxsize */
-				1,			/* nsegments */
-				sizeof(struct ti_ring_data),	/* maxsegsize */
-				0,			/* flags */
-				NULL, NULL,		/* lockfunc, lockarg */
-				&sc->ti_rdata_dmat) != 0) {
-		device_printf(dev, "Failed to allocate rdata dmat\n");
-		error = ENOMEM;
-		goto fail;
-	}
-
-	if (bus_dmamem_alloc(sc->ti_rdata_dmat, (void**)&sc->ti_rdata,
-			     BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
-			     &sc->ti_rdata_dmamap) != 0) {
-		device_printf(dev, "Failed to allocate rdata memory\n");
-		error = ENOMEM;
-		goto fail;
-	}
-
-	if (bus_dmamap_load(sc->ti_rdata_dmat, sc->ti_rdata_dmamap,
-			    sc->ti_rdata, sizeof(struct ti_ring_data),
-			    ti_rdata_cb, sc, BUS_DMA_NOWAIT) != 0) {
-		device_printf(dev, "Failed to load rdata segments\n");
-		error = ENOMEM;
-		goto fail;
-	}
-
-	bzero(sc->ti_rdata, sizeof(struct ti_ring_data));
-
-	/* Try to allocate memory for jumbo buffers. */
-	if (ti_alloc_jumbo_mem(sc)) {
-		device_printf(dev, "jumbo buffer allocation failed\n");
-		error = ENXIO;
-		goto fail;
-	}
-
-	if (bus_dma_tag_create(sc->ti_parent_dmat,	/* parent */
-				1, 0,			/* algnmnt, boundary */
-				BUS_SPACE_MAXADDR,	/* lowaddr */
-				BUS_SPACE_MAXADDR,	/* highaddr */
-				NULL, NULL,		/* filter, filterarg */
-				MCLBYTES * TI_MAXTXSEGS,/* maxsize */
-				TI_MAXTXSEGS,		/* nsegments */
-				MCLBYTES,		/* maxsegsize */
-				0,			/* flags */
-				NULL, NULL,		/* lockfunc, lockarg */
-				&sc->ti_mbuftx_dmat) != 0) {
-		device_printf(dev, "Failed to allocate rdata dmat\n");
+	/* Allocate working area for memory dump. */
+	sc->ti_membuf = malloc(sizeof(uint8_t) * TI_WINLEN, M_DEVBUF, M_NOWAIT);
+	sc->ti_membuf2 = malloc(sizeof(uint8_t) * TI_WINLEN, M_DEVBUF,
+	    M_NOWAIT);
+	if (sc->ti_membuf == NULL || sc->ti_membuf2 == NULL) {
+		device_printf(dev, "cannot allocate memory buffer\n");
 		error = ENOMEM;
 		goto fail;
 	}
-
-	if (bus_dma_tag_create(sc->ti_parent_dmat,	/* parent */
-				1, 0,			/* algnmnt, boundary */
-				BUS_SPACE_MAXADDR,	/* lowaddr */
-				BUS_SPACE_MAXADDR,	/* highaddr */
-				NULL, NULL,		/* filter, filterarg */
-				MCLBYTES,		/* maxsize */
-				1,			/* nsegments */
-				MCLBYTES,		/* maxsegsize */
-				0,			/* flags */
-				NULL, NULL,		/* lockfunc, lockarg */
-				&sc->ti_mbufrx_dmat) != 0) {
-		device_printf(dev, "Failed to allocate rdata dmat\n");
-		error = ENOMEM;
-		goto fail;
-	}
-
-	if (ti_alloc_dmamaps(sc)) {
-		device_printf(dev, "dma map creation failed\n");
-		error = ENXIO;
+	if ((error = ti_dma_alloc(sc)) != 0)
 		goto fail;
-	}
 
 	/*
 	 * We really need a better way to tell a 1000baseTX card
@@ -2467,19 +2487,8 @@ ti_attach(dev)
 	    pci_get_device(dev) == NG_DEVICEID_GA620T)
 		sc->ti_copper = 1;
 
-	/* Set default tuneable values. */
-	sc->ti_stat_ticks = 2 * TI_TICKS_PER_SEC;
-#if 0
-	sc->ti_rx_coal_ticks = TI_TICKS_PER_SEC / 5000;
-#endif
-	sc->ti_rx_coal_ticks = 170;
-	sc->ti_tx_coal_ticks = TI_TICKS_PER_SEC / 500;
-	sc->ti_rx_max_coal_bds = 64;
-#if 0
-	sc->ti_tx_max_coal_bds = 128;
-#endif
-	sc->ti_tx_max_coal_bds = 32;
-	sc->ti_tx_buf_ratio = 21;
+	/* Set default tunable values. */
+	ti_sysctl_node(sc);
 
 	/* Set up ifnet structure */
 	ifp->if_softc = sc;
@@ -2488,8 +2497,10 @@ ti_attach(dev)
 	ifp->if_ioctl = ti_ioctl;
 	ifp->if_start = ti_start;
 	ifp->if_init = ti_init;
-	ifp->if_baudrate = 1000000000;
-	ifp->if_snd.ifq_maxlen = TI_TX_RING_CNT - 1;
+	ifp->if_baudrate = IF_Gbps(1UL);
+	ifp->if_snd.ifq_drv_maxlen = TI_TX_RING_CNT - 1;
+	IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
+	IFQ_SET_READY(&ifp->if_snd);
 
 	/* Set up ifmedia support. */
 	if (sc->ti_copper) {
@@ -2526,8 +2537,8 @@ ti_attach(dev)
 	 */
 
 	/* Register the device */
-	sc->dev = make_dev(&ti_cdevsw, sc->ti_unit, UID_ROOT, GID_OPERATOR,
-			   0600, "ti%d", sc->ti_unit);
+	sc->dev = make_dev(&ti_cdevsw, device_get_unit(dev), UID_ROOT,
+	    GID_OPERATOR, 0600, "ti%d", device_get_unit(dev));
 	sc->dev->si_drv1 = sc;
 
 	/*
@@ -2535,6 +2546,17 @@ ti_attach(dev)
 	 */
 	ether_ifattach(ifp, eaddr);
 
+	/* VLAN capability setup. */
+	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM |
+	    IFCAP_VLAN_HWTAGGING;
+	ifp->if_capenable = ifp->if_capabilities;
+	/* Tell the upper layer we support VLAN over-sized frames. */
+	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
+
+	/* Driver supports link state tracking. */
+	ifp->if_capabilities |= IFCAP_LINKSTATE;
+	ifp->if_capenable |= IFCAP_LINKSTATE;
+
 	/* Hook interrupt last to avoid having to lock softc */
 	error = bus_setup_intr(dev, sc->ti_irq, INTR_TYPE_NET|INTR_MPSAFE,
 	   NULL, ti_intr, sc, &sc->ti_intrhand);
@@ -2559,11 +2581,10 @@ fail:
  * allocated.
  */
 static int
-ti_detach(dev)
-	device_t		dev;
+ti_detach(device_t dev)
 {
-	struct ti_softc		*sc;
-	struct ifnet		*ifp;
+	struct ti_softc *sc;
+	struct ifnet *ifp;
 
 	sc = device_get_softc(dev);
 	if (sc->dev)
@@ -2580,37 +2601,23 @@ ti_detach(dev)
 	/* These should only be active if attach succeeded */
 	callout_drain(&sc->ti_watchdog);
 	bus_generic_detach(dev);
-	ti_free_dmamaps(sc);
+	ti_dma_free(sc);
 	ifmedia_removeall(&sc->ifmedia);
 
-#ifdef TI_PRIVATE_JUMBOS
-	if (sc->ti_cdata.ti_jumbo_buf)
-		bus_dmamem_free(sc->ti_jumbo_dmat, sc->ti_cdata.ti_jumbo_buf,
-		    sc->ti_jumbo_dmamap);
-#endif
-	if (sc->ti_jumbo_dmat)
-		bus_dma_tag_destroy(sc->ti_jumbo_dmat);
-	if (sc->ti_mbuftx_dmat)
-		bus_dma_tag_destroy(sc->ti_mbuftx_dmat);
-	if (sc->ti_mbufrx_dmat)
-		bus_dma_tag_destroy(sc->ti_mbufrx_dmat);
-	if (sc->ti_rdata)
-		bus_dmamem_free(sc->ti_rdata_dmat, sc->ti_rdata,
-				sc->ti_rdata_dmamap);
-	if (sc->ti_rdata_dmat)
-		bus_dma_tag_destroy(sc->ti_rdata_dmat);
-	if (sc->ti_parent_dmat)
-		bus_dma_tag_destroy(sc->ti_parent_dmat);
 	if (sc->ti_intrhand)
 		bus_teardown_intr(dev, sc->ti_irq, sc->ti_intrhand);
 	if (sc->ti_irq)
 		bus_release_resource(dev, SYS_RES_IRQ, 0, sc->ti_irq);
 	if (sc->ti_res) {
-		bus_release_resource(dev, SYS_RES_MEMORY, TI_PCI_LOMEM,
+		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0),
 		    sc->ti_res);
 	}
 	if (ifp)
 		if_free(ifp);
+	if (sc->ti_membuf)
+		free(sc->ti_membuf, M_DEVBUF);
+	if (sc->ti_membuf2)
+		free(sc->ti_membuf2, M_DEVBUF);
 
 	mtx_destroy(&sc->ti_mtx);
 
@@ -2674,6 +2681,59 @@ ti_hdr_split(struct mbuf *top, int hdr_len, int pkt_len, int idx)
 }
 #endif /* TI_JUMBO_HDRSPLIT */
 
+static void
+ti_discard_std(struct ti_softc *sc, int i)
+{
+
+	struct ti_rx_desc *r;
+
+	r = &sc->ti_rdata.ti_rx_std_ring[i];
+	r->ti_len = MCLBYTES - ETHER_ALIGN;
+	r->ti_type = TI_BDTYPE_RECV_BD;
+	r->ti_flags = 0;
+	r->ti_vlan_tag = 0;
+	r->ti_tcp_udp_cksum = 0;
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
+		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM;
+	r->ti_idx = i;
+}
+
+static void
+ti_discard_mini(struct ti_softc *sc, int i)
+{
+
+	struct ti_rx_desc *r;
+
+	r = &sc->ti_rdata.ti_rx_mini_ring[i];
+	r->ti_len = MHLEN - ETHER_ALIGN;
+	r->ti_type = TI_BDTYPE_RECV_BD;
+	r->ti_flags = TI_BDFLAG_MINI_RING;
+	r->ti_vlan_tag = 0;
+	r->ti_tcp_udp_cksum = 0;
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
+		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM;
+	r->ti_idx = i;
+}
+
+#ifndef TI_SF_BUF_JUMBO
+static void
+ti_discard_jumbo(struct ti_softc *sc, int i)
+{
+
+	struct ti_rx_desc *r;
+
+	r = &sc->ti_rdata.ti_rx_jumbo_ring[i];
+	r->ti_len = MJUM9BYTES - ETHER_ALIGN;
+	r->ti_type = TI_BDTYPE_RECV_JUMBO_BD;
+	r->ti_flags = TI_BDFLAG_JUMBO_RING;
+	r->ti_vlan_tag = 0;
+	r->ti_tcp_udp_cksum = 0;
+	if (sc->ti_ifp->if_capenable & IFCAP_RXCSUM)
+		r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM;
+	r->ti_idx = i;
+}
+#endif
+
 /*
  * Frame reception handling. This is called if there's a frame
  * on the receive return list.
@@ -2686,114 +2746,136 @@ ti_hdr_split(struct mbuf *top, int hdr_len, int pkt_len, int idx)
  */
 
 static void
-ti_rxeof(sc)
-	struct ti_softc		*sc;
+ti_rxeof(struct ti_softc *sc)
 {
-	bus_dmamap_t		map;
-	struct ifnet		*ifp;
-	struct ti_cmd_desc	cmd;
+	struct ifnet *ifp;
+#ifdef TI_SF_BUF_JUMBO
+	bus_dmamap_t map;
+#endif
+	struct ti_cmd_desc cmd;
+	int jumbocnt, minicnt, stdcnt, ti_len;
 
 	TI_LOCK_ASSERT(sc);
 
 	ifp = sc->ti_ifp;
 
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_std_ring_tag,
+	    sc->ti_cdata.ti_rx_std_ring_map, BUS_DMASYNC_POSTWRITE);
+	if (ifp->if_mtu > ETHERMTU + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_ring_tag,
+		    sc->ti_cdata.ti_rx_jumbo_ring_map, BUS_DMASYNC_POSTWRITE);
+	if (sc->ti_rdata.ti_rx_mini_ring != NULL)
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_mini_ring_tag,
+		    sc->ti_cdata.ti_rx_mini_ring_map, BUS_DMASYNC_POSTWRITE);
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_return_ring_tag,
+	    sc->ti_cdata.ti_rx_return_ring_map, BUS_DMASYNC_POSTREAD);
+
+	jumbocnt = minicnt = stdcnt = 0;
 	while (sc->ti_rx_saved_considx != sc->ti_return_prodidx.ti_idx) {
-		struct ti_rx_desc	*cur_rx;
-		u_int32_t		rxidx;
-		struct mbuf		*m = NULL;
-		u_int16_t		vlan_tag = 0;
-		int			have_tag = 0;
+		struct ti_rx_desc *cur_rx;
+		uint32_t rxidx;
+		struct mbuf *m = NULL;
+		uint16_t vlan_tag = 0;
+		int have_tag = 0;
 
 		cur_rx =
-		    &sc->ti_rdata->ti_rx_return_ring[sc->ti_rx_saved_considx];
+		    &sc->ti_rdata.ti_rx_return_ring[sc->ti_rx_saved_considx];
 		rxidx = cur_rx->ti_idx;
+		ti_len = cur_rx->ti_len;
 		TI_INC(sc->ti_rx_saved_considx, TI_RETURN_RING_CNT);
 
 		if (cur_rx->ti_flags & TI_BDFLAG_VLAN_TAG) {
 			have_tag = 1;
-			vlan_tag = cur_rx->ti_vlan_tag & 0xfff;
+			vlan_tag = cur_rx->ti_vlan_tag;
 		}
 
 		if (cur_rx->ti_flags & TI_BDFLAG_JUMBO_RING) {
-
+			jumbocnt++;
 			TI_INC(sc->ti_jumbo, TI_JUMBO_RX_RING_CNT);
 			m = sc->ti_cdata.ti_rx_jumbo_chain[rxidx];
+#ifndef TI_SF_BUF_JUMBO
+			if (cur_rx->ti_flags & TI_BDFLAG_ERROR) {
+				ifp->if_ierrors++;
+				ti_discard_jumbo(sc, rxidx);
+				continue;
+			}
+			if (ti_newbuf_jumbo(sc, rxidx, NULL) != 0) {
+				ifp->if_iqdrops++;
+				ti_discard_jumbo(sc, rxidx);
+				continue;
+			}
+			m->m_len = ti_len;
+#else /* !TI_SF_BUF_JUMBO */
 			sc->ti_cdata.ti_rx_jumbo_chain[rxidx] = NULL;
 			map = sc->ti_cdata.ti_rx_jumbo_maps[rxidx];
-			bus_dmamap_sync(sc->ti_jumbo_dmat, map,
+			bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_tag, map,
 			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(sc->ti_jumbo_dmat, map);
+			bus_dmamap_unload(sc->ti_cdata.ti_rx_jumbo_tag, map);
 			if (cur_rx->ti_flags & TI_BDFLAG_ERROR) {
 				ifp->if_ierrors++;
 				ti_newbuf_jumbo(sc, sc->ti_jumbo, m);
 				continue;
 			}
 			if (ti_newbuf_jumbo(sc, sc->ti_jumbo, NULL) == ENOBUFS) {
-				ifp->if_ierrors++;
+				ifp->if_iqdrops++;
 				ti_newbuf_jumbo(sc, sc->ti_jumbo, m);
 				continue;
 			}
-#ifdef TI_PRIVATE_JUMBOS
-			m->m_len = cur_rx->ti_len;
-#else /* TI_PRIVATE_JUMBOS */
 #ifdef TI_JUMBO_HDRSPLIT
 			if (sc->ti_hdrsplit)
 				ti_hdr_split(m, TI_HOSTADDR(cur_rx->ti_addr),
-					     cur_rx->ti_len, rxidx);
+					     ti_len, rxidx);
 			else
 #endif /* TI_JUMBO_HDRSPLIT */
-			m_adj(m, cur_rx->ti_len - m->m_pkthdr.len);
-#endif /* TI_PRIVATE_JUMBOS */
+			m_adj(m, ti_len - m->m_pkthdr.len);
+#endif /* TI_SF_BUF_JUMBO */
 		} else if (cur_rx->ti_flags & TI_BDFLAG_MINI_RING) {
+			minicnt++;
 			TI_INC(sc->ti_mini, TI_MINI_RX_RING_CNT);
 			m = sc->ti_cdata.ti_rx_mini_chain[rxidx];
-			sc->ti_cdata.ti_rx_mini_chain[rxidx] = NULL;
-			map = sc->ti_cdata.ti_rx_mini_maps[rxidx];
-			bus_dmamap_sync(sc->ti_mbufrx_dmat, map,
-			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(sc->ti_mbufrx_dmat, map);
 			if (cur_rx->ti_flags & TI_BDFLAG_ERROR) {
 				ifp->if_ierrors++;
-				ti_newbuf_mini(sc, sc->ti_mini, m);
+				ti_discard_mini(sc, rxidx);
 				continue;
 			}
-			if (ti_newbuf_mini(sc, sc->ti_mini, NULL) == ENOBUFS) {
-				ifp->if_ierrors++;
-				ti_newbuf_mini(sc, sc->ti_mini, m);
+			if (ti_newbuf_mini(sc, rxidx) != 0) {
+				ifp->if_iqdrops++;
+				ti_discard_mini(sc, rxidx);
 				continue;
 			}
-			m->m_len = cur_rx->ti_len;
+			m->m_len = ti_len;
 		} else {
+			stdcnt++;
 			TI_INC(sc->ti_std, TI_STD_RX_RING_CNT);
 			m = sc->ti_cdata.ti_rx_std_chain[rxidx];
-			sc->ti_cdata.ti_rx_std_chain[rxidx] = NULL;
-			map = sc->ti_cdata.ti_rx_std_maps[rxidx];
-			bus_dmamap_sync(sc->ti_mbufrx_dmat, map,
-			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(sc->ti_mbufrx_dmat, map);
 			if (cur_rx->ti_flags & TI_BDFLAG_ERROR) {
 				ifp->if_ierrors++;
-				ti_newbuf_std(sc, sc->ti_std, m);
+				ti_discard_std(sc, rxidx);
 				continue;
 			}
-			if (ti_newbuf_std(sc, sc->ti_std, NULL) == ENOBUFS) {
-				ifp->if_ierrors++;
-				ti_newbuf_std(sc, sc->ti_std, m);
+			if (ti_newbuf_std(sc, rxidx) != 0) {
+				ifp->if_iqdrops++;
+				ti_discard_std(sc, rxidx);
 				continue;
 			}
-			m->m_len = cur_rx->ti_len;
+			m->m_len = ti_len;
 		}
 
-		m->m_pkthdr.len = cur_rx->ti_len;
+		m->m_pkthdr.len = ti_len;
 		ifp->if_ipackets++;
 		m->m_pkthdr.rcvif = ifp;
 
-		if (ifp->if_hwassist) {
-			m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED |
-			    CSUM_DATA_VALID;
-			if ((cur_rx->ti_ip_cksum ^ 0xffff) == 0)
-				m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
-			m->m_pkthdr.csum_data = cur_rx->ti_tcp_udp_cksum;
+		if (ifp->if_capenable & IFCAP_RXCSUM) {
+			if (cur_rx->ti_flags & TI_BDFLAG_IP_CKSUM) {
+				m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
+				if ((cur_rx->ti_ip_cksum ^ 0xffff) == 0)
+					m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
+			}
+			if (cur_rx->ti_flags & TI_BDFLAG_TCP_UDP_CKSUM) {
+				m->m_pkthdr.csum_data =
+				    cur_rx->ti_tcp_udp_cksum;
+				m->m_pkthdr.csum_flags |= CSUM_DATA_VALID;
+			}
 		}
 
 		/*
@@ -2809,31 +2891,48 @@ ti_rxeof(sc)
 		TI_LOCK(sc);
 	}
 
+	bus_dmamap_sync(sc->ti_cdata.ti_rx_return_ring_tag,
+	    sc->ti_cdata.ti_rx_return_ring_map, BUS_DMASYNC_PREREAD);
 	/* Only necessary on the Tigon 1. */
 	if (sc->ti_hwrev == TI_HWREV_TIGON)
 		CSR_WRITE_4(sc, TI_GCR_RXRETURNCONS_IDX,
 		    sc->ti_rx_saved_considx);
 
-	TI_UPDATE_STDPROD(sc, sc->ti_std);
-	TI_UPDATE_MINIPROD(sc, sc->ti_mini);
-	TI_UPDATE_JUMBOPROD(sc, sc->ti_jumbo);
+	if (stdcnt > 0) {
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_std_ring_tag,
+		    sc->ti_cdata.ti_rx_std_ring_map, BUS_DMASYNC_PREWRITE);
+		TI_UPDATE_STDPROD(sc, sc->ti_std);
+	}
+	if (minicnt > 0) {
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_mini_ring_tag,
+		    sc->ti_cdata.ti_rx_mini_ring_map, BUS_DMASYNC_PREWRITE);
+		TI_UPDATE_MINIPROD(sc, sc->ti_mini);
+	}
+	if (jumbocnt > 0) {
+		bus_dmamap_sync(sc->ti_cdata.ti_rx_jumbo_ring_tag,
+		    sc->ti_cdata.ti_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE);
+		TI_UPDATE_JUMBOPROD(sc, sc->ti_jumbo);
+	}
 }
 
 static void
-ti_txeof(sc)
-	struct ti_softc		*sc;
+ti_txeof(struct ti_softc *sc)
 {
-	struct ti_txdesc	*txd;
-	struct ti_tx_desc	txdesc;
-	struct ti_tx_desc	*cur_tx = NULL;
-	struct ifnet		*ifp;
-	int			idx;
+	struct ti_txdesc *txd;
+	struct ti_tx_desc txdesc;
+	struct ti_tx_desc *cur_tx = NULL;
+	struct ifnet *ifp;
+	int idx;
 
 	ifp = sc->ti_ifp;
 
 	txd = STAILQ_FIRST(&sc->ti_cdata.ti_txbusyq);
 	if (txd == NULL)
 		return;
+
+	if (sc->ti_rdata.ti_tx_ring != NULL)
+		bus_dmamap_sync(sc->ti_cdata.ti_tx_ring_tag,
+		    sc->ti_cdata.ti_tx_ring_map, BUS_DMASYNC_POSTWRITE);
 	/*
 	 * Go through our tx ring and free mbufs for those
 	 * frames that have been sent.
@@ -2845,14 +2944,14 @@ ti_txeof(sc)
 			    sizeof(txdesc), &txdesc);
 			cur_tx = &txdesc;
 		} else
-			cur_tx = &sc->ti_rdata->ti_tx_ring[idx];
+			cur_tx = &sc->ti_rdata.ti_tx_ring[idx];
 		sc->ti_txcnt--;
 		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 		if ((cur_tx->ti_flags & TI_BDFLAG_END) == 0)
 			continue;
-		bus_dmamap_sync(sc->ti_mbuftx_dmat, txd->tx_dmamap,
+		bus_dmamap_sync(sc->ti_cdata.ti_tx_tag, txd->tx_dmamap,
 		    BUS_DMASYNC_POSTWRITE);
-		bus_dmamap_unload(sc->ti_mbuftx_dmat, txd->tx_dmamap);
+		bus_dmamap_unload(sc->ti_cdata.ti_tx_tag, txd->tx_dmamap);
 
 		ifp->if_opackets++;
 		m_freem(txd->tx_m);
@@ -2862,72 +2961,73 @@ ti_txeof(sc)
 		txd = STAILQ_FIRST(&sc->ti_cdata.ti_txbusyq);
 	}
 	sc->ti_tx_saved_considx = idx;
-
-	sc->ti_timer = sc->ti_txcnt > 0 ? 5 : 0;
+	if (sc->ti_txcnt == 0)
+		sc->ti_timer = 0;
 }
 
 static void
-ti_intr(xsc)
-	void			*xsc;
+ti_intr(void *xsc)
 {
-	struct ti_softc		*sc;
-	struct ifnet		*ifp;
+	struct ti_softc *sc;
+	struct ifnet *ifp;
 
 	sc = xsc;
 	TI_LOCK(sc);
 	ifp = sc->ti_ifp;
 
-/*#ifdef notdef*/
-	/* Avoid this for now -- checking this register is expensive. */
 	/* Make sure this is really our interrupt. */
 	if (!(CSR_READ_4(sc, TI_MISC_HOST_CTL) & TI_MHC_INTSTATE)) {
 		TI_UNLOCK(sc);
 		return;
 	}
-/*#endif*/
 
 	/* Ack interrupt and stop others from occuring. */
 	CSR_WRITE_4(sc, TI_MB_HOSTINTR, 1);
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		bus_dmamap_sync(sc->ti_cdata.ti_status_tag,
+		    sc->ti_cdata.ti_status_map, BUS_DMASYNC_POSTREAD);
 		/* Check RX return ring producer/consumer */
 		ti_rxeof(sc);
 
 		/* Check TX ring producer/consumer */
 		ti_txeof(sc);
+		bus_dmamap_sync(sc->ti_cdata.ti_status_tag,
+		    sc->ti_cdata.ti_status_map, BUS_DMASYNC_PREREAD);
 	}
 
 	ti_handle_events(sc);
 
-	/* Re-enable interrupts. */
-	CSR_WRITE_4(sc, TI_MB_HOSTINTR, 0);
-
-	if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
-	    ifp->if_snd.ifq_head != NULL)
-		ti_start_locked(ifp);
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		/* Re-enable interrupts. */
+		CSR_WRITE_4(sc, TI_MB_HOSTINTR, 0);
+		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+			ti_start_locked(ifp);
+	}
 
 	TI_UNLOCK(sc);
 }
 
 static void
-ti_stats_update(sc)
-	struct ti_softc		*sc;
+ti_stats_update(struct ti_softc *sc)
 {
-	struct ifnet		*ifp;
+	struct ifnet *ifp;
+	struct ti_stats *s;
 
 	ifp = sc->ti_ifp;
 
-	bus_dmamap_sync(sc->ti_rdata_dmat, sc->ti_rdata_dmamap,
+	if (sc->ti_stat_ticks == 0)
+		return;
+	bus_dmamap_sync(sc->ti_cdata.ti_gib_tag, sc->ti_cdata.ti_gib_map,
 	    BUS_DMASYNC_POSTREAD);
 
-	ifp->if_collisions +=
-	   (sc->ti_rdata->ti_info.ti_stats.dot3StatsSingleCollisionFrames +
-	   sc->ti_rdata->ti_info.ti_stats.dot3StatsMultipleCollisionFrames +
-	   sc->ti_rdata->ti_info.ti_stats.dot3StatsExcessiveCollisions +
-	   sc->ti_rdata->ti_info.ti_stats.dot3StatsLateCollisions) -
-	   ifp->if_collisions;
+	s = &sc->ti_rdata.ti_info->ti_stats;
+	ifp->if_collisions += (s->dot3StatsSingleCollisionFrames +
+	   s->dot3StatsMultipleCollisionFrames +
+	   s->dot3StatsExcessiveCollisions + s->dot3StatsLateCollisions) -
+	    ifp->if_collisions;
 
-	bus_dmamap_sync(sc->ti_rdata_dmat, sc->ti_rdata_dmamap,
+	bus_dmamap_sync(sc->ti_cdata.ti_gib_tag, sc->ti_cdata.ti_gib_map,
 	    BUS_DMASYNC_PREREAD);
 }
 
@@ -2936,22 +3036,20 @@ ti_stats_update(sc)
  * pointers to descriptors.
  */
 static int
-ti_encap(sc, m_head)
-	struct ti_softc		*sc;
-	struct mbuf		**m_head;
-{
-	struct ti_txdesc	*txd;
-	struct ti_tx_desc	*f;
-	struct ti_tx_desc	txdesc;
-	struct mbuf		*m;
-	bus_dma_segment_t	txsegs[TI_MAXTXSEGS];
-	u_int16_t		csum_flags;
-	int			error, frag, i, nseg;
+ti_encap(struct ti_softc *sc, struct mbuf **m_head)
+{
+	struct ti_txdesc *txd;
+	struct ti_tx_desc *f;
+	struct ti_tx_desc txdesc;
+	struct mbuf *m;
+	bus_dma_segment_t txsegs[TI_MAXTXSEGS];
+	uint16_t csum_flags;
+	int error, frag, i, nseg;
 
 	if ((txd = STAILQ_FIRST(&sc->ti_cdata.ti_txfreeq)) == NULL)
 		return (ENOBUFS);
 
-	error = bus_dmamap_load_mbuf_sg(sc->ti_mbuftx_dmat, txd->tx_dmamap,
+	error = bus_dmamap_load_mbuf_sg(sc->ti_cdata.ti_tx_tag, txd->tx_dmamap,
 	    *m_head, txsegs, &nseg, 0);
 	if (error == EFBIG) {
 		m = m_defrag(*m_head, M_DONTWAIT);
@@ -2961,7 +3059,7 @@ ti_encap(sc, m_head)
 			return (ENOMEM);
 		}
 		*m_head = m;
-		error = bus_dmamap_load_mbuf_sg(sc->ti_mbuftx_dmat,
+		error = bus_dmamap_load_mbuf_sg(sc->ti_cdata.ti_tx_tag,
 		    txd->tx_dmamap, *m_head, txsegs, &nseg, 0);
 		if (error) {
 			m_freem(*m_head);
@@ -2977,9 +3075,11 @@ ti_encap(sc, m_head)
 	}
 
 	if (sc->ti_txcnt + nseg >= TI_TX_RING_CNT) {
-		bus_dmamap_unload(sc->ti_mbuftx_dmat, txd->tx_dmamap);
+		bus_dmamap_unload(sc->ti_cdata.ti_tx_tag, txd->tx_dmamap);
 		return (ENOBUFS);
 	}
+	bus_dmamap_sync(sc->ti_cdata.ti_tx_tag, txd->tx_dmamap,
+	    BUS_DMASYNC_PREWRITE);
 
 	m = *m_head;
 	csum_flags = 0;
@@ -2994,24 +3094,19 @@ ti_encap(sc, m_head)
 			csum_flags |= TI_BDFLAG_IP_FRAG;
 	}
 
-	bus_dmamap_sync(sc->ti_mbuftx_dmat, txd->tx_dmamap,
-	    BUS_DMASYNC_PREWRITE);
-	bus_dmamap_sync(sc->ti_rdata_dmat, sc->ti_rdata_dmamap,
-	    BUS_DMASYNC_PREWRITE);
-
 	frag = sc->ti_tx_saved_prodidx;
 	for (i = 0; i < nseg; i++) {
 		if (sc->ti_hwrev == TI_HWREV_TIGON) {
 			bzero(&txdesc, sizeof(txdesc));
 			f = &txdesc;
 		} else
-			f = &sc->ti_rdata->ti_tx_ring[frag];
+			f = &sc->ti_rdata.ti_tx_ring[frag];
 		ti_hostaddr64(&f->ti_addr, txsegs[i].ds_addr);
 		f->ti_len = txsegs[i].ds_len;
 		f->ti_flags = csum_flags;
 		if (m->m_flags & M_VLANTAG) {
 			f->ti_flags |= TI_BDFLAG_VLAN_TAG;
-			f->ti_vlan_tag = m->m_pkthdr.ether_vtag & 0xfff;
+			f->ti_vlan_tag = m->m_pkthdr.ether_vtag;
 		} else {
 			f->ti_vlan_tag = 0;
 		}
@@ -3030,7 +3125,7 @@ ti_encap(sc, m_head)
 		ti_mem_write(sc, TI_TX_RING_BASE + frag * sizeof(txdesc),
 		    sizeof(txdesc), &txdesc);
 	} else
-		sc->ti_rdata->ti_tx_ring[frag].ti_flags |= TI_BDFLAG_END;
+		sc->ti_rdata.ti_tx_ring[frag].ti_flags |= TI_BDFLAG_END;
 
 	STAILQ_REMOVE_HEAD(&sc->ti_cdata.ti_txfreeq, tx_q);
 	STAILQ_INSERT_TAIL(&sc->ti_cdata.ti_txbusyq, txd, tx_q);
@@ -3041,10 +3136,9 @@ ti_encap(sc, m_head)
 }
 
 static void
-ti_start(ifp)
-	struct ifnet		*ifp;
+ti_start(struct ifnet *ifp)
 {
-	struct ti_softc		*sc;
+	struct ti_softc *sc;
 
 	sc = ifp->if_softc;
 	TI_LOCK(sc);
@@ -3057,18 +3151,17 @@ ti_start(ifp)
  * to the mbuf data regions directly in the transmit descriptors.
  */
 static void
-ti_start_locked(ifp)
-	struct ifnet		*ifp;
+ti_start_locked(struct ifnet *ifp)
 {
-	struct ti_softc		*sc;
-	struct mbuf		*m_head = NULL;
-	int			enq = 0;
+	struct ti_softc *sc;
+	struct mbuf *m_head = NULL;
+	int enq = 0;
 
 	sc = ifp->if_softc;
 
-	for (; ifp->if_snd.ifq_head != NULL &&
+	for (; !IFQ_DRV_IS_EMPTY(&ifp->if_snd) &&
 	    sc->ti_txcnt < (TI_TX_RING_CNT - 16);) {
-		IF_DEQUEUE(&ifp->if_snd, m_head);
+		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 		if (m_head == NULL)
 			break;
 
@@ -3084,7 +3177,7 @@ ti_start_locked(ifp)
 		    m_head->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) {
 			if ((TI_TX_RING_CNT - sc->ti_txcnt) <
 			    m_head->m_pkthdr.csum_data + 16) {
-				IF_PREPEND(&ifp->if_snd, m_head);
+				IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 				ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 				break;
 			}
@@ -3098,7 +3191,7 @@ ti_start_locked(ifp)
 		if (ti_encap(sc, &m_head)) {
 			if (m_head == NULL)
 				break;
-			IF_PREPEND(&ifp->if_snd, m_head);
+			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			break;
 		}
@@ -3112,6 +3205,9 @@ ti_start_locked(ifp)
 	}
 
 	if (enq > 0) {
+		if (sc->ti_rdata.ti_tx_ring != NULL)
+			bus_dmamap_sync(sc->ti_cdata.ti_tx_ring_tag,
+			    sc->ti_cdata.ti_tx_ring_map, BUS_DMASYNC_PREWRITE);
 		/* Transmit */
 		CSR_WRITE_4(sc, TI_MB_SENDPROD_IDX, sc->ti_tx_saved_prodidx);
 
@@ -3123,10 +3219,9 @@ ti_start_locked(ifp)
 }
 
 static void
-ti_init(xsc)
-	void			*xsc;
+ti_init(void *xsc)
 {
-	struct ti_softc		*sc;
+	struct ti_softc *sc;
 
 	sc = xsc;
 	TI_LOCK(sc);
@@ -3135,10 +3230,12 @@ ti_init(xsc)
 }
 
 static void
-ti_init_locked(xsc)
-	void			*xsc;
+ti_init_locked(void *xsc)
 {
-	struct ti_softc		*sc = xsc;
+	struct ti_softc *sc = xsc;
+
+	if (sc->ti_ifp->if_drv_flags & IFF_DRV_RUNNING)
+		return;
 
 	/* Cancel pending I/O and flush buffers. */
 	ti_stop(sc);
@@ -3150,21 +3247,20 @@ ti_init_locked(xsc)
 	}
 }
 
-static void ti_init2(sc)
-	struct ti_softc		*sc;
+static void ti_init2(struct ti_softc *sc)
 {
-	struct ti_cmd_desc	cmd;
-	struct ifnet		*ifp;
-	u_int8_t		*ea;
-	struct ifmedia		*ifm;
-	int			tmp;
+	struct ti_cmd_desc cmd;
+	struct ifnet *ifp;
+	uint8_t *ea;
+	struct ifmedia *ifm;
+	int tmp;
 
 	TI_LOCK_ASSERT(sc);
 
 	ifp = sc->ti_ifp;
 
 	/* Specify MTU and interface index. */
-	CSR_WRITE_4(sc, TI_GCR_IFINDEX, sc->ti_unit);
+	CSR_WRITE_4(sc, TI_GCR_IFINDEX, device_get_unit(sc->ti_dev));
 	CSR_WRITE_4(sc, TI_GCR_IFMTU, ifp->if_mtu +
 	    ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN);
 	TI_DO_CMD(TI_CMD_UPDATE_GENCOM, 0, 0);
@@ -3195,18 +3291,34 @@ static void ti_init2(sc)
 	}
 
 	/* Init RX ring. */
-	ti_init_rx_ring_std(sc);
+	if (ti_init_rx_ring_std(sc) != 0) {
+		/* XXX */
+		device_printf(sc->ti_dev, "no memory for std Rx buffers.\n");
+		return;
+	}
 
 	/* Init jumbo RX ring. */
-	if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN))
-		ti_init_rx_ring_jumbo(sc);
+	if (ifp->if_mtu > ETHERMTU + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN) {
+		if (ti_init_rx_ring_jumbo(sc) != 0) {
+			/* XXX */
+			device_printf(sc->ti_dev,
+			    "no memory for jumbo Rx buffers.\n");
+			return;
+		}
+	}
 
 	/*
 	 * If this is a Tigon 2, we can also configure the
 	 * mini ring.
 	 */
-	if (sc->ti_hwrev == TI_HWREV_TIGON_II)
-		ti_init_rx_ring_mini(sc);
+	if (sc->ti_hwrev == TI_HWREV_TIGON_II) {
+		if (ti_init_rx_ring_mini(sc) != 0) {
+			/* XXX */
+			device_printf(sc->ti_dev,
+			    "no memory for mini Rx buffers.\n");
+			return;
+		}
+	}
 
 	CSR_WRITE_4(sc, TI_GCR_RXRETURNCONS_IDX, 0);
 	sc->ti_rx_saved_considx = 0;
@@ -3233,7 +3345,7 @@ static void ti_init2(sc)
 	ifm = &sc->ifmedia;
 	tmp = ifm->ifm_media;
 	ifm->ifm_media = ifm->ifm_cur->ifm_media;
-	ti_ifmedia_upd(ifp);
+	ti_ifmedia_upd_locked(sc);
 	ifm->ifm_media = tmp;
 }
 
@@ -3241,15 +3353,26 @@ static void ti_init2(sc)
  * Set media options.
  */
 static int
-ti_ifmedia_upd(ifp)
-	struct ifnet		*ifp;
+ti_ifmedia_upd(struct ifnet *ifp)
 {
-	struct ti_softc		*sc;
-	struct ifmedia		*ifm;
-	struct ti_cmd_desc	cmd;
-	u_int32_t		flowctl;
+	struct ti_softc *sc;
+	int error;
 
 	sc = ifp->if_softc;
+	TI_LOCK(sc);
+	error = ti_ifmedia_upd(ifp);
+	TI_UNLOCK(sc);
+
+	return (error);
+}
+
+static int
+ti_ifmedia_upd_locked(struct ti_softc *sc)
+{
+	struct ifmedia *ifm;
+	struct ti_cmd_desc cmd;
+	uint32_t flowctl;
+
 	ifm = &sc->ifmedia;
 
 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
@@ -3343,20 +3466,22 @@ ti_ifmedia_upd(ifp)
  * Report current media status.
  */
 static void
-ti_ifmedia_sts(ifp, ifmr)
-	struct ifnet		*ifp;
-	struct ifmediareq	*ifmr;
+ti_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
-	struct ti_softc		*sc;
-	u_int32_t		media = 0;
+	struct ti_softc *sc;
+	uint32_t media = 0;
 
 	sc = ifp->if_softc;
 
+	TI_LOCK(sc);
+
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
 
-	if (sc->ti_linkstat == TI_EV_CODE_LINK_DOWN)
+	if (sc->ti_linkstat == TI_EV_CODE_LINK_DOWN) {
+		TI_UNLOCK(sc);
 		return;
+	}
 
 	ifmr->ifm_status |= IFM_ACTIVE;
 
@@ -3388,27 +3513,28 @@ ti_ifmedia_sts(ifp, ifmr)
 		if (media & TI_LNK_HALF_DUPLEX)
 			ifmr->ifm_active |= IFM_HDX;
 	}
+	TI_UNLOCK(sc);
 }
 
 static int
-ti_ioctl(ifp, command, data)
-	struct ifnet		*ifp;
-	u_long			command;
-	caddr_t			data;
+ti_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 {
-	struct ti_softc		*sc = ifp->if_softc;
-	struct ifreq		*ifr = (struct ifreq *) data;
-	int			mask, error = 0;
-	struct ti_cmd_desc	cmd;
+	struct ti_softc *sc = ifp->if_softc;
+	struct ifreq *ifr = (struct ifreq *) data;
+	struct ti_cmd_desc cmd;
+	int mask, error = 0;
 
 	switch (command) {
 	case SIOCSIFMTU:
 		TI_LOCK(sc);
-		if (ifr->ifr_mtu > TI_JUMBO_MTU)
+		if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > TI_JUMBO_MTU)
 			error = EINVAL;
 		else {
 			ifp->if_mtu = ifr->ifr_mtu;
-			ti_init_locked(sc);
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+				ti_init_locked(sc);
+			}
 		}
 		TI_UNLOCK(sc);
 		break;
@@ -3457,15 +3583,32 @@ ti_ioctl(ifp, command, data)
 	case SIOCSIFCAP:
 		TI_LOCK(sc);
 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
-		if (mask & IFCAP_HWCSUM) {
-			if (IFCAP_HWCSUM & ifp->if_capenable)
-				ifp->if_capenable &= ~IFCAP_HWCSUM;
-			else
-				ifp->if_capenable |= IFCAP_HWCSUM;
-			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		if ((mask & IFCAP_TXCSUM) != 0 &&
+		    (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+			if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
+				ifp->if_hwassist |= TI_CSUM_FEATURES;
+                        else
+				ifp->if_hwassist &= ~TI_CSUM_FEATURES;
+                }
+		if ((mask & IFCAP_RXCSUM) != 0 &&
+		    (ifp->if_capabilities & IFCAP_RXCSUM) != 0)
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+		if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
+		    (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0)
+                        ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
+		if ((mask & IFCAP_VLAN_HWCSUM) != 0 &&
+		    (ifp->if_capabilities & IFCAP_VLAN_HWCSUM) != 0)
+			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
+		if ((mask & (IFCAP_TXCSUM | IFCAP_RXCSUM |
+		    IFCAP_VLAN_HWTAGGING)) != 0) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 				ti_init_locked(sc);
+			}
 		}
 		TI_UNLOCK(sc);
+		VLAN_CAPABILITIES(ifp);
 		break;
 	default:
 		error = ether_ioctl(ifp, command, data);
@@ -3514,8 +3657,8 @@ static int
 ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
     struct thread *td)
 {
-	int error;
 	struct ti_softc *sc;
+	int error;
 
 	sc = dev->si_drv1;
 	if (sc == NULL)
@@ -3531,14 +3674,16 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		outstats = (struct ti_stats *)addr;
 
 		TI_LOCK(sc);
-		bcopy(&sc->ti_rdata->ti_info.ti_stats, outstats,
-		      sizeof(struct ti_stats));
+		bus_dmamap_sync(sc->ti_cdata.ti_gib_tag,
+		    sc->ti_cdata.ti_gib_map, BUS_DMASYNC_POSTREAD);
+		bcopy(&sc->ti_rdata.ti_info->ti_stats, outstats,
+		    sizeof(struct ti_stats));
 		TI_UNLOCK(sc);
 		break;
 	}
 	case TIIOCGETPARAMS:
 	{
-		struct ti_params	*params;
+		struct ti_params *params;
 
 		params = (struct ti_params *)addr;
 
@@ -3551,9 +3696,6 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		params->ti_tx_buf_ratio = sc->ti_tx_buf_ratio;
 		params->param_mask = TI_PARAM_ALL;
 		TI_UNLOCK(sc);
-
-		error = 0;
-
 		break;
 	}
 	case TIIOCSETPARAMS:
@@ -3598,13 +3740,10 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				    sc->ti_tx_buf_ratio);
 		}
 		TI_UNLOCK(sc);
-
-		error = 0;
-
 		break;
 	}
 	case TIIOCSETTRACE: {
-		ti_trace_type	trace_type;
+		ti_trace_type trace_type;
 
 		trace_type = *(ti_trace_type *)addr;
 
@@ -3613,15 +3752,14 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		 * this register to 0 should have the effect of disabling
 		 * tracing.
 		 */
+		TI_LOCK(sc);
 		CSR_WRITE_4(sc, TI_GCR_NIC_TRACING, trace_type);
-
-		error = 0;
-
+		TI_UNLOCK(sc);
 		break;
 	}
 	case TIIOCGETTRACE: {
-		struct ti_trace_buf	*trace_buf;
-		u_int32_t		trace_start, cur_trace_ptr, trace_len;
+		struct ti_trace_buf *trace_buf;
+		uint32_t trace_start, cur_trace_ptr, trace_len;
 
 		trace_buf = (struct ti_trace_buf *)addr;
 
@@ -3629,7 +3767,6 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		trace_start = CSR_READ_4(sc, TI_GCR_NICTRACE_START);
 		cur_trace_ptr = CSR_READ_4(sc, TI_GCR_NICTRACE_PTR);
 		trace_len = CSR_READ_4(sc, TI_GCR_NICTRACE_LEN);
-
 #if 0
 		if_printf(sc->ti_ifp, "trace_start = %#x, cur_trace_ptr = %#x, "
 		       "trace_len = %d\n", trace_start,
@@ -3637,24 +3774,20 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		if_printf(sc->ti_ifp, "trace_buf->buf_len = %d\n",
 		       trace_buf->buf_len);
 #endif
-
 		error = ti_copy_mem(sc, trace_start, min(trace_len,
-				    trace_buf->buf_len),
-				    (caddr_t)trace_buf->buf, 1, 1);
-
+		    trace_buf->buf_len), (caddr_t)trace_buf->buf, 1, 1);
 		if (error == 0) {
 			trace_buf->fill_len = min(trace_len,
-						  trace_buf->buf_len);
+			    trace_buf->buf_len);
 			if (cur_trace_ptr < trace_start)
 				trace_buf->cur_trace_ptr =
-					trace_start - cur_trace_ptr;
+				    trace_start - cur_trace_ptr;
 			else
 				trace_buf->cur_trace_ptr =
-					cur_trace_ptr - trace_start;
+				    cur_trace_ptr - trace_start;
 		} else
 			trace_buf->fill_len = 0;
 		TI_UNLOCK(sc);
-
 		break;
 	}
 
@@ -3676,13 +3809,12 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		 * you're interested in every ioctl, you'll only be
 		 * able to debug one board at a time.
 		 */
-		error = 0;
 		break;
 	case ALT_READ_TG_MEM:
 	case ALT_WRITE_TG_MEM:
 	{
 		struct tg_mem *mem_param;
-		u_int32_t sram_end, scratch_end;
+		uint32_t sram_end, scratch_end;
 
 		mem_param = (struct tg_mem *)addr;
 
@@ -3699,25 +3831,22 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		 * nothing else.
 		 */
 		TI_LOCK(sc);
-		if ((mem_param->tgAddr >= TI_BEG_SRAM)
-		 && ((mem_param->tgAddr + mem_param->len) <= sram_end)) {
+		if (mem_param->tgAddr >= TI_BEG_SRAM &&
+		    mem_param->tgAddr + mem_param->len <= sram_end) {
 			/*
 			 * In this instance, we always copy to/from user
 			 * space, so the user space argument is set to 1.
 			 */
 			error = ti_copy_mem(sc, mem_param->tgAddr,
-					    mem_param->len,
-					    mem_param->userAddr, 1,
-					    (cmd == ALT_READ_TG_MEM) ? 1 : 0);
-		} else if ((mem_param->tgAddr >= TI_BEG_SCRATCH)
-			&& (mem_param->tgAddr <= scratch_end)) {
+			    mem_param->len, mem_param->userAddr, 1,
+			    cmd == ALT_READ_TG_MEM ? 1 : 0);
+		} else if (mem_param->tgAddr >= TI_BEG_SCRATCH &&
+		    mem_param->tgAddr <= scratch_end) {
 			error = ti_copy_scratch(sc, mem_param->tgAddr,
-						mem_param->len,
-						mem_param->userAddr, 1,
-						(cmd == ALT_READ_TG_MEM) ?
-						1 : 0, TI_PROCESSOR_A);
-		} else if ((mem_param->tgAddr >= TI_BEG_SCRATCH_B_DEBUG)
-			&& (mem_param->tgAddr <= TI_BEG_SCRATCH_B_DEBUG)) {
+			    mem_param->len, mem_param->userAddr, 1,
+			    cmd == ALT_READ_TG_MEM ?  1 : 0, TI_PROCESSOR_A);
+		} else if (mem_param->tgAddr >= TI_BEG_SCRATCH_B_DEBUG &&
+		    mem_param->tgAddr <= TI_BEG_SCRATCH_B_DEBUG) {
 			if (sc->ti_hwrev == TI_HWREV_TIGON) {
 				if_printf(sc->ti_ifp,
 				    "invalid memory range for Tigon I\n");
@@ -3725,11 +3854,9 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				break;
 			}
 			error = ti_copy_scratch(sc, mem_param->tgAddr -
-						TI_SCRATCH_DEBUG_OFF,
-						mem_param->len,
-						mem_param->userAddr, 1,
-						(cmd == ALT_READ_TG_MEM) ?
-						1 : 0, TI_PROCESSOR_B);
+			    TI_SCRATCH_DEBUG_OFF, mem_param->len,
+			    mem_param->userAddr, 1,
+			    cmd == ALT_READ_TG_MEM ? 1 : 0, TI_PROCESSOR_B);
 		} else {
 			if_printf(sc->ti_ifp, "memory address %#x len %d is "
 			        "out of supported range\n",
@@ -3737,14 +3864,13 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 			error = EINVAL;
 		}
 		TI_UNLOCK(sc);
-
 		break;
 	}
 	case ALT_READ_TG_REG:
 	case ALT_WRITE_TG_REG:
 	{
-		struct tg_reg	*regs;
-		u_int32_t	tmpval;
+		struct tg_reg *regs;
+		uint32_t tmpval;
 
 		regs = (struct tg_reg *)addr;
 
@@ -3758,7 +3884,7 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		TI_LOCK(sc);
 		if (cmd == ALT_READ_TG_REG) {
 			bus_space_read_region_4(sc->ti_btag, sc->ti_bhandle,
-						regs->addr, &tmpval, 1);
+			    regs->addr, &tmpval, 1);
 			regs->data = ntohl(tmpval);
 #if 0
 			if ((regs->addr == TI_CPU_STATE)
@@ -3770,10 +3896,9 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		} else {
 			tmpval = htonl(regs->data);
 			bus_space_write_region_4(sc->ti_btag, sc->ti_bhandle,
-						 regs->addr, &tmpval, 1);
+			    regs->addr, &tmpval, 1);
 		}
 		TI_UNLOCK(sc);
-
 		break;
 	}
 	default:
@@ -3786,8 +3911,8 @@ ti_ioctl2(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 static void
 ti_watchdog(void *arg)
 {
-	struct ti_softc		*sc;
-	struct ifnet		*ifp;
+	struct ti_softc *sc;
+	struct ifnet *ifp;
 
 	sc = arg;
 	TI_LOCK_ASSERT(sc);
@@ -3805,7 +3930,7 @@ ti_watchdog(void *arg)
 
 	ifp = sc->ti_ifp;
 	if_printf(ifp, "watchdog timeout -- resetting\n");
-	ti_stop(sc);
+	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 	ti_init_locked(sc);
 
 	ifp->if_oerrors++;
@@ -3816,11 +3941,10 @@ ti_watchdog(void *arg)
  * RX and TX lists.
  */
 static void
-ti_stop(sc)
-	struct ti_softc		*sc;
+ti_stop(struct ti_softc *sc)
 {
-	struct ifnet		*ifp;
-	struct ti_cmd_desc	cmd;
+	struct ifnet *ifp;
+	struct ti_cmd_desc cmd;
 
 	TI_LOCK_ASSERT(sc);
 
@@ -3834,11 +3958,11 @@ ti_stop(sc)
 	TI_DO_CMD(TI_CMD_HOST_STATE, TI_CMD_CODE_STACK_DOWN, 0);
 
 	/* Halt and reinitialize. */
-	if (ti_chipinit(sc) != 0)
-		return;
-	ti_mem_zero(sc, 0x2000, 0x100000 - 0x2000);
-	if (ti_chipinit(sc) != 0)
-		return;
+	if (ti_chipinit(sc) == 0) {
+		ti_mem_zero(sc, 0x2000, 0x100000 - 0x2000);
+		/* XXX ignore init errors. */
+		ti_chipinit(sc);
+	}
 
 	/* Free the RX lists. */
 	ti_free_rx_ring_std(sc);
@@ -3866,10 +3990,9 @@ ti_stop(sc)
  * get confused by errant DMAs when rebooting.
  */
 static int
-ti_shutdown(dev)
-	device_t		dev;
+ti_shutdown(device_t dev)
 {
-	struct ti_softc		*sc;
+	struct ti_softc *sc;
 
 	sc = device_get_softc(dev);
 	TI_LOCK(sc);
@@ -3878,3 +4001,65 @@ ti_shutdown(dev)
 
 	return (0);
 }
+
+static void
+ti_sysctl_node(struct ti_softc *sc)
+{
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid_list *child;
+	char tname[32];
+
+	ctx = device_get_sysctl_ctx(sc->ti_dev);
+	child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->ti_dev));
+
+	/* Use DAC */
+	sc->ti_dac = 1;
+	snprintf(tname, sizeof(tname), "dev.ti.%d.dac",
+	    device_get_unit(sc->ti_dev));
+	TUNABLE_INT_FETCH(tname, &sc->ti_dac);
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_coal_ticks", CTLFLAG_RW,
+	    &sc->ti_rx_coal_ticks, 0, "Receive coalcesced ticks");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_max_coal_bds", CTLFLAG_RW,
+	    &sc->ti_rx_max_coal_bds, 0, "Receive max coalcesced BDs");
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_coal_ticks", CTLFLAG_RW,
+	    &sc->ti_tx_coal_ticks, 0, "Send coalcesced ticks");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_max_coal_bds", CTLFLAG_RW,
+	    &sc->ti_tx_max_coal_bds, 0, "Send max coalcesced BDs");
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_buf_ratio", CTLFLAG_RW,
+	    &sc->ti_tx_buf_ratio, 0,
+	    "Ratio of NIC memory devoted to TX buffer");
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "stat_ticks", CTLFLAG_RW,
+	    &sc->ti_stat_ticks, 0,
+	    "Number of clock ticks for statistics update interval");
+
+	/* Pull in device tunables. */
+	sc->ti_rx_coal_ticks = 170;
+	resource_int_value(device_get_name(sc->ti_dev),
+	    device_get_unit(sc->ti_dev), "rx_coal_ticks",
+	    &sc->ti_rx_coal_ticks);
+	sc->ti_rx_max_coal_bds = 64;
+	resource_int_value(device_get_name(sc->ti_dev),
+	    device_get_unit(sc->ti_dev), "rx_max_coal_bds",
+	    &sc->ti_rx_max_coal_bds);
+
+	sc->ti_tx_coal_ticks = TI_TICKS_PER_SEC / 500;
+	resource_int_value(device_get_name(sc->ti_dev),
+	    device_get_unit(sc->ti_dev), "tx_coal_ticks",
+	    &sc->ti_tx_coal_ticks);
+	sc->ti_tx_max_coal_bds = 32;
+	resource_int_value(device_get_name(sc->ti_dev),
+	    device_get_unit(sc->ti_dev), "tx_max_coal_bds",
+	    &sc->ti_tx_max_coal_bds);
+	sc->ti_tx_buf_ratio = 21;
+	resource_int_value(device_get_name(sc->ti_dev),
+	    device_get_unit(sc->ti_dev), "tx_buf_ratio",
+	    &sc->ti_tx_buf_ratio);
+
+	sc->ti_stat_ticks = 2 * TI_TICKS_PER_SEC;
+	resource_int_value(device_get_name(sc->ti_dev),
+	    device_get_unit(sc->ti_dev), "stat_ticks",
+	    &sc->ti_stat_ticks);
+}
diff --git a/sys/dev/ti/if_tireg.h b/sys/dev/ti/if_tireg.h
index 22259b5..63b93f4 100644
--- a/sys/dev/ti/if_tireg.h
+++ b/sys/dev/ti/if_tireg.h
@@ -400,6 +400,8 @@
 #define TI_RETURN_RING_CNT	2048
 
 #define TI_MAXTXSEGS		32
+#define TI_RING_ALIGN		32
+#define TI_JUMBO_RING_ALIGN	64
 
 /*
  * Possible TX ring sizes.
@@ -477,8 +479,8 @@
  */
 
 typedef struct {
-	u_int32_t	ti_addr_hi;
-	u_int32_t	ti_addr_lo;
+	uint32_t	ti_addr_hi;
+	uint32_t	ti_addr_lo;
 } ti_hostaddr;
 
 #define TI_HOSTADDR(x)		x.ti_addr_lo
@@ -516,13 +518,13 @@ ti_hostaddr64(ti_hostaddr *x, bus_addr_t addr)
 struct ti_rcb {
 	ti_hostaddr		ti_hostaddr;
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_max_len;
-	u_int16_t		ti_flags;
+	uint16_t		ti_max_len;
+	uint16_t		ti_flags;
 #else
-	u_int16_t		ti_flags;
-	u_int16_t		ti_max_len;
+	uint16_t		ti_flags;
+	uint16_t		ti_max_len;
 #endif
-	u_int32_t		ti_unused;
+	uint32_t		ti_unused;
 };
 
 #define TI_RCB_FLAG_TCP_UDP_CKSUM	0x00000001
@@ -536,8 +538,8 @@ struct ti_rcb {
 #define TI_RCB_FLAG_RING_DISABLED	0x00000200
 
 struct ti_producer {
-	u_int32_t		ti_idx;
-	u_int32_t		ti_unused;
+	uint32_t		ti_idx;
+	uint32_t		ti_unused;
 };
 
 /*
@@ -571,109 +573,123 @@ struct ti_gib {
 struct ti_rx_desc {
 	ti_hostaddr		ti_addr;
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_idx;
-	u_int16_t		ti_len;
+	uint16_t		ti_idx;
+	uint16_t		ti_len;
 #else
-	u_int16_t		ti_len;
-	u_int16_t		ti_idx;
+	uint16_t		ti_len;
+	uint16_t		ti_idx;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_type;
-	u_int16_t		ti_flags;
+	uint16_t		ti_type;
+	uint16_t		ti_flags;
 #else
-	u_int16_t		ti_flags;
-	u_int16_t		ti_type;
+	uint16_t		ti_flags;
+	uint16_t		ti_type;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_ip_cksum;
-	u_int16_t		ti_tcp_udp_cksum;
+	uint16_t		ti_ip_cksum;
+	uint16_t		ti_tcp_udp_cksum;
 #else
-	u_int16_t		ti_tcp_udp_cksum;
-	u_int16_t		ti_ip_cksum;
+	uint16_t		ti_tcp_udp_cksum;
+	uint16_t		ti_ip_cksum;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_error_flags;
-	u_int16_t		ti_vlan_tag;
+	uint16_t		ti_error_flags;
+	uint16_t		ti_vlan_tag;
 #else
-	u_int16_t		ti_vlan_tag;
-	u_int16_t		ti_error_flags;
+	uint16_t		ti_vlan_tag;
+	uint16_t		ti_error_flags;
 #endif
-	u_int32_t		ti_rsvd;
-	u_int32_t		ti_opaque;
+	uint32_t		ti_rsvd;
+	uint32_t		ti_opaque;
 };
 
+#define	TI_STD_RX_RING_SZ	(sizeof(struct ti_rx_desc) * TI_STD_RX_RING_CNT)
+#define	TI_MINI_RX_RING_SZ	(sizeof(struct ti_rx_desc) * TI_MINI_RX_RING_CNT)
+#define	TI_RX_RETURN_RING_SZ	(sizeof(struct ti_rx_desc) * TI_RETURN_RING_CNT)
+
 struct ti_rx_desc_ext {
 	ti_hostaddr		ti_addr1;
 	ti_hostaddr		ti_addr2;
 	ti_hostaddr		ti_addr3;
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_len1;
-	u_int16_t		ti_len2;
+	uint16_t		ti_len1;
+	uint16_t		ti_len2;
 #else
-	u_int16_t		ti_len2;
-	u_int16_t		ti_len1;
+	uint16_t		ti_len2;
+	uint16_t		ti_len1;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_len3;
-	u_int16_t		ti_rsvd0;
+	uint16_t		ti_len3;
+	uint16_t		ti_rsvd0;
 #else
-	u_int16_t		ti_rsvd0;
-	u_int16_t		ti_len3;
+	uint16_t		ti_rsvd0;
+	uint16_t		ti_len3;
 #endif
 	ti_hostaddr		ti_addr0;
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_idx;
-	u_int16_t		ti_len0;
+	uint16_t		ti_idx;
+	uint16_t		ti_len0;
 #else
-	u_int16_t		ti_len0;
-	u_int16_t		ti_idx;
+	uint16_t		ti_len0;
+	uint16_t		ti_idx;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_type;
-	u_int16_t		ti_flags;
+	uint16_t		ti_type;
+	uint16_t		ti_flags;
 #else
-	u_int16_t		ti_flags;
-	u_int16_t		ti_type;
+	uint16_t		ti_flags;
+	uint16_t		ti_type;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_ip_cksum;
-	u_int16_t		ti_tcp_udp_cksum;
+	uint16_t		ti_ip_cksum;
+	uint16_t		ti_tcp_udp_cksum;
 #else
-	u_int16_t		ti_tcp_udp_cksum;
-	u_int16_t		ti_ip_cksum;
+	uint16_t		ti_tcp_udp_cksum;
+	uint16_t		ti_ip_cksum;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_error_flags;
-	u_int16_t		ti_vlan_tag;
+	uint16_t		ti_error_flags;
+	uint16_t		ti_vlan_tag;
 #else
-	u_int16_t		ti_vlan_tag;
-	u_int16_t		ti_error_flags;
+	uint16_t		ti_vlan_tag;
+	uint16_t		ti_error_flags;
 #endif
-	u_int32_t		ti_rsvd1;
-	u_int32_t		ti_opaque;
+	uint32_t		ti_rsvd1;
+	uint32_t		ti_opaque;
 };
 
+#ifdef TI_SF_BUF_JUMBO
+#define	TI_JUMBO_RX_RING_SZ	\
+	(sizeof(struct ti_rx_desc_ext) * TI_JUMBO_RX_RING_CNT)
+#else
+#define	TI_JUMBO_RX_RING_SZ	\
+	(sizeof(struct ti_rx_desc) * TI_JUMBO_RX_RING_CNT)
+#endif
+
 /*
  * Transmit descriptors are, mercifully, very small.
  */
 struct ti_tx_desc {
 	ti_hostaddr		ti_addr;
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_len;
-	u_int16_t		ti_flags;
+	uint16_t		ti_len;
+	uint16_t		ti_flags;
 #else
-	u_int16_t		ti_flags;
-	u_int16_t		ti_len;
+	uint16_t		ti_flags;
+	uint16_t		ti_len;
 #endif
 #if BYTE_ORDER == BIG_ENDIAN
-	u_int16_t		ti_rsvd;
-	u_int16_t		ti_vlan_tag;
+	uint16_t		ti_rsvd;
+	uint16_t		ti_vlan_tag;
 #else
-	u_int16_t		ti_vlan_tag;
-	u_int16_t		ti_rsvd;
+	uint16_t		ti_vlan_tag;
+	uint16_t		ti_rsvd;
 #endif
 };
 
+#define	TI_TX_RING_SZ		(sizeof(struct ti_tx_desc) * TI_TX_RING_CNT)
+
 /*
  * NOTE!  On the Alpha, we have an alignment constraint.
  * The first thing in the packet is a 14-byte Ethernet header.
@@ -753,7 +769,7 @@ struct ti_tx_desc {
  * Tigon command structure.
  */
 struct ti_cmd_desc {
-	u_int32_t		ti_cmdx;
+	uint32_t		ti_cmdx;
 };
 
 #define TI_CMD_CMD(cmd)		(((((cmd)->ti_cmdx)) >> 24) & 0xff)
@@ -842,9 +858,10 @@ struct ti_cmd_desc {
  * Tigon event structure.
  */
 struct ti_event_desc {
-	u_int32_t		ti_eventx;
-	u_int32_t		ti_rsvd;
+	uint32_t		ti_eventx;
+	uint32_t		ti_rsvd;
 };
+#define	TI_EVENT_RING_SZ	(sizeof(struct ti_event_desc) * TI_EVENT_RING_CNT)
 
 #define TI_EVENT_EVENT(e)	(((((e)->ti_eventx)) >> 24) & 0xff)
 #define TI_EVENT_CODE(e)	(((((e)->ti_eventx)) >> 12) & 0xfff)
@@ -887,23 +904,6 @@ struct ti_event_desc {
 #define TI_CLRBIT(sc, reg, x)	\
 	CSR_WRITE_4((sc), (reg), (CSR_READ_4((sc), (reg)) & ~(x)))
 
-/*
- * Memory management stuff. Note: the SSLOTS, MSLOTS and JSLOTS
- * values are tuneable. They control the actual amount of buffers
- * allocated for the standard, mini and jumbo receive rings.
- */
-
-#define TI_SSLOTS	256
-#define TI_MSLOTS	256
-#define TI_JSLOTS	256
-
-#define TI_JRAWLEN (TI_JUMBO_FRAMELEN + ETHER_ALIGN)
-#define TI_JLEN (TI_JRAWLEN + (sizeof(u_int64_t) - \
-	(TI_JRAWLEN % sizeof(u_int64_t))))
-#define TI_JPAGESZ PAGE_SIZE
-#define TI_RESID (TI_JPAGESZ - (TI_JLEN * TI_JSLOTS) % TI_JPAGESZ)
-#define TI_JMEM ((TI_JLEN * TI_JSLOTS) + TI_RESID)
-
 struct ti_txdesc {
 	struct mbuf	*tx_m;
 	bus_dmamap_t	tx_dmamap;
@@ -912,6 +912,21 @@ struct ti_txdesc {
 
 STAILQ_HEAD(ti_txdq, ti_txdesc);
 
+struct ti_status {
+	/*
+	 * Make sure producer structures are aligned on 32-byte cache
+	 * line boundaries.  We can create separate DMA area for each
+	 * producer/consumer area but it wouldn't get much benefit
+	 * since driver use a global driver lock.
+	 */
+	struct ti_producer	ti_ev_prodidx_r;
+	uint32_t		ti_pad0[6];
+	struct ti_producer	ti_return_prodidx_r;
+	uint32_t		ti_pad1[6];
+	struct ti_producer	ti_tx_considx_r;
+	uint32_t		ti_pad2[6];
+};
+
 /*
  * Ring structures. Most of these reside in host memory and we tell
  * the NIC where they are via the ring control blocks. The exceptions
@@ -919,54 +934,72 @@ STAILQ_HEAD(ti_txdq, ti_txdesc);
  * we access via the shared memory window.
  */
 struct ti_ring_data {
-	struct ti_rx_desc	ti_rx_std_ring[TI_STD_RX_RING_CNT];
-#ifdef TI_PRIVATE_JUMBOS
-	struct ti_rx_desc	ti_rx_jumbo_ring[TI_JUMBO_RX_RING_CNT];
+	struct ti_gib		*ti_info;
+	bus_addr_t		ti_info_paddr;
+	struct ti_status	*ti_status;
+	bus_addr_t		ti_status_paddr;
+	struct ti_rx_desc	*ti_rx_std_ring;
+	bus_addr_t		ti_rx_std_ring_paddr;
+#ifdef TI_SF_BUF_JUMBO
+	struct ti_rx_desc_ext	*ti_rx_jumbo_ring;
 #else
-	struct ti_rx_desc_ext	ti_rx_jumbo_ring[TI_JUMBO_RX_RING_CNT];
+	struct ti_rx_desc	*ti_rx_jumbo_ring;
 #endif
-	struct ti_rx_desc	ti_rx_mini_ring[TI_MINI_RX_RING_CNT];
-	struct ti_rx_desc	ti_rx_return_ring[TI_RETURN_RING_CNT];
-	struct ti_event_desc	ti_event_ring[TI_EVENT_RING_CNT];
-	struct ti_tx_desc	ti_tx_ring[TI_TX_RING_CNT];
-	/*
-	 * Make sure producer structures are aligned on 32-byte cache
-	 * line boundaries.
-	 */
-	struct ti_producer	ti_ev_prodidx_r;
-	u_int32_t		ti_pad0[6];
-	struct ti_producer	ti_return_prodidx_r;
-	u_int32_t		ti_pad1[6];
-	struct ti_producer	ti_tx_considx_r;
-	u_int32_t		ti_pad2[6];
-	struct ti_gib		ti_info;
+	bus_addr_t		ti_rx_jumbo_ring_paddr;
+	struct ti_rx_desc	*ti_rx_mini_ring;
+	bus_addr_t		ti_rx_mini_ring_paddr;
+	struct ti_rx_desc	*ti_rx_return_ring;
+	bus_addr_t		ti_rx_return_ring_paddr;
+	struct ti_event_desc	*ti_event_ring;
+	bus_addr_t		ti_event_ring_paddr;
+	struct ti_tx_desc	*ti_tx_ring;
+	bus_addr_t		ti_tx_ring_paddr;
 };
 
-#define TI_RD_OFF(x)	offsetof(struct ti_ring_data, x)
-
 /*
  * Mbuf pointers. We need these to keep track of the virtual addresses
  * of our mbuf chains since we can only convert from physical to virtual,
  * not the other way around.
  */
 struct ti_chain_data {
+	bus_dma_tag_t		ti_parent_tag;
+	bus_dma_tag_t		ti_gib_tag;
+	bus_dmamap_t		ti_gib_map;
+	bus_dma_tag_t		ti_event_ring_tag;
+	bus_dmamap_t		ti_event_ring_map;
+	bus_dma_tag_t		ti_status_tag;
+	bus_dmamap_t		ti_status_map;
+	bus_dma_tag_t		ti_tx_ring_tag;
+	bus_dmamap_t		ti_tx_ring_map;
+	bus_dma_tag_t		ti_tx_tag;
 	struct ti_txdesc	ti_txdesc[TI_TX_RING_CNT];
 	struct ti_txdq		ti_txfreeq;
 	struct ti_txdq		ti_txbusyq;
+	bus_dma_tag_t		ti_rx_return_ring_tag;
+	bus_dmamap_t		ti_rx_return_ring_map;
+	bus_dma_tag_t		ti_rx_std_ring_tag;
+	bus_dmamap_t		ti_rx_std_ring_map;
+	bus_dma_tag_t		ti_rx_std_tag;
 	struct mbuf		*ti_rx_std_chain[TI_STD_RX_RING_CNT];
 	bus_dmamap_t		ti_rx_std_maps[TI_STD_RX_RING_CNT];
+	bus_dmamap_t		ti_rx_std_sparemap;
+	bus_dma_tag_t		ti_rx_jumbo_ring_tag;
+	bus_dmamap_t		ti_rx_jumbo_ring_map;
+	bus_dma_tag_t		ti_rx_jumbo_tag;
 	struct mbuf		*ti_rx_jumbo_chain[TI_JUMBO_RX_RING_CNT];
 	bus_dmamap_t		ti_rx_jumbo_maps[TI_JUMBO_RX_RING_CNT];
+	bus_dmamap_t		ti_rx_jumbo_sparemap;
+	bus_dma_tag_t		ti_rx_mini_ring_tag;
+	bus_dmamap_t		ti_rx_mini_ring_map;
+	bus_dma_tag_t		ti_rx_mini_tag;
 	struct mbuf		*ti_rx_mini_chain[TI_MINI_RX_RING_CNT];
 	bus_dmamap_t		ti_rx_mini_maps[TI_MINI_RX_RING_CNT];
-	/* Stick the jumbo mem management stuff here too. */
-	caddr_t			ti_jslots[TI_JSLOTS];
-	void			*ti_jumbo_buf;
+	bus_dmamap_t		ti_rx_mini_sparemap;
 };
 
 struct ti_type {
-	u_int16_t		ti_vid;
-	u_int16_t		ti_did;
+	uint16_t		ti_vid;
+	uint16_t		ti_did;
 	const char		*ti_name;
 };
 
@@ -980,11 +1013,6 @@ struct ti_mc_entry {
 	SLIST_ENTRY(ti_mc_entry)	mc_entries;
 };
 
-struct ti_jpool_entry {
-	int                             slot;
-	SLIST_ENTRY(ti_jpool_entry)	jpool_entries;
-};
-
 typedef enum {
 	TI_FLAG_NONE		= 0x00,
 	TI_FLAG_DEBUGING	= 0x01,
@@ -1000,24 +1028,16 @@ struct ti_softc {
 	struct resource		*ti_irq;
 	struct resource		*ti_res;
 	struct ifmedia		ifmedia;	/* media info */
-	u_int8_t		ti_unit;	/* interface number */
-	u_int8_t		ti_hwrev;	/* Tigon rev (1 or 2) */
-	u_int8_t		ti_copper;	/* 1000baseTX card */
-	u_int8_t		ti_linkstat;	/* Link state */
+	uint8_t			ti_hwrev;	/* Tigon rev (1 or 2) */
+	uint8_t			ti_copper;	/* 1000baseTX card */
+	uint8_t			ti_linkstat;	/* Link state */
 	int			ti_hdrsplit;	/* enable header splitting */
-	bus_dma_tag_t		ti_parent_dmat;
-	bus_dma_tag_t		ti_jumbo_dmat;
-	bus_dmamap_t		ti_jumbo_dmamap;
-	bus_dma_tag_t		ti_mbuftx_dmat;
-	bus_dma_tag_t		ti_mbufrx_dmat;
-	bus_dma_tag_t		ti_rdata_dmat;
-	bus_dmamap_t		ti_rdata_dmamap;
-	bus_addr_t		ti_rdata_phys;
-	struct ti_ring_data	*ti_rdata;	/* rings */
+	int			ti_dac;
+	struct ti_ring_data	ti_rdata;	/* rings */
 	struct ti_chain_data	ti_cdata;	/* mbufs */
-#define ti_ev_prodidx		ti_rdata->ti_ev_prodidx_r
-#define ti_return_prodidx	ti_rdata->ti_return_prodidx_r
-#define ti_tx_considx		ti_rdata->ti_tx_considx_r
+#define ti_ev_prodidx		ti_rdata.ti_status->ti_ev_prodidx_r
+#define ti_return_prodidx	ti_rdata.ti_status->ti_return_prodidx_r
+#define ti_tx_considx		ti_rdata.ti_status->ti_tx_considx_r
 	int			ti_tx_saved_prodidx;
 	int			ti_tx_saved_considx;
 	int			ti_rx_saved_considx;
@@ -1027,20 +1047,20 @@ struct ti_softc {
 	int			ti_mini;	/* current mini ring head */
 	int			ti_jumbo;	/* current jumo ring head */
 	SLIST_HEAD(__ti_mchead, ti_mc_entry)	ti_mc_listhead;
-	SLIST_HEAD(__ti_jfreehead, ti_jpool_entry)	ti_jfree_listhead;
-	SLIST_HEAD(__ti_jinusehead, ti_jpool_entry)	ti_jinuse_listhead;
-	u_int32_t		ti_stat_ticks;
-	u_int32_t		ti_rx_coal_ticks;
-	u_int32_t		ti_tx_coal_ticks;
-	u_int32_t		ti_rx_max_coal_bds;
-	u_int32_t		ti_tx_max_coal_bds;
-	u_int32_t		ti_tx_buf_ratio;
+	uint32_t		ti_stat_ticks;
+	uint32_t		ti_rx_coal_ticks;
+	uint32_t		ti_tx_coal_ticks;
+	uint32_t		ti_rx_max_coal_bds;
+	uint32_t		ti_tx_max_coal_bds;
+	uint32_t		ti_tx_buf_ratio;
 	int			ti_if_flags;
 	int			ti_txcnt;
 	struct mtx		ti_mtx;
 	struct callout		ti_watchdog;
 	int			ti_timer;
 	ti_flag_vals		ti_flags;
+	uint8_t			*ti_membuf;
+	uint8_t			*ti_membuf2;
 	struct cdev		 *dev;
 };
 
diff --git a/sys/dev/tl/if_tl.c b/sys/dev/tl/if_tl.c
index a256552..b5ffc2c 100644
--- a/sys/dev/tl/if_tl.c
+++ b/sys/dev/tl/if_tl.c
@@ -203,6 +203,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -228,7 +229,7 @@ MODULE_DEPEND(tl, miibus, 1, 1, 1);
  * Various supported device vendors/types and their names.
  */
 
-static struct tl_type tl_devs[] = {
+static const struct tl_type const tl_devs[] = {
 	{ TI_VENDORID,	TI_DEVICEID_THUNDERLAN,
 		"Texas Instruments ThunderLAN" },
 	{ COMPAQ_VENDORID, COMPAQ_DEVICEID_NETEL_10,
@@ -290,10 +291,6 @@ static u_int8_t tl_eeprom_putbyte(struct tl_softc *, int);
 static u_int8_t	tl_eeprom_getbyte(struct tl_softc *, int, u_int8_t *);
 static int tl_read_eeprom(struct tl_softc *, caddr_t, int, int);
 
-static void tl_mii_sync(struct tl_softc *);
-static void tl_mii_send(struct tl_softc *, u_int32_t, int);
-static int tl_mii_readreg(struct tl_softc *, struct tl_mii_frame *);
-static int tl_mii_writereg(struct tl_softc *, struct tl_mii_frame *);
 static int tl_miibus_readreg(device_t, int, int);
 static int tl_miibus_writereg(device_t, int, int, int);
 static void tl_miibus_statchg(device_t);
@@ -318,6 +315,24 @@ static void tl_dio_clrbit(struct tl_softc *, int, int);
 static void tl_dio_setbit16(struct tl_softc *, int, int);
 static void tl_dio_clrbit16(struct tl_softc *, int, int);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t tl_mii_bitbang_read(device_t);
+static void tl_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops tl_mii_bitbang_ops = {
+	tl_mii_bitbang_read,
+	tl_mii_bitbang_write,
+	{
+		TL_SIO_MDATA,	/* MII_BIT_MDO */
+		TL_SIO_MDATA,	/* MII_BIT_MDI */
+		TL_SIO_MCLK,	/* MII_BIT_MDC */
+		TL_SIO_MTXEN,	/* MII_BIT_DIR_HOST_PHY */
+		0,		/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 #ifdef TL_USEIOSPACE
 #define TL_RES		SYS_RES_IOPORT
 #define TL_RID		TL_PCI_LOIO
@@ -360,7 +375,12 @@ static u_int8_t tl_dio_read8(sc, reg)
 	struct tl_softc		*sc;
 	int			reg;
 {
+
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	return(CSR_READ_1(sc, TL_DIO_DATA + (reg & 3)));
 }
 
@@ -368,7 +388,12 @@ static u_int16_t tl_dio_read16(sc, reg)
 	struct tl_softc		*sc;
 	int			reg;
 {
+
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	return(CSR_READ_2(sc, TL_DIO_DATA + (reg & 3)));
 }
 
@@ -376,7 +401,12 @@ static u_int32_t tl_dio_read32(sc, reg)
 	struct tl_softc		*sc;
 	int			reg;
 {
+
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	return(CSR_READ_4(sc, TL_DIO_DATA + (reg & 3)));
 }
 
@@ -385,9 +415,13 @@ static void tl_dio_write8(sc, reg, val)
 	int			reg;
 	int			val;
 {
+
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_1(sc, TL_DIO_DATA + (reg & 3), val);
-	return;
 }
 
 static void tl_dio_write16(sc, reg, val)
@@ -395,9 +429,13 @@ static void tl_dio_write16(sc, reg, val)
 	int			reg;
 	int			val;
 {
+
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_DATA + (reg & 3), val);
-	return;
 }
 
 static void tl_dio_write32(sc, reg, val)
@@ -405,9 +443,13 @@ static void tl_dio_write32(sc, reg, val)
 	int			reg;
 	int			val;
 {
+
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_4(sc, TL_DIO_DATA + (reg & 3), val);
-	return;
 }
 
 static void
@@ -418,12 +460,16 @@ tl_dio_setbit(sc, reg, bit)
 {
 	u_int8_t			f;
 
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	f = CSR_READ_1(sc, TL_DIO_DATA + (reg & 3));
 	f |= bit;
+	CSR_BARRIER(sc, TL_DIO_DATA + (reg & 3), 1,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_1(sc, TL_DIO_DATA + (reg & 3), f);
-
-	return;
 }
 
 static void
@@ -434,12 +480,16 @@ tl_dio_clrbit(sc, reg, bit)
 {
 	u_int8_t			f;
 
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	f = CSR_READ_1(sc, TL_DIO_DATA + (reg & 3));
 	f &= ~bit;
+	CSR_BARRIER(sc, TL_DIO_DATA + (reg & 3), 1,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_1(sc, TL_DIO_DATA + (reg & 3), f);
-
-	return;
 }
 
 static void tl_dio_setbit16(sc, reg, bit)
@@ -449,12 +499,16 @@ static void tl_dio_setbit16(sc, reg, bit)
 {
 	u_int16_t			f;
 
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	f = CSR_READ_2(sc, TL_DIO_DATA + (reg & 3));
 	f |= bit;
+	CSR_BARRIER(sc, TL_DIO_DATA + (reg & 3), 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_DATA + (reg & 3), f);
-
-	return;
 }
 
 static void tl_dio_clrbit16(sc, reg, bit)
@@ -464,12 +518,16 @@ static void tl_dio_clrbit16(sc, reg, bit)
 {
 	u_int16_t			f;
 
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_ADDR, reg);
+	CSR_BARRIER(sc, TL_DIO_ADDR, 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	f = CSR_READ_2(sc, TL_DIO_DATA + (reg & 3));
 	f &= ~bit;
+	CSR_BARRIER(sc, TL_DIO_DATA + (reg & 3), 2,
+		BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 	CSR_WRITE_2(sc, TL_DIO_DATA + (reg & 3), f);
-
-	return;
 }
 
 /*
@@ -608,61 +666,55 @@ tl_read_eeprom(sc, dest, off, cnt)
 	return(err ? 1 : 0);
 }
 
-static void
-tl_mii_sync(sc)
-	struct tl_softc		*sc;
+#define	TL_SIO_MII	(TL_SIO_MCLK | TL_SIO_MDATA | TL_SIO_MTXEN)
+
+/*
+ * Read the MII serial port for the MII bit-bang module.
+ */
+static uint32_t
+tl_mii_bitbang_read(device_t dev)
 {
-	register int		i;
+	struct tl_softc *sc;
+	uint32_t val;
 
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MTXEN);
+	sc = device_get_softc(dev);
 
-	for (i = 0; i < 32; i++) {
-		tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-		tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	}
+	val = tl_dio_read8(sc, TL_NETSIO) & TL_SIO_MII;
+	CSR_BARRIER(sc, TL_NETSIO, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	return;
+	return (val);
 }
 
+/*
+ * Write the MII serial port for the MII bit-bang module.
+ */
 static void
-tl_mii_send(sc, bits, cnt)
-	struct tl_softc		*sc;
-	u_int32_t		bits;
-	int			cnt;
+tl_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	int			i;
+	struct tl_softc *sc;
 
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-		if (bits & i) {
-			tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MDATA);
-		} else {
-			tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MDATA);
-		}
-		tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	}
+	sc = device_get_softc(dev);
+
+	val = (tl_dio_read8(sc, TL_NETSIO) & ~TL_SIO_MII) | val;
+	CSR_BARRIER(sc, TL_NETSIO, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
+	tl_dio_write8(sc, TL_NETSIO, val);
+	CSR_BARRIER(sc, TL_NETSIO, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
-tl_mii_readreg(sc, frame)
-	struct tl_softc		*sc;
-	struct tl_mii_frame	*frame;
-	
+tl_miibus_readreg(dev, phy, reg)
+	device_t		dev;
+	int			phy, reg;
 {
-	int			i, ack;
-	int			minten = 0;
+	struct tl_softc		*sc;
+	int			minten, val;
 
-	tl_mii_sync(sc);
+	sc = device_get_softc(dev);
 
 	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = TL_MII_STARTDELIM;
-	frame->mii_opcode = TL_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-	
-	/*
 	 * Turn off MII interrupt by forcing MINTEN low.
 	 */
 	minten = tl_dio_read8(sc, TL_NETSIO) & TL_SIO_MINTEN;
@@ -670,89 +722,26 @@ tl_mii_readreg(sc, frame)
 		tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MINTEN);
 	}
 
-	/*
- 	 * Turn on data xmit.
-	 */
-	tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MTXEN);
-
-	/*
-	 * Send command/address info.
-	 */
-	tl_mii_send(sc, frame->mii_stdelim, 2);
-	tl_mii_send(sc, frame->mii_opcode, 2);
-	tl_mii_send(sc, frame->mii_phyaddr, 5);
-	tl_mii_send(sc, frame->mii_regaddr, 5);
-
-	/*
-	 * Turn off xmit.
-	 */
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MTXEN);
-
-	/* Idle bit */
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-
-	/* Check for ack */
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	ack = tl_dio_read8(sc, TL_NETSIO) & TL_SIO_MDATA;
-
-	/* Complete the cycle */
-	tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHYs in sync.
-	 */
-	if (ack) {
-		for(i = 0; i < 16; i++) {
-			tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-			tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-		if (!ack) {
-			if (tl_dio_read8(sc, TL_NETSIO) & TL_SIO_MDATA)
-				frame->mii_data |= i;
-		}
-		tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	}
-
-fail:
-
-	tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
+	val = mii_bitbang_readreg(dev, &tl_mii_bitbang_ops, phy, reg);
 
-	/* Reenable interrupts */
+	/* Reenable interrupts. */
 	if (minten) {
 		tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MINTEN);
 	}
 
-	if (ack)
-		return(1);
-	return(0);
+	return (val);
 }
 
 static int
-tl_mii_writereg(sc, frame)
-	struct tl_softc		*sc;
-	struct tl_mii_frame	*frame;
-	
+tl_miibus_writereg(dev, phy, reg, data)
+	device_t		dev;
+	int			phy, reg, data;
 {
+	struct tl_softc		*sc;
 	int			minten;
 
-	tl_mii_sync(sc);
-
-	/*
-	 * Set up frame for TX.
-	 */
+	sc = device_get_softc(dev);
 
-	frame->mii_stdelim = TL_MII_STARTDELIM;
-	frame->mii_opcode = TL_MII_WRITEOP;
-	frame->mii_turnaround = TL_MII_TURNAROUND;
-	
 	/*
 	 * Turn off MII interrupt by forcing MINTEN low.
 	 */
@@ -761,67 +750,12 @@ tl_mii_writereg(sc, frame)
 		tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MINTEN);
 	}
 
-	/*
- 	 * Turn on data output.
-	 */
-	tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MTXEN);
-
-	tl_mii_send(sc, frame->mii_stdelim, 2);
-	tl_mii_send(sc, frame->mii_opcode, 2);
-	tl_mii_send(sc, frame->mii_phyaddr, 5);
-	tl_mii_send(sc, frame->mii_regaddr, 5);
-	tl_mii_send(sc, frame->mii_turnaround, 2);
-	tl_mii_send(sc, frame->mii_data, 16);
-
-	tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MCLK);
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MCLK);
-
-	/*
-	 * Turn off xmit.
-	 */
-	tl_dio_clrbit(sc, TL_NETSIO, TL_SIO_MTXEN);
+	mii_bitbang_writereg(dev, &tl_mii_bitbang_ops, phy, reg, data);
 
-	/* Reenable interrupts */
-	if (minten)
+	/* Reenable interrupts. */
+	if (minten) {
 		tl_dio_setbit(sc, TL_NETSIO, TL_SIO_MINTEN);
-
-	return(0);
-}
-
-static int
-tl_miibus_readreg(dev, phy, reg)
-	device_t		dev;
-	int			phy, reg;
-{
-	struct tl_softc		*sc;
-	struct tl_mii_frame	frame;
-
-	sc = device_get_softc(dev);
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	tl_mii_readreg(sc, &frame);
-
-	return(frame.mii_data);
-}
-
-static int
-tl_miibus_writereg(dev, phy, reg, data)
-	device_t		dev;
-	int			phy, reg, data;
-{
-	struct tl_softc		*sc;
-	struct tl_mii_frame	frame;
-
-	sc = device_get_softc(dev);
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
-
-	tl_mii_writereg(sc, &frame);
+	}
 
 	return(0);
 }
@@ -841,8 +775,6 @@ tl_miibus_statchg(dev)
 	} else {
 		tl_dio_clrbit(sc, TL_NETCMD, TL_CMD_DUPLEX);
 	}
-
-	return;
 }
 
 /*
@@ -865,8 +797,6 @@ tl_setmode(sc, media)
 			tl_dio_clrbit(sc, TL_NETCMD, TL_CMD_DUPLEX);
 		}
 	}
-
-	return;
 }
 
 /*
@@ -909,8 +839,6 @@ tl_setfilt(sc, addr, slot)
 
 	for (i = 0; i < ETHER_ADDR_LEN; i++)
 		tl_dio_write8(sc, regaddr + i, *(addr + i));
-
-	return;
 }
 
 /*
@@ -980,8 +908,6 @@ tl_setmulti(sc)
 
 	tl_dio_write32(sc, TL_HASH1, hashes[0]);
 	tl_dio_write32(sc, TL_HASH2, hashes[1]);
-
-	return;
 }
 
 /*
@@ -994,13 +920,10 @@ static void
 tl_hardreset(dev)
 	device_t		dev;
 {
-	struct tl_softc		*sc;
 	int			i;
 	u_int16_t		flags;
 
-	sc = device_get_softc(dev);
-
-	tl_mii_sync(sc);
+	mii_bitbang_sync(dev, &tl_mii_bitbang_ops);
 
 	flags = BMCR_LOOP|BMCR_ISO|BMCR_PDOWN;
 
@@ -1010,11 +933,10 @@ tl_hardreset(dev)
 	tl_miibus_writereg(dev, 31, MII_BMCR, BMCR_ISO);
 	DELAY(50000);
 	tl_miibus_writereg(dev, 31, MII_BMCR, BMCR_LOOP|BMCR_ISO);
-	tl_mii_sync(sc);
+	mii_bitbang_sync(dev, &tl_mii_bitbang_ops);
 	while(tl_miibus_readreg(dev, 31, MII_BMCR) & BMCR_RESET);
 
 	DELAY(50000);
-	return;
 }
 
 static void
@@ -1072,8 +994,6 @@ tl_softreset(sc, internal)
 
 	/* Wait for things to settle down a little. */
 	DELAY(500);
-
-        return;
 }
 
 /*
@@ -1084,7 +1004,7 @@ static int
 tl_probe(dev)
 	device_t		dev;
 {
-	struct tl_type		*t;
+	const struct tl_type	*t;
 
 	t = tl_devs;
 
@@ -1105,7 +1025,7 @@ tl_attach(dev)
 	device_t		dev;
 {
 	u_int16_t		did, vid;
-	struct tl_type		*t;
+	const struct tl_type	*t;
 	struct ifnet		*ifp;
 	struct tl_softc		*sc;
 	int			error, flags, i, rid, unit;
@@ -1415,9 +1335,9 @@ static int
 tl_list_rx_init(sc)
 	struct tl_softc		*sc;
 {
-	struct tl_chain_data	*cd;
-	struct tl_list_data	*ld;
-	int			i;
+	struct tl_chain_data		*cd;
+	struct tl_list_data		*ld;
+	int				i;
 
 	cd = &sc->tl_cdata;
 	ld = sc->tl_ldata;
@@ -1783,8 +1703,6 @@ tl_intr(xsc)
 		tl_start_locked(ifp);
 
 	TL_UNLOCK(sc);
-
-	return;
 }
 
 static void
@@ -1843,8 +1761,6 @@ tl_stats_update(xsc)
 		mii = device_get_softc(sc->tl_miibus);
 		mii_tick(mii);
 	}
-
-	return;
 }
 
 /*
@@ -2046,8 +1962,6 @@ tl_start_locked(ifp)
 	 * Set a timeout in case the chip goes out to lunch.
 	 */
 	sc->tl_timer = 5;
-
-	return;
 }
 
 static void
@@ -2143,8 +2057,6 @@ tl_init_locked(sc)
 
 	/* Start the stats update counter */
 	callout_reset(&sc->tl_stat_callout, hz, tl_stats_update, sc);
-
-	return;
 }
 
 /*
@@ -2204,8 +2116,6 @@ tl_ifmedia_sts(ifp, ifmr)
 		ifmr->ifm_status = mii->mii_media_status;
 	}
 	TL_UNLOCK(sc);
-
-	return;
 }
 
 static int
@@ -2284,8 +2194,6 @@ tl_watchdog(sc)
 
 	tl_softreset(sc, 1);
 	tl_init_locked(sc);
-
-	return;
 }
 
 /*
@@ -2351,8 +2259,6 @@ tl_stop(sc)
 		sizeof(sc->tl_ldata->tl_tx_list));
 
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-
-	return;
 }
 
 /*
diff --git a/sys/dev/tl/if_tlreg.h b/sys/dev/tl/if_tlreg.h
index 4e340b1..4a82cc5 100644
--- a/sys/dev/tl/if_tlreg.h
+++ b/sys/dev/tl/if_tlreg.h
@@ -32,11 +32,10 @@
  * $FreeBSD$
  */
 
-
 struct tl_type {
 	u_int16_t		tl_vid;
 	u_int16_t		tl_did;
-	char			*tl_name;
+	const char		*tl_name;
 };
 
 /*
@@ -203,6 +202,7 @@ struct tl_softc {
 
 #define TL_INT_MASK		0x001C
 #define TL_VEC_MASK		0x1FE0
+
 /*
  * Host command register bits
  */
@@ -390,36 +390,6 @@ struct tl_softc {
 #define TL_MASK_MASK5		0x20
 #define TL_MASK_MASK4		0x10
 
-/*
- * MII frame format
- */
-#ifdef ANSI_DOESNT_ALLOW_BITFIELDS
-struct tl_mii_frame {
-	u_int16_t		mii_stdelim:2,
-				mii_opcode:2,
-				mii_phyaddr:5,
-				mii_regaddr:5,
-				mii_turnaround:2;
-	u_int16_t		mii_data;
-};
-#else
-struct tl_mii_frame {
-	u_int8_t		mii_stdelim;
-	u_int8_t		mii_opcode;
-	u_int8_t		mii_phyaddr;
-	u_int8_t		mii_regaddr;
-	u_int8_t		mii_turnaround;
-	u_int16_t		mii_data;
-};
-#endif
-/*
- * MII constants
- */
-#define TL_MII_STARTDELIM	0x01
-#define TL_MII_READOP		0x02
-#define TL_MII_WRITEOP		0x01
-#define TL_MII_TURNAROUND	0x02
-
 #define TL_LAST_FRAG		0x80000000
 #define TL_CSTAT_UNUSED		0x8000
 #define TL_CSTAT_FRAMECMP	0x4000
@@ -499,6 +469,9 @@ struct tl_stats {
 #define CSR_READ_2(sc, reg)		bus_read_2(sc->tl_res, reg)
 #define CSR_READ_1(sc, reg)		bus_read_1(sc->tl_res, reg)
 
+#define	CSR_BARRIER(sc, reg, length, flags)				\
+	bus_barrier(sc->tl_res, reg, length, flags)
+
 #define CMD_PUT(sc, x) CSR_WRITE_4(sc, TL_HOSTCMD, x)
 #define CMD_SET(sc, x)	\
 	CSR_WRITE_4(sc, TL_HOSTCMD, CSR_READ_4(sc, TL_HOSTCMD) | (x))
diff --git a/sys/dev/twa/tw_osl_freebsd.c b/sys/dev/twa/tw_osl_freebsd.c
index caf18f1..5651b7f 100644
--- a/sys/dev/twa/tw_osl_freebsd.c
+++ b/sys/dev/twa/tw_osl_freebsd.c
@@ -54,7 +54,7 @@ TW_INT32	TW_DEBUG_LEVEL_FOR_OSL = TW_OSL_DEBUG;
 TW_INT32	TW_OSL_DEBUG_LEVEL_FOR_CL = TW_OSL_DEBUG;
 #endif /* TW_OSL_DEBUG */
 
-MALLOC_DEFINE(TW_OSLI_MALLOC_CLASS, "twa_commands", "twa commands");
+static MALLOC_DEFINE(TW_OSLI_MALLOC_CLASS, "twa_commands", "twa commands");
 
 
 static	d_open_t		twa_open;
diff --git a/sys/dev/twe/twe_freebsd.c b/sys/dev/twe/twe_freebsd.c
index 0328599..a7b9c91 100644
--- a/sys/dev/twe/twe_freebsd.c
+++ b/sys/dev/twe/twe_freebsd.c
@@ -872,7 +872,7 @@ twed_detach(device_t dev)
 /********************************************************************************
  * Allocate a command buffer
  */
-MALLOC_DEFINE(TWE_MALLOC_CLASS, "twe_commands", "twe commands");
+static MALLOC_DEFINE(TWE_MALLOC_CLASS, "twe_commands", "twe commands");
 
 struct twe_request *
 twe_allocate_request(struct twe_softc *sc, int tag)
diff --git a/sys/dev/tws/tws_services.c b/sys/dev/tws/tws_services.c
index 07200b5..d2a52cd 100644
--- a/sys/dev/tws/tws_services.c
+++ b/sys/dev/tws/tws_services.c
@@ -53,7 +53,7 @@ struct tws_sense *tws_find_sense_from_mfa(struct tws_softc *sc, u_int64_t mfa);
 
 
 
-struct error_desc array[] = {
+static struct error_desc array[] = {
     { "Cannot add sysctl tree node", 0x2000, ERROR,
        "%s: (0x%02X: 0x%04X): %s:\n", "ERROR" },
     { "Register window not available", 0x2001, ERROR,
diff --git a/sys/dev/tws/tws_services.h b/sys/dev/tws/tws_services.h
index 643d720..f9016da 100644
--- a/sys/dev/tws/tws_services.h
+++ b/sys/dev/tws/tws_services.h
@@ -114,7 +114,6 @@ struct error_desc {
     char *error_str;
 };
 
-extern struct error_desc array[];
 /* ----------- q services ------------- */
 
 #define TWS_FREE_Q        0
diff --git a/sys/dev/uart/uart_core.c b/sys/dev/uart/uart_core.c
index 9260d06..fb1bb64 100644
--- a/sys/dev/uart/uart_core.c
+++ b/sys/dev/uart/uart_core.c
@@ -56,7 +56,7 @@ char uart_driver_name[] = "uart";
 SLIST_HEAD(uart_devinfo_list, uart_devinfo) uart_sysdevs =
     SLIST_HEAD_INITIALIZER(uart_sysdevs);
 
-MALLOC_DEFINE(M_UART, "UART", "UART driver");
+static MALLOC_DEFINE(M_UART, "UART", "UART driver");
 
 void
 uart_add_sysdev(struct uart_devinfo *di)
diff --git a/sys/dev/uart/uart_dev_ns8250.c b/sys/dev/uart/uart_dev_ns8250.c
index 489be29..5d56a68 100644
--- a/sys/dev/uart/uart_dev_ns8250.c
+++ b/sys/dev/uart/uart_dev_ns8250.c
@@ -582,9 +582,11 @@ static int
 ns8250_bus_ipend(struct uart_softc *sc)
 {
 	struct uart_bas *bas;
+	struct ns8250_softc *ns8250;
 	int ipend;
 	uint8_t iir, lsr;
 
+	ns8250 = (struct ns8250_softc *)sc;
 	bas = &sc->sc_bas;
 	uart_lock(sc->sc_hwmtx);
 	iir = uart_getreg(bas, REG_IIR);
@@ -602,9 +604,10 @@ ns8250_bus_ipend(struct uart_softc *sc)
 		if (lsr & LSR_RXRDY)
 			ipend |= SER_INT_RXREADY;
 	} else {
-		if (iir & IIR_TXRDY)
+		if (iir & IIR_TXRDY) {
 			ipend |= SER_INT_TXIDLE;
-		else
+			uart_setreg(bas, REG_IER, ns8250->ier);
+		} else
 			ipend |= SER_INT_SIGCHG;
 	}
 	if (ipend == 0)
diff --git a/sys/dev/ubsec/ubsec.c b/sys/dev/ubsec/ubsec.c
index 1ea150d..49e9dad 100644
--- a/sys/dev/ubsec/ubsec.c
+++ b/sys/dev/ubsec/ubsec.c
@@ -177,7 +177,8 @@ static	int ubsec_ksigbits(struct crparam *);
 static	void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 static	void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 
-SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0,
+    "Broadcom driver parameters");
 
 #ifdef UBSEC_DEBUG
 static	void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
diff --git a/sys/dev/usb/controller/at91dci.c b/sys/dev/usb/controller/at91dci.c
index 449aaa3..f831115 100644
--- a/sys/dev/usb/controller/at91dci.c
+++ b/sys/dev/usb/controller/at91dci.c
@@ -91,7 +91,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int at91dcidebug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, at91dci, CTLFLAG_RW, 0, "USB at91dci");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, at91dci, CTLFLAG_RW, 0, "USB at91dci");
 SYSCTL_INT(_hw_usb_at91dci, OID_AUTO, debug, CTLFLAG_RW,
     &at91dcidebug, 0, "at91dci debug level");
 #endif
@@ -2123,7 +2123,7 @@ tr_handle_get_port_status:
 		if (sc->sc_flags.status_vbus &&
 		    sc->sc_flags.status_bus_reset) {
 			/* reset endpoint flags */
-			bzero(sc->sc_ep_flags, sizeof(sc->sc_ep_flags));
+			memset(sc->sc_ep_flags, 0, sizeof(sc->sc_ep_flags));
 		}
 	}
 	if (sc->sc_flags.change_suspend) {
diff --git a/sys/dev/usb/controller/atmegadci.c b/sys/dev/usb/controller/atmegadci.c
index a3134d8..ad53fc3 100644
--- a/sys/dev/usb/controller/atmegadci.c
+++ b/sys/dev/usb/controller/atmegadci.c
@@ -83,7 +83,8 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int atmegadci_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, atmegadci, CTLFLAG_RW, 0, "USB ATMEGA DCI");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, atmegadci, CTLFLAG_RW, 0,
+    "USB ATMEGA DCI");
 SYSCTL_INT(_hw_usb_atmegadci, OID_AUTO, debug, CTLFLAG_RW,
     &atmegadci_debug, 0, "ATMEGA DCI debug level");
 #endif
diff --git a/sys/dev/usb/controller/avr32dci.c b/sys/dev/usb/controller/avr32dci.c
index d8b3934..26785a3 100644
--- a/sys/dev/usb/controller/avr32dci.c
+++ b/sys/dev/usb/controller/avr32dci.c
@@ -83,7 +83,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int avr32dci_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, avr32dci, CTLFLAG_RW, 0, "USB AVR32 DCI");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, avr32dci, CTLFLAG_RW, 0, "USB AVR32 DCI");
 SYSCTL_INT(_hw_usb_avr32dci, OID_AUTO, debug, CTLFLAG_RW,
     &avr32dci_debug, 0, "AVR32 DCI debug level");
 #endif
@@ -415,12 +415,11 @@ repeat:
 			buf_res.length = count;
 		}
 		/* receive data */
-		bcopy(sc->physdata +
+		memcpy(buf_res.buffer, sc->physdata +
 		    (AVR32_EPTSTA_CURRENT_BANK(temp) << td->bank_shift) +
-		    (td->ep_no << 16) + (td->offset % td->max_packet_size),
-		    buf_res.buffer, buf_res.length)
+		    (td->ep_no << 16) + (td->offset % td->max_packet_size), buf_res.length);
 		/* update counters */
-		    count -= buf_res.length;
+		count -= buf_res.length;
 		td->offset += buf_res.length;
 		td->remainder -= buf_res.length;
 	}
@@ -491,12 +490,12 @@ repeat:
 			buf_res.length = count;
 		}
 		/* transmit data */
-		bcopy(buf_res.buffer, sc->physdata +
+		memcpy(sc->physdata +
 		    (AVR32_EPTSTA_CURRENT_BANK(temp) << td->bank_shift) +
 		    (td->ep_no << 16) + (td->offset % td->max_packet_size),
-		    buf_res.length)
+		    buf_res.buffer, buf_res.length);
 		/* update counters */
-		    count -= buf_res.length;
+		count -= buf_res.length;
 		td->offset += buf_res.length;
 		td->remainder -= buf_res.length;
 	}
diff --git a/sys/dev/usb/controller/ehci.c b/sys/dev/usb/controller/ehci.c
index b7cd3cd..7e682f4 100644
--- a/sys/dev/usb/controller/ehci.c
+++ b/sys/dev/usb/controller/ehci.c
@@ -94,7 +94,7 @@ static int ehcinohighspeed = 0;
 static int ehciiaadbug = 0;
 static int ehcilostintrbug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
     &ehcidebug, 0, "Debug level");
 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
@@ -3369,7 +3369,7 @@ ehci_roothub_exec(struct usb_device *udev,
 		break;
 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
 		len = 16;
-		bzero(sc->sc_hub_desc.temp, 16);
+		memset(sc->sc_hub_desc.temp, 0, 16);
 		break;
 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
 		DPRINTFN(9, "get port status i=%d\n",
diff --git a/sys/dev/usb/controller/musb_otg.c b/sys/dev/usb/controller/musb_otg.c
index 7c9b02e..be32b2b 100644
--- a/sys/dev/usb/controller/musb_otg.c
+++ b/sys/dev/usb/controller/musb_otg.c
@@ -85,7 +85,7 @@
 #ifdef USB_DEBUG
 static int musbotgdebug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, musbotg, CTLFLAG_RW, 0, "USB musbotg");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, musbotg, CTLFLAG_RW, 0, "USB musbotg");
 SYSCTL_INT(_hw_usb_musbotg, OID_AUTO, debug, CTLFLAG_RW,
     &musbotgdebug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/controller/ohci.c b/sys/dev/usb/controller/ohci.c
index 0d63a6c..fa2d607 100644
--- a/sys/dev/usb/controller/ohci.c
+++ b/sys/dev/usb/controller/ohci.c
@@ -80,7 +80,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int ohcidebug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ohci, CTLFLAG_RW, 0, "USB ohci");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ohci, CTLFLAG_RW, 0, "USB ohci");
 SYSCTL_INT(_hw_usb_ohci, OID_AUTO, debug, CTLFLAG_RW,
     &ohcidebug, 0, "ohci debug level");
 
@@ -2347,7 +2347,7 @@ ohci_roothub_exec(struct usb_device *udev,
 
 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
 		len = 16;
-		bzero(sc->sc_hub_desc.temp, 16);
+		memset(sc->sc_hub_desc.temp, 0, 16);
 		break;
 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
 		DPRINTFN(9, "get port status i=%d\n",
diff --git a/sys/dev/usb/controller/uhci.c b/sys/dev/usb/controller/uhci.c
index b78342f..c365e01 100644
--- a/sys/dev/usb/controller/uhci.c
+++ b/sys/dev/usb/controller/uhci.c
@@ -85,7 +85,7 @@ __FBSDID("$FreeBSD$");
 static int uhcidebug = 0;
 static int uhcinoloop = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW, 0, "USB uhci");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW, 0, "USB uhci");
 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RW,
     &uhcidebug, 0, "uhci debug level");
 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RW,
@@ -2702,7 +2702,7 @@ uhci_roothub_exec(struct usb_device *udev,
 		break;
 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
 		len = 16;
-		bzero(sc->sc_hub_desc.temp, 16);
+		memset(sc->sc_hub_desc.temp, 0, 16);
 		break;
 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
 		if (index == 1)
diff --git a/sys/dev/usb/controller/usb_controller.c b/sys/dev/usb/controller/usb_controller.c
index dade0ad..7ff0bc0 100644
--- a/sys/dev/usb/controller/usb_controller.c
+++ b/sys/dev/usb/controller/usb_controller.c
@@ -75,7 +75,7 @@ static void	usb_attach_sub(device_t, struct usb_bus *);
 #ifdef USB_DEBUG
 static int usb_ctrl_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ctrl, CTLFLAG_RW, 0, "USB controller");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ctrl, CTLFLAG_RW, 0, "USB controller");
 SYSCTL_INT(_hw_usb_ctrl, OID_AUTO, debug, CTLFLAG_RW, &usb_ctrl_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/controller/uss820dci.c b/sys/dev/usb/controller/uss820dci.c
index b524529..b98043e 100644
--- a/sys/dev/usb/controller/uss820dci.c
+++ b/sys/dev/usb/controller/uss820dci.c
@@ -79,7 +79,8 @@
 #ifdef USB_DEBUG
 static int uss820dcidebug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uss820dci, CTLFLAG_RW, 0, "USB uss820dci");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uss820dci, CTLFLAG_RW, 0,
+    "USB uss820dci");
 SYSCTL_INT(_hw_usb_uss820dci, OID_AUTO, debug, CTLFLAG_RW,
     &uss820dcidebug, 0, "uss820dci debug level");
 #endif
diff --git a/sys/dev/usb/controller/xhci.c b/sys/dev/usb/controller/xhci.c
index 0df4c4d..9bc7d21 100644
--- a/sys/dev/usb/controller/xhci.c
+++ b/sys/dev/usb/controller/xhci.c
@@ -86,7 +86,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int xhcidebug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI");
 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW,
     &xhcidebug, 0, "Debug level");
 
@@ -292,7 +292,7 @@ xhci_start_controller(struct xhci_softc *sc)
 	XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST);
 
 	for (i = 0; i != 100; i++) {
-		usb_pause_mtx(NULL, hz / 1000);
+		usb_pause_mtx(NULL, hz / 100);
 		temp = XREAD4(sc, oper, XHCI_USBCMD) &
 		    (XHCI_CMD_HCRST | XHCI_STS_CNR);
 		if (!temp)
@@ -453,7 +453,7 @@ xhci_start_controller(struct xhci_softc *sc)
 	    XHCI_CMD_INTE | XHCI_CMD_HSEE);
 
 	for (i = 0; i != 100; i++) {
-		usb_pause_mtx(NULL, hz / 1000);
+		usb_pause_mtx(NULL, hz / 100);
 		temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
 		if (!temp)
 			break;
@@ -487,7 +487,7 @@ xhci_halt_controller(struct xhci_softc *sc)
 	XWRITE4(sc, oper, XHCI_USBCMD, 0);
 
 	for (i = 0; i != 100; i++) {
-		usb_pause_mtx(NULL, hz / 1000);
+		usb_pause_mtx(NULL, hz / 100);
 		temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
 		if (temp)
 			break;
@@ -1110,7 +1110,7 @@ xhci_cmd_nop(struct xhci_softc *sc)
 
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 #endif
 
@@ -1127,7 +1127,7 @@ xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
 	trb.dwTrb2 = 0;
 	trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT));
 
-	err = xhci_do_command(sc, &trb, 50 /* ms */);
+	err = xhci_do_command(sc, &trb, 100 /* ms */);
 	if (err)
 		goto done;
 
@@ -1154,7 +1154,7 @@ xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
 
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 static usb_error_t
@@ -1310,7 +1310,7 @@ xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx,
 
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 static usb_error_t
@@ -1328,7 +1328,7 @@ xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx,
 	    XHCI_TRB_3_SLOT_SET(slot_id);
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 static usb_error_t
@@ -1351,7 +1351,7 @@ xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve,
 
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 static usb_error_t
@@ -1373,7 +1373,7 @@ xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr,
 	    XHCI_TRB_3_EP_SET(ep_id);
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 static usb_error_t
@@ -1396,7 +1396,7 @@ xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend,
 
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 static usb_error_t
@@ -1414,7 +1414,7 @@ xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
 
 	trb.dwTrb3 = htole32(temp);
 
-	return (xhci_do_command(sc, &trb, 50 /* ms */));
+	return (xhci_do_command(sc, &trb, 100 /* ms */));
 }
 
 /*------------------------------------------------------------------------*
@@ -2831,7 +2831,7 @@ struct xhci_bos_desc xhci_bosd = {
 		.bLength = sizeof(xhci_bosd.usb2extd),
 		.bDescriptorType = 1,
 		.bDevCapabilityType = 2,
-		.bmAttributes = 2,
+		.bmAttributes[0] = 2,
 	},
 	.usbdcd = {
 		.bLength = sizeof(xhci_bosd.usbdcd),
@@ -2841,7 +2841,8 @@ struct xhci_bos_desc xhci_bosd = {
 		HSETW(.wSpeedsSupported, 0x000C),
 		.bFunctionalitySupport = 8,
 		.bU1DevExitLat = 255,	/* dummy - not used */
-		.bU2DevExitLat = 255,	/* dummy - not used */
+		.wU2DevExitLat[0] = 0x00,
+		.wU2DevExitLat[1] = 0x08,
 	},
 	.cidd = {
 		.bLength = sizeof(xhci_bosd.cidd),
diff --git a/sys/dev/usb/input/atp.c b/sys/dev/usb/input/atp.c
index 9a17950..6cfba78 100644
--- a/sys/dev/usb/input/atp.c
+++ b/sys/dev/usb/input/atp.c
@@ -114,7 +114,7 @@ __FBSDID("$FreeBSD$");
 
 
 /* Tunables */
-SYSCTL_NODE(_hw_usb, OID_AUTO, atp, CTLFLAG_RW, 0, "USB atp");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, atp, CTLFLAG_RW, 0, "USB atp");
 
 #ifdef USB_DEBUG
 enum atp_log_level {
diff --git a/sys/dev/usb/input/uep.c b/sys/dev/usb/input/uep.c
index e90298b..e534b27 100644
--- a/sys/dev/usb/input/uep.c
+++ b/sys/dev/usb/input/uep.c
@@ -57,7 +57,7 @@
 #ifdef USB_DEBUG
 static int uep_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uep, CTLFLAG_RW, 0, "USB uep");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uep, CTLFLAG_RW, 0, "USB uep");
 SYSCTL_INT(_hw_usb_uep, OID_AUTO, debug, CTLFLAG_RW,
     &uep_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/input/uhid.c b/sys/dev/usb/input/uhid.c
index 4723142..929227b 100644
--- a/sys/dev/usb/input/uhid.c
+++ b/sys/dev/usb/input/uhid.c
@@ -78,7 +78,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int uhid_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uhid, CTLFLAG_RW, 0, "USB uhid");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uhid, CTLFLAG_RW, 0, "USB uhid");
 SYSCTL_INT(_hw_usb_uhid, OID_AUTO, debug, CTLFLAG_RW,
     &uhid_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/input/ukbd.c b/sys/dev/usb/input/ukbd.c
index bf3ecd7..688f17b 100644
--- a/sys/dev/usb/input/ukbd.c
+++ b/sys/dev/usb/input/ukbd.c
@@ -93,7 +93,7 @@ __FBSDID("$FreeBSD$");
 static int ukbd_debug = 0;
 static int ukbd_no_leds = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ukbd, CTLFLAG_RW, 0, "USB ukbd");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ukbd, CTLFLAG_RW, 0, "USB ukbd");
 SYSCTL_INT(_hw_usb_ukbd, OID_AUTO, debug, CTLFLAG_RW,
     &ukbd_debug, 0, "Debug level");
 SYSCTL_INT(_hw_usb_ukbd, OID_AUTO, no_leds, CTLFLAG_RW,
diff --git a/sys/dev/usb/input/ums.c b/sys/dev/usb/input/ums.c
index 12d9e89..a910df4 100644
--- a/sys/dev/usb/input/ums.c
+++ b/sys/dev/usb/input/ums.c
@@ -76,7 +76,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int ums_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ums, CTLFLAG_RW, 0, "USB ums");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ums, CTLFLAG_RW, 0, "USB ums");
 SYSCTL_INT(_hw_usb_ums, OID_AUTO, debug, CTLFLAG_RW,
     &ums_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/misc/udbp.c b/sys/dev/usb/misc/udbp.c
index 1a72cae..150985e 100644
--- a/sys/dev/usb/misc/udbp.c
+++ b/sys/dev/usb/misc/udbp.c
@@ -96,7 +96,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int udbp_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, udbp, CTLFLAG_RW, 0, "USB udbp");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, udbp, CTLFLAG_RW, 0, "USB udbp");
 SYSCTL_INT(_hw_usb_udbp, OID_AUTO, debug, CTLFLAG_RW,
     &udbp_debug, 0, "udbp debug level");
 #endif
diff --git a/sys/dev/usb/net/if_aue.c b/sys/dev/usb/net/if_aue.c
index adf47f4..a14f233 100644
--- a/sys/dev/usb/net/if_aue.c
+++ b/sys/dev/usb/net/if_aue.c
@@ -102,7 +102,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int aue_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, aue, CTLFLAG_RW, 0, "USB aue");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, aue, CTLFLAG_RW, 0, "USB aue");
 SYSCTL_INT(_hw_usb_aue, OID_AUTO, debug, CTLFLAG_RW, &aue_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/net/if_axe.c b/sys/dev/usb/net/if_axe.c
index 62adda4..0d18f2b 100644
--- a/sys/dev/usb/net/if_axe.c
+++ b/sys/dev/usb/net/if_axe.c
@@ -133,7 +133,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int axe_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, axe, CTLFLAG_RW, 0, "USB axe");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, axe, CTLFLAG_RW, 0, "USB axe");
 SYSCTL_INT(_hw_usb_axe, OID_AUTO, debug, CTLFLAG_RW, &axe_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/net/if_cdce.c b/sys/dev/usb/net/if_cdce.c
index 12e6f67..eabf9c6 100644
--- a/sys/dev/usb/net/if_cdce.c
+++ b/sys/dev/usb/net/if_cdce.c
@@ -111,7 +111,7 @@ static uint32_t	cdce_m_crc32(struct mbuf *, uint32_t, uint32_t);
 static int cdce_debug = 0;
 static int cdce_tx_interval = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, cdce, CTLFLAG_RW, 0, "USB CDC-Ethernet");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, cdce, CTLFLAG_RW, 0, "USB CDC-Ethernet");
 SYSCTL_INT(_hw_usb_cdce, OID_AUTO, debug, CTLFLAG_RW, &cdce_debug, 0,
     "Debug level");
 SYSCTL_INT(_hw_usb_cdce, OID_AUTO, interval, CTLFLAG_RW, &cdce_tx_interval, 0,
diff --git a/sys/dev/usb/net/if_cue.c b/sys/dev/usb/net/if_cue.c
index 90a18f3..7466ba9 100644
--- a/sys/dev/usb/net/if_cue.c
+++ b/sys/dev/usb/net/if_cue.c
@@ -124,7 +124,7 @@ static void	cue_reset(struct cue_softc *);
 #ifdef USB_DEBUG
 static int cue_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, cue, CTLFLAG_RW, 0, "USB cue");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, cue, CTLFLAG_RW, 0, "USB cue");
 SYSCTL_INT(_hw_usb_cue, OID_AUTO, debug, CTLFLAG_RW, &cue_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/net/if_ipheth.c b/sys/dev/usb/net/if_ipheth.c
index d666835..ad4b6f1 100644
--- a/sys/dev/usb/net/if_ipheth.c
+++ b/sys/dev/usb/net/if_ipheth.c
@@ -81,7 +81,7 @@ static uether_fn_t ipheth_setpromisc;
 #ifdef USB_DEBUG
 static int ipheth_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ipheth, CTLFLAG_RW, 0, "USB iPhone ethernet");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ipheth, CTLFLAG_RW, 0, "USB iPhone ethernet");
 SYSCTL_INT(_hw_usb_ipheth, OID_AUTO, debug, CTLFLAG_RW, &ipheth_debug, 0, "Debug level");
 #endif
 
diff --git a/sys/dev/usb/net/if_kue.c b/sys/dev/usb/net/if_kue.c
index 5480a5d..2d3fc42 100644
--- a/sys/dev/usb/net/if_kue.c
+++ b/sys/dev/usb/net/if_kue.c
@@ -165,7 +165,7 @@ static void	kue_reset(struct kue_softc *);
 #ifdef USB_DEBUG
 static int kue_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, kue, CTLFLAG_RW, 0, "USB kue");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, kue, CTLFLAG_RW, 0, "USB kue");
 SYSCTL_INT(_hw_usb_kue, OID_AUTO, debug, CTLFLAG_RW, &kue_debug, 0,
     "Debug level");
 #endif
@@ -380,8 +380,9 @@ kue_setmulti(struct usb_ether *ue)
 		 */
 		if (i == KUE_MCFILTCNT(sc))
 			break;
-		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
-		    KUE_MCFILT(sc, i), ETHER_ADDR_LEN);
+		memcpy(KUE_MCFILT(sc, i),
+		    LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
+		    ETHER_ADDR_LEN);
 		i++;
 	}
 	if_maddr_runlock(ifp);
diff --git a/sys/dev/usb/net/if_mos.c b/sys/dev/usb/net/if_mos.c
index 5fbbb39..11cb962 100644
--- a/sys/dev/usb/net/if_mos.c
+++ b/sys/dev/usb/net/if_mos.c
@@ -132,7 +132,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int mos_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, mos, CTLFLAG_RW, 0, "USB mos");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, mos, CTLFLAG_RW, 0, "USB mos");
 SYSCTL_INT(_hw_usb_mos, OID_AUTO, debug, CTLFLAG_RW, &mos_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/net/if_rue.c b/sys/dev/usb/net/if_rue.c
index c874c1d..e4fd035 100644
--- a/sys/dev/usb/net/if_rue.c
+++ b/sys/dev/usb/net/if_rue.c
@@ -99,7 +99,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int rue_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, rue, CTLFLAG_RW, 0, "USB rue");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, rue, CTLFLAG_RW, 0, "USB rue");
 SYSCTL_INT(_hw_usb_rue, OID_AUTO, debug, CTLFLAG_RW,
     &rue_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/net/if_udav.c b/sys/dev/usb/net/if_udav.c
index e9a0504..3fa3d15 100644
--- a/sys/dev/usb/net/if_udav.c
+++ b/sys/dev/usb/net/if_udav.c
@@ -187,7 +187,7 @@ static const struct usb_ether_methods udav_ue_methods = {
 #ifdef USB_DEBUG
 static int udav_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, udav, CTLFLAG_RW, 0, "USB udav");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, udav, CTLFLAG_RW, 0, "USB udav");
 SYSCTL_INT(_hw_usb_udav, OID_AUTO, debug, CTLFLAG_RW, &udav_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/net/if_usie.c b/sys/dev/usb/net/if_usie.c
index cda1d3d..ed9054f 100644
--- a/sys/dev/usb/net/if_usie.c
+++ b/sys/dev/usb/net/if_usie.c
@@ -77,7 +77,7 @@ __FBSDID("$FreeBSD$");
 #ifdef	USB_DEBUG
 static int usie_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, usie, CTLFLAG_RW, 0, "sierra USB modem");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, usie, CTLFLAG_RW, 0, "sierra USB modem");
 SYSCTL_INT(_hw_usb_usie, OID_AUTO, debug, CTLFLAG_RW, &usie_debug, 0,
     "usie debug level");
 #endif
diff --git a/sys/dev/usb/net/uhso.c b/sys/dev/usb/net/uhso.c
index 6b42de8..8211170 100644
--- a/sys/dev/usb/net/uhso.c
+++ b/sys/dev/usb/net/uhso.c
@@ -283,7 +283,7 @@ static const STRUCT_USB_HOST_ID uhso_devs[] = {
 #undef UHSO_DEV
 };
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uhso, CTLFLAG_RW, 0, "USB uhso");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uhso, CTLFLAG_RW, 0, "USB uhso");
 static int uhso_autoswitch = 1;
 SYSCTL_INT(_hw_usb_uhso, OID_AUTO, auto_switch, CTLFLAG_RW,
     &uhso_autoswitch, 0, "Automatically switch to modem mode");
@@ -1153,7 +1153,7 @@ uhso_mux_read_callback(struct usb_xfer *xfer, usb_error_t error)
 		/* FALLTHROUGH */
 	case USB_ST_SETUP:
 tr_setup:
-		bzero(&req, sizeof(struct usb_device_request));
+		memset(&req, 0, sizeof(struct usb_device_request));
 		req.bmRequestType = UT_READ_CLASS_INTERFACE;
 		req.bRequest = UCDC_GET_ENCAPSULATED_RESPONSE;
 		USETW(req.wValue, 0);
@@ -1206,7 +1206,7 @@ uhso_mux_write_callback(struct usb_xfer *xfer, usb_error_t error)
 
 			usbd_get_page(pc, 0, &res);
 
-			bzero(&req, sizeof(struct usb_device_request));
+			memset(&req, 0, sizeof(struct usb_device_request));
 			req.bmRequestType = UT_WRITE_CLASS_INTERFACE;
 			req.bRequest = UCDC_SEND_ENCAPSULATED_COMMAND;
 			USETW(req.wValue, 0);
@@ -1731,7 +1731,7 @@ uhso_if_rxflush(void *arg)
 			 * copy the IP-packet into it.
 			 */
 			m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
-			bcopy(mtod(m0, uint8_t *), mtod(m, uint8_t *), iplen);
+			memcpy(mtod(m, uint8_t *), mtod(m0, uint8_t *), iplen);
 			m->m_pkthdr.len = m->m_len = iplen;
 
 			/* Adjust the size of the original mbuf */
diff --git a/sys/dev/usb/net/usb_ethernet.c b/sys/dev/usb/net/usb_ethernet.c
index bc73d70..e405ae8 100644
--- a/sys/dev/usb/net/usb_ethernet.c
+++ b/sys/dev/usb/net/usb_ethernet.c
@@ -57,7 +57,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_process.h>
 #include <dev/usb/net/usb_ethernet.h>
 
-SYSCTL_NODE(_net, OID_AUTO, ue, CTLFLAG_RD, 0, "USB Ethernet parameters");
+static SYSCTL_NODE(_net, OID_AUTO, ue, CTLFLAG_RD, 0,
+    "USB Ethernet parameters");
 
 #define	UE_LOCK(_ue)		mtx_lock((_ue)->ue_mtx)
 #define	UE_UNLOCK(_ue)		mtx_unlock((_ue)->ue_mtx)
diff --git a/sys/dev/usb/quirk/usb_quirk.c b/sys/dev/usb/quirk/usb_quirk.c
index 128fe2a..b0d7a7d 100644
--- a/sys/dev/usb/quirk/usb_quirk.c
+++ b/sys/dev/usb/quirk/usb_quirk.c
@@ -148,10 +148,6 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = {
 	    UQ_MSC_FORCE_PROTO_SCSI),
 	USB_QUIRK(AIPTEK, POCKETCAM3M, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_BBB,
 	    UQ_MSC_FORCE_PROTO_SCSI),
-	USB_QUIRK(ALCOR, SDCR_6335, 0x0000, 0xffff, UQ_MSC_NO_TEST_UNIT_READY,
-	    UQ_MSC_NO_SYNC_CACHE),
-	USB_QUIRK(ALCOR, SDCR_6362, 0x0000, 0xffff, UQ_MSC_NO_TEST_UNIT_READY,
-	    UQ_MSC_NO_SYNC_CACHE),
 	USB_QUIRK(ALCOR, UMCR_9361, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_BBB,
 	    UQ_MSC_FORCE_PROTO_SCSI, UQ_MSC_NO_GETMAXLUN),
 	USB_QUIRK(ALCOR, TRANSCEND, 0x0000, 0xffff, UQ_MSC_NO_GETMAXLUN,
@@ -462,6 +458,7 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = {
 	 * Quirks for manufacturers which USB devices does not respond
 	 * after issuing non-supported commands:
 	 */
+	USB_QUIRK(ALCOR, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MSC_NO_TEST_UNIT_READY, UQ_MATCH_VENDOR_ONLY),
 	USB_QUIRK(FEIYA, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MATCH_VENDOR_ONLY),
 	USB_QUIRK(REALTEK, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MATCH_VENDOR_ONLY),
 	USB_QUIRK(INITIO, DUMMY, 0x0000, 0xffff, UQ_MSC_NO_SYNC_CACHE, UQ_MATCH_VENDOR_ONLY),
diff --git a/sys/dev/usb/serial/u3g.c b/sys/dev/usb/serial/u3g.c
index 46a153f..6db2b55 100644
--- a/sys/dev/usb/serial/u3g.c
+++ b/sys/dev/usb/serial/u3g.c
@@ -66,7 +66,7 @@
 #ifdef USB_DEBUG
 static int u3g_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, u3g, CTLFLAG_RW, 0, "USB 3g");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, u3g, CTLFLAG_RW, 0, "USB 3g");
 SYSCTL_INT(_hw_usb_u3g, OID_AUTO, debug, CTLFLAG_RW,
     &u3g_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/serial/ubsa.c b/sys/dev/usb/serial/ubsa.c
index 6afe05b..c29d75d 100644
--- a/sys/dev/usb/serial/ubsa.c
+++ b/sys/dev/usb/serial/ubsa.c
@@ -95,7 +95,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int ubsa_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ubsa, CTLFLAG_RW, 0, "USB ubsa");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ubsa, CTLFLAG_RW, 0, "USB ubsa");
 SYSCTL_INT(_hw_usb_ubsa, OID_AUTO, debug, CTLFLAG_RW,
     &ubsa_debug, 0, "ubsa debug level");
 #endif
diff --git a/sys/dev/usb/serial/ubser.c b/sys/dev/usb/serial/ubser.c
index 0d70ba9..86278cc 100644
--- a/sys/dev/usb/serial/ubser.c
+++ b/sys/dev/usb/serial/ubser.c
@@ -116,7 +116,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int ubser_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ubser, CTLFLAG_RW, 0, "USB ubser");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ubser, CTLFLAG_RW, 0, "USB ubser");
 SYSCTL_INT(_hw_usb_ubser, OID_AUTO, debug, CTLFLAG_RW,
     &ubser_debug, 0, "ubser debug level");
 #endif
diff --git a/sys/dev/usb/serial/uchcom.c b/sys/dev/usb/serial/uchcom.c
index 3f69c4d..2b12029 100644
--- a/sys/dev/usb/serial/uchcom.c
+++ b/sys/dev/usb/serial/uchcom.c
@@ -103,7 +103,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int uchcom_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uchcom, CTLFLAG_RW, 0, "USB uchcom");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uchcom, CTLFLAG_RW, 0, "USB uchcom");
 SYSCTL_INT(_hw_usb_uchcom, OID_AUTO, debug, CTLFLAG_RW,
     &uchcom_debug, 0, "uchcom debug level");
 #endif
diff --git a/sys/dev/usb/serial/ufoma.c b/sys/dev/usb/serial/ufoma.c
index 31be85c..7233f90 100644
--- a/sys/dev/usb/serial/ufoma.c
+++ b/sys/dev/usb/serial/ufoma.c
@@ -438,7 +438,7 @@ ufoma_attach(device_t dev)
 		goto detach;
 	}
 	sc->sc_modetable[0] = (elements + 1);
-	bcopy(mad->bMode, &sc->sc_modetable[1], elements);
+	memcpy(&sc->sc_modetable[1], mad->bMode, elements);
 
 	sc->sc_currentmode = UMCPC_ACM_MODE_UNLINKED;
 	sc->sc_modetoactivate = mad->bMode[0];
@@ -968,7 +968,7 @@ ufoma_cfg_param(struct ucom_softc *ucom, struct termios *t)
 	}
 	DPRINTF("\n");
 
-	bzero(&ls, sizeof(ls));
+	memset(&ls, 0, sizeof(ls));
 
 	USETDW(ls.dwDTERate, t->c_ospeed);
 
diff --git a/sys/dev/usb/serial/uftdi.c b/sys/dev/usb/serial/uftdi.c
index 1c88063..843a56a 100644
--- a/sys/dev/usb/serial/uftdi.c
+++ b/sys/dev/usb/serial/uftdi.c
@@ -75,7 +75,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int uftdi_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uftdi, CTLFLAG_RW, 0, "USB uftdi");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uftdi, CTLFLAG_RW, 0, "USB uftdi");
 SYSCTL_INT(_hw_usb_uftdi, OID_AUTO, debug, CTLFLAG_RW,
     &uftdi_debug, 0, "Debug level");
 #endif
@@ -560,7 +560,7 @@ static int
 uftdi_set_parm_soft(struct termios *t,
     struct uftdi_param_config *cfg, uint8_t type)
 {
-	bzero(cfg, sizeof(*cfg));
+	memset(cfg, 0, sizeof(*cfg));
 
 	switch (type) {
 	case UFTDI_TYPE_SIO:
diff --git a/sys/dev/usb/serial/ulpt.c b/sys/dev/usb/serial/ulpt.c
index def2ae5..063e297 100644
--- a/sys/dev/usb/serial/ulpt.c
+++ b/sys/dev/usb/serial/ulpt.c
@@ -74,7 +74,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int ulpt_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ulpt, CTLFLAG_RW, 0, "USB ulpt");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ulpt, CTLFLAG_RW, 0, "USB ulpt");
 SYSCTL_INT(_hw_usb_ulpt, OID_AUTO, debug, CTLFLAG_RW,
     &ulpt_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/serial/umcs.c b/sys/dev/usb/serial/umcs.c
index 94ed4d9..a2b7852 100644
--- a/sys/dev/usb/serial/umcs.c
+++ b/sys/dev/usb/serial/umcs.c
@@ -79,7 +79,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int umcs_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, umcs, CTLFLAG_RW, 0, "USB umcs quadport serial adapter");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, umcs, CTLFLAG_RW, 0, "USB umcs quadport serial adapter");
 SYSCTL_INT(_hw_usb_umcs, OID_AUTO, debug, CTLFLAG_RW, &umcs_debug, 0, "Debug level");
 #endif					/* USB_DEBUG */
 
diff --git a/sys/dev/usb/serial/umodem.c b/sys/dev/usb/serial/umodem.c
index ed5162f..9289eec 100644
--- a/sys/dev/usb/serial/umodem.c
+++ b/sys/dev/usb/serial/umodem.c
@@ -118,7 +118,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int umodem_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, umodem, CTLFLAG_RW, 0, "USB umodem");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, umodem, CTLFLAG_RW, 0, "USB umodem");
 SYSCTL_INT(_hw_usb_umodem, OID_AUTO, debug, CTLFLAG_RW,
     &umodem_debug, 0, "Debug level");
 #endif
@@ -127,7 +127,11 @@ static const STRUCT_USB_HOST_ID umodem_devs[] = {
 	/* Generic Modem class match */
 	{USB_IFACE_CLASS(UICLASS_CDC),
 		USB_IFACE_SUBCLASS(UISUBCLASS_ABSTRACT_CONTROL_MODEL),
-	USB_IFACE_PROTOCOL(UIPROTO_CDC_AT)},
+		USB_IFACE_PROTOCOL(UIPROTO_CDC_AT)},
+	/* Huawei Modem class match */
+	{USB_IFACE_CLASS(UICLASS_CDC),
+		USB_IFACE_SUBCLASS(UISUBCLASS_ABSTRACT_CONTROL_MODEL),
+		USB_IFACE_PROTOCOL(0xFF)},
 	/* Kyocera AH-K3001V */
 	{USB_VPI(USB_VENDOR_KYOCERA, USB_PRODUCT_KYOCERA_AHK3001V, 1)},
 	{USB_VPI(USB_VENDOR_SIERRA, USB_PRODUCT_SIERRA_MC5720, 1)},
@@ -536,7 +540,7 @@ umodem_cfg_param(struct ucom_softc *ucom, struct termios *t)
 
 	DPRINTF("sc=%p\n", sc);
 
-	bzero(&ls, sizeof(ls));
+	memset(&ls, 0, sizeof(ls));
 
 	USETDW(ls.dwDTERate, t->c_ospeed);
 
diff --git a/sys/dev/usb/serial/umoscom.c b/sys/dev/usb/serial/umoscom.c
index c346ae6..eef2f47 100644
--- a/sys/dev/usb/serial/umoscom.c
+++ b/sys/dev/usb/serial/umoscom.c
@@ -50,7 +50,7 @@
 #ifdef USB_DEBUG
 static int umoscom_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, umoscom, CTLFLAG_RW, 0, "USB umoscom");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, umoscom, CTLFLAG_RW, 0, "USB umoscom");
 SYSCTL_INT(_hw_usb_umoscom, OID_AUTO, debug, CTLFLAG_RW,
     &umoscom_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/serial/uplcom.c b/sys/dev/usb/serial/uplcom.c
index 4af0537..f868a53 100644
--- a/sys/dev/usb/serial/uplcom.c
+++ b/sys/dev/usb/serial/uplcom.c
@@ -118,7 +118,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int uplcom_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uplcom, CTLFLAG_RW, 0, "USB uplcom");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uplcom, CTLFLAG_RW, 0, "USB uplcom");
 SYSCTL_INT(_hw_usb_uplcom, OID_AUTO, debug, CTLFLAG_RW,
     &uplcom_debug, 0, "Debug level");
 #endif
@@ -659,7 +659,7 @@ uplcom_cfg_param(struct ucom_softc *ucom, struct termios *t)
 
 	DPRINTF("sc = %p\n", sc);
 
-	bzero(&ls, sizeof(ls));
+	memset(&ls, 0, sizeof(ls));
 
 	USETDW(ls.dwDTERate, t->c_ospeed);
 
diff --git a/sys/dev/usb/serial/usb_serial.c b/sys/dev/usb/serial/usb_serial.c
index 61885ea..a3e6ffb 100644
--- a/sys/dev/usb/serial/usb_serial.c
+++ b/sys/dev/usb/serial/usb_serial.c
@@ -101,7 +101,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_gdb.h"
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ucom, CTLFLAG_RW, 0, "USB ucom");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ucom, CTLFLAG_RW, 0, "USB ucom");
 
 #ifdef USB_DEBUG
 static int ucom_debug = 0;
diff --git a/sys/dev/usb/serial/uslcom.c b/sys/dev/usb/serial/uslcom.c
index 6eaec83..848f2d5 100644
--- a/sys/dev/usb/serial/uslcom.c
+++ b/sys/dev/usb/serial/uslcom.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
 #include <dev/usb/usbdi_util.h>
+#include <dev/usb/usb_ioctl.h>
 #include "usbdevs.h"
 
 #define	USB_DEBUG_VAR uslcom_debug
@@ -52,7 +53,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int uslcom_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uslcom, CTLFLAG_RW, 0, "USB uslcom");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uslcom, CTLFLAG_RW, 0, "USB uslcom");
 SYSCTL_INT(_hw_usb_uslcom, OID_AUTO, debug, CTLFLAG_RW,
     &uslcom_debug, 0, "Debug level");
 #endif
@@ -63,49 +64,72 @@ SYSCTL_INT(_hw_usb_uslcom, OID_AUTO, debug, CTLFLAG_RW,
 
 #define	USLCOM_SET_DATA_BITS(x)	((x) << 8)
 
+/* Request types */
 #define	USLCOM_WRITE		0x41
 #define	USLCOM_READ		0xc1
 
+/* Request codes */
 #define	USLCOM_UART		0x00
 #define	USLCOM_BAUD_RATE	0x01	
 #define	USLCOM_DATA		0x03
 #define	USLCOM_BREAK		0x05
 #define	USLCOM_CTRL		0x07
+#define	USLCOM_RCTRL            0x08
+#define	USLCOM_SET_FLOWCTRL     0x13
+#define	USLCOM_VENDOR_SPECIFIC	0xff
 
+/* USLCOM_UART values */
 #define	USLCOM_UART_DISABLE	0x00
 #define	USLCOM_UART_ENABLE	0x01
 
+/* USLCOM_CTRL/USLCOM_RCTRL values */
 #define	USLCOM_CTRL_DTR_ON	0x0001	
 #define	USLCOM_CTRL_DTR_SET	0x0100
 #define	USLCOM_CTRL_RTS_ON	0x0002
 #define	USLCOM_CTRL_RTS_SET	0x0200
 #define	USLCOM_CTRL_CTS		0x0010
 #define	USLCOM_CTRL_DSR		0x0020
+#define	USLCOM_CTRL_RI          0x0040
 #define	USLCOM_CTRL_DCD		0x0080
 
+/* USLCOM_BAUD_RATE values */
 #define	USLCOM_BAUD_REF		0x384000
 
+/* USLCOM_DATA values */
 #define	USLCOM_STOP_BITS_1	0x00
 #define	USLCOM_STOP_BITS_2	0x02
-
 #define	USLCOM_PARITY_NONE	0x00
 #define	USLCOM_PARITY_ODD	0x10
 #define	USLCOM_PARITY_EVEN	0x20
 
-#define	USLCOM_PORT_NO		0xFFFF /* XXX think this should be 0 --hps */
+#define	USLCOM_PORT_NO		0x0000
 
+/* USLCOM_BREAK values */
 #define	USLCOM_BREAK_OFF	0x00
 #define	USLCOM_BREAK_ON		0x01
 
+/* USLCOM_SET_FLOWCTRL values - 1st word */
+#define	USLCOM_FLOW_DTR_ON      0x00000001 /* DTR static active */
+#define	USLCOM_FLOW_CTS_HS      0x00000008 /* CTS handshake */
+/* USLCOM_SET_FLOWCTRL values - 2nd word */
+#define	USLCOM_FLOW_RTS_ON      0x00000040 /* RTS static active */
+#define	USLCOM_FLOW_RTS_HS      0x00000080 /* RTS handshake */
+
+/* USLCOM_VENDOR_SPECIFIC values */
+#define	USLCOM_WRITE_LATCH	0x37E1
+#define	USLCOM_READ_LATCH	0x00C2
+
 enum {
 	USLCOM_BULK_DT_WR,
 	USLCOM_BULK_DT_RD,
+	USLCOM_CTRL_DT_RD,
 	USLCOM_N_TRANSFER,
 };
 
 struct uslcom_softc {
 	struct ucom_super_softc sc_super_ucom;
 	struct ucom_softc sc_ucom;
+	struct usb_callout sc_watchdog;
 
 	struct usb_xfer *sc_xfer[USLCOM_N_TRANSFER];
 	struct usb_device *sc_udev;
@@ -121,12 +145,15 @@ static device_detach_t uslcom_detach;
 
 static usb_callback_t uslcom_write_callback;
 static usb_callback_t uslcom_read_callback;
+static usb_callback_t uslcom_control_callback;
 
 static void uslcom_open(struct ucom_softc *);
 static void uslcom_close(struct ucom_softc *);
 static void uslcom_set_dtr(struct ucom_softc *, uint8_t);
 static void uslcom_set_rts(struct ucom_softc *, uint8_t);
 static void uslcom_set_break(struct ucom_softc *, uint8_t);
+static int uslcom_ioctl(struct ucom_softc *, uint32_t, caddr_t, int,
+		struct thread *);
 static int uslcom_pre_param(struct ucom_softc *, struct termios *);
 static void uslcom_param(struct ucom_softc *, struct termios *);
 static void uslcom_get_status(struct ucom_softc *, uint8_t *, uint8_t *);
@@ -143,7 +170,7 @@ static const struct usb_config uslcom_config[USLCOM_N_TRANSFER] = {
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
 		.bufsize = USLCOM_BULK_BUF_SIZE,
-		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+               .flags = {.pipe_bof = 1,},
 		.callback = &uslcom_write_callback,
 	},
 
@@ -155,6 +182,15 @@ static const struct usb_config uslcom_config[USLCOM_N_TRANSFER] = {
 		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
 		.callback = &uslcom_read_callback,
 	},
+	[USLCOM_CTRL_DT_RD] = {
+		.type = UE_CONTROL,
+		.endpoint = 0x00,
+		.direction = UE_DIR_ANY,
+		.bufsize = sizeof(struct usb_device_request) + 8,
+		.flags = {.pipe_bof = 1,},
+		.callback = &uslcom_control_callback,
+		.timeout = 1000,	/* 1 second timeout */
+	},
 };
 
 static struct ucom_callback uslcom_callback = {
@@ -164,6 +200,7 @@ static struct ucom_callback uslcom_callback = {
 	.ucom_cfg_set_dtr = &uslcom_set_dtr,
 	.ucom_cfg_set_rts = &uslcom_set_rts,
 	.ucom_cfg_set_break = &uslcom_set_break,
+	.ucom_ioctl = &uslcom_ioctl,
 	.ucom_cfg_param = &uslcom_param,
 	.ucom_pre_param = &uslcom_pre_param,
 	.ucom_start_read = &uslcom_start_read,
@@ -280,6 +317,19 @@ MODULE_DEPEND(uslcom, ucom, 1, 1, 1);
 MODULE_DEPEND(uslcom, usb, 1, 1, 1);
 MODULE_VERSION(uslcom, 1);
 
+static void
+uslcom_watchdog(void *arg)
+{
+	struct uslcom_softc *sc = arg;
+
+	mtx_assert(&sc->sc_mtx, MA_OWNED);
+
+	usbd_transfer_start(sc->sc_xfer[USLCOM_CTRL_DT_RD]);
+
+	usb_callout_reset(&sc->sc_watchdog,
+	    hz / 4, &uslcom_watchdog, sc);
+}
+
 static int
 uslcom_probe(device_t dev)
 {
@@ -310,6 +360,7 @@ uslcom_attach(device_t dev)
 
 	device_set_usb_desc(dev);
 	mtx_init(&sc->sc_mtx, "uslcom", NULL, MTX_DEF);
+	usb_callout_init_mtx(&sc->sc_watchdog, &sc->sc_mtx, 0);
 
 	sc->sc_udev = uaa->device;
 
@@ -350,6 +401,8 @@ uslcom_detach(device_t dev)
 
 	ucom_detach(&sc->sc_super_ucom, &sc->sc_ucom);
 	usbd_transfer_unsetup(sc->sc_xfer, USLCOM_N_TRANSFER);
+
+	usb_callout_drain(&sc->sc_watchdog);
 	mtx_destroy(&sc->sc_mtx);
 
 	return (0);
@@ -371,6 +424,9 @@ uslcom_open(struct ucom_softc *ucom)
 	    &req, NULL, 0, 1000)) {
 		DPRINTF("UART enable failed (ignored)\n");
 	}
+
+	/* start polling status */
+	uslcom_watchdog(sc);
 }
 
 static void
@@ -379,13 +435,16 @@ uslcom_close(struct ucom_softc *ucom)
 	struct uslcom_softc *sc = ucom->sc_parent;
 	struct usb_device_request req;
 
+	/* stop polling status */
+	usb_callout_stop(&sc->sc_watchdog);
+
 	req.bmRequestType = USLCOM_WRITE;
 	req.bRequest = USLCOM_UART;
 	USETW(req.wValue, USLCOM_UART_DISABLE);
 	USETW(req.wIndex, USLCOM_PORT_NO);
 	USETW(req.wLength, 0);
 
-        if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, 
+	if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, 
 	    &req, NULL, 0, 1000)) {
 		DPRINTF("UART disable failed (ignored)\n");
 	}
@@ -452,6 +511,7 @@ uslcom_param(struct ucom_softc *ucom, struct termios *t)
 {
 	struct uslcom_softc *sc = ucom->sc_parent;
 	struct usb_device_request req;
+	uint32_t flowctrl[4];
 	uint16_t data;
 
 	DPRINTF("\n");
@@ -503,7 +563,28 @@ uslcom_param(struct ucom_softc *ucom, struct termios *t)
 	    &req, NULL, 0, 1000)) {
 		DPRINTF("Set format failed (ignored)\n");
 	}
-	return;
+       
+	if (t->c_cflag & CRTSCTS) {
+		flowctrl[0] = htole32(USLCOM_FLOW_DTR_ON | USLCOM_FLOW_CTS_HS);
+		flowctrl[1] = htole32(USLCOM_FLOW_RTS_HS);
+		flowctrl[2] = 0;
+		flowctrl[3] = 0;
+	} else {
+		flowctrl[0] = htole32(USLCOM_FLOW_DTR_ON);
+		flowctrl[1] = htole32(USLCOM_FLOW_RTS_ON);
+		flowctrl[2] = 0;
+		flowctrl[3] = 0;
+	}
+	req.bmRequestType = USLCOM_WRITE;
+	req.bRequest = USLCOM_SET_FLOWCTRL;
+	USETW(req.wValue, 0);
+	USETW(req.wIndex, USLCOM_PORT_NO);
+	USETW(req.wLength, sizeof(flowctrl));
+
+	if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, 
+	    &req, flowctrl, 0, 1000)) {
+		DPRINTF("Set flowcontrol failed (ignored)\n");
+	}
 }
 
 static void
@@ -536,6 +617,55 @@ uslcom_set_break(struct ucom_softc *ucom, uint8_t onoff)
 	}
 }
 
+static int
+uslcom_ioctl(struct ucom_softc *ucom, uint32_t cmd, caddr_t data,
+    int flag, struct thread *td)
+{
+	struct uslcom_softc *sc = ucom->sc_parent;
+	struct usb_device_request req;
+	int error = 0;
+	uint8_t latch;
+
+	DPRINTF("cmd=0x%08x\n", cmd);
+
+	switch (cmd) {
+	case USB_GET_GPIO:
+		req.bmRequestType = USLCOM_READ;
+		req.bRequest = USLCOM_VENDOR_SPECIFIC;
+		USETW(req.wValue, USLCOM_READ_LATCH);
+		USETW(req.wIndex, 0);
+		USETW(req.wLength, sizeof(latch));
+
+		if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, 
+		    &req, &latch, 0, 1000)) {
+			DPRINTF("Get LATCH failed\n");
+			error = EIO;
+		}
+		*(int *)data = latch;
+		break;
+
+	case USB_SET_GPIO:
+		req.bmRequestType = USLCOM_WRITE;
+		req.bRequest = USLCOM_VENDOR_SPECIFIC;
+		USETW(req.wValue, USLCOM_WRITE_LATCH);
+		USETW(req.wIndex, (*(int *)data));
+		USETW(req.wLength, 0);
+		
+		if (ucom_cfg_do_request(sc->sc_udev, &sc->sc_ucom, 
+		    &req, NULL, 0, 1000)) {
+			DPRINTF("Set LATCH failed\n");
+			error = EIO;
+		}
+		break;
+
+	default:
+		DPRINTF("Unknown IOCTL\n");
+		error = ENOIOCTL;
+		break;
+	}
+	return (error);
+}
+
 static void
 uslcom_write_callback(struct usb_xfer *xfer, usb_error_t error)
 {
@@ -599,6 +729,59 @@ tr_setup:
 }
 
 static void
+uslcom_control_callback(struct usb_xfer *xfer, usb_error_t error)
+{
+	struct uslcom_softc *sc = usbd_xfer_softc(xfer);
+	struct usb_page_cache *pc;
+	struct usb_device_request req;
+	uint8_t msr = 0;
+	uint8_t buf;
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		pc = usbd_xfer_get_frame(xfer, 1);
+		usbd_copy_out(pc, 0, &buf, sizeof(buf));
+		if (buf & USLCOM_CTRL_CTS)
+			msr |= SER_CTS;
+		if (buf & USLCOM_CTRL_DSR)
+			msr |= SER_DSR;
+		if (buf & USLCOM_CTRL_RI)
+			msr |= SER_RI;
+		if (buf & USLCOM_CTRL_DCD)
+			msr |= SER_DCD;
+
+		if (msr != sc->sc_msr) {
+			DPRINTF("status change msr=0x%02x "
+			    "(was 0x%02x)\n", msr, sc->sc_msr);
+			sc->sc_msr = msr;
+			ucom_status_change(&sc->sc_ucom);
+		}
+		break;
+
+	case USB_ST_SETUP:
+		req.bmRequestType = USLCOM_READ;
+		req.bRequest = USLCOM_RCTRL;
+		USETW(req.wValue, 0);
+		USETW(req.wIndex, USLCOM_PORT_NO);
+		USETW(req.wLength, sizeof(buf));
+               
+		usbd_xfer_set_frames(xfer, 2);
+		usbd_xfer_set_frame_len(xfer, 0, sizeof(req));
+		usbd_xfer_set_frame_len(xfer, 1, sizeof(buf));
+
+		pc = usbd_xfer_get_frame(xfer, 0);
+		usbd_copy_in(pc, 0, &req, sizeof(req));
+		usbd_transfer_submit(xfer);
+		break;
+
+	default:		/* error */
+		if (error != USB_ERR_CANCELLED)
+			DPRINTF("error=%s\n", usbd_errstr(error));
+		break;
+	}
+}
+
+static void
 uslcom_start_read(struct ucom_softc *ucom)
 {
 	struct uslcom_softc *sc = ucom->sc_parent;
diff --git a/sys/dev/usb/serial/uvisor.c b/sys/dev/usb/serial/uvisor.c
index 976ea19..df96958 100644
--- a/sys/dev/usb/serial/uvisor.c
+++ b/sys/dev/usb/serial/uvisor.c
@@ -80,7 +80,7 @@
 #ifdef USB_DEBUG
 static int uvisor_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uvisor, CTLFLAG_RW, 0, "USB uvisor");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uvisor, CTLFLAG_RW, 0, "USB uvisor");
 SYSCTL_INT(_hw_usb_uvisor, OID_AUTO, debug, CTLFLAG_RW,
     &uvisor_debug, 0, "Debug level");
 #endif
@@ -311,8 +311,9 @@ uvisor_attach(device_t dev)
 	int error;
 
 	DPRINTF("sc=%p\n", sc);
-	bcopy(uvisor_config, uvisor_config_copy,
+	memcpy(uvisor_config_copy, uvisor_config,
 	    sizeof(uvisor_config_copy));
+
 	device_set_usb_desc(dev);
 
 	mtx_init(&sc->sc_mtx, "uvisor", NULL, MTX_DEF);
diff --git a/sys/dev/usb/serial/uvscom.c b/sys/dev/usb/serial/uvscom.c
index 52e02ad..f5dc2d5 100644
--- a/sys/dev/usb/serial/uvscom.c
+++ b/sys/dev/usb/serial/uvscom.c
@@ -70,7 +70,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int uvscom_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uvscom, CTLFLAG_RW, 0, "USB uvscom");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uvscom, CTLFLAG_RW, 0, "USB uvscom");
 SYSCTL_INT(_hw_usb_uvscom, OID_AUTO, debug, CTLFLAG_RW,
     &uvscom_debug, 0, "Debug level");
 #endif
diff --git a/sys/dev/usb/storage/umass.c b/sys/dev/usb/storage/umass.c
index b932cfa..3f2bd11 100644
--- a/sys/dev/usb/storage/umass.c
+++ b/sys/dev/usb/storage/umass.c
@@ -173,7 +173,7 @@ __FBSDID("$FreeBSD$");
 #define	UDMASS_ALL	0xffff0000	/* all of the above */
 static int umass_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, umass, CTLFLAG_RW, 0, "USB umass");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, umass, CTLFLAG_RW, 0, "USB umass");
 SYSCTL_INT(_hw_usb_umass, OID_AUTO, debug, CTLFLAG_RW,
     &umass_debug, 0, "umass debug level");
 
@@ -891,7 +891,7 @@ umass_attach(device_t dev)
 	int32_t err;
 
 	/*
-	 * NOTE: the softc struct is bzero-ed in device_set_driver.
+	 * NOTE: the softc struct is cleared in device_set_driver.
 	 * We can safely call umass_detach without specifically
 	 * initializing the struct.
 	 */
@@ -1305,11 +1305,13 @@ umass_t_bbb_command_callback(struct usb_xfer *xfer, usb_error_t error)
 			}
 			sc->cbw.bCDBLength = sc->sc_transfer.cmd_len;
 
-			bcopy(sc->sc_transfer.cmd_data, sc->cbw.CBWCDB,
+			memcpy(sc->cbw.CBWCDB, sc->sc_transfer.cmd_data,
 			    sc->sc_transfer.cmd_len);
 
-			bzero(sc->sc_transfer.cmd_data + sc->sc_transfer.cmd_len,
-			    sizeof(sc->cbw.CBWCDB) - sc->sc_transfer.cmd_len);
+			memset(sc->sc_transfer.cmd_data +
+			    sc->sc_transfer.cmd_len, 0,
+			    sizeof(sc->cbw.CBWCDB) -
+			    sc->sc_transfer.cmd_len);
 
 			DIF(UDMASS_BBB, umass_bbb_dump_cbw(sc, &sc->cbw));
 
@@ -1480,9 +1482,9 @@ umass_t_bbb_status_callback(struct usb_xfer *xfer, usb_error_t error)
 
 		/* Zero missing parts of the CSW: */
 
-		if (actlen < sizeof(sc->csw)) {
-			bzero(&sc->csw, sizeof(sc->csw));
-		}
+		if (actlen < sizeof(sc->csw))
+			memset(&sc->csw, 0, sizeof(sc->csw));
+
 		pc = usbd_xfer_get_frame(xfer, 0);
 		usbd_copy_out(pc, 0, &sc->csw, actlen);
 
@@ -2755,7 +2757,7 @@ umass_scsi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 		if (sc->sc_quirks & NO_TEST_UNIT_READY) {
 			DPRINTF(sc, UDMASS_SCSI, "Converted TEST_UNIT_READY "
 			    "to START_UNIT\n");
-			bzero(sc->sc_transfer.cmd_data, cmd_len);
+			memset(sc->sc_transfer.cmd_data, 0, cmd_len);
 			sc->sc_transfer.cmd_data[0] = START_STOP_UNIT;
 			sc->sc_transfer.cmd_data[4] = SSS_START;
 			return (1);
@@ -2768,14 +2770,14 @@ umass_scsi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 		 * information.
 		 */
 		if (sc->sc_quirks & FORCE_SHORT_INQUIRY) {
-			bcopy(cmd_ptr, sc->sc_transfer.cmd_data, cmd_len);
+			memcpy(sc->sc_transfer.cmd_data, cmd_ptr, cmd_len);
 			sc->sc_transfer.cmd_data[4] = SHORT_INQUIRY_LENGTH;
 			return (1);
 		}
 		break;
 	}
 
-	bcopy(cmd_ptr, sc->sc_transfer.cmd_data, cmd_len);
+	memcpy(sc->sc_transfer.cmd_data, cmd_ptr, cmd_len);
 	return (1);
 }
 
@@ -2810,10 +2812,11 @@ umass_rbc_transform(struct umass_softc *sc, uint8_t *cmd_ptr, uint8_t cmd_len)
 	case REQUEST_SENSE:
 	case PREVENT_ALLOW:
 
-		bcopy(cmd_ptr, sc->sc_transfer.cmd_data, cmd_len);
+		memcpy(sc->sc_transfer.cmd_data, cmd_ptr, cmd_len);
 
 		if ((sc->sc_quirks & RBC_PAD_TO_12) && (cmd_len < 12)) {
-			bzero(sc->sc_transfer.cmd_data + cmd_len, 12 - cmd_len);
+			memset(sc->sc_transfer.cmd_data + cmd_len,
+			    0, 12 - cmd_len);
 			cmd_len = 12;
 		}
 		sc->sc_transfer.cmd_len = cmd_len;
@@ -2841,7 +2844,7 @@ umass_ufi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 	sc->sc_transfer.cmd_len = UFI_COMMAND_LENGTH;
 
 	/* Zero the command data */
-	bzero(sc->sc_transfer.cmd_data, UFI_COMMAND_LENGTH);
+	memset(sc->sc_transfer.cmd_data, 0, UFI_COMMAND_LENGTH);
 
 	switch (cmd_ptr[0]) {
 		/*
@@ -2898,7 +2901,7 @@ umass_ufi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 		return (0);		/* failure */
 	}
 
-	bcopy(cmd_ptr, sc->sc_transfer.cmd_data, cmd_len);
+	memcpy(sc->sc_transfer.cmd_data, cmd_ptr, cmd_len);
 	return (1);			/* success */
 }
 
@@ -2919,7 +2922,7 @@ umass_atapi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 	sc->sc_transfer.cmd_len = ATAPI_COMMAND_LENGTH;
 
 	/* Zero the command data */
-	bzero(sc->sc_transfer.cmd_data, ATAPI_COMMAND_LENGTH);
+	memset(sc->sc_transfer.cmd_data, 0, ATAPI_COMMAND_LENGTH);
 
 	switch (cmd_ptr[0]) {
 		/*
@@ -2933,7 +2936,7 @@ umass_atapi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 		 * information.
 		 */
 		if (sc->sc_quirks & FORCE_SHORT_INQUIRY) {
-			bcopy(cmd_ptr, sc->sc_transfer.cmd_data, cmd_len);
+			memcpy(sc->sc_transfer.cmd_data, cmd_ptr, cmd_len);
 
 			sc->sc_transfer.cmd_data[4] = SHORT_INQUIRY_LENGTH;
 			return (1);
@@ -2994,7 +2997,7 @@ umass_atapi_transform(struct umass_softc *sc, uint8_t *cmd_ptr,
 		break;
 	}
 
-	bcopy(cmd_ptr, sc->sc_transfer.cmd_data, cmd_len);
+	memcpy(sc->sc_transfer.cmd_data, cmd_ptr, cmd_len);
 	return (1);			/* success */
 }
 
diff --git a/sys/dev/usb/storage/urio.c b/sys/dev/usb/storage/urio.c
index 6687173..e79ae87 100644
--- a/sys/dev/usb/storage/urio.c
+++ b/sys/dev/usb/storage/urio.c
@@ -80,7 +80,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int urio_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, urio, CTLFLAG_RW, 0, "USB urio");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, urio, CTLFLAG_RW, 0, "USB urio");
 SYSCTL_INT(_hw_usb_urio, OID_AUTO, debug, CTLFLAG_RW,
     &urio_debug, 0, "urio debug level");
 #endif
@@ -440,7 +440,7 @@ urio_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr,
 			error = EPERM;
 			goto done;
 		}
-		bzero(&ur, sizeof(ur));
+		memset(&ur, 0, sizeof(ur));
 		rio_cmd = addr;
 		ur.ucr_request.bmRequestType =
 		    rio_cmd->requesttype | UT_READ_VENDOR_DEVICE;
@@ -451,7 +451,7 @@ urio_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr,
 			error = EPERM;
 			goto done;
 		}
-		bzero(&ur, sizeof(ur));
+		memset(&ur, 0, sizeof(ur));
 		rio_cmd = addr;
 		ur.ucr_request.bmRequestType =
 		    rio_cmd->requesttype | UT_WRITE_VENDOR_DEVICE;
diff --git a/sys/dev/usb/storage/ustorage_fs.c b/sys/dev/usb/storage/ustorage_fs.c
index dbf025a..dd6b413 100644
--- a/sys/dev/usb/storage/ustorage_fs.c
+++ b/sys/dev/usb/storage/ustorage_fs.c
@@ -355,7 +355,7 @@ ustorage_fs_attach(device_t dev)
 	int unit;
 
 	/*
-	 * NOTE: the softc struct is bzero-ed in device_set_driver.
+	 * NOTE: the softc struct is cleared in device_set_driver.
 	 * We can safely call ustorage_fs_detach without specifically
 	 * initializing the struct.
 	 */
@@ -364,6 +364,9 @@ ustorage_fs_attach(device_t dev)
 	sc->sc_udev = uaa->device;
 	unit = device_get_unit(dev);
 
+	/* enable power saving mode */
+	usbd_set_power_mode(uaa->device, USB_POWER_MODE_SAVE);
+
 	if (unit == 0) {
 		if (ustorage_fs_ramdisk == NULL) {
 			/*
@@ -371,7 +374,9 @@ ustorage_fs_attach(device_t dev)
 			 * further
 			 */
 			ustorage_fs_ramdisk =
-			    malloc(USTORAGE_FS_RAM_SECT << 9, M_USB, M_ZERO | M_WAITOK);
+			    malloc(USTORAGE_FS_RAM_SECT << 9, M_USB,
+			    M_ZERO | M_WAITOK);
+
 			if (ustorage_fs_ramdisk == NULL) {
 				return (ENOMEM);
 			}
diff --git a/sys/dev/usb/template/usb_template.c b/sys/dev/usb/template/usb_template.c
index cf97482..209016a 100644
--- a/sys/dev/usb/template/usb_template.c
+++ b/sys/dev/usb/template/usb_template.c
@@ -913,7 +913,7 @@ usb_hw_ep_resolve(struct usb_device *udev,
 	}
 	ues = udev->bus->scratch[0].hw_ep_scratch;
 
-	bzero(ues, sizeof(*ues));
+	memset(ues, 0, sizeof(*ues));
 
 	ues->ep_max = ues->ep;
 	ues->cd = (void *)desc;
@@ -1240,7 +1240,7 @@ usb_temp_setup(struct usb_device *udev,
 	}
 	uts = udev->bus->scratch[0].temp_setup;
 
-	bzero(uts, sizeof(*uts));
+	memset(uts, 0, sizeof(*uts));
 
 	uts->usb_speed = udev->speed;
 	uts->self_powered = udev->flags.self_powered;
diff --git a/sys/dev/usb/usb.h b/sys/dev/usb/usb.h
index 6a9b126..fb1c8e6 100644
--- a/sys/dev/usb/usb.h
+++ b/sys/dev/usb/usb.h
@@ -323,7 +323,7 @@ struct usb_devcap_usb2ext_descriptor {
 	uByte	bLength;
 	uByte	bDescriptorType;
 	uByte	bDevCapabilityType;
-	uByte	bmAttributes;
+	uDWord	bmAttributes;
 #define	USB_V2EXT_LPM 0x02
 } __packed;
 typedef struct usb_devcap_usb2ext_descriptor usb_devcap_usb2ext_descriptor_t;
@@ -336,7 +336,7 @@ struct usb_devcap_ss_descriptor {
 	uWord	wSpeedsSupported;
 	uByte	bFunctionalitySupport;
 	uByte	bU1DevExitLat;
-	uByte	bU2DevExitLat;
+	uWord	wU2DevExitLat;
 } __packed;
 typedef struct usb_devcap_ss_descriptor usb_devcap_ss_descriptor_t;
 
diff --git a/sys/dev/usb/usb_busdma.c b/sys/dev/usb/usb_busdma.c
index f550dd0..d31aeec 100644
--- a/sys/dev/usb/usb_busdma.c
+++ b/sys/dev/usb/usb_busdma.c
@@ -80,9 +80,9 @@ void
 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
     struct usb_page_search *res)
 {
+#if USB_HAVE_BUSDMA
 	struct usb_page *page;
 
-#if USB_HAVE_BUSDMA
 	if (pc->page_start) {
 
 		/* Case 1 - something has been loaded into DMA */
@@ -145,7 +145,7 @@ usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
 		if (buf_res.length > len) {
 			buf_res.length = len;
 		}
-		bcopy(ptr, buf_res.buffer, buf_res.length);
+		memcpy(buf_res.buffer, ptr, buf_res.length);
 
 		offset += buf_res.length;
 		len -= buf_res.length;
@@ -267,7 +267,7 @@ usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
 		if (res.length > len) {
 			res.length = len;
 		}
-		bcopy(res.buffer, ptr, res.length);
+		memcpy(ptr, res.buffer, res.length);
 
 		offset += res.length;
 		len -= res.length;
@@ -325,7 +325,7 @@ usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
 		if (res.length > len) {
 			res.length = len;
 		}
-		bzero(res.buffer, res.length);
+		memset(res.buffer, 0, res.length);
 
 		offset += res.length;
 		len -= res.length;
@@ -560,7 +560,7 @@ usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
 		bus_dmamem_free(utag->tag, ptr, map);
 		goto error;
 	}
-	bzero(ptr, size);
+	memset(ptr, 0, size);
 
 	usb_pc_cpu_flush(pc);
 
@@ -797,7 +797,7 @@ usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
     struct mtx *mtx, usb_dma_callback_t *func,
     uint8_t ndmabits, uint8_t nudt)
 {
-	bzero(udpt, sizeof(*udpt));
+	memset(udpt, 0, sizeof(*udpt));
 
 	/* sanity checking */
 	if ((nudt == 0) ||
@@ -818,7 +818,7 @@ usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
 	udpt->dma_bits = ndmabits;
 
 	while (nudt--) {
-		bzero(udt, sizeof(*udt));
+		memset(udt, 0, sizeof(*udt));
 		udt->tag_parent = udpt;
 		udt++;
 	}
diff --git a/sys/dev/usb/usb_compat_linux.c b/sys/dev/usb/usb_compat_linux.c
index 8e20d20..2f78d9f 100644
--- a/sys/dev/usb/usb_compat_linux.c
+++ b/sys/dev/usb/usb_compat_linux.c
@@ -564,7 +564,7 @@ usb_clear_halt(struct usb_device *dev, struct usb_host_endpoint *uhe)
 	type = uhe->desc.bmAttributes & UE_XFERTYPE;
 	addr = uhe->desc.bEndpointAddress;
 
-	bzero(cfg, sizeof(cfg));
+	memset(cfg, 0, sizeof(cfg));
 
 	cfg[0].type = type;
 	cfg[0].endpoint = addr & UE_ADDR;
@@ -709,12 +709,12 @@ usb_control_msg(struct usb_device *dev, struct usb_host_endpoint *uhe,
 	urb->dev = dev;
 	urb->endpoint = uhe;
 
-	bcopy(&req, urb->setup_packet, sizeof(req));
+	memcpy(urb->setup_packet, &req, sizeof(req));
 
 	if (size && (!(req.bmRequestType & UT_READ))) {
 		/* move the data to a real buffer */
-		bcopy(data, USB_ADD_BYTES(urb->setup_packet,
-		    sizeof(req)), size);
+		memcpy(USB_ADD_BYTES(urb->setup_packet, sizeof(req)),
+		    data, size);
 	}
 	err = usb_start_wait_urb(urb, timeout, &actlen);
 
@@ -789,7 +789,7 @@ usb_setup_endpoint(struct usb_device *dev,
 	if (bufsize == 0) {
 		return (0);
 	}
-	bzero(cfg, sizeof(cfg));
+	memset(cfg, 0, sizeof(cfg));
 
 	if (type == UE_ISOCHRONOUS) {
 
@@ -1251,7 +1251,7 @@ usb_init_urb(struct urb *urb)
 	if (urb == NULL) {
 		return;
 	}
-	bzero(urb, sizeof(*urb));
+	memset(urb, 0, sizeof(*urb));
 }
 
 /*------------------------------------------------------------------------*
diff --git a/sys/dev/usb/usb_dev.c b/sys/dev/usb/usb_dev.c
index e5d98fe..e2c934e 100644
--- a/sys/dev/usb/usb_dev.c
+++ b/sys/dev/usb/usb_dev.c
@@ -81,7 +81,7 @@
 #ifdef USB_DEBUG
 static int usb_fifo_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RW,
     &usb_fifo_debug, 0, "Debug Level");
 
@@ -1809,8 +1809,8 @@ usb_fifo_free_buffer(struct usb_fifo *f)
 	}
 	/* reset queues */
 
-	bzero(&f->free_q, sizeof(f->free_q));
-	bzero(&f->used_q, sizeof(f->used_q));
+	memset(&f->free_q, 0, sizeof(f->free_q));
+	memset(&f->used_q, 0, sizeof(f->used_q));
 }
 
 void
@@ -1909,7 +1909,7 @@ usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
 
 			io_len = MIN(len, m->cur_data_len);
 
-			bcopy(ptr, m->cur_data_ptr, io_len);
+			memcpy(m->cur_data_ptr, ptr, io_len);
 
 			m->cur_data_len = io_len;
 			ptr = USB_ADD_BYTES(ptr, io_len);
@@ -2052,7 +2052,7 @@ usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
 
 			io_len = MIN(len, m->cur_data_len);
 
-			bcopy(m->cur_data_ptr, ptr, io_len);
+			memcpy(ptr, m->cur_data_ptr, io_len);
 
 			len -= io_len;
 			ptr = USB_ADD_BYTES(ptr, io_len);
diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c
index 7e0de33..726d7a7 100644
--- a/sys/dev/usb/usb_device.c
+++ b/sys/dev/usb/usb_device.c
@@ -1851,7 +1851,8 @@ repeat_set_config:
 		}
 	}
 	if (set_config_failed == 0 && config_index == 0 &&
-	    usb_test_quirk(&uaa, UQ_MSC_NO_SYNC_CACHE) == 0) {
+	    usb_test_quirk(&uaa, UQ_MSC_NO_SYNC_CACHE) == 0 &&
+	    usb_test_quirk(&uaa, UQ_MSC_NO_GETMAXLUN) == 0) {
 
 		/*
 		 * Try to figure out if there are any MSC quirks we
diff --git a/sys/dev/usb/usb_generic.c b/sys/dev/usb/usb_generic.c
index d62f8f9..f175eb9 100644
--- a/sys/dev/usb/usb_generic.c
+++ b/sys/dev/usb/usb_generic.c
@@ -126,7 +126,7 @@ struct usb_fifo_methods usb_ugen_methods = {
 #ifdef USB_DEBUG
 static int ugen_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ugen, CTLFLAG_RW, 0, "USB generic");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ugen, CTLFLAG_RW, 0, "USB generic");
 SYSCTL_INT(_hw_usb_ugen, OID_AUTO, debug, CTLFLAG_RW, &ugen_debug,
     0, "Debug level");
 
@@ -240,7 +240,7 @@ ugen_open_pipe_write(struct usb_fifo *f)
 		/* transfers are already opened */
 		return (0);
 	}
-	bzero(usb_config, sizeof(usb_config));
+	memset(usb_config, 0, sizeof(usb_config));
 
 	usb_config[1].type = UE_CONTROL;
 	usb_config[1].endpoint = 0;
@@ -308,7 +308,7 @@ ugen_open_pipe_read(struct usb_fifo *f)
 		/* transfers are already opened */
 		return (0);
 	}
-	bzero(usb_config, sizeof(usb_config));
+	memset(usb_config, 0, sizeof(usb_config));
 
 	usb_config[1].type = UE_CONTROL;
 	usb_config[1].endpoint = 0;
diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c
index 5795d56..8652661 100644
--- a/sys/dev/usb/usb_hub.c
+++ b/sys/dev/usb/usb_hub.c
@@ -76,7 +76,7 @@
 #ifdef USB_DEBUG
 static int uhub_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uhub, CTLFLAG_RW, 0, "USB HUB");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uhub, CTLFLAG_RW, 0, "USB HUB");
 SYSCTL_INT(_hw_usb_uhub, OID_AUTO, debug, CTLFLAG_RW, &uhub_debug, 0,
     "Debug level");
 
diff --git a/sys/dev/usb/usb_ioctl.h b/sys/dev/usb/usb_ioctl.h
index d35fa10..9af6ee5 100644
--- a/sys/dev/usb/usb_ioctl.h
+++ b/sys/dev/usb/usb_ioctl.h
@@ -289,6 +289,10 @@ struct usb_gen_quirk {
 #define	USB_GET_CM_OVER_DATA	_IOR ('U', 180, int)
 #define	USB_SET_CM_OVER_DATA	_IOW ('U', 181, int)
 
+/* GPIO control */
+#define	USB_GET_GPIO		_IOR ('U', 182, int)
+#define	USB_SET_GPIO		_IOW ('U', 183, int)
+
 /* USB file system interface */
 #define	USB_FS_START		_IOW ('U', 192, struct usb_fs_start)
 #define	USB_FS_STOP		_IOW ('U', 193, struct usb_fs_stop)
diff --git a/sys/dev/usb/usb_msctest.c b/sys/dev/usb/usb_msctest.c
index 0355653..7f33014 100644
--- a/sys/dev/usb/usb_msctest.c
+++ b/sys/dev/usb/usb_msctest.c
@@ -475,8 +475,8 @@ bbb_command_start(struct bbb_transfer *sc, uint8_t dir, uint8_t lun,
 	sc->data_timeout = (data_timeout + USB_MS_HZ);
 	sc->actlen = 0;
 	sc->cmd_len = cmd_len;
-	bzero(&sc->cbw.CBWCDB, sizeof(sc->cbw.CBWCDB));
-	bcopy(cmd_ptr, &sc->cbw.CBWCDB, cmd_len);
+	memset(&sc->cbw.CBWCDB, 0, sizeof(sc->cbw.CBWCDB));
+	memcpy(&sc->cbw.CBWCDB, cmd_ptr, cmd_len);
 	DPRINTFN(1, "SCSI cmd = %*D\n", (int)cmd_len, &sc->cbw.CBWCDB, ":");
 
 	mtx_lock(&sc->mtx);
@@ -603,6 +603,29 @@ usb_iface_is_cdrom(struct usb_device *udev, uint8_t iface_index)
 	return (is_cdrom);
 }
 
+static uint8_t
+usb_msc_get_max_lun(struct usb_device *udev, uint8_t iface_index)
+{
+	struct usb_device_request req;
+	usb_error_t err;
+	uint8_t buf = 0;
+
+
+	/* The Get Max Lun command is a class-specific request. */
+	req.bmRequestType = UT_READ_CLASS_INTERFACE;
+	req.bRequest = 0xFE;		/* GET_MAX_LUN */
+	USETW(req.wValue, 0);
+	req.wIndex[0] = iface_index;
+	req.wIndex[1] = 0;
+	USETW(req.wLength, 1);
+
+	err = usbd_do_request(udev, NULL, &req, &buf);
+	if (err)
+		buf = 0;
+
+	return (buf);
+}
+
 usb_error_t
 usb_msc_auto_quirk(struct usb_device *udev, uint8_t iface_index)
 {
@@ -622,6 +645,11 @@ usb_msc_auto_quirk(struct usb_device *udev, uint8_t iface_index)
 	 */
 	usb_pause_mtx(NULL, hz);
 
+	if (usb_msc_get_max_lun(udev, iface_index) == 0) {
+		DPRINTF("Device has only got one LUN.\n");
+		usbd_add_dynamic_quirk(udev, UQ_MSC_NO_GETMAXLUN);
+	}
+
 	is_no_direct = 1;
 	for (timeout = 4; timeout; timeout--) {
 		err = bbb_command_start(sc, DIR_IN, 0, sc->buffer,
diff --git a/sys/dev/usb/usb_process.c b/sys/dev/usb/usb_process.c
index 051ded9..ab579f2 100644
--- a/sys/dev/usb/usb_process.c
+++ b/sys/dev/usb/usb_process.c
@@ -67,11 +67,17 @@ static int usb_pcount;
 #define	USB_THREAD_CREATE(f, s, p, ...) \
 		kproc_kthread_add((f), (s), &usbproc, (p), RFHIGHPID, \
 		    0, "usb", __VA_ARGS__)
+#if (__FreeBSD_version >= 900000)
+#define	USB_THREAD_SUSPEND_CHECK() kthread_suspend_check()
+#else
+#define	USB_THREAD_SUSPEND_CHECK() kthread_suspend_check(curthread)
+#endif
 #define	USB_THREAD_SUSPEND(p)   kthread_suspend(p,0)
 #define	USB_THREAD_EXIT(err)	kthread_exit()
 #else
 #define	USB_THREAD_CREATE(f, s, p, ...) \
 		kthread_create((f), (s), (p), RFHIGHPID, 0, __VA_ARGS__)
+#define	USB_THREAD_SUSPEND_CHECK() kthread_suspend_check(curproc)
 #define	USB_THREAD_SUSPEND(p)   kthread_suspend(p,0)
 #define	USB_THREAD_EXIT(err)	kthread_exit(err)
 #endif
@@ -79,7 +85,7 @@ static int usb_pcount;
 #ifdef USB_DEBUG
 static int usb_proc_debug;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, proc, CTLFLAG_RW, 0, "USB process");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, proc, CTLFLAG_RW, 0, "USB process");
 SYSCTL_INT(_hw_usb_proc, OID_AUTO, debug, CTLFLAG_RW, &usb_proc_debug, 0,
     "Debug level");
 
@@ -98,6 +104,9 @@ usb_process(void *arg)
 	struct usb_proc_msg *pm;
 	struct thread *td;
 
+	/* in case of attach error, check for suspended */
+	USB_THREAD_SUSPEND_CHECK();
+
 	/* adjust priority */
 	td = curthread;
 	thread_lock(td);
diff --git a/sys/dev/usb/usb_request.c b/sys/dev/usb/usb_request.c
index 347f946..d692b93 100644
--- a/sys/dev/usb/usb_request.c
+++ b/sys/dev/usb/usb_request.c
@@ -517,7 +517,7 @@ usbd_do_request_flags(struct usb_device *udev, struct mtx *mtx,
 				}
 			} else
 #endif
-				bcopy(desc, data, length);
+				memcpy(data, desc, length);
 		}
 		goto done;		/* success */
 	}
diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c
index cd7b2e2..6f4b678 100644
--- a/sys/dev/usb/usb_transfer.c
+++ b/sys/dev/usb/usb_transfer.c
@@ -858,7 +858,7 @@ usbd_transfer_setup(struct usb_device *udev,
 	if (parm.err) {
 		goto done;
 	}
-	bzero(&parm, sizeof(parm));
+	memset(&parm, 0, sizeof(parm));
 
 	parm.udev = udev;
 	parm.speed = usbd_get_speed(udev);
@@ -982,7 +982,7 @@ usbd_transfer_setup(struct usb_device *udev,
 				 * memory:
 				 */
 				xfer = &dummy;
-				bzero(&dummy, sizeof(dummy));
+				memset(&dummy, 0, sizeof(dummy));
 				refcount++;
 			}
 
diff --git a/sys/dev/usb/usb_util.c b/sys/dev/usb/usb_util.c
index 1c357a3..4fe79c5 100644
--- a/sys/dev/usb/usb_util.c
+++ b/sys/dev/usb/usb_util.c
@@ -58,31 +58,6 @@
 #include <dev/usb/usb_bus.h>
 
 /*------------------------------------------------------------------------*
- * device_delete_all_children - delete all children of a device
- *------------------------------------------------------------------------*/
-#ifndef device_delete_all_children
-int
-device_delete_all_children(device_t dev)
-{
-	device_t *devlist;
-	int devcount;
-	int error;
-
-	error = device_get_children(dev, &devlist, &devcount);
-	if (error == 0) {
-		while (devcount-- > 0) {
-			error = device_delete_child(dev, devlist[devcount]);
-			if (error) {
-				break;
-			}
-		}
-		free(devlist, M_TEMP);
-	}
-	return (error);
-}
-#endif
-
-/*------------------------------------------------------------------------*
  *	device_set_usb_desc
  *
  * This function can be called at probe or attach to set the USB
@@ -140,33 +115,21 @@ device_set_usb_desc(device_t dev)
  *
  * This function will delay the code by the passed number of system
  * ticks. The passed mutex "mtx" will be dropped while waiting, if
- * "mtx" is not NULL.
+ * "mtx" is different from NULL.
  *------------------------------------------------------------------------*/
 void
-usb_pause_mtx(struct mtx *mtx, int _ticks)
+usb_pause_mtx(struct mtx *mtx, int timo)
 {
 	if (mtx != NULL)
 		mtx_unlock(mtx);
 
-	if (cold) {
-		/* convert to milliseconds */
-		_ticks = (_ticks * 1000) / hz;
-		/* convert to microseconds, rounded up */
-		_ticks = (_ticks + 1) * 1000;
-		DELAY(_ticks);
-
-	} else {
+	/*
+	 * Add one tick to the timeout so that we don't return too
+	 * early! Note that pause() will assert that the passed
+	 * timeout is positive and non-zero!
+	 */
+	pause("USBWAIT", timo + 1);
 
-		/*
-		 * Add one to the number of ticks so that we don't return
-		 * too early!
-		 */
-		_ticks++;
-
-		if (pause("USBWAIT", _ticks)) {
-			/* ignore */
-		}
-	}
 	if (mtx != NULL)
 		mtx_lock(mtx);
 }
diff --git a/sys/dev/usb/usb_util.h b/sys/dev/usb/usb_util.h
index 35abedd..7e52404 100644
--- a/sys/dev/usb/usb_util.h
+++ b/sys/dev/usb/usb_util.h
@@ -27,7 +27,6 @@
 #ifndef _USB_UTIL_H_
 #define	_USB_UTIL_H_
 
-int	device_delete_all_children(device_t dev);
 uint8_t	usb_make_str_desc(void *ptr, uint16_t max_len, const char *s);
 void	usb_printbcd(char *p, uint16_t p_len, uint16_t bcd);
 void	usb_trim_spaces(char *p);
diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs
index 5a4d3f4..4e3af74 100644
--- a/sys/dev/usb/usbdevs
+++ b/sys/dev/usb/usbdevs
@@ -664,7 +664,7 @@ vendor LINKSYS3		0x1915	Linksys
 vendor QUALCOMMINC	0x19d2	Qualcomm, Incorporated
 vendor WCH2		0x1a86	QinHeng Electronics
 vendor STELERA		0x1a8d	Stelera Wireless
-vendor MATRIXORBITAL	0x1b3d	Matrix Orbital 
+vendor MATRIXORBITAL	0x1b3d	Matrix Orbital
 vendor OVISLINK		0x1b75	OvisLink
 vendor TCTMOBILE	0x1bbb  TCT Mobile
 vendor WAGO		0x1be3  WAGO Kontakttechnik GmbH.
@@ -915,8 +915,10 @@ product ALCATEL OT535		0x02df	One Touch 535/735
 /* Alcor Micro, Inc. products */
 product ALCOR2 KBD_HUB		0x2802	Kbd Hub
 
+product ALCOR DUMMY		0x0000	Dummy product
 product ALCOR SDCR_6335		0x6335	SD/MMC Card Reader
 product ALCOR SDCR_6362		0x6362	SD/MMC Card Reader
+product ALCOR SDCR_6366		0x6366	SD/MMC Card Reader
 product ALCOR TRANSCEND		0x6387	Transcend JetFlash Drive
 product ALCOR MA_KBD_HUB	0x9213	MacAlly Kbd Hub
 product ALCOR AU9814		0x9215	AU9814 Hub
@@ -2084,11 +2086,13 @@ product LINKSYS4 RT3070		0x0078	RT3070
 product LINKSYS4 WUSB600NV2	0x0079	WUSB600N v2
 
 /* Logitech products */
+product LOGITECH LANW300NU2	0x0166  LAN-W300N/U2
 product LOGITECH M2452		0x0203	M2452 keyboard
 product LOGITECH M4848		0x0301	M4848 mouse
 product LOGITECH PAGESCAN	0x040f	PageScan
 product LOGITECH QUICKCAMWEB	0x0801	QuickCam Web
 product LOGITECH QUICKCAMPRO	0x0810	QuickCam Pro
+product LOGITECH WEBCAMC100	0X0817	Webcam C100
 product LOGITECH QUICKCAMEXP	0x0840	QuickCam Express
 product LOGITECH QUICKCAM	0x0850	QuickCam
 product LOGITECH QUICKCAMPRO3	0x0990	QuickCam Pro 9000
@@ -2134,7 +2138,7 @@ product MACALLY MOUSE1		0x0101	mouse
 
 /* Marvell Technology Group, Ltd. products */
 product MARVELL SHEEVAPLUG	0x9e8f	SheevaPlug serial interface
-  
+
 /* Matrix Orbital products */
 product MATRIXORBITAL MOUA	0x0153	Martrix Orbital MOU-Axxxx LCD displays
 
@@ -2179,6 +2183,7 @@ product MELCO RT2870_1		0x0148	RT2870
 product MELCO RT2870_2		0x0150	RT2870
 product MELCO WLIUCGN		0x015d	WLI-UC-GN
 product MELCO WLIUCG301N	0x016f	WLI-UC-G301N
+product MELCO WLIUCGNM		0x01a2  WLI-UC-GNM
 
 /* Merlin products */
 product MERLIN V620             0x1110  Merlin V620
diff --git a/sys/dev/usb/wlan/if_rum.c b/sys/dev/usb/wlan/if_rum.c
index c51d485..8cf9f47 100644
--- a/sys/dev/usb/wlan/if_rum.c
+++ b/sys/dev/usb/wlan/if_rum.c
@@ -80,7 +80,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int rum_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, rum, CTLFLAG_RW, 0, "USB rum");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, rum, CTLFLAG_RW, 0, "USB rum");
 SYSCTL_INT(_hw_usb_rum, OID_AUTO, debug, CTLFLAG_RW, &rum_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/wlan/if_run.c b/sys/dev/usb/wlan/if_run.c
index a0ac26a..ad6c736 100644
--- a/sys/dev/usb/wlan/if_run.c
+++ b/sys/dev/usb/wlan/if_run.c
@@ -82,7 +82,7 @@ __FBSDID("$FreeBSD$");
 
 #ifdef	RUN_DEBUG
 int run_debug = 0;
-SYSCTL_NODE(_hw_usb, OID_AUTO, run, CTLFLAG_RW, 0, "USB run");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, run, CTLFLAG_RW, 0, "USB run");
 SYSCTL_INT(_hw_usb_run, OID_AUTO, debug, CTLFLAG_RW, &run_debug, 0,
     "run debug level");
 #endif
@@ -208,12 +208,14 @@ static const STRUCT_USB_HOST_ID run_devs[] = {
     RUN_DEV(LOGITEC,		RT2870_1),
     RUN_DEV(LOGITEC,		RT2870_2),
     RUN_DEV(LOGITEC,		RT2870_3),
+    RUN_DEV(LOGITECH,		LANW300NU2),
     RUN_DEV(MELCO,		RT2870_1),
     RUN_DEV(MELCO,		RT2870_2),
     RUN_DEV(MELCO,		WLIUCAG300N),
     RUN_DEV(MELCO,		WLIUCG300N),
     RUN_DEV(MELCO,		WLIUCG301N),
     RUN_DEV(MELCO,		WLIUCGN),
+    RUN_DEV(MELCO,		WLIUCGNM),
     RUN_DEV(MOTOROLA4,		RT2770),
     RUN_DEV(MOTOROLA4,		RT3070),
     RUN_DEV(MSI,		RT3070_1),
diff --git a/sys/dev/usb/wlan/if_uath.c b/sys/dev/usb/wlan/if_uath.c
index 328dc4f..8bc821b 100644
--- a/sys/dev/usb/wlan/if_uath.c
+++ b/sys/dev/usb/wlan/if_uath.c
@@ -111,7 +111,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/wlan/if_uathreg.h>
 #include <dev/usb/wlan/if_uathvar.h>
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, uath, CTLFLAG_RW, 0, "USB Atheros");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, uath, CTLFLAG_RW, 0, "USB Atheros");
 
 static	int uath_countrycode = CTRY_DEFAULT;	/* country code */
 SYSCTL_INT(_hw_usb_uath, OID_AUTO, countrycode, CTLFLAG_RW, &uath_countrycode,
@@ -710,12 +710,12 @@ uath_cmdsend(struct uath_softc *sc, uint32_t code, const void *idata, int ilen,
 	cmd->buflen = roundup2(sizeof(struct uath_cmd_hdr) + ilen, 4);
 
 	hdr = (struct uath_cmd_hdr *)cmd->buf;
-	bzero(hdr, sizeof (struct uath_cmd_hdr));	/* XXX not needed */
+	memset(hdr, 0, sizeof(struct uath_cmd_hdr));
 	hdr->len   = htobe32(cmd->buflen);
 	hdr->code  = htobe32(code);
 	hdr->msgid = cmd->msgid;	/* don't care about endianness */
 	hdr->magic = htobe32((cmd->flags & UATH_CMD_FLAG_MAGIC) ? 1 << 24 : 0);
-	bcopy(idata, (uint8_t *)(hdr + 1), ilen);
+	memcpy((uint8_t *)(hdr + 1), idata, ilen);
 
 #ifdef UATH_DEBUG
 	if (sc->sc_debug & UATH_DEBUG_CMDS) {
@@ -1403,7 +1403,7 @@ uath_dataflush(struct uath_softc *sc)
 	chunk->flags = UATH_CFLAGS_FINAL;
 	chunk->length = htobe16(sizeof (struct uath_tx_desc));
 
-	bzero(desc, sizeof(struct uath_tx_desc));
+	memset(desc, 0, sizeof(struct uath_tx_desc));
 	desc->msglen = htobe32(sizeof(struct uath_tx_desc));
 	desc->msgid  = (sc->sc_msgid++) + 1; /* don't care about endianness */
 	desc->type   = htobe32(WDCMSG_FLUSH);
@@ -1482,7 +1482,7 @@ uath_set_chan(struct uath_softc *sc, struct ieee80211_channel *c)
 #endif
 	struct uath_cmd_reset reset;
 
-	bzero(&reset, sizeof reset);
+	memset(&reset, 0, sizeof(reset));
 	if (IEEE80211_IS_CHAN_2GHZ(c))
 		reset.flags |= htobe32(UATH_CHAN_2GHZ);
 	if (IEEE80211_IS_CHAN_5GHZ(c))
@@ -1971,7 +1971,7 @@ uath_create_connection(struct uath_softc *sc, uint32_t connid)
 	struct uath_cmd_create_connection create;
 
 	ni = ieee80211_ref_node(vap->iv_bss);
-	bzero(&create, sizeof create);
+	memset(&create, 0, sizeof(create));
 	create.connid = htobe32(connid);
 	create.bssid = htobe32(0);
 	/* XXX packed or not?  */
@@ -2000,7 +2000,7 @@ uath_set_rates(struct uath_softc *sc, const struct ieee80211_rateset *rs)
 {
 	struct uath_cmd_rates rates;
 
-	bzero(&rates, sizeof rates);
+	memset(&rates, 0, sizeof(rates));
 	rates.connid = htobe32(UATH_ID_BSS);		/* XXX */
 	rates.size   = htobe32(sizeof(struct uath_cmd_rateset));
 	/* XXX bounds check rs->rs_nrates */
@@ -2022,7 +2022,7 @@ uath_write_associd(struct uath_softc *sc)
 	struct uath_cmd_set_associd associd;
 
 	ni = ieee80211_ref_node(vap->iv_bss);
-	bzero(&associd, sizeof associd);
+	memset(&associd, 0, sizeof(associd));
 	associd.defaultrateix = htobe32(1);	/* XXX */
 	associd.associd = htobe32(ni->ni_associd);
 	associd.timoffset = htobe32(0x3b);	/* XXX */
@@ -2168,7 +2168,7 @@ uath_set_key(struct uath_softc *sc, const struct ieee80211_key *wk,
 	struct uath_cmd_crypto crypto;
 	int i;
 
-	bzero(&crypto, sizeof crypto);
+	memset(&crypto, 0, sizeof(crypto));
 	crypto.keyidx = htobe32(index);
 	crypto.magic1 = htobe32(1);
 	crypto.size   = htobe32(368);
@@ -2176,7 +2176,7 @@ uath_set_key(struct uath_softc *sc, const struct ieee80211_key *wk,
 	crypto.flags  = htobe32(0x80000068);
 	if (index != UATH_DEFAULT_KEY)
 		crypto.flags |= htobe32(index << 16);
-	memset(crypto.magic2, 0xff, sizeof crypto.magic2);
+	memset(crypto.magic2, 0xff, sizeof(crypto.magic2));
 
 	/*
 	 * Each byte of the key must be XOR'ed with 10101010 before being
diff --git a/sys/dev/usb/wlan/if_upgt.c b/sys/dev/usb/wlan/if_upgt.c
index b9381a2..78ac2d9 100644
--- a/sys/dev/usb/wlan/if_upgt.c
+++ b/sys/dev/usb/wlan/if_upgt.c
@@ -70,7 +70,7 @@
  * Sebastien Bourdeauducq <lekernel@prism54.org>.
  */
 
-SYSCTL_NODE(_hw, OID_AUTO, upgt, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_hw, OID_AUTO, upgt, CTLFLAG_RD, 0,
     "USB PrismGT GW3887 driver parameters");
 
 #ifdef UPGT_DEBUG
@@ -432,7 +432,7 @@ upgt_get_stats(struct upgt_softc *sc)
 	/*
 	 * Transmit the URB containing the CMD data.
 	 */
-	bzero(data_cmd->buf, MCLBYTES);
+	memset(data_cmd->buf, 0, MCLBYTES);
 
 	mem = (struct upgt_lmac_mem *)data_cmd->buf;
 	mem->addr = htole32(sc->sc_memaddr_frame_start +
@@ -540,7 +540,7 @@ upgt_set_led(struct upgt_softc *sc, int action)
 	/*
 	 * Transmit the URB containing the CMD data.
 	 */
-	bzero(data_cmd->buf, MCLBYTES);
+	memset(data_cmd->buf, 0, MCLBYTES);
 
 	mem = (struct upgt_lmac_mem *)data_cmd->buf;
 	mem->addr = htole32(sc->sc_memaddr_frame_start +
@@ -670,7 +670,7 @@ upgt_set_macfilter(struct upgt_softc *sc, uint8_t state)
 	/*
 	 * Transmit the URB containing the CMD data.
 	 */
-	bzero(data_cmd->buf, MCLBYTES);
+	memset(data_cmd->buf, 0, MCLBYTES);
 
 	mem = (struct upgt_lmac_mem *)data_cmd->buf;
 	mem->addr = htole32(sc->sc_memaddr_frame_start +
@@ -785,11 +785,11 @@ upgt_setup_rates(struct ieee80211vap *vap, struct ieee80211com *ic)
 		 * will pickup a rate.
 		 */
 		if (ic->ic_curmode == IEEE80211_MODE_11B)
-			bcopy(rateset_auto_11b, sc->sc_cur_rateset,
+			memcpy(sc->sc_cur_rateset, rateset_auto_11b,
 			    sizeof(sc->sc_cur_rateset));
 		if (ic->ic_curmode == IEEE80211_MODE_11G ||
 		    ic->ic_curmode == IEEE80211_MODE_AUTO)
-			bcopy(rateset_auto_11g, sc->sc_cur_rateset,
+			memcpy(sc->sc_cur_rateset, rateset_auto_11g,
 			    sizeof(sc->sc_cur_rateset));
 	} else {
 		/* set a fixed rate */
@@ -975,7 +975,7 @@ upgt_set_chan(struct upgt_softc *sc, struct ieee80211_channel *c)
 	/*
 	 * Transmit the URB containing the CMD data.
 	 */
-	bzero(data_cmd->buf, MCLBYTES);
+	memset(data_cmd->buf, 0, MCLBYTES);
 
 	mem = (struct upgt_lmac_mem *)data_cmd->buf;
 	mem->addr = htole32(sc->sc_memaddr_frame_start +
@@ -998,11 +998,11 @@ upgt_set_chan(struct upgt_softc *sc, struct ieee80211_channel *c)
 	chan->settings = sc->sc_eeprom_freq6_settings;
 	chan->unknown3 = UPGT_CHANNEL_UNKNOWN3;
 
-	bcopy(&sc->sc_eeprom_freq3[channel].data, chan->freq3_1,
+	memcpy(chan->freq3_1, &sc->sc_eeprom_freq3[channel].data,
 	    sizeof(chan->freq3_1));
-	bcopy(&sc->sc_eeprom_freq4[channel], chan->freq4,
+	memcpy(chan->freq4, &sc->sc_eeprom_freq4[channel],
 	    sizeof(sc->sc_eeprom_freq4[channel]));
-	bcopy(&sc->sc_eeprom_freq3[channel].data, chan->freq3_2,
+	memcpy(chan->freq3_2, &sc->sc_eeprom_freq3[channel].data,
 	    sizeof(chan->freq3_2));
 
 	data_cmd->buflen = sizeof(*mem) + sizeof(*chan);
@@ -1331,7 +1331,7 @@ upgt_eeprom_read(struct upgt_softc *sc)
 		/*
 		 * Transmit the URB containing the CMD data.
 		 */
-		bzero(data_cmd->buf, MCLBYTES);
+		memset(data_cmd->buf, 0, MCLBYTES);
 
 		mem = (struct upgt_lmac_mem *)data_cmd->buf;
 		mem->addr = htole32(sc->sc_memaddr_frame_start +
@@ -1423,8 +1423,9 @@ upgt_rxeof(struct usb_xfer *xfer, struct upgt_data *data, int *rssi)
 		    "received EEPROM block (offset=%d, len=%d)\n",
 		    eeprom_offset, eeprom_len);
 
-		bcopy(data->buf + sizeof(struct upgt_lmac_eeprom) + 4,
-			sc->sc_eeprom + eeprom_offset, eeprom_len);
+		memcpy(sc->sc_eeprom + eeprom_offset,
+		    data->buf + sizeof(struct upgt_lmac_eeprom) + 4,
+		    eeprom_len);
 
 		/* EEPROM data has arrived in time, wakeup.  */
 		wakeup(sc);
@@ -1498,7 +1499,7 @@ upgt_rx(struct upgt_softc *sc, uint8_t *data, int pkglen, int *rssi)
 		return (NULL);
 	}
 	m_adj(m, ETHER_ALIGN);
-	bcopy(rxdesc->data, mtod(m, char *), pkglen);
+	memcpy(mtod(m, char *), rxdesc->data, pkglen);
 	/* trim FCS */
 	m->m_len = m->m_pkthdr.len = pkglen - IEEE80211_CRC_LEN;
 	m->m_pkthdr.rcvif = ifp;
@@ -1620,7 +1621,7 @@ upgt_fw_load(struct upgt_softc *sc)
 		goto fail;
 	}
 	data_cmd->buflen = sizeof(start_fwload_cmd);
-	bcopy(start_fwload_cmd, data_cmd->buf, data_cmd->buflen);
+	memcpy(data_cmd->buf, start_fwload_cmd, data_cmd->buflen);
 	upgt_bulk_tx(sc, data_cmd);
 
 	/* send X2 header */
@@ -1631,7 +1632,7 @@ upgt_fw_load(struct upgt_softc *sc)
 	}
 	data_cmd->buflen = sizeof(struct upgt_fw_x2_header);
 	x2 = (struct upgt_fw_x2_header *)data_cmd->buf;
-	bcopy(UPGT_X2_SIGNATURE, x2->signature, UPGT_X2_SIGNATURE_SIZE);
+	memcpy(x2->signature, UPGT_X2_SIGNATURE, UPGT_X2_SIGNATURE_SIZE);
 	x2->startaddr = htole32(UPGT_MEMADDR_FIRMWARE_START);
 	x2->len = htole32(fw->datasize);
 	x2->crc = upgt_crc32_le((uint8_t *)data_cmd->buf +
@@ -1925,7 +1926,7 @@ upgt_device_reset(struct upgt_softc *sc)
 		UPGT_UNLOCK(sc);
 		return (ENOBUFS);
 	}
-	bcopy(init_cmd, data->buf, sizeof(init_cmd));
+	memcpy(data->buf, init_cmd, sizeof(init_cmd));
 	data->buflen = sizeof(init_cmd);
 	upgt_bulk_tx(sc, data);
 	usb_pause_mtx(&sc->sc_mtx, 100);
@@ -2178,7 +2179,7 @@ upgt_tx_start(struct upgt_softc *sc, struct mbuf *m, struct ieee80211_node *ni,
 	}
 
 	/* Transmit the URB containing the TX data.  */
-	bzero(data->buf, MCLBYTES);
+	memset(data->buf, 0, MCLBYTES);
 	mem = (struct upgt_lmac_mem *)data->buf;
 	mem->addr = htole32(data->addr);
 	txdesc = (struct upgt_lmac_tx_desc *)(mem + 1);
@@ -2192,7 +2193,7 @@ upgt_tx_start(struct upgt_softc *sc, struct mbuf *m, struct ieee80211_node *ni,
 	} else {
 		/* data frames  */
 		txdesc->header1.flags = UPGT_H1_FLAGS_TX_DATA;
-		bcopy(sc->sc_cur_rateset, txdesc->rates, sizeof(txdesc->rates));
+		memcpy(txdesc->rates, sc->sc_cur_rateset, sizeof(txdesc->rates));
 	}
 	txdesc->header1.type = UPGT_H1_TYPE_TX_DATA;
 	txdesc->header1.len = htole16(m->m_pkthdr.len);
diff --git a/sys/dev/usb/wlan/if_ural.c b/sys/dev/usb/wlan/if_ural.c
index 048392d..a72da83 100644
--- a/sys/dev/usb/wlan/if_ural.c
+++ b/sys/dev/usb/wlan/if_ural.c
@@ -81,7 +81,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int ural_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, ural, CTLFLAG_RW, 0, "USB ural");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, ural, CTLFLAG_RW, 0, "USB ural");
 SYSCTL_INT(_hw_usb_ural, OID_AUTO, debug, CTLFLAG_RW, &ural_debug, 0,
     "Debug level");
 #endif
diff --git a/sys/dev/usb/wlan/if_urtw.c b/sys/dev/usb/wlan/if_urtw.c
index 6ae7e16..2a52a8c 100644
--- a/sys/dev/usb/wlan/if_urtw.c
+++ b/sys/dev/usb/wlan/if_urtw.c
@@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/wlan/if_urtwreg.h>
 #include <dev/usb/wlan/if_urtwvar.h>
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, urtw, CTLFLAG_RW, 0, "USB Realtek 8187L");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, urtw, CTLFLAG_RW, 0, "USB Realtek 8187L");
 #ifdef URTW_DEBUG
 int urtw_debug = 0;
 SYSCTL_INT(_hw_usb_urtw, OID_AUTO, debug, CTLFLAG_RW, &urtw_debug, 0,
@@ -1745,7 +1745,7 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0,
 	if ((0 == xferlen % 64) || (0 == xferlen % 512))
 		xferlen += 1;
 
-	bzero(data->buf, URTW_TX_MAXSIZE);
+	memset(data->buf, 0, URTW_TX_MAXSIZE);
 	flags = m0->m_pkthdr.len & 0xfff;
 	flags |= URTW_TX_FLAG_NO_ENC;
 	if ((ic->ic_flags & IEEE80211_F_SHPREAMBLE) &&
diff --git a/sys/dev/usb/wlan/if_zyd.c b/sys/dev/usb/wlan/if_zyd.c
index f8d905e..835cdc5 100644
--- a/sys/dev/usb/wlan/if_zyd.c
+++ b/sys/dev/usb/wlan/if_zyd.c
@@ -78,7 +78,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int zyd_debug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, zyd, CTLFLAG_RW, 0, "USB zyd");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, zyd, CTLFLAG_RW, 0, "USB zyd");
 SYSCTL_INT(_hw_usb_zyd, OID_AUTO, debug, CTLFLAG_RW, &zyd_debug, 0,
     "zyd debug level");
 
@@ -683,7 +683,7 @@ zyd_intr_read_callback(struct usb_xfer *xfer, usb_error_t error)
 				if (i != cnt)
 					continue;
 				/* copy answer into caller-supplied buffer */
-				bcopy(cmd->data, rqp->odata, rqp->olen);
+				memcpy(rqp->odata, cmd->data, rqp->olen);
 				DPRINTF(sc, ZYD_DEBUG_CMD,
 				    "command %p complete, data = %*D \n",
 				    rqp, rqp->olen, rqp->odata, ":");
@@ -783,7 +783,7 @@ zyd_cmd(struct zyd_softc *sc, uint16_t code, const void *idata, int ilen,
 		return (EINVAL);
 
 	cmd.code = htole16(code);
-	bcopy(idata, cmd.data, ilen);
+	memcpy(cmd.data, idata, ilen);
 	DPRINTF(sc, ZYD_DEBUG_CMD, "sending cmd %p = %*D\n",
 	    &rq, ilen, idata, ":");
 
diff --git a/sys/dev/virtio/balloon/virtio_balloon.c b/sys/dev/virtio/balloon/virtio_balloon.c
new file mode 100644
index 0000000..ef7aca9
--- /dev/null
+++ b/sys/dev/virtio/balloon/virtio_balloon.c
@@ -0,0 +1,569 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Driver for VirtIO memory balloon devices. */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/endian.h>
+#include <sys/kthread.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/sglist.h>
+#include <sys/sysctl.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+#include <dev/virtio/balloon/virtio_balloon.h>
+
+#include "virtio_if.h"
+
+struct vtballoon_softc {
+	device_t		 vtballoon_dev;
+	struct mtx		 vtballoon_mtx;
+	uint64_t		 vtballoon_features;
+	uint32_t		 vtballoon_flags;
+#define VTBALLOON_FLAG_DETACH	 0x01
+
+	struct virtqueue	*vtballoon_inflate_vq;
+	struct virtqueue	*vtballoon_deflate_vq;
+
+	uint32_t		 vtballoon_desired_npages;
+	uint32_t		 vtballoon_current_npages;
+	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
+
+	struct proc		*vtballoon_kproc;
+	uint32_t		*vtballoon_page_frames;
+	int			 vtballoon_timeout;
+};
+
+static struct virtio_feature_desc vtballoon_feature_desc[] = {
+	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
+	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
+
+	{ 0, NULL }
+};
+
+static int	vtballoon_probe(device_t);
+static int	vtballoon_attach(device_t);
+static int	vtballoon_detach(device_t);
+static int	vtballoon_config_change(device_t);
+
+static void	vtballoon_negotiate_features(struct vtballoon_softc *);
+static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
+
+static int	vtballoon_vq_intr(void *);
+
+static void	vtballoon_inflate(struct vtballoon_softc *, int);
+static void	vtballoon_deflate(struct vtballoon_softc *, int);
+
+static void	vtballoon_send_page_frames(struct vtballoon_softc *,
+		    struct virtqueue *, int);
+
+static void	vtballoon_pop(struct vtballoon_softc *);
+static void	vtballoon_stop(struct vtballoon_softc *);
+
+static vm_page_t
+		vtballoon_alloc_page(struct vtballoon_softc *);
+static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
+
+static int	vtballoon_sleep(struct vtballoon_softc *);
+static void	vtballoon_thread(void *);
+static void	vtballoon_add_sysctl(struct vtballoon_softc *);
+
+/* Features desired/implemented by this driver. */
+#define VTBALLOON_FEATURES		0
+
+/* Timeout between retries when the balloon needs inflating. */
+#define VTBALLOON_LOWMEM_TIMEOUT	hz
+
+/*
+ * Maximum number of pages we'll request to inflate or deflate
+ * the balloon in one virtqueue request. Both Linux and NetBSD
+ * have settled on 256, doing up to 1MB at a time.
+ */
+#define VTBALLOON_PAGES_PER_REQUEST	256
+
+#define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
+#define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
+					    "VirtIO Balloon Lock", MTX_SPIN)
+#define VTBALLOON_LOCK(_sc)		mtx_lock_spin(VTBALLOON_MTX((_sc)))
+#define VTBALLOON_UNLOCK(_sc)		mtx_unlock_spin(VTBALLOON_MTX((_sc)))
+#define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
+
+static device_method_t vtballoon_methods[] = {
+	/* Device methods. */
+	DEVMETHOD(device_probe,		vtballoon_probe),
+	DEVMETHOD(device_attach,	vtballoon_attach),
+	DEVMETHOD(device_detach,	vtballoon_detach),
+
+	/* VirtIO methods. */
+	DEVMETHOD(virtio_config_change, vtballoon_config_change),
+
+	{ 0, 0 }
+};
+
+static driver_t vtballoon_driver = {
+	"vtballoon",
+	vtballoon_methods,
+	sizeof(struct vtballoon_softc)
+};
+static devclass_t vtballoon_devclass;
+
+DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
+    vtballoon_devclass, 0, 0);
+MODULE_VERSION(virtio_balloon, 1);
+MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
+
+static int
+vtballoon_probe(device_t dev)
+{
+
+	if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
+		return (ENXIO);
+
+	device_set_desc(dev, "VirtIO Balloon Adapter");
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtballoon_attach(device_t dev)
+{
+	struct vtballoon_softc *sc;
+	int error;
+
+	sc = device_get_softc(dev);
+	sc->vtballoon_dev = dev;
+
+	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
+	TAILQ_INIT(&sc->vtballoon_pages);
+
+	vtballoon_add_sysctl(sc);
+
+	virtio_set_feature_desc(dev, vtballoon_feature_desc);
+	vtballoon_negotiate_features(sc);
+
+	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
+	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (sc->vtballoon_page_frames == NULL) {
+		error = ENOMEM;
+		device_printf(dev,
+		    "cannot allocate page frame request array\n");
+		goto fail;
+	}
+
+	error = vtballoon_alloc_virtqueues(sc);
+	if (error) {
+		device_printf(dev, "cannot allocate virtqueues\n");
+		goto fail;
+	}
+
+	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
+	if (error) {
+		device_printf(dev, "cannot setup virtqueue interrupts\n");
+		goto fail;
+	}
+
+	error = kproc_create(vtballoon_thread, sc, &sc->vtballoon_kproc,
+	    0, 0, "virtio_balloon");
+	if (error) {
+		device_printf(dev, "cannot create balloon kproc\n");
+		goto fail;
+	}
+
+	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
+	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
+
+fail:
+	if (error)
+		vtballoon_detach(dev);
+
+	return (error);
+}
+
+static int
+vtballoon_detach(device_t dev)
+{
+	struct vtballoon_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->vtballoon_kproc != NULL) {
+		VTBALLOON_LOCK(sc);
+		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
+		wakeup_one(sc);
+		msleep_spin(sc->vtballoon_kproc, VTBALLOON_MTX(sc),
+		    "vtbdth", 0);
+		VTBALLOON_UNLOCK(sc);
+
+		sc->vtballoon_kproc = NULL;
+	}
+
+	if (device_is_attached(dev)) {
+		vtballoon_pop(sc);
+		vtballoon_stop(sc);
+	}
+
+	if (sc->vtballoon_page_frames != NULL) {
+		free(sc->vtballoon_page_frames, M_DEVBUF);
+		sc->vtballoon_page_frames = NULL;
+	}
+
+	VTBALLOON_LOCK_DESTROY(sc);
+
+	return (0);
+}
+
+static int
+vtballoon_config_change(device_t dev)
+{
+	struct vtballoon_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	VTBALLOON_LOCK(sc);
+	wakeup_one(sc);
+	VTBALLOON_UNLOCK(sc);
+
+	return (1);
+}
+
+static void
+vtballoon_negotiate_features(struct vtballoon_softc *sc)
+{
+	device_t dev;
+	uint64_t features;
+
+	dev = sc->vtballoon_dev;
+	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
+	sc->vtballoon_features = features;
+}
+
+static int
+vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
+{
+	device_t dev;
+	struct vq_alloc_info vq_info[2];
+	int nvqs;
+
+	dev = sc->vtballoon_dev;
+	nvqs = 2;
+
+	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
+	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
+
+	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
+	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
+
+	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
+}
+
+static int
+vtballoon_vq_intr(void *xsc)
+{
+	struct vtballoon_softc *sc;
+
+	sc = xsc;
+
+	VTBALLOON_LOCK(sc);
+	wakeup_one(sc);
+	VTBALLOON_UNLOCK(sc);
+
+	return (1);
+}
+
+static void
+vtballoon_inflate(struct vtballoon_softc *sc, int npages)
+{
+	struct virtqueue *vq;
+	vm_page_t m;
+	int i;
+
+	vq = sc->vtballoon_inflate_vq;
+	m = NULL;
+
+	if (npages > VTBALLOON_PAGES_PER_REQUEST)
+		npages = VTBALLOON_PAGES_PER_REQUEST;
+	KASSERT(npages > 0, ("balloon doesn't need inflating?"));
+
+	for (i = 0; i < npages; i++) {
+		if ((m = vtballoon_alloc_page(sc)) == NULL)
+			break;
+
+		sc->vtballoon_page_frames[i] =
+		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
+
+		KASSERT(m->queue == PQ_NONE, ("allocated page on queue"));
+		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
+	}
+
+	if (i > 0)
+		vtballoon_send_page_frames(sc, vq, i);
+
+	if (m == NULL)
+		sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
+}
+
+static void
+vtballoon_deflate(struct vtballoon_softc *sc, int npages)
+{
+	TAILQ_HEAD(, vm_page) free_pages;
+	struct virtqueue *vq;
+	vm_page_t m;
+	int i;
+
+	vq = sc->vtballoon_deflate_vq;
+	TAILQ_INIT(&free_pages);
+
+	if (npages > VTBALLOON_PAGES_PER_REQUEST)
+		npages = VTBALLOON_PAGES_PER_REQUEST;
+	KASSERT(npages > 0, ("balloon doesn't need deflating?"));
+
+	for (i = 0; i < npages; i++) {
+		m = TAILQ_FIRST(&sc->vtballoon_pages);
+		KASSERT(m != NULL, ("no more pages to deflate"));
+
+		sc->vtballoon_page_frames[i] =
+		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
+
+		TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
+		TAILQ_INSERT_TAIL(&free_pages, m, pageq);
+	}
+
+	if (i > 0) {
+		/* Always tell host first before freeing the pages. */
+		vtballoon_send_page_frames(sc, vq, i);
+
+		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
+			TAILQ_REMOVE(&free_pages, m, pageq);
+			vtballoon_free_page(sc, m);
+		}
+	}
+
+	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
+	    sc->vtballoon_current_npages == 0) ||
+	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
+	    sc->vtballoon_current_npages != 0), ("balloon empty?"));
+}
+
+static void
+vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
+    int npages)
+{
+	struct sglist sg;
+	struct sglist_seg segs[1];
+	void *c;
+	int error;
+
+	sglist_init(&sg, 1, segs);
+
+	error = sglist_append(&sg, sc->vtballoon_page_frames,
+	    npages * sizeof(uint32_t));
+	KASSERT(error == 0, ("error adding page frames to sglist"));
+
+	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
+	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
+
+	/*
+	 * Inflate and deflate operations are done synchronously. The
+	 * interrupt handler will wake us up.
+	 */
+	VTBALLOON_LOCK(sc);
+	virtqueue_notify(vq);
+
+	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
+		msleep_spin(sc, VTBALLOON_MTX(sc), "vtbspf", 0);
+	VTBALLOON_UNLOCK(sc);
+
+	KASSERT(c == vq, ("unexpected balloon operation response"));
+}
+
+static void
+vtballoon_pop(struct vtballoon_softc *sc)
+{
+
+	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
+		vtballoon_deflate(sc, sc->vtballoon_current_npages);
+}
+
+static void
+vtballoon_stop(struct vtballoon_softc *sc)
+{
+
+	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
+	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
+
+	virtio_stop(sc->vtballoon_dev);
+}
+
+static vm_page_t
+vtballoon_alloc_page(struct vtballoon_softc *sc)
+{
+	vm_page_t m;
+
+	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
+	    VM_ALLOC_NOOBJ);
+	if (m != NULL)
+		sc->vtballoon_current_npages++;
+
+	return (m);
+}
+
+static void
+vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
+{
+
+	vm_page_unwire(m, 0);
+	vm_page_free(m);
+	sc->vtballoon_current_npages--;
+}
+
+static uint32_t
+vtballoon_desired_size(struct vtballoon_softc *sc)
+{
+	uint32_t desired;
+
+	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
+	    offsetof(struct virtio_balloon_config, num_pages));
+
+	return (le32toh(desired));
+}
+
+static void
+vtballoon_update_size(struct vtballoon_softc *sc)
+{
+
+	virtio_write_dev_config_4(sc->vtballoon_dev,
+	    offsetof(struct virtio_balloon_config, actual),
+	    htole32(sc->vtballoon_current_npages));
+
+}
+
+static int
+vtballoon_sleep(struct vtballoon_softc *sc)
+{
+	int rc, timeout;
+	uint32_t current, desired;
+
+	rc = 0;
+	current = sc->vtballoon_current_npages;
+
+	VTBALLOON_LOCK(sc);
+	for (;;) {
+		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
+			rc = 1;
+			break;
+		}
+
+		desired = vtballoon_desired_size(sc);
+		sc->vtballoon_desired_npages = desired;
+
+		/*
+		 * If given, use non-zero timeout on the first time through
+		 * the loop. On subsequent times, timeout will be zero so
+		 * we will reevaluate the desired size of the balloon and
+		 * break out to retry if needed.
+		 */
+		timeout = sc->vtballoon_timeout;
+		sc->vtballoon_timeout = 0;
+
+		if (current > desired)
+			break;
+		if (current < desired && timeout == 0)
+			break;
+
+		msleep_spin(sc, VTBALLOON_MTX(sc), "vtbslp", timeout);
+	}
+	VTBALLOON_UNLOCK(sc);
+
+	return (rc);
+}
+
+static void
+vtballoon_thread(void *xsc)
+{
+	struct vtballoon_softc *sc;
+	uint32_t current, desired;
+
+	sc = xsc;
+
+	for (;;) {
+		if (vtballoon_sleep(sc) != 0)
+			break;
+
+		current = sc->vtballoon_current_npages;
+		desired = sc->vtballoon_desired_npages;
+
+		if (desired != current) {
+			if (desired > current)
+				vtballoon_inflate(sc, desired - current);
+			else
+				vtballoon_deflate(sc, current - desired);
+
+			vtballoon_update_size(sc);
+		}
+	}
+
+	kproc_exit(0);
+}
+
+static void
+vtballoon_add_sysctl(struct vtballoon_softc *sc)
+{
+	device_t dev;
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *tree;
+	struct sysctl_oid_list *child;
+
+	dev = sc->vtballoon_dev;
+	ctx = device_get_sysctl_ctx(dev);
+	tree = device_get_sysctl_tree(dev);
+	child = SYSCTL_CHILDREN(tree);
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
+	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
+	    "Desired balloon size in pages");
+
+	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
+	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
+	    "Current balloon size in pages");
+}
diff --git a/sys/dev/virtio/balloon/virtio_balloon.h b/sys/dev/virtio/balloon/virtio_balloon.h
new file mode 100644
index 0000000..cea84ba
--- /dev/null
+++ b/sys/dev/virtio/balloon/virtio_balloon.h
@@ -0,0 +1,41 @@
+/*
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VIRTIO_BALLOON_H
+#define _VIRTIO_BALLOON_H
+
+#include <sys/types.h>
+
+/* Feature bits. */
+#define VIRTIO_BALLOON_F_MUST_TELL_HOST	0x1 /* Tell before reclaiming pages */
+#define VIRTIO_BALLOON_F_STATS_VQ	0x2 /* Memory stats virtqueue */
+
+/* Size of a PFN in the balloon interface. */
+#define VIRTIO_BALLOON_PFN_SHIFT 12
+
+struct virtio_balloon_config {
+	/* Number of pages host wants Guest to give up. */
+	uint32_t num_pages;
+
+	/* Number of pages we've actually got in balloon. */
+	uint32_t actual;
+};
+
+#define VIRTIO_BALLOON_S_SWAP_IN  0   /* Amount of memory swapped in */
+#define VIRTIO_BALLOON_S_SWAP_OUT 1   /* Amount of memory swapped out */
+#define VIRTIO_BALLOON_S_MAJFLT   2   /* Number of major faults */
+#define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
+#define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
+#define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
+#define VIRTIO_BALLOON_S_NR       6
+
+struct virtio_balloon_stat {
+	uint16_t tag;
+	uint64_t val;
+} __packed;
+
+#endif /* _VIRTIO_BALLOON_H */
diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c
new file mode 100644
index 0000000..09783a8
--- /dev/null
+++ b/sys/dev/virtio/block/virtio_blk.c
@@ -0,0 +1,1149 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Driver for VirtIO block devices. */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bio.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/sglist.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/taskqueue.h>
+
+#include <geom/geom_disk.h>
+#include <vm/uma.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+#include <dev/virtio/block/virtio_blk.h>
+
+#include "virtio_if.h"
+
+struct vtblk_request {
+	struct virtio_blk_outhdr	 vbr_hdr;
+	struct bio			*vbr_bp;
+	uint8_t				 vbr_ack;
+
+	TAILQ_ENTRY(vtblk_request)	 vbr_link;
+};
+
+struct vtblk_softc {
+	device_t		 vtblk_dev;
+	struct mtx		 vtblk_mtx;
+	uint64_t		 vtblk_features;
+	uint32_t		 vtblk_flags;
+#define VTBLK_FLAG_INDIRECT	0x0001
+#define VTBLK_FLAG_READONLY	0x0002
+#define VTBLK_FLAG_DETACHING	0x0004
+#define VTBLK_FLAG_SUSPENDED	0x0008
+#define VTBLK_FLAG_DUMPING	0x0010
+
+	struct virtqueue	*vtblk_vq;
+	struct sglist		*vtblk_sglist;
+	struct disk		*vtblk_disk;
+
+	struct bio_queue_head	 vtblk_bioq;
+	TAILQ_HEAD(, vtblk_request)
+				 vtblk_req_free;
+	TAILQ_HEAD(, vtblk_request)
+				 vtblk_req_ready;
+
+	struct taskqueue	*vtblk_tq;
+	struct task		 vtblk_intr_task;
+
+	int			 vtblk_sector_size;
+	int			 vtblk_max_nsegs;
+	int			 vtblk_unit;
+	int			 vtblk_request_count;
+
+	struct vtblk_request	 vtblk_dump_request;
+};
+
+static struct virtio_feature_desc vtblk_feature_desc[] = {
+	{ VIRTIO_BLK_F_BARRIER,		"HostBarrier"	},
+	{ VIRTIO_BLK_F_SIZE_MAX,	"MaxSegSize"	},
+	{ VIRTIO_BLK_F_SEG_MAX,		"MaxNumSegs"	},
+	{ VIRTIO_BLK_F_GEOMETRY,	"DiskGeometry"	},
+	{ VIRTIO_BLK_F_RO,		"ReadOnly"	},
+	{ VIRTIO_BLK_F_BLK_SIZE,	"BlockSize"	},
+	{ VIRTIO_BLK_F_SCSI,		"SCSICmds"	},
+	{ VIRTIO_BLK_F_FLUSH,		"FlushCmd"	},
+	{ VIRTIO_BLK_F_TOPOLOGY,	"Topology"	},
+
+	{ 0, NULL }
+};
+
+static int	vtblk_modevent(module_t, int, void *);
+
+static int	vtblk_probe(device_t);
+static int	vtblk_attach(device_t);
+static int	vtblk_detach(device_t);
+static int	vtblk_suspend(device_t);
+static int	vtblk_resume(device_t);
+static int	vtblk_shutdown(device_t);
+
+static void	vtblk_negotiate_features(struct vtblk_softc *);
+static int	vtblk_maximum_segments(struct vtblk_softc *,
+		    struct virtio_blk_config *);
+static int	vtblk_alloc_virtqueue(struct vtblk_softc *);
+static void	vtblk_alloc_disk(struct vtblk_softc *,
+		    struct virtio_blk_config *);
+static void	vtblk_create_disk(struct vtblk_softc *);
+
+static int	vtblk_open(struct disk *);
+static int	vtblk_close(struct disk *);
+static int	vtblk_ioctl(struct disk *, u_long, void *, int,
+		    struct thread *);
+static int	vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
+static void	vtblk_strategy(struct bio *);
+
+static void	vtblk_startio(struct vtblk_softc *);
+static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *);
+static int	vtblk_execute_request(struct vtblk_softc *,
+		    struct vtblk_request *);
+
+static int	vtblk_vq_intr(void *);
+static void	vtblk_intr_task(void *, int);
+
+static void	vtblk_stop(struct vtblk_softc *);
+
+static void	vtblk_get_ident(struct vtblk_softc *);
+static void	vtblk_prepare_dump(struct vtblk_softc *);
+static int	vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
+static int	vtblk_flush_dump(struct vtblk_softc *);
+static int	vtblk_poll_request(struct vtblk_softc *,
+		    struct vtblk_request *);
+
+static void	vtblk_drain_vq(struct vtblk_softc *, int);
+static void	vtblk_drain(struct vtblk_softc *);
+
+static int	vtblk_alloc_requests(struct vtblk_softc *);
+static void	vtblk_free_requests(struct vtblk_softc *);
+static struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *);
+static void	vtblk_enqueue_request(struct vtblk_softc *,
+		    struct vtblk_request *);
+
+static struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *);
+static void	vtblk_enqueue_ready(struct vtblk_softc *,
+		    struct vtblk_request *);
+
+static void	vtblk_bio_error(struct bio *, int);
+
+/* Tunables. */
+static int vtblk_no_ident = 0;
+TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
+
+/* Features desired/implemented by this driver. */
+#define VTBLK_FEATURES \
+    (VIRTIO_BLK_F_BARRIER		| \
+     VIRTIO_BLK_F_SIZE_MAX		| \
+     VIRTIO_BLK_F_SEG_MAX		| \
+     VIRTIO_BLK_F_GEOMETRY		| \
+     VIRTIO_BLK_F_RO			| \
+     VIRTIO_BLK_F_BLK_SIZE		| \
+     VIRTIO_BLK_F_FLUSH			| \
+     VIRTIO_RING_F_INDIRECT_DESC)
+
+#define VTBLK_MTX(_sc)		&(_sc)->vtblk_mtx
+#define VTBLK_LOCK_INIT(_sc, _name) \
+				mtx_init(VTBLK_MTX((_sc)), (_name), \
+				    "VTBLK Lock", MTX_DEF)
+#define VTBLK_LOCK(_sc)		mtx_lock(VTBLK_MTX((_sc)))
+#define VTBLK_TRYLOCK(_sc)	mtx_trylock(VTBLK_MTX((_sc)))
+#define VTBLK_UNLOCK(_sc)	mtx_unlock(VTBLK_MTX((_sc)))
+#define VTBLK_LOCK_DESTROY(_sc)	mtx_destroy(VTBLK_MTX((_sc)))
+#define VTBLK_LOCK_ASSERT(_sc)	mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
+#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
+				mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
+
+#define VTBLK_BIO_SEGMENTS(_bp)	sglist_count((_bp)->bio_data, (_bp)->bio_bcount)
+
+#define VTBLK_DISK_NAME		"vtbd"
+
+/*
+ * Each block request uses at least two segments - one for the header
+ * and one for the status.
+ */
+#define VTBLK_MIN_SEGMENTS	2
+
+static uma_zone_t vtblk_req_zone;
+
+static device_method_t vtblk_methods[] = {
+	/* Device methods. */
+	DEVMETHOD(device_probe,		vtblk_probe),
+	DEVMETHOD(device_attach,	vtblk_attach),
+	DEVMETHOD(device_detach,	vtblk_detach),
+	DEVMETHOD(device_suspend,	vtblk_suspend),
+	DEVMETHOD(device_resume,	vtblk_resume),
+	DEVMETHOD(device_shutdown,	vtblk_shutdown),
+
+	{ 0, 0 }
+};
+
+static driver_t vtblk_driver = {
+	"vtblk",
+	vtblk_methods,
+	sizeof(struct vtblk_softc)
+};
+static devclass_t vtblk_devclass;
+
+DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
+    vtblk_modevent, 0);
+MODULE_VERSION(virtio_blk, 1);
+MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
+
+static int
+vtblk_modevent(module_t mod, int type, void *unused)
+{
+	int error;
+
+	error = 0;
+
+	switch (type) {
+	case MOD_LOAD:
+		vtblk_req_zone = uma_zcreate("vtblk_request",
+		    sizeof(struct vtblk_request),
+		    NULL, NULL, NULL, NULL, 0, 0);
+		break;
+	case MOD_QUIESCE:
+	case MOD_UNLOAD:
+		if (uma_zone_get_cur(vtblk_req_zone) > 0)
+			error = EBUSY;
+		else if (type == MOD_UNLOAD) {
+			uma_zdestroy(vtblk_req_zone);
+			vtblk_req_zone = NULL;
+		}
+		break;
+	case MOD_SHUTDOWN:
+		break;
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+
+	return (error);
+}
+
+static int
+vtblk_probe(device_t dev)
+{
+
+	if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK)
+		return (ENXIO);
+
+	device_set_desc(dev, "VirtIO Block Adapter");
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtblk_attach(device_t dev)
+{
+	struct vtblk_softc *sc;
+	struct virtio_blk_config blkcfg;
+	int error;
+
+	sc = device_get_softc(dev);
+	sc->vtblk_dev = dev;
+	sc->vtblk_unit = device_get_unit(dev);
+
+	VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
+
+	bioq_init(&sc->vtblk_bioq);
+	TAILQ_INIT(&sc->vtblk_req_free);
+	TAILQ_INIT(&sc->vtblk_req_ready);
+
+	virtio_set_feature_desc(dev, vtblk_feature_desc);
+	vtblk_negotiate_features(sc);
+
+	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
+		sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
+
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
+		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
+
+	/* Get local copy of config. */
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY) == 0) {
+		bzero(&blkcfg, sizeof(struct virtio_blk_config));
+		virtio_read_device_config(dev, 0, &blkcfg,
+		    offsetof(struct virtio_blk_config, physical_block_exp));
+	} else
+		virtio_read_device_config(dev, 0, &blkcfg,
+		    sizeof(struct virtio_blk_config));
+
+	/*
+	 * With the current sglist(9) implementation, it is not easy
+	 * for us to support a maximum segment size as adjacent
+	 * segments are coalesced. For now, just make sure it's larger
+	 * than the maximum supported transfer size.
+	 */
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
+		if (blkcfg.size_max < MAXPHYS) {
+			error = ENOTSUP;
+			device_printf(dev, "host requires unsupported "
+			    "maximum segment size feature\n");
+			goto fail;
+		}
+	}
+
+	sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
+
+	/*
+	 * Allocate working sglist. The number of segments may be too
+	 * large to safely store on the stack.
+	 */
+	sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
+	if (sc->vtblk_sglist == NULL) {
+		error = ENOMEM;
+		device_printf(dev, "cannot allocate sglist\n");
+		goto fail;
+	}
+
+	error = vtblk_alloc_virtqueue(sc);
+	if (error) {
+		device_printf(dev, "cannot allocate virtqueue\n");
+		goto fail;
+	}
+
+	error = vtblk_alloc_requests(sc);
+	if (error) {
+		device_printf(dev, "cannot preallocate requests\n");
+		goto fail;
+	}
+
+	vtblk_alloc_disk(sc, &blkcfg);
+
+	TASK_INIT(&sc->vtblk_intr_task, 0, vtblk_intr_task, sc);
+	sc->vtblk_tq = taskqueue_create_fast("vtblk_taskq", M_NOWAIT,
+	    taskqueue_thread_enqueue, &sc->vtblk_tq);
+	if (sc->vtblk_tq == NULL) {
+		error = ENOMEM;
+		device_printf(dev, "cannot allocate taskqueue\n");
+		goto fail;
+	}
+	taskqueue_start_threads(&sc->vtblk_tq, 1, PI_DISK, "%s taskq",
+	    device_get_nameunit(dev));
+
+	error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY);
+	if (error) {
+		device_printf(dev, "cannot setup virtqueue interrupt\n");
+		goto fail;
+	}
+
+	vtblk_create_disk(sc);
+
+	virtqueue_enable_intr(sc->vtblk_vq);
+
+fail:
+	if (error)
+		vtblk_detach(dev);
+
+	return (error);
+}
+
+static int
+vtblk_detach(device_t dev)
+{
+	struct vtblk_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	VTBLK_LOCK(sc);
+	sc->vtblk_flags |= VTBLK_FLAG_DETACHING;
+	if (device_is_attached(dev))
+		vtblk_stop(sc);
+	VTBLK_UNLOCK(sc);
+
+	if (sc->vtblk_tq != NULL) {
+		taskqueue_drain(sc->vtblk_tq, &sc->vtblk_intr_task);
+		taskqueue_free(sc->vtblk_tq);
+		sc->vtblk_tq = NULL;
+	}
+
+	vtblk_drain(sc);
+
+	if (sc->vtblk_disk != NULL) {
+		disk_destroy(sc->vtblk_disk);
+		sc->vtblk_disk = NULL;
+	}
+
+	if (sc->vtblk_sglist != NULL) {
+		sglist_free(sc->vtblk_sglist);
+		sc->vtblk_sglist = NULL;
+	}
+
+	VTBLK_LOCK_DESTROY(sc);
+
+	return (0);
+}
+
+static int
+vtblk_suspend(device_t dev)
+{
+	struct vtblk_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	VTBLK_LOCK(sc);
+	sc->vtblk_flags |= VTBLK_FLAG_SUSPENDED;
+	/* TODO Wait for any inflight IO to complete? */
+	VTBLK_UNLOCK(sc);
+
+	return (0);
+}
+
+static int
+vtblk_resume(device_t dev)
+{
+	struct vtblk_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	VTBLK_LOCK(sc);
+	sc->vtblk_flags &= ~VTBLK_FLAG_SUSPENDED;
+	/* TODO Resume IO? */
+	VTBLK_UNLOCK(sc);
+
+	return (0);
+}
+
+static int
+vtblk_shutdown(device_t dev)
+{
+
+	return (0);
+}
+
+static int
+vtblk_open(struct disk *dp)
+{
+	struct vtblk_softc *sc;
+
+	if ((sc = dp->d_drv1) == NULL)
+		return (ENXIO);
+
+	return (sc->vtblk_flags & VTBLK_FLAG_DETACHING ? ENXIO : 0);
+}
+
+static int
+vtblk_close(struct disk *dp)
+{
+	struct vtblk_softc *sc;
+
+	if ((sc = dp->d_drv1) == NULL)
+		return (ENXIO);
+
+	return (0);
+}
+
+static int
+vtblk_ioctl(struct disk *dp, u_long cmd, void *addr, int flag,
+    struct thread *td)
+{
+	struct vtblk_softc *sc;
+
+	if ((sc = dp->d_drv1) == NULL)
+		return (ENXIO);
+
+	return (ENOTTY);
+}
+
+static int
+vtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset,
+    size_t length)
+{
+	struct disk *dp;
+	struct vtblk_softc *sc;
+	int error;
+
+	dp = arg;
+	error = 0;
+
+	if ((sc = dp->d_drv1) == NULL)
+		return (ENXIO);
+
+	if (VTBLK_TRYLOCK(sc) == 0) {
+		device_printf(sc->vtblk_dev,
+		    "softc already locked, cannot dump...\n");
+		return (EBUSY);
+	}
+
+	if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
+		vtblk_prepare_dump(sc);
+		sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
+	}
+
+	if (length > 0)
+		error = vtblk_write_dump(sc, virtual, offset, length);
+	else if (virtual == NULL && offset == 0)
+		error = vtblk_flush_dump(sc);
+
+	VTBLK_UNLOCK(sc);
+
+	return (error);
+}
+
+static void
+vtblk_strategy(struct bio *bp)
+{
+	struct vtblk_softc *sc;
+
+	if ((sc = bp->bio_disk->d_drv1) == NULL) {
+		vtblk_bio_error(bp, EINVAL);
+		return;
+	}
+
+	/*
+	 * Fail any write if RO. Unfortunately, there does not seem to
+	 * be a better way to report our readonly'ness to GEOM above.
+	 */
+	if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
+	    (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
+		vtblk_bio_error(bp, EROFS);
+		return;
+	}
+
+	/*
+	 * Prevent read/write buffers spanning too many segments from
+	 * getting into the queue. This should only trip if d_maxsize
+	 * was incorrectly set.
+	 */
+	if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
+		KASSERT(VTBLK_BIO_SEGMENTS(bp) <= sc->vtblk_max_nsegs -
+		    VTBLK_MIN_SEGMENTS,
+		    ("bio spanned too many segments: %d, max: %d",
+		    VTBLK_BIO_SEGMENTS(bp),
+		    sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS));
+	}
+
+	VTBLK_LOCK(sc);
+	if ((sc->vtblk_flags & VTBLK_FLAG_DETACHING) == 0) {
+		bioq_disksort(&sc->vtblk_bioq, bp);
+		vtblk_startio(sc);
+	} else
+		vtblk_bio_error(bp, ENXIO);
+	VTBLK_UNLOCK(sc);
+}
+
+static void
+vtblk_negotiate_features(struct vtblk_softc *sc)
+{
+	device_t dev;
+	uint64_t features;
+
+	dev = sc->vtblk_dev;
+	features = VTBLK_FEATURES;
+
+	sc->vtblk_features = virtio_negotiate_features(dev, features);
+}
+
+static int
+vtblk_maximum_segments(struct vtblk_softc *sc,
+    struct virtio_blk_config *blkcfg)
+{
+	device_t dev;
+	int nsegs;
+
+	dev = sc->vtblk_dev;
+	nsegs = VTBLK_MIN_SEGMENTS;
+
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
+		nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1);
+		if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT)
+			nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);
+	} else
+		nsegs += 1;
+
+	return (nsegs);
+}
+
+static int
+vtblk_alloc_virtqueue(struct vtblk_softc *sc)
+{
+	device_t dev;
+	struct vq_alloc_info vq_info;
+
+	dev = sc->vtblk_dev;
+
+	VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs,
+	    vtblk_vq_intr, sc, &sc->vtblk_vq,
+	    "%s request", device_get_nameunit(dev));
+
+	return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
+}
+
+static void
+vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
+{
+	device_t dev;
+	struct disk *dp;
+
+	dev = sc->vtblk_dev;
+
+	sc->vtblk_disk = dp = disk_alloc();
+	dp->d_open = vtblk_open;
+	dp->d_close = vtblk_close;
+	dp->d_ioctl = vtblk_ioctl;
+	dp->d_strategy = vtblk_strategy;
+	dp->d_name = VTBLK_DISK_NAME;
+	dp->d_unit = sc->vtblk_unit;
+	dp->d_drv1 = sc;
+
+	if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0)
+		dp->d_dump = vtblk_dump;
+
+	/* Capacity is always in 512-byte units. */
+	dp->d_mediasize = blkcfg->capacity * 512;
+
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_BLK_SIZE))
+		sc->vtblk_sector_size = blkcfg->blk_size;
+	else
+		sc->vtblk_sector_size = 512;
+	dp->d_sectorsize = sc->vtblk_sector_size;
+
+	/*
+	 * The VirtIO maximum I/O size is given in terms of segments.
+	 * However, FreeBSD limits I/O size by logical buffer size, not
+	 * by physically contiguous pages. Therefore, we have to assume
+	 * no pages are contiguous. This may impose an artificially low
+	 * maximum I/O size. But in practice, since QEMU advertises 128
+	 * segments, this gives us a maximum IO size of 125 * PAGE_SIZE,
+	 * which is typically greater than MAXPHYS. Eventually we should
+	 * just advertise MAXPHYS and split buffers that are too big.
+	 *
+	 * Note we must subtract one additional segment in case of non
+	 * page aligned buffers.
+	 */
+	dp->d_maxsize = (sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS - 1) *
+	    PAGE_SIZE;
+	if (dp->d_maxsize < PAGE_SIZE)
+		dp->d_maxsize = PAGE_SIZE; /* XXX */
+
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_GEOMETRY)) {
+		dp->d_fwsectors = blkcfg->geometry.sectors;
+		dp->d_fwheads = blkcfg->geometry.heads;
+	}
+
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH))
+		dp->d_flags |= DISKFLAG_CANFLUSHCACHE;
+}
+
+static void
+vtblk_create_disk(struct vtblk_softc *sc)
+{
+	struct disk *dp;
+
+	dp = sc->vtblk_disk;
+
+	/*
+	 * Retrieving the identification string must be done after
+	 * the virtqueue interrupt is setup otherwise it will hang.
+	 */
+	vtblk_get_ident(sc);
+
+	device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
+	    (uintmax_t) dp->d_mediasize >> 20,
+	    (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
+	    dp->d_sectorsize);
+
+	disk_create(dp, DISK_VERSION);
+}
+
+static void
+vtblk_startio(struct vtblk_softc *sc)
+{
+	struct virtqueue *vq;
+	struct vtblk_request *req;
+	int enq;
+
+	vq = sc->vtblk_vq;
+	enq = 0;
+
+	VTBLK_LOCK_ASSERT(sc);
+
+	if (sc->vtblk_flags & VTBLK_FLAG_SUSPENDED)
+		return;
+
+	while (!virtqueue_full(vq)) {
+		if ((req = vtblk_dequeue_ready(sc)) == NULL)
+			req = vtblk_bio_request(sc);
+		if (req == NULL)
+			break;
+
+		if (vtblk_execute_request(sc, req) != 0) {
+			vtblk_enqueue_ready(sc, req);
+			break;
+		}
+
+		enq++;
+	}
+
+	if (enq > 0)
+		virtqueue_notify(vq);
+}
+
+static struct vtblk_request *
+vtblk_bio_request(struct vtblk_softc *sc)
+{
+	struct bio_queue_head *bioq;
+	struct vtblk_request *req;
+	struct bio *bp;
+
+	bioq = &sc->vtblk_bioq;
+
+	if (bioq_first(bioq) == NULL)
+		return (NULL);
+
+	req = vtblk_dequeue_request(sc);
+	if (req == NULL)
+		return (NULL);
+
+	bp = bioq_takefirst(bioq);
+	req->vbr_bp = bp;
+	req->vbr_ack = -1;
+	req->vbr_hdr.ioprio = 1;
+
+	switch (bp->bio_cmd) {
+	case BIO_FLUSH:
+		req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
+		break;
+	case BIO_READ:
+		req->vbr_hdr.type = VIRTIO_BLK_T_IN;
+		req->vbr_hdr.sector = bp->bio_offset / 512;
+		break;
+	case BIO_WRITE:
+		req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
+		req->vbr_hdr.sector = bp->bio_offset / 512;
+		break;
+	default:
+		KASSERT(0, ("bio with unhandled cmd: %d", bp->bio_cmd));
+		req->vbr_hdr.type = -1;
+		break;
+	}
+
+	if (bp->bio_flags & BIO_ORDERED)
+		req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
+
+	return (req);
+}
+
+static int
+vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+	struct sglist *sg;
+	struct bio *bp;
+	int writable, error;
+
+	sg = sc->vtblk_sglist;
+	bp = req->vbr_bp;
+	writable = 0;
+
+	VTBLK_LOCK_ASSERT(sc);
+
+	sglist_reset(sg);
+	error = sglist_append(sg, &req->vbr_hdr,
+	    sizeof(struct virtio_blk_outhdr));
+	KASSERT(error == 0, ("error adding header to sglist"));
+	KASSERT(sg->sg_nseg == 1,
+	    ("header spanned multiple segments: %d", sg->sg_nseg));
+
+	if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
+		error = sglist_append(sg, bp->bio_data, bp->bio_bcount);
+		KASSERT(error == 0, ("error adding buffer to sglist"));
+
+		/* BIO_READ means the host writes into our buffer. */
+		if (bp->bio_cmd == BIO_READ)
+			writable += sg->sg_nseg - 1;
+	}
+
+	error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
+	KASSERT(error == 0, ("error adding ack to sglist"));
+	writable++;
+
+	KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
+	    ("fewer than min segments: %d", sg->sg_nseg));
+
+	error = virtqueue_enqueue(sc->vtblk_vq, req, sg,
+	    sg->sg_nseg - writable, writable);
+
+	return (error);
+}
+
+static int
+vtblk_vq_intr(void *xsc)
+{
+	struct vtblk_softc *sc;
+
+	sc = xsc;
+
+	virtqueue_disable_intr(sc->vtblk_vq);
+	taskqueue_enqueue_fast(sc->vtblk_tq, &sc->vtblk_intr_task);
+
+	return (1);
+}
+
+static void
+vtblk_intr_task(void *arg, int pending)
+{
+	struct vtblk_softc *sc;
+	struct vtblk_request *req;
+	struct virtqueue *vq;
+	struct bio *bp;
+
+	sc = arg;
+	vq = sc->vtblk_vq;
+
+	VTBLK_LOCK(sc);
+	if (sc->vtblk_flags & VTBLK_FLAG_DETACHING) {
+		VTBLK_UNLOCK(sc);
+		return;
+	}
+
+	while ((req = virtqueue_dequeue(vq, NULL)) != NULL) {
+		bp = req->vbr_bp;
+
+		if (req->vbr_ack == VIRTIO_BLK_S_OK)
+			bp->bio_resid = 0;
+		else {
+			bp->bio_flags |= BIO_ERROR;
+			if (req->vbr_ack == VIRTIO_BLK_S_UNSUPP)
+				bp->bio_error = ENOTSUP;
+			else
+				bp->bio_error = EIO;
+		}
+
+		biodone(bp);
+		vtblk_enqueue_request(sc, req);
+	}
+
+	vtblk_startio(sc);
+
+	if (virtqueue_enable_intr(vq) != 0) {
+		virtqueue_disable_intr(vq);
+		VTBLK_UNLOCK(sc);
+		taskqueue_enqueue_fast(sc->vtblk_tq,
+		    &sc->vtblk_intr_task);
+		return;
+	}
+
+	VTBLK_UNLOCK(sc);
+}
+
+static void
+vtblk_stop(struct vtblk_softc *sc)
+{
+
+	virtqueue_disable_intr(sc->vtblk_vq);
+	virtio_stop(sc->vtblk_dev);
+}
+
+static void
+vtblk_get_ident(struct vtblk_softc *sc)
+{
+	struct bio buf;
+	struct disk *dp;
+	struct vtblk_request *req;
+	int len, error;
+
+	dp = sc->vtblk_disk;
+	len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE);
+
+	if (vtblk_no_ident != 0)
+		return;
+
+	req = vtblk_dequeue_request(sc);
+	if (req == NULL)
+		return;
+
+	req->vbr_ack = -1;
+	req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID;
+	req->vbr_hdr.ioprio = 1;
+	req->vbr_hdr.sector = 0;
+
+	req->vbr_bp = &buf;
+	bzero(&buf, sizeof(struct bio));
+
+	buf.bio_cmd = BIO_READ;
+	buf.bio_data = dp->d_ident;
+	buf.bio_bcount = len;
+
+	VTBLK_LOCK(sc);
+	error = vtblk_poll_request(sc, req);
+	vtblk_enqueue_request(sc, req);
+	VTBLK_UNLOCK(sc);
+
+	if (error) {
+		device_printf(sc->vtblk_dev,
+		    "error getting device identifier: %d\n", error);
+	}
+}
+
+static void
+vtblk_prepare_dump(struct vtblk_softc *sc)
+{
+	device_t dev;
+	struct virtqueue *vq;
+
+	dev = sc->vtblk_dev;
+	vq = sc->vtblk_vq;
+
+	vtblk_stop(sc);
+
+	/*
+	 * Drain all requests caught in-flight in the virtqueue,
+	 * skipping biodone(). When dumping, only one request is
+	 * outstanding at a time, and we just poll the virtqueue
+	 * for the response.
+	 */
+	vtblk_drain_vq(sc, 1);
+
+	if (virtio_reinit(dev, sc->vtblk_features) != 0)
+		panic("cannot reinit VirtIO block device during dump");
+
+	virtqueue_disable_intr(vq);
+	virtio_reinit_complete(dev);
+}
+
+static int
+vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset,
+    size_t length)
+{
+	struct bio buf;
+	struct vtblk_request *req;
+
+	req = &sc->vtblk_dump_request;
+	req->vbr_ack = -1;
+	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
+	req->vbr_hdr.ioprio = 1;
+	req->vbr_hdr.sector = offset / 512;
+
+	req->vbr_bp = &buf;
+	bzero(&buf, sizeof(struct bio));
+
+	buf.bio_cmd = BIO_WRITE;
+	buf.bio_data = virtual;
+	buf.bio_bcount = length;
+
+	return (vtblk_poll_request(sc, req));
+}
+
+static int
+vtblk_flush_dump(struct vtblk_softc *sc)
+{
+	struct bio buf;
+	struct vtblk_request *req;
+
+	req = &sc->vtblk_dump_request;
+	req->vbr_ack = -1;
+	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
+	req->vbr_hdr.ioprio = 1;
+	req->vbr_hdr.sector = 0;
+
+	req->vbr_bp = &buf;
+	bzero(&buf, sizeof(struct bio));
+
+	buf.bio_cmd = BIO_FLUSH;
+
+	return (vtblk_poll_request(sc, req));
+}
+
+static int
+vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+	device_t dev;
+	struct virtqueue *vq;
+	struct vtblk_request *r;
+	int error;
+
+	dev = sc->vtblk_dev;
+	vq = sc->vtblk_vq;
+
+	if (!virtqueue_empty(vq))
+		return (EBUSY);
+
+	error = vtblk_execute_request(sc, req);
+	if (error)
+		return (error);
+
+	virtqueue_notify(vq);
+
+	r = virtqueue_poll(vq, NULL);
+	KASSERT(r == req, ("unexpected request response"));
+
+	if (req->vbr_ack != VIRTIO_BLK_S_OK) {
+		error = req->vbr_ack == VIRTIO_BLK_S_UNSUPP ? ENOTSUP : EIO;
+		if (bootverbose)
+			device_printf(dev,
+			    "vtblk_poll_request: IO error: %d\n", error);
+	}
+
+	return (error);
+}
+
+static void
+vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
+{
+	struct virtqueue *vq;
+	struct vtblk_request *req;
+	int last;
+
+	vq = sc->vtblk_vq;
+	last = 0;
+
+	while ((req = virtqueue_drain(vq, &last)) != NULL) {
+		if (!skip_done)
+			vtblk_bio_error(req->vbr_bp, ENXIO);
+
+		vtblk_enqueue_request(sc, req);
+	}
+
+	KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
+}
+
+static void
+vtblk_drain(struct vtblk_softc *sc)
+{
+	struct bio_queue_head *bioq;
+	struct vtblk_request *req;
+	struct bio *bp;
+
+	bioq = &sc->vtblk_bioq;
+
+	if (sc->vtblk_vq != NULL)
+		vtblk_drain_vq(sc, 0);
+
+	while ((req = vtblk_dequeue_ready(sc)) != NULL) {
+		vtblk_bio_error(req->vbr_bp, ENXIO);
+		vtblk_enqueue_request(sc, req);
+	}
+
+	while (bioq_first(bioq) != NULL) {
+		bp = bioq_takefirst(bioq);
+		vtblk_bio_error(bp, ENXIO);
+	}
+
+	vtblk_free_requests(sc);
+}
+
+static int
+vtblk_alloc_requests(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
+	int i, size;
+
+	size = virtqueue_size(sc->vtblk_vq);
+
+	/*
+	 * Preallocate sufficient requests to keep the virtqueue full. Each
+	 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
+	 * the number allocated when indirect descriptors are not available.
+	 */
+	if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
+		size /= VTBLK_MIN_SEGMENTS;
+
+	for (i = 0; i < size; i++) {
+		req = uma_zalloc(vtblk_req_zone, M_NOWAIT);
+		if (req == NULL)
+			return (ENOMEM);
+
+		sc->vtblk_request_count++;
+		vtblk_enqueue_request(sc, req);
+	}
+
+	return (0);
+}
+
+static void
+vtblk_free_requests(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
+
+	while ((req = vtblk_dequeue_request(sc)) != NULL) {
+		sc->vtblk_request_count--;
+		uma_zfree(vtblk_req_zone, req);
+	}
+
+	KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
+}
+
+static struct vtblk_request *
+vtblk_dequeue_request(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
+
+	req = TAILQ_FIRST(&sc->vtblk_req_free);
+	if (req != NULL)
+		TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
+
+	return (req);
+}
+
+static void
+vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+
+	bzero(req, sizeof(struct vtblk_request));
+	TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
+}
+
+static struct vtblk_request *
+vtblk_dequeue_ready(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
+
+	req = TAILQ_FIRST(&sc->vtblk_req_ready);
+	if (req != NULL)
+		TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
+
+	return (req);
+}
+
+static void
+vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+
+	TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
+}
+
+static void
+vtblk_bio_error(struct bio *bp, int error)
+{
+
+	biofinish(bp, NULL, error);
+}
diff --git a/sys/dev/virtio/block/virtio_blk.h b/sys/dev/virtio/block/virtio_blk.h
new file mode 100644
index 0000000..4fb32e0
--- /dev/null
+++ b/sys/dev/virtio/block/virtio_blk.h
@@ -0,0 +1,106 @@
+/*
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VIRTIO_BLK_H
+#define _VIRTIO_BLK_H
+
+#include <sys/types.h>
+
+/* Feature bits */
+#define VIRTIO_BLK_F_BARRIER	0x0001	/* Does host support barriers? */
+#define VIRTIO_BLK_F_SIZE_MAX	0x0002	/* Indicates maximum segment size */
+#define VIRTIO_BLK_F_SEG_MAX	0x0004	/* Indicates maximum # of segments */
+#define VIRTIO_BLK_F_GEOMETRY	0x0010	/* Legacy geometry available  */
+#define VIRTIO_BLK_F_RO		0x0020	/* Disk is read-only */
+#define VIRTIO_BLK_F_BLK_SIZE	0x0040	/* Block size of disk is available*/
+#define VIRTIO_BLK_F_SCSI	0x0080	/* Supports scsi command passthru */
+#define VIRTIO_BLK_F_FLUSH	0x0200	/* Cache flush command support */
+#define VIRTIO_BLK_F_TOPOLOGY	0x0400	/* Topology information is available */
+
+#define VIRTIO_BLK_ID_BYTES	20	/* ID string length */
+
+struct virtio_blk_config {
+	/* The capacity (in 512-byte sectors). */
+	uint64_t capacity;
+	/* The maximum segment size (if VIRTIO_BLK_F_SIZE_MAX) */
+	uint32_t size_max;
+	/* The maximum number of segments (if VIRTIO_BLK_F_SEG_MAX) */
+	uint32_t seg_max;
+	/* geometry the device (if VIRTIO_BLK_F_GEOMETRY) */
+	struct virtio_blk_geometry {
+		uint16_t cylinders;
+		uint8_t heads;
+		uint8_t sectors;
+	} geometry;
+
+	/* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
+	uint32_t blk_size;
+
+	/* the next 4 entries are guarded by VIRTIO_BLK_F_TOPOLOGY  */
+	/* exponent for physical block per logical block. */
+	uint8_t physical_block_exp;
+	/* alignment offset in logical blocks. */
+	uint8_t alignment_offset;
+	/* minimum I/O size without performance penalty in logical blocks. */
+	uint16_t min_io_size;
+	/* optimal sustained I/O size in logical blocks. */
+	uint32_t opt_io_size;
+} __packed;
+
+/*
+ * Command types
+ *
+ * Usage is a bit tricky as some bits are used as flags and some are not.
+ *
+ * Rules:
+ *   VIRTIO_BLK_T_OUT may be combined with VIRTIO_BLK_T_SCSI_CMD or
+ *   VIRTIO_BLK_T_BARRIER.  VIRTIO_BLK_T_FLUSH is a command of its own
+ *   and may not be combined with any of the other flags.
+ */
+
+/* These two define direction. */
+#define VIRTIO_BLK_T_IN		0
+#define VIRTIO_BLK_T_OUT	1
+
+/* This bit says it's a scsi command, not an actual read or write. */
+#define VIRTIO_BLK_T_SCSI_CMD	2
+
+/* Cache flush command */
+#define VIRTIO_BLK_T_FLUSH	4
+
+/* Get device ID command */
+#define VIRTIO_BLK_T_GET_ID	8
+
+/* Barrier before this op. */
+#define VIRTIO_BLK_T_BARRIER	0x80000000
+
+/* ID string length */
+#define VIRTIO_BLK_ID_BYTES	20
+
+/* This is the first element of the read scatter-gather list. */
+struct virtio_blk_outhdr {
+	/* VIRTIO_BLK_T* */
+	uint32_t type;
+	/* io priority. */
+	uint32_t ioprio;
+	/* Sector (ie. 512 byte offset) */
+	uint64_t sector;
+};
+
+struct virtio_scsi_inhdr {
+	uint32_t errors;
+	uint32_t data_len;
+	uint32_t sense_len;
+	uint32_t residual;
+};
+
+/* And this is the final byte of the write scatter-gather list. */
+#define VIRTIO_BLK_S_OK		0
+#define VIRTIO_BLK_S_IOERR	1
+#define VIRTIO_BLK_S_UNSUPP	2
+
+#endif /* _VIRTIO_BLK_H */
diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c
new file mode 100644
index 0000000..22becb1
--- /dev/null
+++ b/sys/dev/virtio/network/if_vtnet.c
@@ -0,0 +1,2746 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Driver for VirtIO network devices. */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#ifdef HAVE_KERNEL_OPTION_HEADERS
+#include "opt_device_polling.h"
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/sockio.h>
+#include <sys/mbuf.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+#include <sys/random.h>
+#include <sys/sglist.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <vm/uma.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <net/if_dl.h>
+#include <net/if_types.h>
+#include <net/if_media.h>
+#include <net/if_vlan_var.h>
+
+#include <net/bpf.h>
+
+#include <netinet/in_systm.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/udp.h>
+#include <netinet/tcp.h>
+#include <netinet/sctp.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+#include <dev/virtio/network/virtio_net.h>
+#include <dev/virtio/network/if_vtnetvar.h>
+
+#include "virtio_if.h"
+
+static int	vtnet_modevent(module_t, int, void *);
+
+static int	vtnet_probe(device_t);
+static int	vtnet_attach(device_t);
+static int	vtnet_detach(device_t);
+static int	vtnet_suspend(device_t);
+static int	vtnet_resume(device_t);
+static int	vtnet_shutdown(device_t);
+static int	vtnet_config_change(device_t);
+
+static void	vtnet_negotiate_features(struct vtnet_softc *);
+static int	vtnet_alloc_virtqueues(struct vtnet_softc *);
+static void	vtnet_get_hwaddr(struct vtnet_softc *);
+static void	vtnet_set_hwaddr(struct vtnet_softc *);
+static int	vtnet_is_link_up(struct vtnet_softc *);
+static void	vtnet_update_link_status(struct vtnet_softc *);
+static void	vtnet_watchdog(struct vtnet_softc *);
+static void	vtnet_config_change_task(void *, int);
+static int	vtnet_change_mtu(struct vtnet_softc *, int);
+static int	vtnet_ioctl(struct ifnet *, u_long, caddr_t);
+
+static int	vtnet_init_rx_vq(struct vtnet_softc *);
+static void	vtnet_free_rx_mbufs(struct vtnet_softc *);
+static void	vtnet_free_tx_mbufs(struct vtnet_softc *);
+static void	vtnet_free_ctrl_vq(struct vtnet_softc *);
+
+#ifdef DEVICE_POLLING
+static poll_handler_t vtnet_poll;
+#endif
+
+static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
+		    struct mbuf **);
+static int	vtnet_replace_rxbuf(struct vtnet_softc *,
+		    struct mbuf *, int);
+static int	vtnet_newbuf(struct vtnet_softc *);
+static void	vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
+static void	vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
+static int	vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
+static void	vtnet_vlan_tag_remove(struct mbuf *);
+static int	vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
+		    struct virtio_net_hdr *);
+static int	vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
+static int	vtnet_rxeof(struct vtnet_softc *, int, int *);
+static void	vtnet_rx_intr_task(void *, int);
+static int	vtnet_rx_vq_intr(void *);
+
+static void	vtnet_txeof(struct vtnet_softc *);
+static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
+		    struct virtio_net_hdr *);
+static int	vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
+		    struct vtnet_tx_header *);
+static int	vtnet_encap(struct vtnet_softc *, struct mbuf **);
+static void	vtnet_start_locked(struct ifnet *);
+static void	vtnet_start(struct ifnet *);
+static void	vtnet_tick(void *);
+static void	vtnet_tx_intr_task(void *, int);
+static int	vtnet_tx_vq_intr(void *);
+
+static void	vtnet_stop(struct vtnet_softc *);
+static int	vtnet_reinit(struct vtnet_softc *);
+static void	vtnet_init_locked(struct vtnet_softc *);
+static void	vtnet_init(void *);
+
+static void	vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
+		    struct sglist *, int, int);
+
+static void	vtnet_rx_filter(struct vtnet_softc *sc);
+static int	vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
+static int	vtnet_set_promisc(struct vtnet_softc *, int);
+static int	vtnet_set_allmulti(struct vtnet_softc *, int);
+static void	vtnet_rx_filter_mac(struct vtnet_softc *);
+
+static int	vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
+static void	vtnet_rx_filter_vlan(struct vtnet_softc *);
+static void	vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
+static void	vtnet_register_vlan(void *, struct ifnet *, uint16_t);
+static void	vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
+
+static int	vtnet_ifmedia_upd(struct ifnet *);
+static void	vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
+
+static void	vtnet_add_statistics(struct vtnet_softc *);
+
+static int	vtnet_enable_rx_intr(struct vtnet_softc *);
+static int	vtnet_enable_tx_intr(struct vtnet_softc *);
+static void	vtnet_disable_rx_intr(struct vtnet_softc *);
+static void	vtnet_disable_tx_intr(struct vtnet_softc *);
+
+/* Tunables. */
+static int vtnet_csum_disable = 0;
+TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
+static int vtnet_tso_disable = 0;
+TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
+static int vtnet_lro_disable = 0;
+TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
+
+/*
+ * Reducing the number of transmit completed interrupts can
+ * improve performance. To do so, the define below keeps the
+ * Tx vq interrupt disabled and adds calls to vtnet_txeof()
+ * in the start and watchdog paths. The price to pay for this
+ * is the m_free'ing of transmitted mbufs may be delayed until
+ * the watchdog fires.
+ */
+#define VTNET_TX_INTR_MODERATION
+
+static uma_zone_t vtnet_tx_header_zone;
+
+static struct virtio_feature_desc vtnet_feature_desc[] = {
+	{ VIRTIO_NET_F_CSUM,		"TxChecksum"	},
+	{ VIRTIO_NET_F_GUEST_CSUM,	"RxChecksum"	},
+	{ VIRTIO_NET_F_MAC,		"MacAddress"	},
+	{ VIRTIO_NET_F_GSO,		"TxAllGSO"	},
+	{ VIRTIO_NET_F_GUEST_TSO4,	"RxTSOv4"	},
+	{ VIRTIO_NET_F_GUEST_TSO6,	"RxTSOv6"	},
+	{ VIRTIO_NET_F_GUEST_ECN,	"RxECN"		},
+	{ VIRTIO_NET_F_GUEST_UFO,	"RxUFO"		},
+	{ VIRTIO_NET_F_HOST_TSO4,	"TxTSOv4"	},
+	{ VIRTIO_NET_F_HOST_TSO6,	"TxTSOv6"	},
+	{ VIRTIO_NET_F_HOST_ECN,	"TxTSOECN"	},
+	{ VIRTIO_NET_F_HOST_UFO,	"TxUFO"		},
+	{ VIRTIO_NET_F_MRG_RXBUF,	"MrgRxBuf"	},
+	{ VIRTIO_NET_F_STATUS,		"Status"	},
+	{ VIRTIO_NET_F_CTRL_VQ,		"ControlVq"	},
+	{ VIRTIO_NET_F_CTRL_RX,		"RxMode"	},
+	{ VIRTIO_NET_F_CTRL_VLAN,	"VLanFilter"	},
+	{ VIRTIO_NET_F_CTRL_RX_EXTRA,	"RxModeExtra"	},
+
+	{ 0, NULL }
+};
+
+static device_method_t vtnet_methods[] = {
+	/* Device methods. */
+	DEVMETHOD(device_probe,		vtnet_probe),
+	DEVMETHOD(device_attach,	vtnet_attach),
+	DEVMETHOD(device_detach,	vtnet_detach),
+	DEVMETHOD(device_suspend,	vtnet_suspend),
+	DEVMETHOD(device_resume,	vtnet_resume),
+	DEVMETHOD(device_shutdown,	vtnet_shutdown),
+
+	/* VirtIO methods. */
+	DEVMETHOD(virtio_config_change, vtnet_config_change),
+
+	{ 0, 0 }
+};
+
+static driver_t vtnet_driver = {
+	"vtnet",
+	vtnet_methods,
+	sizeof(struct vtnet_softc)
+};
+static devclass_t vtnet_devclass;
+
+DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
+    vtnet_modevent, 0);
+MODULE_VERSION(vtnet, 1);
+MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
+
+static int
+vtnet_modevent(module_t mod, int type, void *unused)
+{
+	int error;
+
+	error = 0;
+
+	switch (type) {
+	case MOD_LOAD:
+		vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr",
+		    sizeof(struct vtnet_tx_header),
+		    NULL, NULL, NULL, NULL, 0, 0);
+		break;
+	case MOD_QUIESCE:
+	case MOD_UNLOAD:
+		if (uma_zone_get_cur(vtnet_tx_header_zone) > 0)
+			error = EBUSY;
+		else if (type == MOD_UNLOAD) {
+			uma_zdestroy(vtnet_tx_header_zone);
+			vtnet_tx_header_zone = NULL;
+		}
+		break;
+	case MOD_SHUTDOWN:
+		break;
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+
+	return (error);
+}
+
+static int
+vtnet_probe(device_t dev)
+{
+
+	if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
+		return (ENXIO);
+
+	device_set_desc(dev, "VirtIO Networking Adapter");
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtnet_attach(device_t dev)
+{
+	struct vtnet_softc *sc;
+	struct ifnet *ifp;
+	int tx_size, error;
+
+	sc = device_get_softc(dev);
+	sc->vtnet_dev = dev;
+
+	VTNET_LOCK_INIT(sc);
+	callout_init_mtx(&sc->vtnet_tick_ch, VTNET_MTX(sc), 0);
+
+	ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
+	    vtnet_ifmedia_sts);
+	ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
+	ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
+
+	vtnet_add_statistics(sc);
+
+	virtio_set_feature_desc(dev, vtnet_feature_desc);
+	vtnet_negotiate_features(sc);
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
+		sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
+		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
+	} else
+		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
+
+	sc->vtnet_rx_mbuf_size = MCLBYTES;
+	sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
+		sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
+
+		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
+			sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
+		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
+			sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
+	}
+
+	vtnet_get_hwaddr(sc);
+
+	error = vtnet_alloc_virtqueues(sc);
+	if (error) {
+		device_printf(dev, "cannot allocate virtqueues\n");
+		goto fail;
+	}
+
+	ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		device_printf(dev, "cannot allocate ifnet structure\n");
+		error = ENOSPC;
+		goto fail;
+	}
+
+	ifp->if_softc = sc;
+	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_init = vtnet_init;
+	ifp->if_start = vtnet_start;
+	ifp->if_ioctl = vtnet_ioctl;
+
+	sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
+	sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
+
+	tx_size = virtqueue_size(sc->vtnet_tx_vq);
+	sc->vtnet_tx_size = tx_size;
+	IFQ_SET_MAXLEN(&ifp->if_snd, tx_size - 1);
+	ifp->if_snd.ifq_drv_maxlen = tx_size - 1;
+	IFQ_SET_READY(&ifp->if_snd);
+
+	ether_ifattach(ifp, sc->vtnet_hwaddr);
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
+		ifp->if_capabilities |= IFCAP_LINKSTATE;
+
+	/* Tell the upper layer(s) we support long frames. */
+	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
+	ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
+		ifp->if_capabilities |= IFCAP_TXCSUM;
+
+		if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
+			ifp->if_capabilities |= IFCAP_TSO4;
+		if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
+			ifp->if_capabilities |= IFCAP_TSO6;
+		if (ifp->if_capabilities & IFCAP_TSO)
+			ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
+
+		if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
+			sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
+	}
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
+		ifp->if_capabilities |= IFCAP_RXCSUM;
+
+		if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
+		    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
+			ifp->if_capabilities |= IFCAP_LRO;
+	}
+
+	if (ifp->if_capabilities & IFCAP_HWCSUM) {
+		/*
+		 * VirtIO does not support VLAN tagging, but we can fake
+		 * it by inserting and removing the 802.1Q header during
+		 * transmit and receive. We are then able to do checksum
+		 * offloading of VLAN frames.
+		 */
+		ifp->if_capabilities |=
+		    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
+	}
+
+	ifp->if_capenable = ifp->if_capabilities;
+
+	/*
+	 * Capabilities after here are not enabled by default.
+	 */
+
+	if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
+		ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
+
+		sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
+		    vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
+		sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
+		    vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
+	}
+
+#ifdef DEVICE_POLLING
+	ifp->if_capabilities |= IFCAP_POLLING;
+#endif
+
+	TASK_INIT(&sc->vtnet_rx_intr_task, 0, vtnet_rx_intr_task, sc);
+	TASK_INIT(&sc->vtnet_tx_intr_task, 0, vtnet_tx_intr_task, sc);
+	TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
+
+	sc->vtnet_tq = taskqueue_create_fast("vtnet_taskq", M_NOWAIT,
+	    taskqueue_thread_enqueue, &sc->vtnet_tq);
+	if (sc->vtnet_tq == NULL) {
+		error = ENOMEM;
+		device_printf(dev, "cannot allocate taskqueue\n");
+		ether_ifdetach(ifp);
+		goto fail;
+	}
+	taskqueue_start_threads(&sc->vtnet_tq, 1, PI_NET, "%s taskq",
+	    device_get_nameunit(dev));
+
+	error = virtio_setup_intr(dev, INTR_TYPE_NET);
+	if (error) {
+		device_printf(dev, "cannot setup virtqueue interrupts\n");
+		taskqueue_free(sc->vtnet_tq);
+		sc->vtnet_tq = NULL;
+		ether_ifdetach(ifp);
+		goto fail;
+	}
+
+	/*
+	 * Device defaults to promiscuous mode for backwards
+	 * compatibility. Turn it off if possible.
+	 */
+	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
+		VTNET_LOCK(sc);
+		if (vtnet_set_promisc(sc, 0) != 0) {
+			ifp->if_flags |= IFF_PROMISC;
+			device_printf(dev,
+			    "cannot disable promiscuous mode\n");
+		}
+		VTNET_UNLOCK(sc);
+	} else
+		ifp->if_flags |= IFF_PROMISC;
+
+fail:
+	if (error)
+		vtnet_detach(dev);
+
+	return (error);
+}
+
+static int
+vtnet_detach(device_t dev)
+{
+	struct vtnet_softc *sc;
+	struct ifnet *ifp;
+
+	sc = device_get_softc(dev);
+	ifp = sc->vtnet_ifp;
+
+	KASSERT(mtx_initialized(VTNET_MTX(sc)),
+	    ("vtnet mutex not initialized"));
+
+#ifdef DEVICE_POLLING
+	if (ifp != NULL && ifp->if_capenable & IFCAP_POLLING)
+		ether_poll_deregister(ifp);
+#endif
+
+	if (device_is_attached(dev)) {
+		VTNET_LOCK(sc);
+		vtnet_stop(sc);
+		VTNET_UNLOCK(sc);
+
+		callout_drain(&sc->vtnet_tick_ch);
+		taskqueue_drain(taskqueue_fast, &sc->vtnet_cfgchg_task);
+
+		ether_ifdetach(ifp);
+	}
+
+	if (sc->vtnet_tq != NULL) {
+		taskqueue_drain(sc->vtnet_tq, &sc->vtnet_rx_intr_task);
+		taskqueue_drain(sc->vtnet_tq, &sc->vtnet_tx_intr_task);
+		taskqueue_free(sc->vtnet_tq);
+		sc->vtnet_tq = NULL;
+	}
+
+	if (sc->vtnet_vlan_attach != NULL) {
+		EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
+		sc->vtnet_vlan_attach = NULL;
+	}
+	if (sc->vtnet_vlan_detach != NULL) {
+		EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach);
+		sc->vtnet_vlan_detach = NULL;
+	}
+
+	if (ifp) {
+		if_free(ifp);
+		sc->vtnet_ifp = NULL;
+	}
+
+	if (sc->vtnet_rx_vq != NULL)
+		vtnet_free_rx_mbufs(sc);
+	if (sc->vtnet_tx_vq != NULL)
+		vtnet_free_tx_mbufs(sc);
+	if (sc->vtnet_ctrl_vq != NULL)
+		vtnet_free_ctrl_vq(sc);
+
+	ifmedia_removeall(&sc->vtnet_media);
+	VTNET_LOCK_DESTROY(sc);
+
+	return (0);
+}
+
+static int
+vtnet_suspend(device_t dev)
+{
+	struct vtnet_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	VTNET_LOCK(sc);
+	vtnet_stop(sc);
+	sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
+	VTNET_UNLOCK(sc);
+
+	return (0);
+}
+
+static int
+vtnet_resume(device_t dev)
+{
+	struct vtnet_softc *sc;
+	struct ifnet *ifp;
+
+	sc = device_get_softc(dev);
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK(sc);
+	if (ifp->if_flags & IFF_UP)
+		vtnet_init_locked(sc);
+	sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
+	VTNET_UNLOCK(sc);
+
+	return (0);
+}
+
+static int
+vtnet_shutdown(device_t dev)
+{
+
+	/*
+	 * Suspend already does all of what we need to
+	 * do here; we just never expect to be resumed.
+	 */
+	return (vtnet_suspend(dev));
+}
+
+static int
+vtnet_config_change(device_t dev)
+{
+	struct vtnet_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	taskqueue_enqueue_fast(taskqueue_fast, &sc->vtnet_cfgchg_task);
+
+	return (1);
+}
+
+static void
+vtnet_negotiate_features(struct vtnet_softc *sc)
+{
+	device_t dev;
+	uint64_t mask, features;
+
+	dev = sc->vtnet_dev;
+	mask = 0;
+
+	if (vtnet_csum_disable)
+		mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
+
+	/*
+	 * TSO and LRO are only available when their corresponding
+	 * checksum offload feature is also negotiated.
+	 */
+
+	if (vtnet_csum_disable || vtnet_tso_disable)
+		mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
+		    VIRTIO_NET_F_HOST_ECN;
+
+	if (vtnet_csum_disable || vtnet_lro_disable)
+		mask |= VTNET_LRO_FEATURES;
+
+	features = VTNET_FEATURES & ~mask;
+#ifdef VTNET_TX_INTR_MODERATION
+	features |= VIRTIO_F_NOTIFY_ON_EMPTY;
+#endif
+	sc->vtnet_features = virtio_negotiate_features(dev, features);
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0 &&
+	    virtio_with_feature(dev, VTNET_LRO_FEATURES)) {
+		/*
+		 * LRO without mergeable buffers requires special care. This
+		 * is not ideal because every receive buffer must be large
+		 * enough to hold the maximum TCP packet, the Ethernet header,
+		 * and the vtnet_rx_header. This requires up to 34 descriptors
+		 * when using MCLBYTES clusters. If we do not have indirect
+		 * descriptors, LRO is disabled since the virtqueue will not
+		 * be able to contain very many receive buffers.
+		 */
+		if (virtio_with_feature(dev,
+		    VIRTIO_RING_F_INDIRECT_DESC) == 0) {
+			device_printf(dev,
+			    "LRO disabled due to lack of both mergeable "
+			    "buffers and indirect descriptors\n");
+
+			sc->vtnet_features = virtio_negotiate_features(dev,
+			    features & ~VTNET_LRO_FEATURES);
+		} else
+			sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
+	}
+}
+
+static int
+vtnet_alloc_virtqueues(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct vq_alloc_info vq_info[3];
+	int nvqs, rxsegs;
+
+	dev = sc->vtnet_dev;
+	nvqs = 2;
+
+	/*
+	 * Indirect descriptors are not needed for the Rx
+	 * virtqueue when mergeable buffers are negotiated.
+	 * The header is placed inline with the data, not
+	 * in a separate descriptor, and mbuf clusters are
+	 * always physically contiguous.
+	 */
+	if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
+		rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
+		    VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
+	} else
+		rxsegs = 0;
+
+	VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
+	    vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
+	    "%s receive", device_get_nameunit(dev));
+
+	VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
+	    vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
+	    "%s transmit", device_get_nameunit(dev));
+
+	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
+		nvqs++;
+
+		VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
+		    &sc->vtnet_ctrl_vq, "%s control",
+		    device_get_nameunit(dev));
+	}
+
+	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
+}
+
+static void
+vtnet_get_hwaddr(struct vtnet_softc *sc)
+{
+	device_t dev;
+
+	dev = sc->vtnet_dev;
+
+	if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
+		virtio_read_device_config(dev,
+		    offsetof(struct virtio_net_config, mac),
+		    sc->vtnet_hwaddr, ETHER_ADDR_LEN);
+	} else {
+		/* Generate random locally administered unicast address. */
+		sc->vtnet_hwaddr[0] = 0xB2;
+		arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
+
+		vtnet_set_hwaddr(sc);
+	}
+}
+
+static void
+vtnet_set_hwaddr(struct vtnet_softc *sc)
+{
+	device_t dev;
+
+	dev = sc->vtnet_dev;
+
+	virtio_write_device_config(dev,
+	    offsetof(struct virtio_net_config, mac),
+	    sc->vtnet_hwaddr, ETHER_ADDR_LEN);
+}
+
+static int
+vtnet_is_link_up(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct ifnet *ifp;
+	uint16_t status;
+
+	dev = sc->vtnet_dev;
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK_ASSERT(sc);
+
+	if ((ifp->if_capenable & IFCAP_LINKSTATE) == 0)
+		return (1);
+
+	status = virtio_read_dev_config_2(dev,
+	    offsetof(struct virtio_net_config, status));
+
+	return ((status & VIRTIO_NET_S_LINK_UP) != 0);
+}
+
+static void
+vtnet_update_link_status(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct ifnet *ifp;
+	int link;
+
+	dev = sc->vtnet_dev;
+	ifp = sc->vtnet_ifp;
+
+	link = vtnet_is_link_up(sc);
+
+	if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
+		sc->vtnet_flags |= VTNET_FLAG_LINK;
+		if (bootverbose)
+			device_printf(dev, "Link is up\n");
+
+		if_link_state_change(ifp, LINK_STATE_UP);
+		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+			vtnet_start_locked(ifp);
+	} else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
+		sc->vtnet_flags &= ~VTNET_FLAG_LINK;
+		if (bootverbose)
+			device_printf(dev, "Link is down\n");
+
+		if_link_state_change(ifp, LINK_STATE_DOWN);
+	}
+}
+
+static void
+vtnet_watchdog(struct vtnet_softc *sc)
+{
+	struct ifnet *ifp;
+
+	ifp = sc->vtnet_ifp;
+
+#ifdef VTNET_TX_INTR_MODERATION
+	vtnet_txeof(sc);
+#endif
+
+	if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
+		return;
+
+	if_printf(ifp, "watchdog timeout -- resetting\n");
+#ifdef VTNET_DEBUG
+	virtqueue_dump(sc->vtnet_tx_vq);
+#endif
+	ifp->if_oerrors++;
+	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+	vtnet_init_locked(sc);
+}
+
+static void
+vtnet_config_change_task(void *arg, int pending)
+{
+	struct vtnet_softc *sc;
+
+	sc = arg;
+
+	VTNET_LOCK(sc);
+	vtnet_update_link_status(sc);
+	VTNET_UNLOCK(sc);
+}
+
+static int
+vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+	struct vtnet_softc *sc;
+	struct ifreq *ifr;
+	int reinit, mask, error;
+
+	sc = ifp->if_softc;
+	ifr = (struct ifreq *) data;
+	reinit = 0;
+	error = 0;
+
+	switch (cmd) {
+	case SIOCSIFMTU:
+		if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
+			error = EINVAL;
+		else if (ifp->if_mtu != ifr->ifr_mtu) {
+			VTNET_LOCK(sc);
+			error = vtnet_change_mtu(sc, ifr->ifr_mtu);
+			VTNET_UNLOCK(sc);
+		}
+		break;
+
+	case SIOCSIFFLAGS:
+		VTNET_LOCK(sc);
+		if ((ifp->if_flags & IFF_UP) == 0) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+				vtnet_stop(sc);
+		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			if ((ifp->if_flags ^ sc->vtnet_if_flags) &
+			    (IFF_PROMISC | IFF_ALLMULTI)) {
+				if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
+					vtnet_rx_filter(sc);
+				else
+					error = ENOTSUP;
+			}
+		} else
+			vtnet_init_locked(sc);
+
+		if (error == 0)
+			sc->vtnet_if_flags = ifp->if_flags;
+		VTNET_UNLOCK(sc);
+		break;
+
+	case SIOCADDMULTI:
+	case SIOCDELMULTI:
+		VTNET_LOCK(sc);
+		if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
+		    (ifp->if_drv_flags & IFF_DRV_RUNNING))
+			vtnet_rx_filter_mac(sc);
+		VTNET_UNLOCK(sc);
+		break;
+
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd);
+		break;
+
+	case SIOCSIFCAP:
+		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+
+#ifdef DEVICE_POLLING
+		if (mask & IFCAP_POLLING) {
+			if (ifr->ifr_reqcap & IFCAP_POLLING) {
+				error = ether_poll_register(vtnet_poll, ifp);
+				if (error)
+					break;
+
+				VTNET_LOCK(sc);
+				vtnet_disable_rx_intr(sc);
+				vtnet_disable_tx_intr(sc);
+				ifp->if_capenable |= IFCAP_POLLING;
+				VTNET_UNLOCK(sc);
+			} else {
+				error = ether_poll_deregister(ifp);
+
+				/* Enable interrupts even in error case. */
+				VTNET_LOCK(sc);
+				vtnet_enable_tx_intr(sc);
+				vtnet_enable_rx_intr(sc);
+				ifp->if_capenable &= ~IFCAP_POLLING;
+				VTNET_UNLOCK(sc);
+			}
+		}
+#endif
+		VTNET_LOCK(sc);
+
+		if (mask & IFCAP_TXCSUM) {
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+			if (ifp->if_capenable & IFCAP_TXCSUM)
+				ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
+			else
+				ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
+		}
+
+		if (mask & IFCAP_TSO4) {
+			ifp->if_capenable ^= IFCAP_TSO4;
+			if (ifp->if_capenable & IFCAP_TSO4)
+				ifp->if_hwassist |= CSUM_TSO;
+			else
+				ifp->if_hwassist &= ~CSUM_TSO;
+		}
+
+		if (mask & IFCAP_RXCSUM) {
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+			reinit = 1;
+		}
+
+		if (mask & IFCAP_LRO) {
+			ifp->if_capenable ^= IFCAP_LRO;
+			reinit = 1;
+		}
+
+		if (mask & IFCAP_VLAN_HWFILTER) {
+			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
+			reinit = 1;
+		}
+
+		if (mask & IFCAP_VLAN_HWTSO)
+			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
+
+		if (mask & IFCAP_VLAN_HWTAGGING)
+			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
+
+		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+			vtnet_init_locked(sc);
+		}
+		VLAN_CAPABILITIES(ifp);
+
+		VTNET_UNLOCK(sc);
+		break;
+
+	default:
+		error = ether_ioctl(ifp, cmd, data);
+		break;
+	}
+
+	VTNET_LOCK_ASSERT_NOTOWNED(sc);
+
+	return (error);
+}
+
+static int
+vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
+{
+	struct ifnet *ifp;
+	int new_frame_size, clsize;
+
+	ifp = sc->vtnet_ifp;
+
+	if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
+		new_frame_size = sizeof(struct vtnet_rx_header) +
+		    sizeof(struct ether_vlan_header) + new_mtu;
+
+		if (new_frame_size > MJUM9BYTES)
+			return (EINVAL);
+
+		if (new_frame_size <= MCLBYTES)
+			clsize = MCLBYTES;
+		else
+			clsize = MJUM9BYTES;
+	} else {
+		new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
+		    sizeof(struct ether_vlan_header) + new_mtu;
+
+		if (new_frame_size <= MCLBYTES)
+			clsize = MCLBYTES;
+		else
+			clsize = MJUMPAGESIZE;
+	}
+
+	sc->vtnet_rx_mbuf_size = clsize;
+	sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
+	KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
+	    ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
+
+	ifp->if_mtu = new_mtu;
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+		vtnet_init_locked(sc);
+	}
+
+	return (0);
+}
+
+static int
+vtnet_init_rx_vq(struct vtnet_softc *sc)
+{
+	struct virtqueue *vq;
+	int nbufs, error;
+
+	vq = sc->vtnet_rx_vq;
+	nbufs = 0;
+	error = ENOSPC;
+
+	while (!virtqueue_full(vq)) {
+		if ((error = vtnet_newbuf(sc)) != 0)
+			break;
+		nbufs++;
+	}
+
+	if (nbufs > 0) {
+		virtqueue_notify(vq);
+
+		/*
+		 * EMSGSIZE signifies the virtqueue did not have enough
+		 * entries available to hold the last mbuf. This is not
+		 * an error. We should not get ENOSPC since we check if
+		 * the virtqueue is full before attempting to add a
+		 * buffer.
+		 */
+		if (error == EMSGSIZE)
+			error = 0;
+	}
+
+	return (error);
+}
+
+static void
+vtnet_free_rx_mbufs(struct vtnet_softc *sc)
+{
+	struct virtqueue *vq;
+	struct mbuf *m;
+	int last;
+
+	vq = sc->vtnet_rx_vq;
+	last = 0;
+
+	while ((m = virtqueue_drain(vq, &last)) != NULL)
+		m_freem(m);
+
+	KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
+}
+
+static void
+vtnet_free_tx_mbufs(struct vtnet_softc *sc)
+{
+	struct virtqueue *vq;
+	struct vtnet_tx_header *txhdr;
+	int last;
+
+	vq = sc->vtnet_tx_vq;
+	last = 0;
+
+	while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
+		m_freem(txhdr->vth_mbuf);
+		uma_zfree(vtnet_tx_header_zone, txhdr);
+	}
+
+	KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
+}
+
+static void
+vtnet_free_ctrl_vq(struct vtnet_softc *sc)
+{
+
+	/*
+	 * The control virtqueue is only polled, therefore
+	 * it should already be empty.
+	 */
+	KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
+	    ("Ctrl Vq not empty"));
+}
+
+#ifdef DEVICE_POLLING
+static int
+vtnet_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
+{
+	struct vtnet_softc *sc;
+	int rx_done;
+
+	sc = ifp->if_softc;
+	rx_done = 0;
+
+	VTNET_LOCK(sc);
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		if (cmd == POLL_AND_CHECK_STATUS)
+			vtnet_update_link_status(sc);
+
+		if (virtqueue_nused(sc->vtnet_rx_vq) > 0)
+			vtnet_rxeof(sc, count, &rx_done);
+
+		vtnet_txeof(sc);
+		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+			vtnet_start_locked(ifp);
+	}
+	VTNET_UNLOCK(sc);
+
+	return (rx_done);
+}
+#endif /* DEVICE_POLLING */
+
+static struct mbuf *
+vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
+{
+	struct mbuf *m_head, *m_tail, *m;
+	int i, clsize;
+
+	clsize = sc->vtnet_rx_mbuf_size;
+
+	m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize);
+	if (m_head == NULL)
+		goto fail;
+
+	m_head->m_len = clsize;
+	m_tail = m_head;
+
+	if (nbufs > 1) {
+		KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
+		    ("chained Rx mbuf requested without LRO_NOMRG"));
+
+		for (i = 0; i < nbufs - 1; i++) {
+			m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize);
+			if (m == NULL)
+				goto fail;
+
+			m->m_len = clsize;
+			m_tail->m_next = m;
+			m_tail = m;
+		}
+	}
+
+	if (m_tailp != NULL)
+		*m_tailp = m_tail;
+
+	return (m_head);
+
+fail:
+	sc->vtnet_stats.mbuf_alloc_failed++;
+	m_freem(m_head);
+
+	return (NULL);
+}
+
+static int
+vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
+{
+	struct mbuf *m, *m_prev;
+	struct mbuf *m_new, *m_tail;
+	int len, clsize, nreplace, error;
+
+	m = m0;
+	m_prev = NULL;
+	len = len0;
+
+	m_tail = NULL;
+	clsize = sc->vtnet_rx_mbuf_size;
+	nreplace = 0;
+
+	if (m->m_next != NULL)
+		KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
+		    ("chained Rx mbuf without LRO_NOMRG"));
+
+	/*
+	 * Since LRO_NOMRG mbuf chains are so large, we want to avoid
+	 * allocating an entire chain for each received frame. When
+	 * the received frame's length is less than that of the chain,
+	 * the unused mbufs are reassigned to the new chain.
+	 */
+	while (len > 0) {
+		/*
+		 * Something is seriously wrong if we received
+		 * a frame larger than the mbuf chain. Drop it.
+		 */
+		if (m == NULL) {
+			sc->vtnet_stats.rx_frame_too_large++;
+			return (EMSGSIZE);
+		}
+
+		KASSERT(m->m_len == clsize,
+		    ("mbuf length not expected cluster size: %d",
+		    m->m_len));
+
+		m->m_len = MIN(m->m_len, len);
+		len -= m->m_len;
+
+		m_prev = m;
+		m = m->m_next;
+		nreplace++;
+	}
+
+	KASSERT(m_prev != NULL, ("m_prev == NULL"));
+	KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
+	    ("too many replacement mbufs: %d/%d", nreplace,
+	    sc->vtnet_rx_mbuf_count));
+
+	m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
+	if (m_new == NULL) {
+		m_prev->m_len = clsize;
+		return (ENOBUFS);
+	}
+
+	/*
+	 * Move unused mbufs, if any, from the original chain
+	 * onto the end of the new chain.
+	 */
+	if (m_prev->m_next != NULL) {
+		m_tail->m_next = m_prev->m_next;
+		m_prev->m_next = NULL;
+	}
+
+	error = vtnet_enqueue_rxbuf(sc, m_new);
+	if (error) {
+		/*
+		 * BAD! We could not enqueue the replacement mbuf chain. We
+		 * must restore the m0 chain to the original state if it was
+		 * modified so we can subsequently discard it.
+		 *
+		 * NOTE: The replacement is suppose to be an identical copy
+		 * to the one just dequeued so this is an unexpected error.
+		 */
+		sc->vtnet_stats.rx_enq_replacement_failed++;
+
+		if (m_tail->m_next != NULL) {
+			m_prev->m_next = m_tail->m_next;
+			m_tail->m_next = NULL;
+		}
+
+		m_prev->m_len = clsize;
+		m_freem(m_new);
+	}
+
+	return (error);
+}
+
+static int
+vtnet_newbuf(struct vtnet_softc *sc)
+{
+	struct mbuf *m;
+	int error;
+
+	m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
+	if (m == NULL)
+		return (ENOBUFS);
+
+	error = vtnet_enqueue_rxbuf(sc, m);
+	if (error)
+		m_freem(m);
+
+	return (error);
+}
+
+static void
+vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
+{
+	struct virtqueue *vq;
+	struct mbuf *m;
+
+	vq = sc->vtnet_rx_vq;
+
+	while (--nbufs > 0) {
+		if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
+			break;
+		vtnet_discard_rxbuf(sc, m);
+	}
+}
+
+static void
+vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
+{
+	int error;
+
+	/*
+	 * Requeue the discarded mbuf. This should always be
+	 * successful since it was just dequeued.
+	 */
+	error = vtnet_enqueue_rxbuf(sc, m);
+	KASSERT(error == 0, ("cannot requeue discarded mbuf"));
+}
+
+static int
+vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
+{
+	struct sglist sg;
+	struct sglist_seg segs[VTNET_MAX_RX_SEGS];
+	struct vtnet_rx_header *rxhdr;
+	struct virtio_net_hdr *hdr;
+	uint8_t *mdata;
+	int offset, error;
+
+	VTNET_LOCK_ASSERT(sc);
+	if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0)
+		KASSERT(m->m_next == NULL, ("chained Rx mbuf"));
+
+	sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
+
+	mdata = mtod(m, uint8_t *);
+	offset = 0;
+
+	if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
+		rxhdr = (struct vtnet_rx_header *) mdata;
+		hdr = &rxhdr->vrh_hdr;
+		offset += sizeof(struct vtnet_rx_header);
+
+		error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
+		KASSERT(error == 0, ("cannot add header to sglist"));
+	}
+
+	error = sglist_append(&sg, mdata + offset, m->m_len - offset);
+	if (error)
+		return (error);
+
+	if (m->m_next != NULL) {
+		error = sglist_append_mbuf(&sg, m->m_next);
+		if (error)
+			return (error);
+	}
+
+	return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
+}
+
+static void
+vtnet_vlan_tag_remove(struct mbuf *m)
+{
+	struct ether_vlan_header *evl;
+
+	evl = mtod(m, struct ether_vlan_header *);
+
+	m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag);
+	m->m_flags |= M_VLANTAG;
+
+	/* Strip the 802.1Q header. */
+	bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
+	    ETHER_HDR_LEN - ETHER_TYPE_LEN);
+	m_adj(m, ETHER_VLAN_ENCAP_LEN);
+}
+
+#ifdef notyet
+static int
+vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
+    struct virtio_net_hdr *hdr)
+{
+	struct ether_header *eh;
+	struct ether_vlan_header *evh;
+	struct ip *ip;
+	struct ip6_hdr *ip6;
+	struct udphdr *udp;
+	int ip_offset, csum_start, csum_offset, hlen;
+	uint16_t eth_type;
+	uint8_t ip_proto;
+
+	/*
+	 * Convert the VirtIO checksum interface to FreeBSD's interface.
+	 * The host only provides us with the offset at which to start
+	 * checksumming, and the offset from that to place the completed
+	 * checksum. While this maps well with how Linux does checksums,
+	 * for FreeBSD, we must parse the received packet in order to set
+	 * the appropriate CSUM_* flags.
+	 */
+
+	/*
+	 * Every mbuf added to the receive virtqueue is always at least
+	 * MCLBYTES big, so assume something is amiss if the first mbuf
+	 * does not contain both the Ethernet and protocol headers.
+	 */
+	ip_offset = sizeof(struct ether_header);
+	if (m->m_len < ip_offset)
+		return (1);
+
+	eh = mtod(m, struct ether_header *);
+	eth_type = ntohs(eh->ether_type);
+	if (eth_type == ETHERTYPE_VLAN) {
+		ip_offset = sizeof(struct ether_vlan_header);
+		if (m->m_len < ip_offset)
+			return (1);
+		evh = mtod(m, struct ether_vlan_header *);
+		eth_type = ntohs(evh->evl_proto);
+	}
+
+	switch (eth_type) {
+	case ETHERTYPE_IP:
+		if (m->m_len < ip_offset + sizeof(struct ip))
+			return (1);
+
+		ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
+		 /* Sanity check the IP header. */
+		if (ip->ip_v != IPVERSION)
+			return (1);
+		hlen = ip->ip_hl << 2;
+		if (hlen < sizeof(struct ip))
+			return (1);
+		if (ntohs(ip->ip_len) < hlen)
+			return (1);
+		if (ntohs(ip->ip_len) != (m->m_pkthdr.len - ip_offset))
+			return (1);
+
+		ip_proto = ip->ip_p;
+		csum_start = ip_offset + hlen;
+		break;
+
+	case ETHERTYPE_IPV6:
+		if (m->m_len < ip_offset + sizeof(struct ip6_hdr))
+			return (1);
+
+		/*
+		 * XXX FreeBSD does not handle any IPv6 checksum offloading
+		 * at the moment.
+		 */
+
+		ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
+		/* XXX Assume no extension headers are present. */
+		ip_proto = ip6->ip6_nxt;
+		csum_start = ip_offset + sizeof(struct ip6_hdr);
+		break;
+
+	default:
+		sc->vtnet_stats.rx_csum_bad_ethtype++;
+		return (1);
+	}
+
+	/* Assume checksum begins right after the IP header. */
+	if (hdr->csum_start != csum_start) {
+		sc->vtnet_stats.rx_csum_bad_start++;
+		return (1);
+	}
+
+	switch (ip_proto) {
+	case IPPROTO_TCP:
+		csum_offset = offsetof(struct tcphdr, th_sum);
+		break;
+
+	case IPPROTO_UDP:
+		csum_offset = offsetof(struct udphdr, uh_sum);
+		break;
+
+	case IPPROTO_SCTP:
+		csum_offset = offsetof(struct sctphdr, checksum);
+		break;
+
+	default:
+		sc->vtnet_stats.rx_csum_bad_ipproto++;
+		return (1);
+	}
+
+	if (hdr->csum_offset != csum_offset) {
+		sc->vtnet_stats.rx_csum_bad_offset++;
+		return (1);
+	}
+
+	/*
+	 * The IP header checksum is almost certainly valid but I'm
+	 * uncertain if that is guaranteed.
+	 *
+	 * m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
+	 */
+
+	switch (ip_proto) {
+	case IPPROTO_UDP:
+		if (m->m_len < csum_start + sizeof(struct udphdr))
+			return (1);
+
+		udp = (struct udphdr *)(mtod(m, uint8_t *) + csum_start);
+		if (udp->uh_sum == 0)
+			return (0);
+
+		/* FALLTHROUGH */
+
+	case IPPROTO_TCP:
+		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+		m->m_pkthdr.csum_data = 0xFFFF;
+		break;
+
+	case IPPROTO_SCTP:
+		m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
+		break;
+	}
+
+	sc->vtnet_stats.rx_csum_offloaded++;
+
+	return (0);
+}
+#endif
+
+/*
+ * Alternative method of doing receive checksum offloading. Rather
+ * than parsing the received frame down to the IP header, use the
+ * csum_offset to determine which CSUM_* flags are appropriate. We
+ * can get by with doing this only because the checksum offsets are
+ * unique for the things we care about.
+ */
+static int
+vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
+    struct virtio_net_hdr *hdr)
+{
+	struct ether_header *eh;
+	struct ether_vlan_header *evh;
+	struct udphdr *udp;
+	int csum_len;
+	uint16_t eth_type;
+
+	csum_len = hdr->csum_start + hdr->csum_offset;
+
+	if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
+		return (1);
+	if (m->m_len < csum_len)
+		return (1);
+
+	eh = mtod(m, struct ether_header *);
+	eth_type = ntohs(eh->ether_type);
+	if (eth_type == ETHERTYPE_VLAN) {
+		evh = mtod(m, struct ether_vlan_header *);
+		eth_type = ntohs(evh->evl_proto);
+	}
+
+	if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
+		sc->vtnet_stats.rx_csum_bad_ethtype++;
+		return (1);
+	}
+
+	/* Use the offset to determine the appropriate CSUM_* flags. */
+	switch (hdr->csum_offset) {
+	case offsetof(struct udphdr, uh_sum):
+		if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
+			return (1);
+		udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
+		if (udp->uh_sum == 0)
+			return (0);
+
+		/* FALLTHROUGH */
+
+	case offsetof(struct tcphdr, th_sum):
+		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+		m->m_pkthdr.csum_data = 0xFFFF;
+		break;
+
+	case offsetof(struct sctphdr, checksum):
+		m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
+		break;
+
+	default:
+		sc->vtnet_stats.rx_csum_bad_offset++;
+		return (1);
+	}
+
+	sc->vtnet_stats.rx_csum_offloaded++;
+
+	return (0);
+}
+
+static int
+vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
+{
+	struct ifnet *ifp;
+	struct virtqueue *vq;
+	struct mbuf *m, *m_tail;
+	int len;
+
+	ifp = sc->vtnet_ifp;
+	vq = sc->vtnet_rx_vq;
+	m_tail = m_head;
+
+	while (--nbufs > 0) {
+		m = virtqueue_dequeue(vq, &len);
+		if (m == NULL) {
+			ifp->if_ierrors++;
+			goto fail;
+		}
+
+		if (vtnet_newbuf(sc) != 0) {
+			ifp->if_iqdrops++;
+			vtnet_discard_rxbuf(sc, m);
+			if (nbufs > 1)
+				vtnet_discard_merged_rxbuf(sc, nbufs);
+			goto fail;
+		}
+
+		if (m->m_len < len)
+			len = m->m_len;
+
+		m->m_len = len;
+		m->m_flags &= ~M_PKTHDR;
+
+		m_head->m_pkthdr.len += len;
+		m_tail->m_next = m;
+		m_tail = m;
+	}
+
+	return (0);
+
+fail:
+	sc->vtnet_stats.rx_mergeable_failed++;
+	m_freem(m_head);
+
+	return (1);
+}
+
+static int
+vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
+{
+	struct virtio_net_hdr lhdr;
+	struct ifnet *ifp;
+	struct virtqueue *vq;
+	struct mbuf *m;
+	struct ether_header *eh;
+	struct virtio_net_hdr *hdr;
+	struct virtio_net_hdr_mrg_rxbuf *mhdr;
+	int len, deq, nbufs, adjsz, rx_npkts;
+
+	ifp = sc->vtnet_ifp;
+	vq = sc->vtnet_rx_vq;
+	hdr = &lhdr;
+	deq = 0;
+	rx_npkts = 0;
+
+	VTNET_LOCK_ASSERT(sc);
+
+	while (--count >= 0) {
+		m = virtqueue_dequeue(vq, &len);
+		if (m == NULL)
+			break;
+		deq++;
+
+		if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
+			ifp->if_ierrors++;
+			vtnet_discard_rxbuf(sc, m);
+			continue;
+		}
+
+		if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
+			nbufs = 1;
+			adjsz = sizeof(struct vtnet_rx_header);
+			/*
+			 * Account for our pad between the header and
+			 * the actual start of the frame.
+			 */
+			len += VTNET_RX_HEADER_PAD;
+		} else {
+			mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
+			nbufs = mhdr->num_buffers;
+			adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
+		}
+
+		if (vtnet_replace_rxbuf(sc, m, len) != 0) {
+			ifp->if_iqdrops++;
+			vtnet_discard_rxbuf(sc, m);
+			if (nbufs > 1)
+				vtnet_discard_merged_rxbuf(sc, nbufs);
+			continue;
+		}
+
+		m->m_pkthdr.len = len;
+		m->m_pkthdr.rcvif = ifp;
+		m->m_pkthdr.csum_flags = 0;
+
+		if (nbufs > 1) {
+			if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
+				continue;
+		}
+
+		ifp->if_ipackets++;
+
+		/*
+		 * Save copy of header before we strip it. For both mergeable
+		 * and non-mergeable, the VirtIO header is placed first in the
+		 * mbuf's data. We no longer need num_buffers, so always use a
+		 * virtio_net_hdr.
+		 */
+		memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
+		m_adj(m, adjsz);
+
+		if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
+			eh = mtod(m, struct ether_header *);
+			if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
+				vtnet_vlan_tag_remove(m);
+
+				/*
+				 * With the 802.1Q header removed, update the
+				 * checksum starting location accordingly.
+				 */
+				if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
+					hdr->csum_start -=
+					    ETHER_VLAN_ENCAP_LEN;
+			}
+		}
+
+		if (ifp->if_capenable & IFCAP_RXCSUM &&
+		    hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+			if (vtnet_rx_csum(sc, m, hdr) != 0)
+				sc->vtnet_stats.rx_csum_failed++;
+		}
+
+		VTNET_UNLOCK(sc);
+		rx_npkts++;
+		(*ifp->if_input)(ifp, m);
+		VTNET_LOCK(sc);
+
+		/*
+		 * The interface may have been stopped while we were
+		 * passing the packet up the network stack.
+		 */
+		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+			break;
+	}
+
+	virtqueue_notify(vq);
+
+	if (rx_npktsp != NULL)
+		*rx_npktsp = rx_npkts;
+
+	return (count > 0 ? 0 : EAGAIN);
+}
+
+static void
+vtnet_rx_intr_task(void *arg, int pending)
+{
+	struct vtnet_softc *sc;
+	struct ifnet *ifp;
+	int more;
+
+	sc = arg;
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK(sc);
+
+#ifdef DEVICE_POLLING
+	if (ifp->if_capenable & IFCAP_POLLING) {
+		VTNET_UNLOCK(sc);
+		return;
+	}
+#endif
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+		vtnet_enable_rx_intr(sc);
+		VTNET_UNLOCK(sc);
+		return;
+	}
+
+	more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
+	if (!more && vtnet_enable_rx_intr(sc) != 0) {
+		vtnet_disable_rx_intr(sc);
+		more = 1;
+	}
+
+	VTNET_UNLOCK(sc);
+
+	if (more) {
+		sc->vtnet_stats.rx_task_rescheduled++;
+		taskqueue_enqueue_fast(sc->vtnet_tq,
+		    &sc->vtnet_rx_intr_task);
+	}
+}
+
+static int
+vtnet_rx_vq_intr(void *xsc)
+{
+	struct vtnet_softc *sc;
+
+	sc = xsc;
+
+	vtnet_disable_rx_intr(sc);
+	taskqueue_enqueue_fast(sc->vtnet_tq, &sc->vtnet_rx_intr_task);
+
+	return (1);
+}
+
+static void
+vtnet_txeof(struct vtnet_softc *sc)
+{
+	struct virtqueue *vq;
+	struct ifnet *ifp;
+	struct vtnet_tx_header *txhdr;
+	int deq;
+
+	vq = sc->vtnet_tx_vq;
+	ifp = sc->vtnet_ifp;
+	deq = 0;
+
+	VTNET_LOCK_ASSERT(sc);
+
+	while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
+		deq++;
+		ifp->if_opackets++;
+		m_freem(txhdr->vth_mbuf);
+		uma_zfree(vtnet_tx_header_zone, txhdr);
+	}
+
+	if (deq > 0) {
+		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+		if (virtqueue_empty(vq))
+			sc->vtnet_watchdog_timer = 0;
+	}
+}
+
+static struct mbuf *
+vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
+    struct virtio_net_hdr *hdr)
+{
+	struct ifnet *ifp;
+	struct ether_header *eh;
+	struct ether_vlan_header *evh;
+	struct ip *ip;
+	struct ip6_hdr *ip6;
+	struct tcphdr *tcp;
+	int ip_offset;
+	uint16_t eth_type, csum_start;
+	uint8_t ip_proto, gso_type;
+
+	ifp = sc->vtnet_ifp;
+	M_ASSERTPKTHDR(m);
+
+	ip_offset = sizeof(struct ether_header);
+	if (m->m_len < ip_offset) {
+		if ((m = m_pullup(m, ip_offset)) == NULL)
+			return (NULL);
+	}
+
+	eh = mtod(m, struct ether_header *);
+	eth_type = ntohs(eh->ether_type);
+	if (eth_type == ETHERTYPE_VLAN) {
+		ip_offset = sizeof(struct ether_vlan_header);
+		if (m->m_len < ip_offset) {
+			if ((m = m_pullup(m, ip_offset)) == NULL)
+				return (NULL);
+		}
+		evh = mtod(m, struct ether_vlan_header *);
+		eth_type = ntohs(evh->evl_proto);
+	}
+
+	switch (eth_type) {
+	case ETHERTYPE_IP:
+		if (m->m_len < ip_offset + sizeof(struct ip)) {
+			m = m_pullup(m, ip_offset + sizeof(struct ip));
+			if (m == NULL)
+				return (NULL);
+		}
+
+		ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
+		ip_proto = ip->ip_p;
+		csum_start = ip_offset + (ip->ip_hl << 2);
+		gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
+		break;
+
+	case ETHERTYPE_IPV6:
+		if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
+			m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
+			if (m == NULL)
+				return (NULL);
+		}
+
+		ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
+		/*
+		 * XXX Assume no extension headers are present. Presently,
+		 * this will always be true in the case of TSO, and FreeBSD
+		 * does not perform checksum offloading of IPv6 yet.
+		 */
+		ip_proto = ip6->ip6_nxt;
+		csum_start = ip_offset + sizeof(struct ip6_hdr);
+		gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
+		break;
+
+	default:
+		return (m);
+	}
+
+	if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
+		hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
+		hdr->csum_start = csum_start;
+		hdr->csum_offset = m->m_pkthdr.csum_data;
+
+		sc->vtnet_stats.tx_csum_offloaded++;
+	}
+
+	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
+		if (ip_proto != IPPROTO_TCP)
+			return (m);
+
+		if (m->m_len < csum_start + sizeof(struct tcphdr)) {
+			m = m_pullup(m, csum_start + sizeof(struct tcphdr));
+			if (m == NULL)
+				return (NULL);
+		}
+
+		tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
+		hdr->gso_type = gso_type;
+		hdr->hdr_len = csum_start + (tcp->th_off << 2);
+		hdr->gso_size = m->m_pkthdr.tso_segsz;
+
+		if (tcp->th_flags & TH_CWR) {
+			/*
+			 * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
+			 * ECN support is only configurable globally with the
+			 * net.inet.tcp.ecn.enable sysctl knob.
+			 */
+			if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
+				if_printf(ifp, "TSO with ECN not supported "
+				    "by host\n");
+				m_freem(m);
+				return (NULL);
+			}
+
+			hdr->flags |= VIRTIO_NET_HDR_GSO_ECN;
+		}
+
+		sc->vtnet_stats.tx_tso_offloaded++;
+	}
+
+	return (m);
+}
+
+static int
+vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
+    struct vtnet_tx_header *txhdr)
+{
+	struct sglist sg;
+	struct sglist_seg segs[VTNET_MAX_TX_SEGS];
+	struct virtqueue *vq;
+	struct mbuf *m;
+	int collapsed, error;
+
+	vq = sc->vtnet_tx_vq;
+	m = *m_head;
+	collapsed = 0;
+
+	sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
+	error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
+	KASSERT(error == 0 && sg.sg_nseg == 1,
+	    ("cannot add header to sglist"));
+
+again:
+	error = sglist_append_mbuf(&sg, m);
+	if (error) {
+		if (collapsed)
+			goto fail;
+
+		m = m_collapse(m, M_DONTWAIT, VTNET_MAX_TX_SEGS - 1);
+		if (m == NULL)
+			goto fail;
+
+		*m_head = m;
+		collapsed = 1;
+		goto again;
+	}
+
+	txhdr->vth_mbuf = m;
+
+	return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
+
+fail:
+	m_freem(*m_head);
+	*m_head = NULL;
+
+	return (ENOBUFS);
+}
+
+static int
+vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
+{
+	struct vtnet_tx_header *txhdr;
+	struct virtio_net_hdr *hdr;
+	struct mbuf *m;
+	int error;
+
+	txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO);
+	if (txhdr == NULL)
+		return (ENOMEM);
+
+	/*
+	 * Always use the non-mergeable header to simplify things. When
+	 * the mergeable feature is negotiated, the num_buffers field
+	 * must be set to zero. We use vtnet_hdr_size later to enqueue
+	 * the correct header size to the host.
+	 */
+	hdr = &txhdr->vth_uhdr.hdr;
+	m = *m_head;
+
+	error = ENOBUFS;
+
+	if (m->m_flags & M_VLANTAG) {
+		m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
+		if ((*m_head = m) == NULL)
+			goto fail;
+		m->m_flags &= ~M_VLANTAG;
+	}
+
+	if (m->m_pkthdr.csum_flags != 0) {
+		m = vtnet_tx_offload(sc, m, hdr);
+		if ((*m_head = m) == NULL)
+			goto fail;
+	}
+
+	error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
+fail:
+	if (error)
+		uma_zfree(vtnet_tx_header_zone, txhdr);
+
+	return (error);
+}
+
+static void
+vtnet_start(struct ifnet *ifp)
+{
+	struct vtnet_softc *sc;
+
+	sc = ifp->if_softc;
+
+	VTNET_LOCK(sc);
+	vtnet_start_locked(ifp);
+	VTNET_UNLOCK(sc);
+}
+
+static void
+vtnet_start_locked(struct ifnet *ifp)
+{
+	struct vtnet_softc *sc;
+	struct virtqueue *vq;
+	struct mbuf *m0;
+	int enq;
+
+	sc = ifp->if_softc;
+	vq = sc->vtnet_tx_vq;
+	enq = 0;
+
+	VTNET_LOCK_ASSERT(sc);
+
+	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
+	    IFF_DRV_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
+		return;
+
+#ifdef VTNET_TX_INTR_MODERATION
+	if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
+		vtnet_txeof(sc);
+#endif
+
+	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
+		if (virtqueue_full(vq)) {
+			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+			break;
+		}
+
+		IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
+		if (m0 == NULL)
+			break;
+
+		if (vtnet_encap(sc, &m0) != 0) {
+			if (m0 == NULL)
+				break;
+			IFQ_DRV_PREPEND(&ifp->if_snd, m0);
+			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+			break;
+		}
+
+		enq++;
+		ETHER_BPF_MTAP(ifp, m0);
+	}
+
+	if (enq > 0) {
+		virtqueue_notify(vq);
+		sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
+	}
+}
+
+static void
+vtnet_tick(void *xsc)
+{
+	struct vtnet_softc *sc;
+
+	sc = xsc;
+
+	VTNET_LOCK_ASSERT(sc);
+#ifdef VTNET_DEBUG
+	virtqueue_dump(sc->vtnet_rx_vq);
+	virtqueue_dump(sc->vtnet_tx_vq);
+#endif
+
+	vtnet_watchdog(sc);
+	callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
+}
+
+static void
+vtnet_tx_intr_task(void *arg, int pending)
+{
+	struct vtnet_softc *sc;
+	struct ifnet *ifp;
+
+	sc = arg;
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK(sc);
+
+#ifdef DEVICE_POLLING
+	if (ifp->if_capenable & IFCAP_POLLING) {
+		VTNET_UNLOCK(sc);
+		return;
+	}
+#endif
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+		vtnet_enable_tx_intr(sc);
+		VTNET_UNLOCK(sc);
+		return;
+	}
+
+	vtnet_txeof(sc);
+
+	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+		vtnet_start_locked(ifp);
+
+	if (vtnet_enable_tx_intr(sc) != 0) {
+		vtnet_disable_tx_intr(sc);
+		sc->vtnet_stats.tx_task_rescheduled++;
+		VTNET_UNLOCK(sc);
+		taskqueue_enqueue_fast(sc->vtnet_tq, &sc->vtnet_tx_intr_task);
+		return;
+	}
+
+	VTNET_UNLOCK(sc);
+}
+
+static int
+vtnet_tx_vq_intr(void *xsc)
+{
+	struct vtnet_softc *sc;
+
+	sc = xsc;
+
+	vtnet_disable_tx_intr(sc);
+	taskqueue_enqueue_fast(sc->vtnet_tq, &sc->vtnet_tx_intr_task);
+
+	return (1);
+}
+
+static void
+vtnet_stop(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct ifnet *ifp;
+
+	dev = sc->vtnet_dev;
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK_ASSERT(sc);
+
+	sc->vtnet_watchdog_timer = 0;
+	callout_stop(&sc->vtnet_tick_ch);
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+
+	vtnet_disable_rx_intr(sc);
+	vtnet_disable_tx_intr(sc);
+
+	/*
+	 * Stop the host VirtIO adapter. Note this will reset the host
+	 * adapter's state back to the pre-initialized state, so in
+	 * order to make the device usable again, we must drive it
+	 * through virtio_reinit() and virtio_reinit_complete().
+	 */
+	virtio_stop(dev);
+
+	sc->vtnet_flags &= ~VTNET_FLAG_LINK;
+
+	vtnet_free_rx_mbufs(sc);
+	vtnet_free_tx_mbufs(sc);
+}
+
+static int
+vtnet_reinit(struct vtnet_softc *sc)
+{
+	struct ifnet *ifp;
+	uint64_t features;
+
+	ifp = sc->vtnet_ifp;
+	features = sc->vtnet_features;
+
+	/*
+	 * Re-negotiate with the host, removing any disabled receive
+	 * features. Transmit features are disabled only on our side
+	 * via if_capenable and if_hwassist.
+	 */
+
+	if (ifp->if_capabilities & IFCAP_RXCSUM) {
+		if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
+			features &= ~VIRTIO_NET_F_GUEST_CSUM;
+	}
+
+	if (ifp->if_capabilities & IFCAP_LRO) {
+		if ((ifp->if_capenable & IFCAP_LRO) == 0)
+			features &= ~VTNET_LRO_FEATURES;
+	}
+
+	if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
+		if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
+			features &= ~VIRTIO_NET_F_CTRL_VLAN;
+	}
+
+	return (virtio_reinit(sc->vtnet_dev, features));
+}
+
+static void
+vtnet_init_locked(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct ifnet *ifp;
+	int error;
+
+	dev = sc->vtnet_dev;
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK_ASSERT(sc);
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		return;
+
+	/* Stop host's adapter, cancel any pending I/O. */
+	vtnet_stop(sc);
+
+	/* Reinitialize the host device. */
+	error = vtnet_reinit(sc);
+	if (error) {
+		device_printf(dev,
+		    "reinitialization failed, stopping device...\n");
+		vtnet_stop(sc);
+		return;
+	}
+
+	/* Update host with assigned MAC address. */
+	bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
+	vtnet_set_hwaddr(sc);
+
+	ifp->if_hwassist = 0;
+	if (ifp->if_capenable & IFCAP_TXCSUM)
+		ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
+	if (ifp->if_capenable & IFCAP_TSO4)
+		ifp->if_hwassist |= CSUM_TSO;
+
+	error = vtnet_init_rx_vq(sc);
+	if (error) {
+		device_printf(dev,
+		    "cannot allocate mbufs for Rx virtqueue\n");
+		vtnet_stop(sc);
+		return;
+	}
+
+	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
+		if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
+			/* Restore promiscuous and all-multicast modes. */
+			vtnet_rx_filter(sc);
+
+			/* Restore filtered MAC addresses. */
+			vtnet_rx_filter_mac(sc);
+		}
+
+		/* Restore VLAN filters. */
+		if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
+			vtnet_rx_filter_vlan(sc);
+	}
+
+#ifdef DEVICE_POLLING
+	if (ifp->if_capenable & IFCAP_POLLING) {
+		vtnet_disable_rx_intr(sc);
+		vtnet_disable_tx_intr(sc);
+	} else
+#endif
+	{
+		vtnet_enable_rx_intr(sc);
+		vtnet_enable_tx_intr(sc);
+	}
+
+	ifp->if_drv_flags |= IFF_DRV_RUNNING;
+	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+
+	virtio_reinit_complete(dev);
+
+	vtnet_update_link_status(sc);
+	callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
+}
+
+static void
+vtnet_init(void *xsc)
+{
+	struct vtnet_softc *sc;
+
+	sc = xsc;
+
+	VTNET_LOCK(sc);
+	vtnet_init_locked(sc);
+	VTNET_UNLOCK(sc);
+}
+
+static void
+vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
+    struct sglist *sg, int readable, int writable)
+{
+	struct virtqueue *vq;
+	void *c;
+
+	vq = sc->vtnet_ctrl_vq;
+
+	VTNET_LOCK_ASSERT(sc);
+	KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
+	    ("no control virtqueue"));
+	KASSERT(virtqueue_empty(vq),
+	    ("control command already enqueued"));
+
+	if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
+		return;
+
+	virtqueue_notify(vq);
+
+	/*
+	 * Poll until the command is complete. Previously, we would
+	 * sleep until the control virtqueue interrupt handler woke
+	 * us up, but dropping the VTNET_MTX leads to serialization
+	 * difficulties.
+	 *
+	 * Furthermore, it appears QEMU/KVM only allocates three MSIX
+	 * vectors. Two of those vectors are needed for the Rx and Tx
+	 * virtqueues. We do not support sharing both a Vq and config
+	 * changed notification on the same MSIX vector.
+	 */
+	c = virtqueue_poll(vq, NULL);
+	KASSERT(c == cookie, ("unexpected control command response"));
+}
+
+static void
+vtnet_rx_filter(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct ifnet *ifp;
+
+	dev = sc->vtnet_dev;
+	ifp = sc->vtnet_ifp;
+
+	VTNET_LOCK_ASSERT(sc);
+	KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
+	    ("CTRL_RX feature not negotiated"));
+
+	if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
+		device_printf(dev, "cannot %s promiscuous mode\n",
+		    ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
+
+	if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
+		device_printf(dev, "cannot %s all-multicast mode\n",
+		    ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
+}
+
+static int
+vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
+{
+	struct virtio_net_ctrl_hdr hdr;
+	struct sglist_seg segs[3];
+	struct sglist sg;
+	uint8_t onoff, ack;
+	int error;
+
+	if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
+		return (ENOTSUP);
+
+	error = 0;
+
+	hdr.class = VIRTIO_NET_CTRL_RX;
+	hdr.cmd = cmd;
+	onoff = !!on;
+	ack = VIRTIO_NET_ERR;
+
+	sglist_init(&sg, 3, segs);
+	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
+	error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
+	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
+	KASSERT(error == 0 && sg.sg_nseg == 3,
+	    ("error adding Rx filter message to sglist"));
+
+	vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
+
+	return (ack == VIRTIO_NET_OK ? 0 : EIO);
+}
+
+static int
+vtnet_set_promisc(struct vtnet_softc *sc, int on)
+{
+
+	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
+}
+
+static int
+vtnet_set_allmulti(struct vtnet_softc *sc, int on)
+{
+
+	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
+}
+
+static void
+vtnet_rx_filter_mac(struct vtnet_softc *sc)
+{
+	struct virtio_net_ctrl_hdr hdr;
+	struct vtnet_mac_filter *filter;
+	struct sglist_seg segs[4];
+	struct sglist sg;
+	struct ifnet *ifp;
+	struct ifaddr *ifa;
+	struct ifmultiaddr *ifma;
+	int ucnt, mcnt, promisc, allmulti, error;
+	uint8_t ack;
+
+	ifp = sc->vtnet_ifp;
+	ucnt = 0;
+	mcnt = 0;
+	promisc = 0;
+	allmulti = 0;
+	error = 0;
+
+	VTNET_LOCK_ASSERT(sc);
+	KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
+	    ("CTRL_RX feature not negotiated"));
+
+	/*
+	 * Allocate the MAC filtering table. Note we could do this
+	 * at attach time, but it is probably not worth keeping it
+	 * around for an infrequent occurrence.
+	 */
+	filter = malloc(sizeof(struct vtnet_mac_filter), M_DEVBUF,
+	    M_NOWAIT | M_ZERO);
+	if (filter == NULL) {
+		device_printf(sc->vtnet_dev,
+		    "cannot allocate MAC address filtering table\n");
+		return;
+	}
+
+	/* Unicast MAC addresses: */
+	if_addr_rlock(ifp);
+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		if (ifa->ifa_addr->sa_family != AF_LINK)
+			continue;
+		else if (ucnt == VTNET_MAX_MAC_ENTRIES)
+			break;
+
+		bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
+		    &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
+		ucnt++;
+	}
+	if_addr_runlock(ifp);
+
+	if (ucnt >= VTNET_MAX_MAC_ENTRIES) {
+		promisc = 1;
+		filter->vmf_unicast.nentries = 0;
+
+		if_printf(ifp, "more than %d MAC addresses assigned, "
+		    "falling back to promiscuous mode\n",
+		    VTNET_MAX_MAC_ENTRIES);
+	} else
+		filter->vmf_unicast.nentries = ucnt;
+
+	/* Multicast MAC addresses: */
+	if_maddr_rlock(ifp);
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+		else if (mcnt == VTNET_MAX_MAC_ENTRIES)
+			break;
+
+		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
+		    &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
+		mcnt++;
+	}
+	if_maddr_runlock(ifp);
+
+	if (mcnt >= VTNET_MAX_MAC_ENTRIES) {
+		allmulti = 1;
+		filter->vmf_multicast.nentries = 0;
+
+		if_printf(ifp, "more than %d multicast MAC addresses "
+		    "assigned, falling back to all-multicast mode\n",
+		    VTNET_MAX_MAC_ENTRIES);
+	} else
+		filter->vmf_multicast.nentries = mcnt;
+
+	if (promisc && allmulti)
+		goto out;
+
+	hdr.class = VIRTIO_NET_CTRL_MAC;
+	hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
+	ack = VIRTIO_NET_ERR;
+
+	sglist_init(&sg, 4, segs);
+	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
+	error |= sglist_append(&sg, &filter->vmf_unicast,
+	    sizeof(struct vtnet_mac_table));
+	error |= sglist_append(&sg, &filter->vmf_multicast,
+	    sizeof(struct vtnet_mac_table));
+	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
+	KASSERT(error == 0 && sg.sg_nseg == 4,
+	    ("error adding MAC filtering message to sglist"));
+
+	vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
+
+	if (ack != VIRTIO_NET_OK)
+		if_printf(ifp, "error setting host MAC filter table\n");
+
+out:
+	free(filter, M_DEVBUF);
+
+	if (promisc)
+		if (vtnet_set_promisc(sc, 1) != 0)
+			if_printf(ifp, "cannot enable promiscuous mode\n");
+	if (allmulti)
+		if (vtnet_set_allmulti(sc, 1) != 0)
+			if_printf(ifp, "cannot enable all-multicast mode\n");
+}
+
+static int
+vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
+{
+	struct virtio_net_ctrl_hdr hdr;
+	struct sglist_seg segs[3];
+	struct sglist sg;
+	uint8_t ack;
+	int error;
+
+	hdr.class = VIRTIO_NET_CTRL_VLAN;
+	hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
+	ack = VIRTIO_NET_ERR;
+	error = 0;
+
+	sglist_init(&sg, 3, segs);
+	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
+	error |= sglist_append(&sg, &tag, sizeof(uint16_t));
+	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
+	KASSERT(error == 0 && sg.sg_nseg == 3,
+	    ("error adding VLAN control message to sglist"));
+
+	vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
+
+	return (ack == VIRTIO_NET_OK ? 0 : EIO);
+}
+
+static void
+vtnet_rx_filter_vlan(struct vtnet_softc *sc)
+{
+	device_t dev;
+	uint32_t w, mask;
+	uint16_t tag;
+	int i, nvlans, error;
+
+	VTNET_LOCK_ASSERT(sc);
+	KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
+	    ("VLAN_FILTER feature not negotiated"));
+
+	dev = sc->vtnet_dev;
+	nvlans = sc->vtnet_nvlans;
+	error = 0;
+
+	/* Enable filtering for each configured VLAN. */
+	for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
+		w = sc->vtnet_vlan_shadow[i];
+		for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
+			if ((w & mask) != 0) {
+				w &= ~mask;
+				nvlans--;
+				if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
+					error++;
+			}
+		}
+	}
+
+	KASSERT(nvlans == 0, ("VLAN count incorrect"));
+	if (error)
+		device_printf(dev, "cannot restore VLAN filter table\n");
+}
+
+static void
+vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
+{
+	struct ifnet *ifp;
+	int idx, bit;
+
+	KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
+	    ("VLAN_FILTER feature not negotiated"));
+
+	if ((tag == 0) || (tag > 4095))
+		return;
+
+	ifp = sc->vtnet_ifp;
+	idx = (tag >> 5) & 0x7F;
+	bit = tag & 0x1F;
+
+	VTNET_LOCK(sc);
+
+	/* Update shadow VLAN table. */
+	if (add) {
+		sc->vtnet_nvlans++;
+		sc->vtnet_vlan_shadow[idx] |= (1 << bit);
+	} else {
+		sc->vtnet_nvlans--;
+		sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
+	}
+
+	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
+		if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
+			device_printf(sc->vtnet_dev,
+			    "cannot %s VLAN %d %s the host filter table\n",
+			    add ? "add" : "remove", tag,
+			    add ? "to" : "from");
+		}
+	}
+
+	VTNET_UNLOCK(sc);
+}
+
+static void
+vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
+{
+
+	if (ifp->if_softc != arg)
+		return;
+
+	vtnet_set_vlan_filter(arg, 1, tag);
+}
+
+static void
+vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag)
+{
+
+	if (ifp->if_softc != arg)
+		return;
+
+	vtnet_set_vlan_filter(arg, 0, tag);
+}
+
+static int
+vtnet_ifmedia_upd(struct ifnet *ifp)
+{
+	struct vtnet_softc *sc;
+	struct ifmedia *ifm;
+
+	sc = ifp->if_softc;
+	ifm = &sc->vtnet_media;
+
+	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
+		return (EINVAL);
+
+	return (0);
+}
+
+static void
+vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	struct vtnet_softc *sc;
+
+	sc = ifp->if_softc;
+
+	ifmr->ifm_status = IFM_AVALID;
+	ifmr->ifm_active = IFM_ETHER;
+
+	VTNET_LOCK(sc);
+	if (vtnet_is_link_up(sc) != 0) {
+		ifmr->ifm_status |= IFM_ACTIVE;
+		ifmr->ifm_active |= VTNET_MEDIATYPE;
+	} else
+		ifmr->ifm_active |= IFM_NONE;
+	VTNET_UNLOCK(sc);
+}
+
+static void
+vtnet_add_statistics(struct vtnet_softc *sc)
+{
+	device_t dev;
+	struct vtnet_statistics *stats;
+        struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *tree;
+	struct sysctl_oid_list *child;
+
+	dev = sc->vtnet_dev;
+	stats = &sc->vtnet_stats;
+	ctx = device_get_sysctl_ctx(dev);
+	tree = device_get_sysctl_tree(dev);
+	child = SYSCTL_CHILDREN(tree);
+
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_failed",
+	    CTLFLAG_RD, &stats->mbuf_alloc_failed,
+	    "Mbuf cluster allocation failures");
+
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_frame_too_large",
+	    CTLFLAG_RD, &stats->rx_frame_too_large,
+	    "Received frame larger than the mbuf chain");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
+	    CTLFLAG_RD, &stats->rx_enq_replacement_failed,
+	    "Enqueuing the replacement receive mbuf failed");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_mergeable_failed",
+	    CTLFLAG_RD, &stats->rx_mergeable_failed,
+	    "Mergeable buffers receive failures");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
+	    CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
+	    "Received checksum offloaded buffer with unsupported "
+	    "Ethernet type");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_start",
+	    CTLFLAG_RD, &stats->rx_csum_bad_start,
+	    "Received checksum offloaded buffer with incorrect start offset");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
+	    CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
+	    "Received checksum offloaded buffer with incorrect IP protocol");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_offset",
+	    CTLFLAG_RD, &stats->rx_csum_bad_offset,
+	    "Received checksum offloaded buffer with incorrect offset");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_failed",
+	    CTLFLAG_RD, &stats->rx_csum_failed,
+	    "Received buffer checksum offload failed");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_offloaded",
+	    CTLFLAG_RD, &stats->rx_csum_offloaded,
+	    "Received buffer checksum offload succeeded");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_task_rescheduled",
+	    CTLFLAG_RD, &stats->rx_task_rescheduled,
+	    "Times the receive interrupt task rescheduled itself");
+
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_offloaded",
+	    CTLFLAG_RD, &stats->tx_csum_offloaded,
+	    "Offloaded checksum of transmitted buffer");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_offloaded",
+	    CTLFLAG_RD, &stats->tx_tso_offloaded,
+	    "Segmentation offload of transmitted buffer");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
+	    CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
+	    "Aborted transmit of checksum offloaded buffer with unknown "
+	    "Ethernet type");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
+	    CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
+	    "Aborted transmit of TSO buffer with unknown Ethernet type");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_task_rescheduled",
+	    CTLFLAG_RD, &stats->tx_task_rescheduled,
+	    "Times the transmit interrupt task rescheduled itself");
+}
+
+static int
+vtnet_enable_rx_intr(struct vtnet_softc *sc)
+{
+
+	return (virtqueue_enable_intr(sc->vtnet_rx_vq));
+}
+
+static void
+vtnet_disable_rx_intr(struct vtnet_softc *sc)
+{
+
+	virtqueue_disable_intr(sc->vtnet_rx_vq);
+}
+
+static int
+vtnet_enable_tx_intr(struct vtnet_softc *sc)
+{
+
+#ifdef VTNET_TX_INTR_MODERATION
+	return (0);
+#else
+	return (virtqueue_enable_intr(sc->vtnet_tx_vq));
+#endif
+}
+
+static void
+vtnet_disable_tx_intr(struct vtnet_softc *sc)
+{
+
+	virtqueue_disable_intr(sc->vtnet_tx_vq);
+}
diff --git a/sys/dev/virtio/network/if_vtnetvar.h b/sys/dev/virtio/network/if_vtnetvar.h
new file mode 100644
index 0000000..613b2b0
--- /dev/null
+++ b/sys/dev/virtio/network/if_vtnetvar.h
@@ -0,0 +1,240 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _IF_VTNETVAR_H
+#define _IF_VTNETVAR_H
+
+struct vtnet_statistics {
+	unsigned long	mbuf_alloc_failed;
+
+	unsigned long	rx_frame_too_large;
+	unsigned long	rx_enq_replacement_failed;
+	unsigned long	rx_mergeable_failed;
+	unsigned long	rx_csum_bad_ethtype;
+	unsigned long	rx_csum_bad_start;
+	unsigned long	rx_csum_bad_ipproto;
+	unsigned long	rx_csum_bad_offset;
+	unsigned long	rx_csum_failed;
+	unsigned long	rx_csum_offloaded;
+	unsigned long	rx_task_rescheduled;
+
+	unsigned long	tx_csum_offloaded;
+	unsigned long	tx_tso_offloaded;
+	unsigned long	tx_csum_bad_ethtype;
+	unsigned long	tx_tso_bad_ethtype;
+	unsigned long	tx_task_rescheduled;
+};
+
+struct vtnet_softc {
+	device_t		 vtnet_dev;
+	struct ifnet		*vtnet_ifp;
+	struct mtx		 vtnet_mtx;
+
+	uint32_t		 vtnet_flags;
+#define VTNET_FLAG_LINK		 0x0001
+#define VTNET_FLAG_SUSPENDED	 0x0002
+#define VTNET_FLAG_CTRL_VQ	 0x0004
+#define VTNET_FLAG_CTRL_RX	 0x0008
+#define VTNET_FLAG_VLAN_FILTER	 0x0010
+#define VTNET_FLAG_TSO_ECN	 0x0020
+#define VTNET_FLAG_MRG_RXBUFS	 0x0040
+#define VTNET_FLAG_LRO_NOMRG	 0x0080
+
+	struct virtqueue	*vtnet_rx_vq;
+	struct virtqueue	*vtnet_tx_vq;
+	struct virtqueue	*vtnet_ctrl_vq;
+
+	int			 vtnet_hdr_size;
+	int			 vtnet_tx_size;
+	int			 vtnet_rx_size;
+	int			 vtnet_rx_process_limit;
+	int			 vtnet_rx_mbuf_size;
+	int			 vtnet_rx_mbuf_count;
+	int			 vtnet_if_flags;
+	int			 vtnet_watchdog_timer;
+	uint64_t		 vtnet_features;
+
+	struct taskqueue	*vtnet_tq;
+	struct task		 vtnet_rx_intr_task;
+	struct task		 vtnet_tx_intr_task;
+	struct task		 vtnet_cfgchg_task;
+
+	struct vtnet_statistics	 vtnet_stats;
+
+	struct callout		 vtnet_tick_ch;
+
+	eventhandler_tag	 vtnet_vlan_attach;
+	eventhandler_tag	 vtnet_vlan_detach;
+
+	struct ifmedia		 vtnet_media;
+	/*
+	 * Fake media type; the host does not provide us with
+	 * any real media information.
+	 */
+#define VTNET_MEDIATYPE		 (IFM_ETHER | IFM_1000_T | IFM_FDX)
+	char			 vtnet_hwaddr[ETHER_ADDR_LEN];
+
+	/*
+	 * During reset, the host's VLAN filtering table is lost. The
+	 * array below is used to restore all the VLANs configured on
+	 * this interface after a reset.
+	 */
+#define VTNET_VLAN_SHADOW_SIZE	 (4096 / 32)
+	int			 vtnet_nvlans;
+	uint32_t		 vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
+
+	char			 vtnet_mtx_name[16];
+};
+
+/*
+ * When mergeable buffers are not negotiated, the vtnet_rx_header structure
+ * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
+ * both keep the VirtIO header and the data non-contiguous and to keep the
+ * frame's payload 4 byte aligned.
+ *
+ * When mergeable buffers are negotiated, the host puts the VirtIO header in
+ * the beginning of the first mbuf's data.
+ */
+#define VTNET_RX_HEADER_PAD	4
+struct vtnet_rx_header {
+	struct virtio_net_hdr	vrh_hdr;
+	char			vrh_pad[VTNET_RX_HEADER_PAD];
+} __packed;
+
+/*
+ * For each outgoing frame, the vtnet_tx_header below is allocated from
+ * the vtnet_tx_header_zone.
+ */
+struct vtnet_tx_header {
+	union {
+		struct virtio_net_hdr		hdr;
+		struct virtio_net_hdr_mrg_rxbuf	mhdr;
+	} vth_uhdr;
+
+	struct mbuf *vth_mbuf;
+};
+
+/*
+ * The VirtIO specification does not place a limit on the number of MAC
+ * addresses the guest driver may request to be filtered. In practice,
+ * the host is constrained by available resources. To simplify this driver,
+ * impose a reasonably high limit of MAC addresses we will filter before
+ * falling back to promiscuous or all-multicast modes.
+ */
+#define VTNET_MAX_MAC_ENTRIES	128
+
+struct vtnet_mac_table {
+	uint32_t	nentries;
+	uint8_t		macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
+} __packed;
+
+struct vtnet_mac_filter {
+	struct vtnet_mac_table	vmf_unicast;
+	uint32_t		vmf_pad; /* Make tables non-contiguous. */
+	struct vtnet_mac_table	vmf_multicast;
+};
+
+/*
+ * The MAC filter table is malloc(9)'d when needed. Ensure it will
+ * always fit in one segment.
+ */
+CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
+
+#define VTNET_WATCHDOG_TIMEOUT	5
+#define VTNET_CSUM_OFFLOAD	(CSUM_TCP | CSUM_UDP | CSUM_SCTP)
+
+/* Features desired/implemented by this driver. */
+#define VTNET_FEATURES \
+    (VIRTIO_NET_F_MAC			| \
+     VIRTIO_NET_F_STATUS		| \
+     VIRTIO_NET_F_CTRL_VQ		| \
+     VIRTIO_NET_F_CTRL_RX		| \
+     VIRTIO_NET_F_CTRL_VLAN		| \
+     VIRTIO_NET_F_CSUM			| \
+     VIRTIO_NET_F_HOST_TSO4		| \
+     VIRTIO_NET_F_HOST_TSO6		| \
+     VIRTIO_NET_F_HOST_ECN		| \
+     VIRTIO_NET_F_GUEST_CSUM		| \
+     VIRTIO_NET_F_GUEST_TSO4		| \
+     VIRTIO_NET_F_GUEST_TSO6		| \
+     VIRTIO_NET_F_GUEST_ECN		| \
+     VIRTIO_NET_F_MRG_RXBUF		| \
+     VIRTIO_RING_F_INDIRECT_DESC)
+
+/*
+ * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
+ * frames larger than 1514 bytes. We do not yet support software LRO
+ * via tcp_lro_rx().
+ */
+#define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
+    VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
+
+#define VTNET_MAX_MTU		65536
+#define VTNET_MAX_RX_SIZE	65550
+
+/*
+ * Used to preallocate the Vq indirect descriptors. The first segment
+ * is reserved for the header.
+ */
+#define VTNET_MIN_RX_SEGS	2
+#define VTNET_MAX_RX_SEGS	34
+#define VTNET_MAX_TX_SEGS	34
+
+/*
+ * Assert we can receive and transmit the maximum with regular
+ * size clusters.
+ */
+CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
+CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
+
+/*
+ * Determine how many mbufs are in each receive buffer. For LRO without
+ * mergeable descriptors, we must allocate an mbuf chain large enough to
+ * hold both the vtnet_rx_header and the maximum receivable data.
+ */
+#define VTNET_NEEDED_RX_MBUFS(_sc)					\
+	((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :		\
+	    howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,	\
+	        (_sc)->vtnet_rx_mbuf_size)
+
+#define VTNET_MTX(_sc)		&(_sc)->vtnet_mtx
+#define VTNET_LOCK(_sc)		mtx_lock(VTNET_MTX((_sc)))
+#define VTNET_UNLOCK(_sc)	mtx_unlock(VTNET_MTX((_sc)))
+#define VTNET_LOCK_DESTROY(_sc)	mtx_destroy(VTNET_MTX((_sc)))
+#define VTNET_LOCK_ASSERT(_sc)	mtx_assert(VTNET_MTX((_sc)), MA_OWNED)
+#define VTNET_LOCK_ASSERT_NOTOWNED(_sc)	\
+	 			mtx_assert(VTNET_MTX((_sc)), MA_NOTOWNED)
+
+#define VTNET_LOCK_INIT(_sc) do {					\
+    snprintf((_sc)->vtnet_mtx_name, sizeof((_sc)->vtnet_mtx_name),	\
+        "%s", device_get_nameunit((_sc)->vtnet_dev));			\
+    mtx_init(VTNET_MTX((_sc)), (_sc)->vtnet_mtx_name,			\
+        "VTNET Core Lock", MTX_DEF);					\
+} while (0)
+
+#endif /* _IF_VTNETVAR_H */
diff --git a/sys/dev/virtio/network/virtio_net.h b/sys/dev/virtio/network/virtio_net.h
new file mode 100644
index 0000000..7361aa1
--- /dev/null
+++ b/sys/dev/virtio/network/virtio_net.h
@@ -0,0 +1,138 @@
+/*
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VIRTIO_NET_H
+#define _VIRTIO_NET_H
+
+#include <sys/types.h>
+
+/* The feature bitmap for virtio net */
+#define VIRTIO_NET_F_CSUM	0x00001 /* Host handles pkts w/ partial csum */
+#define VIRTIO_NET_F_GUEST_CSUM 0x00002 /* Guest handles pkts w/ partial csum*/
+#define VIRTIO_NET_F_MAC	0x00020 /* Host has given MAC address. */
+#define VIRTIO_NET_F_GSO	0x00040 /* Host handles pkts w/ any GSO type */
+#define VIRTIO_NET_F_GUEST_TSO4	0x00080 /* Guest can handle TSOv4 in. */
+#define VIRTIO_NET_F_GUEST_TSO6	0x00100 /* Guest can handle TSOv6 in. */
+#define VIRTIO_NET_F_GUEST_ECN	0x00200 /* Guest can handle TSO[6] w/ ECN in.*/
+#define VIRTIO_NET_F_GUEST_UFO	0x00400 /* Guest can handle UFO in. */
+#define VIRTIO_NET_F_HOST_TSO4	0x00800 /* Host can handle TSOv4 in. */
+#define VIRTIO_NET_F_HOST_TSO6	0x01000 /* Host can handle TSOv6 in. */
+#define VIRTIO_NET_F_HOST_ECN	0x02000 /* Host can handle TSO[6] w/ ECN in. */
+#define VIRTIO_NET_F_HOST_UFO	0x04000 /* Host can handle UFO in. */
+#define VIRTIO_NET_F_MRG_RXBUF	0x08000 /* Host can merge receive buffers. */
+#define VIRTIO_NET_F_STATUS	0x10000 /* virtio_net_config.status available*/
+#define VIRTIO_NET_F_CTRL_VQ	0x20000 /* Control channel available */
+#define VIRTIO_NET_F_CTRL_RX	0x40000 /* Control channel RX mode support */
+#define VIRTIO_NET_F_CTRL_VLAN	0x80000 /* Control channel VLAN filtering */
+#define VIRTIO_NET_F_CTRL_RX_EXTRA 0x100000 /* Extra RX mode control support */
+
+#define VIRTIO_NET_S_LINK_UP	1	/* Link is up */
+
+struct virtio_net_config {
+	/* The config defining mac address (if VIRTIO_NET_F_MAC) */
+	uint8_t		mac[ETHER_ADDR_LEN]; 
+	/* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */
+	uint16_t	status;
+} __packed;
+
+/*
+ * This is the first element of the scatter-gather list.  If you don't
+ * specify GSO or CSUM features, you can simply ignore the header.
+ */
+struct virtio_net_hdr {
+#define VIRTIO_NET_HDR_F_NEEDS_CSUM	1	/* Use csum_start,csum_offset*/
+	uint8_t	flags;
+#define VIRTIO_NET_HDR_GSO_NONE		0	/* Not a GSO frame */
+#define VIRTIO_NET_HDR_GSO_TCPV4	1	/* GSO frame, IPv4 TCP (TSO) */
+#define VIRTIO_NET_HDR_GSO_UDP		3	/* GSO frame, IPv4 UDP (UFO) */
+#define VIRTIO_NET_HDR_GSO_TCPV6	4	/* GSO frame, IPv6 TCP */
+#define VIRTIO_NET_HDR_GSO_ECN		0x80	/* TCP has ECN set */
+	uint8_t gso_type;
+	uint16_t hdr_len;	/* Ethernet + IP + tcp/udp hdrs */
+	uint16_t gso_size;	/* Bytes to append to hdr_len per frame */
+	uint16_t csum_start;	/* Position to start checksumming from */
+	uint16_t csum_offset;	/* Offset after that to place checksum */
+};
+
+/*
+ * This is the version of the header to use when the MRG_RXBUF
+ * feature has been negotiated.
+ */
+struct virtio_net_hdr_mrg_rxbuf {
+	struct virtio_net_hdr hdr;
+	uint16_t num_buffers;	/* Number of merged rx buffers */
+};
+
+/*
+ * Control virtqueue data structures
+ *
+ * The control virtqueue expects a header in the first sg entry
+ * and an ack/status response in the last entry.  Data for the
+ * command goes in between.
+ */
+struct virtio_net_ctrl_hdr {
+	uint8_t class;
+	uint8_t cmd;
+} __packed;
+
+typedef uint8_t virtio_net_ctrl_ack;
+
+#define VIRTIO_NET_OK	0
+#define VIRTIO_NET_ERR	1
+
+/*
+ * Control the RX mode, ie. promiscuous, allmulti, etc...
+ * All commands require an "out" sg entry containing a 1 byte
+ * state value, zero = disable, non-zero = enable.  Commands
+ * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
+ * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
+ */
+#define VIRTIO_NET_CTRL_RX	0
+#define VIRTIO_NET_CTRL_RX_PROMISC	0
+#define VIRTIO_NET_CTRL_RX_ALLMULTI	1
+#define VIRTIO_NET_CTRL_RX_ALLUNI	2
+#define VIRTIO_NET_CTRL_RX_NOMULTI	3
+#define VIRTIO_NET_CTRL_RX_NOUNI	4
+#define VIRTIO_NET_CTRL_RX_NOBCAST	5
+
+/*
+ * Control the MAC filter table.
+ *
+ * The MAC filter table is managed by the hypervisor, the guest should
+ * assume the size is infinite.  Filtering should be considered
+ * non-perfect, ie. based on hypervisor resources, the guest may
+ * received packets from sources not specified in the filter list.
+ *
+ * In addition to the class/cmd header, the TABLE_SET command requires
+ * two out scatterlists.  Each contains a 4 byte count of entries followed
+ * by a concatenated byte stream of the ETH_ALEN MAC addresses.  The
+ * first sg list contains unicast addresses, the second is for multicast.
+ * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
+ * is available.
+ */
+struct virtio_net_ctrl_mac {
+	uint32_t	entries;
+	uint8_t		macs[][ETHER_ADDR_LEN];
+} __packed;
+
+#define VIRTIO_NET_CTRL_MAC	1
+#define VIRTIO_NET_CTRL_MAC_TABLE_SET	0
+
+/*
+ * Control VLAN filtering
+ *
+ * The VLAN filter table is controlled via a simple ADD/DEL interface.
+ * VLAN IDs not added may be filtered by the hypervisor.  Del is the
+ * opposite of add.  Both commands expect an out entry containing a 2
+ * byte VLAN ID.  VLAN filtering is available with the
+ * VIRTIO_NET_F_CTRL_VLAN feature bit.
+ */
+#define VIRTIO_NET_CTRL_VLAN	2
+#define VIRTIO_NET_CTRL_VLAN_ADD	0
+#define VIRTIO_NET_CTRL_VLAN_DEL	1
+
+#endif /* _VIRTIO_NET_H */
diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c
new file mode 100644
index 0000000..dd348a5
--- /dev/null
+++ b/sys/dev/virtio/pci/virtio_pci.c
@@ -0,0 +1,1081 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Driver for the VirtIO PCI interface. */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+#include <dev/virtio/pci/virtio_pci.h>
+
+#include "virtio_bus_if.h"
+#include "virtio_if.h"
+
+struct vtpci_softc {
+	device_t			 vtpci_dev;
+	struct resource			*vtpci_res;
+	struct resource			*vtpci_msix_res;
+	uint64_t			 vtpci_features;
+	uint32_t			 vtpci_flags;
+#define VIRTIO_PCI_FLAG_NO_MSI		 0x0001
+#define VIRTIO_PCI_FLAG_MSI		 0x0002
+#define VIRTIO_PCI_FLAG_NO_MSIX		 0x0010
+#define VIRTIO_PCI_FLAG_MSIX		 0x0020
+#define VIRTIO_PCI_FLAG_SHARED_MSIX	 0x0040
+
+	device_t			 vtpci_child_dev;
+	struct virtio_feature_desc	*vtpci_child_feat_desc;
+
+	/*
+	 * Ideally, each virtqueue that the driver provides a callback for
+	 * will receive its own MSIX vector. If there are not sufficient
+	 * vectors available, we will then attempt to have all the VQs
+	 * share one vector. Note that when using MSIX, the configuration
+	 * changed notifications must be on their own vector.
+	 *
+	 * If MSIX is not available, we will attempt to have the whole
+	 * device share one MSI vector, and then, finally, one legacy
+	 * interrupt.
+	 */
+	int				 vtpci_nvqs;
+	struct vtpci_virtqueue {
+		struct virtqueue *vq;
+
+		/* Index into vtpci_intr_res[] below. Unused, then -1. */
+		int		  ires_idx;
+	} vtpci_vqx[VIRTIO_MAX_VIRTQUEUES];
+
+	/*
+	 * When using MSIX interrupts, the first element of vtpci_intr_res[]
+	 * is always the configuration changed notifications. The remaining
+	 * element(s) are used for the virtqueues.
+	 *
+	 * With MSI and legacy interrupts, only the first element of
+	 * vtpci_intr_res[] is used.
+	 */
+	int				 vtpci_nintr_res;
+	struct vtpci_intr_resource {
+		struct resource	*irq;
+		int		 rid;
+		void		*intrhand;
+	} vtpci_intr_res[1 + VIRTIO_MAX_VIRTQUEUES];
+};
+
+static int	vtpci_probe(device_t);
+static int	vtpci_attach(device_t);
+static int	vtpci_detach(device_t);
+static int	vtpci_suspend(device_t);
+static int	vtpci_resume(device_t);
+static int	vtpci_shutdown(device_t);
+static void	vtpci_driver_added(device_t, driver_t *);
+static void	vtpci_child_detached(device_t, device_t);
+static int	vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
+static int	vtpci_write_ivar(device_t, device_t, int, uintptr_t);
+
+static uint64_t	vtpci_negotiate_features(device_t, uint64_t);
+static int	vtpci_with_feature(device_t, uint64_t);
+static int	vtpci_alloc_virtqueues(device_t, int, int,
+		    struct vq_alloc_info *);
+static int	vtpci_setup_intr(device_t, enum intr_type);
+static void	vtpci_stop(device_t);
+static int	vtpci_reinit(device_t, uint64_t);
+static void	vtpci_reinit_complete(device_t);
+static void	vtpci_notify_virtqueue(device_t, uint16_t);
+static uint8_t	vtpci_get_status(device_t);
+static void	vtpci_set_status(device_t, uint8_t);
+static void	vtpci_read_dev_config(device_t, bus_size_t, void *, int);
+static void	vtpci_write_dev_config(device_t, bus_size_t, void *, int);
+
+static void	vtpci_describe_features(struct vtpci_softc *, const char *,
+		    uint64_t);
+static void	vtpci_probe_and_attach_child(struct vtpci_softc *);
+
+static int	vtpci_alloc_interrupts(struct vtpci_softc *, int, int,
+		    struct vq_alloc_info *);
+static int	vtpci_alloc_intr_resources(struct vtpci_softc *, int,
+		    struct vq_alloc_info *);
+static int	vtpci_alloc_msi(struct vtpci_softc *);
+static int	vtpci_alloc_msix(struct vtpci_softc *, int);
+static int	vtpci_register_msix_vector(struct vtpci_softc *, int, int);
+
+static void	vtpci_free_interrupts(struct vtpci_softc *);
+static void	vtpci_free_virtqueues(struct vtpci_softc *);
+static void	vtpci_release_child_resources(struct vtpci_softc *);
+static void	vtpci_reset(struct vtpci_softc *);
+
+static int	vtpci_legacy_intr(void *);
+static int	vtpci_vq_shared_intr(void *);
+static int	vtpci_vq_intr(void *);
+static int	vtpci_config_intr(void *);
+
+/*
+ * I/O port read/write wrappers.
+ */
+#define vtpci_read_config_1(sc, o)	bus_read_1((sc)->vtpci_res, (o))
+#define vtpci_read_config_2(sc, o)	bus_read_2((sc)->vtpci_res, (o))
+#define vtpci_read_config_4(sc, o)	bus_read_4((sc)->vtpci_res, (o))
+#define vtpci_write_config_1(sc, o, v)	bus_write_1((sc)->vtpci_res, (o), (v))
+#define vtpci_write_config_2(sc, o, v)	bus_write_2((sc)->vtpci_res, (o), (v))
+#define vtpci_write_config_4(sc, o, v)	bus_write_4((sc)->vtpci_res, (o), (v))
+
+/* Tunables. */
+static int vtpci_disable_msix = 0;
+TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
+
+static device_method_t vtpci_methods[] = {
+	/* Device interface. */
+	DEVMETHOD(device_probe,			  vtpci_probe),
+	DEVMETHOD(device_attach,		  vtpci_attach),
+	DEVMETHOD(device_detach,		  vtpci_detach),
+	DEVMETHOD(device_suspend,		  vtpci_suspend),
+	DEVMETHOD(device_resume,		  vtpci_resume),
+	DEVMETHOD(device_shutdown,		  vtpci_shutdown),
+
+	/* Bus interface. */
+	DEVMETHOD(bus_driver_added,		  vtpci_driver_added),
+	DEVMETHOD(bus_child_detached,		  vtpci_child_detached),
+	DEVMETHOD(bus_read_ivar,		  vtpci_read_ivar),
+	DEVMETHOD(bus_write_ivar,		  vtpci_write_ivar),
+
+	/* VirtIO bus interface. */
+	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_negotiate_features),
+	DEVMETHOD(virtio_bus_with_feature,	  vtpci_with_feature),
+	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_alloc_virtqueues),
+	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_setup_intr),
+	DEVMETHOD(virtio_bus_stop,		  vtpci_stop),
+	DEVMETHOD(virtio_bus_reinit,		  vtpci_reinit),
+	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_reinit_complete),
+	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_notify_virtqueue),
+	DEVMETHOD(virtio_bus_read_device_config,  vtpci_read_dev_config),
+	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
+
+	{ 0, 0 }
+};
+
+static driver_t vtpci_driver = {
+	"virtio_pci",
+	vtpci_methods,
+	sizeof(struct vtpci_softc)
+};
+
+devclass_t vtpci_devclass;
+
+DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
+MODULE_VERSION(virtio_pci, 1);
+MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
+MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
+
+static int
+vtpci_probe(device_t dev)
+{
+	char desc[36];
+	const char *name;
+
+	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
+		return (ENXIO);
+
+	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
+	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
+		return (ENXIO);
+
+	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
+		return (ENXIO);
+
+	name = virtio_device_name(pci_get_subdevice(dev));
+	if (name == NULL)
+		name = "Unknown";
+
+	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
+	device_set_desc_copy(dev, desc);
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtpci_attach(device_t dev)
+{
+	struct vtpci_softc *sc;
+	device_t child;
+	int rid;
+
+	sc = device_get_softc(dev);
+	sc->vtpci_dev = dev;
+
+	pci_enable_busmaster(dev);
+
+	rid = PCIR_BAR(0);
+	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
+	    RF_ACTIVE);
+	if (sc->vtpci_res == NULL) {
+		device_printf(dev, "cannot map I/O space\n");
+		return (ENXIO);
+	}
+
+	if (pci_find_extcap(dev, PCIY_MSI, NULL) != 0)
+		sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSI;
+
+	if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) {
+		rid = PCIR_BAR(1);
+		sc->vtpci_msix_res = bus_alloc_resource_any(dev,
+		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	}
+
+	if (sc->vtpci_msix_res == NULL)
+		sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSIX;
+
+	vtpci_reset(sc);
+
+	/* Tell the host we've noticed this device. */
+	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
+
+	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
+		device_printf(dev, "cannot create child device\n");
+		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
+		vtpci_detach(dev);
+		return (ENOMEM);
+	}
+
+	sc->vtpci_child_dev = child;
+	vtpci_probe_and_attach_child(sc);
+
+	return (0);
+}
+
+static int
+vtpci_detach(device_t dev)
+{
+	struct vtpci_softc *sc;
+	device_t child;
+	int error;
+
+	sc = device_get_softc(dev);
+
+	if ((child = sc->vtpci_child_dev) != NULL) {
+		error = device_delete_child(dev, child);
+		if (error)
+			return (error);
+		sc->vtpci_child_dev = NULL;
+	}
+
+	vtpci_reset(sc);
+
+	if (sc->vtpci_msix_res != NULL) {
+		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
+		    sc->vtpci_msix_res);
+		sc->vtpci_msix_res = NULL;
+	}
+
+	if (sc->vtpci_res != NULL) {
+		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
+		    sc->vtpci_res);
+		sc->vtpci_res = NULL;
+	}
+
+	return (0);
+}
+
+static int
+vtpci_suspend(device_t dev)
+{
+
+	return (bus_generic_suspend(dev));
+}
+
+static int
+vtpci_resume(device_t dev)
+{
+
+	return (bus_generic_resume(dev));
+}
+
+static int
+vtpci_shutdown(device_t dev)
+{
+
+	(void) bus_generic_shutdown(dev);
+	/* Forcibly stop the host device. */
+	vtpci_stop(dev);
+
+	return (0);
+}
+
+static void
+vtpci_driver_added(device_t dev, driver_t *driver)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	vtpci_probe_and_attach_child(sc);
+}
+
+static void
+vtpci_child_detached(device_t dev, device_t child)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	vtpci_reset(sc);
+	vtpci_release_child_resources(sc);
+}
+
+static int
+vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->vtpci_child_dev != child)
+		return (ENOENT);
+
+	switch (index) {
+	case VIRTIO_IVAR_DEVTYPE:
+		*result = pci_get_subdevice(dev);
+		break;
+	default:
+		return (ENOENT);
+	}
+
+	return (0);
+}
+
+static int
+vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->vtpci_child_dev != child)
+		return (ENOENT);
+
+	switch (index) {
+	case VIRTIO_IVAR_FEATURE_DESC:
+		sc->vtpci_child_feat_desc = (void *) value;
+		break;
+	default:
+		return (ENOENT);
+	}
+
+	return (0);
+}
+
+static uint64_t
+vtpci_negotiate_features(device_t dev, uint64_t child_features)
+{
+	struct vtpci_softc *sc;
+	uint64_t host_features, features;
+
+	sc = device_get_softc(dev);
+
+	host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
+	vtpci_describe_features(sc, "host", host_features);
+
+	/*
+	 * Limit negotiated features to what the driver, virtqueue, and
+	 * host all support.
+	 */
+	features = host_features & child_features;
+	features = virtqueue_filter_features(features);
+	sc->vtpci_features = features;
+
+	vtpci_describe_features(sc, "negotiated", features);
+	vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
+
+	return (features);
+}
+
+static int
+vtpci_with_feature(device_t dev, uint64_t feature)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	return ((sc->vtpci_features & feature) != 0);
+}
+
+static int
+vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
+    struct vq_alloc_info *vq_info)
+{
+	struct vtpci_softc *sc;
+	struct vtpci_virtqueue *vqx;
+	struct vq_alloc_info *info;
+	int queue, error;
+	uint16_t vq_size;
+
+	sc = device_get_softc(dev);
+
+	if (sc->vtpci_nvqs != 0 || nvqs <= 0 ||
+	    nvqs > VIRTIO_MAX_VIRTQUEUES)
+		return (EINVAL);
+
+	error = vtpci_alloc_interrupts(sc, flags, nvqs, vq_info);
+	if (error) {
+		device_printf(dev, "cannot allocate interrupts\n");
+		return (error);
+	}
+
+	if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
+		error = vtpci_register_msix_vector(sc,
+		    VIRTIO_MSI_CONFIG_VECTOR, 0);
+		if (error)
+			return (error);
+	}
+
+	for (queue = 0; queue < nvqs; queue++) {
+		vqx = &sc->vtpci_vqx[queue];
+		info = &vq_info[queue];
+
+		vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue);
+
+		vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
+		error = virtqueue_alloc(dev, queue, vq_size,
+		    VIRTIO_PCI_VRING_ALIGN, 0xFFFFFFFFUL, info, &vqx->vq);
+		if (error)
+			return (error);
+
+		if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
+			error = vtpci_register_msix_vector(sc,
+			    VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx);
+			if (error)
+				return (error);
+		}
+
+		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
+		    virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
+
+		*info->vqai_vq = vqx->vq;
+		sc->vtpci_nvqs++;
+	}
+
+	return (0);
+}
+
+static int
+vtpci_setup_intr(device_t dev, enum intr_type type)
+{
+	struct vtpci_softc *sc;
+	struct vtpci_intr_resource *ires;
+	struct vtpci_virtqueue *vqx;
+	int i, flags, error;
+
+	sc = device_get_softc(dev);
+	flags = type | INTR_MPSAFE;
+	ires = &sc->vtpci_intr_res[0];
+
+	if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) == 0) {
+		error = bus_setup_intr(dev, ires->irq, flags,
+		    vtpci_legacy_intr, NULL, sc, &ires->intrhand);
+
+		return (error);
+	}
+
+	error = bus_setup_intr(dev, ires->irq, flags, vtpci_config_intr,
+	    NULL, sc, &ires->intrhand);
+	if (error)
+		return (error);
+
+	if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) {
+		ires = &sc->vtpci_intr_res[1];
+		error = bus_setup_intr(dev, ires->irq, flags,
+		    vtpci_vq_shared_intr, NULL, sc, &ires->intrhand);
+
+		return (error);
+	}
+
+	/* Setup an interrupt handler for each virtqueue. */
+	for (i = 0; i < sc->vtpci_nvqs; i++) {
+		vqx = &sc->vtpci_vqx[i];
+		if (vqx->ires_idx < 1)
+			continue;
+
+		ires = &sc->vtpci_intr_res[vqx->ires_idx];
+		error = bus_setup_intr(dev, ires->irq, flags,
+		    vtpci_vq_intr, NULL, vqx->vq, &ires->intrhand);
+		if (error)
+			return (error);
+	}
+
+	return (0);
+}
+
+static void
+vtpci_stop(device_t dev)
+{
+
+	vtpci_reset(device_get_softc(dev));
+}
+
+static int
+vtpci_reinit(device_t dev, uint64_t features)
+{
+	struct vtpci_softc *sc;
+	struct vtpci_virtqueue *vqx;
+	struct virtqueue *vq;
+	int queue, error;
+	uint16_t vq_size;
+
+	sc = device_get_softc(dev);
+
+	/*
+	 * Redrive the device initialization. This is a bit of an abuse
+	 * of the specification, but both VirtualBox and QEMU/KVM seem
+	 * to play nice. We do not allow the host device to change from
+	 * what was originally negotiated beyond what the guest driver
+	 * changed (MSIX state should not change, number of virtqueues
+	 * and their size remain the same, etc).
+	 */
+
+	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
+		vtpci_stop(dev);
+
+	/*
+	 * Quickly drive the status through ACK and DRIVER. The device
+	 * does not become usable again until vtpci_reinit_complete().
+	 */
+	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
+	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
+
+	vtpci_negotiate_features(dev, features);
+
+	if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
+		error = vtpci_register_msix_vector(sc,
+		    VIRTIO_MSI_CONFIG_VECTOR, 0);
+		if (error)
+			return (error);
+	}
+
+	for (queue = 0; queue < sc->vtpci_nvqs; queue++) {
+		vqx = &sc->vtpci_vqx[queue];
+		vq = vqx->vq;
+
+		KASSERT(vq != NULL, ("vq %d not allocated", queue));
+		vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue);
+
+		vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
+		error = virtqueue_reinit(vq, vq_size);
+		if (error)
+			return (error);
+
+		if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
+			error = vtpci_register_msix_vector(sc,
+			    VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx);
+			if (error)
+				return (error);
+		}
+
+		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
+		    virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
+	}
+
+	return (0);
+}
+
+static void
+vtpci_reinit_complete(device_t dev)
+{
+
+	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
+}
+
+static void
+vtpci_notify_virtqueue(device_t dev, uint16_t queue)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
+}
+
+static uint8_t
+vtpci_get_status(device_t dev)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
+}
+
+static void
+vtpci_set_status(device_t dev, uint8_t status)
+{
+	struct vtpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (status != VIRTIO_CONFIG_STATUS_RESET)
+		status |= vtpci_get_status(dev);
+
+	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
+}
+
+static void
+vtpci_read_dev_config(device_t dev, bus_size_t offset,
+    void *dst, int length)
+{
+	struct vtpci_softc *sc;
+	bus_size_t off;
+	uint8_t *d;
+	int size;
+
+	sc = device_get_softc(dev);
+	off = VIRTIO_PCI_CONFIG(sc) + offset;
+
+	for (d = dst; length > 0; d += size, off += size, length -= size) {
+		if (length >= 4) {
+			size = 4;
+			*(uint32_t *)d = vtpci_read_config_4(sc, off);
+		} else if (length >= 2) {
+			size = 2;
+			*(uint16_t *)d = vtpci_read_config_2(sc, off);
+		} else {
+			size = 1;
+			*d = vtpci_read_config_1(sc, off);
+		}
+	}
+}
+
+static void
+vtpci_write_dev_config(device_t dev, bus_size_t offset,
+    void *src, int length)
+{
+	struct vtpci_softc *sc;
+	bus_size_t off;
+	uint8_t *s;
+	int size;
+
+	sc = device_get_softc(dev);
+	off = VIRTIO_PCI_CONFIG(sc) + offset;
+
+	for (s = src; length > 0; s += size, off += size, length -= size) {
+		if (length >= 4) {
+			size = 4;
+			vtpci_write_config_4(sc, off, *(uint32_t *)s);
+		} else if (length >= 2) {
+			size = 2;
+			vtpci_write_config_2(sc, off, *(uint16_t *)s);
+		} else {
+			size = 1;
+			vtpci_write_config_1(sc, off, *s);
+		}
+	}
+}
+
+static void
+vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
+    uint64_t features)
+{
+	device_t dev, child;
+
+	dev = sc->vtpci_dev;
+	child = sc->vtpci_child_dev;
+
+	if (device_is_attached(child) && bootverbose == 0)
+		return;
+
+	virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
+}
+
+static void
+vtpci_probe_and_attach_child(struct vtpci_softc *sc)
+{
+	device_t dev, child;
+
+	dev = sc->vtpci_dev;
+	child = sc->vtpci_child_dev;
+
+	if (child == NULL)
+		return;
+
+	if (device_get_state(child) != DS_NOTPRESENT)
+		return;
+
+	if (device_probe(child) != 0)
+		return;
+
+	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
+	if (device_attach(child) != 0) {
+		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
+		vtpci_reset(sc);
+		vtpci_release_child_resources(sc);
+
+		/* Reset status for future attempt. */
+		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
+	} else
+		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
+}
+
+static int
+vtpci_alloc_interrupts(struct vtpci_softc *sc, int flags, int nvqs,
+    struct vq_alloc_info *vq_info)
+{
+	int i, nvectors, error;
+
+	/*
+	 * Only allocate a vector for virtqueues that are actually
+	 * expecting an interrupt.
+	 */
+	for (nvectors = 0, i = 0; i < nvqs; i++)
+		if (vq_info[i].vqai_intr != NULL)
+			nvectors++;
+
+	if (vtpci_disable_msix != 0 ||
+	    sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSIX ||
+	    flags & VIRTIO_ALLOC_VQS_DISABLE_MSIX ||
+	    vtpci_alloc_msix(sc, nvectors) != 0) {
+		/*
+		 * Use MSI interrupts if available. Otherwise, we fallback
+		 * to legacy interrupts.
+		 */
+		if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSI) == 0 &&
+		    vtpci_alloc_msi(sc) == 0)
+			sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSI;
+
+		sc->vtpci_nintr_res = 1;
+	}
+
+	error = vtpci_alloc_intr_resources(sc, nvqs, vq_info);
+
+	return (error);
+}
+
+static int
+vtpci_alloc_intr_resources(struct vtpci_softc *sc, int nvqs,
+    struct vq_alloc_info *vq_info)
+{
+	device_t dev;
+	struct resource *irq;
+	struct vtpci_virtqueue *vqx;
+	int i, rid, flags, res_idx;
+
+	dev = sc->vtpci_dev;
+	flags = RF_ACTIVE;
+
+	if ((sc->vtpci_flags &
+	    (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) == 0) {
+		rid = 0;
+		flags |= RF_SHAREABLE;
+	} else
+		rid = 1;
+
+	for (i = 0; i < sc->vtpci_nintr_res; i++) {
+		irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, flags);
+		if (irq == NULL)
+			return (ENXIO);
+
+		sc->vtpci_intr_res[i].irq = irq;
+		sc->vtpci_intr_res[i].rid = rid++;
+	}
+
+	/*
+	 * Map the virtqueue into the correct index in vq_intr_res[]. Note the
+	 * first index is reserved for configuration changes notifications.
+	 */
+	for (i = 0, res_idx = 1; i < nvqs; i++) {
+		vqx = &sc->vtpci_vqx[i];
+
+		if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
+			if (vq_info[i].vqai_intr == NULL)
+				vqx->ires_idx = -1;
+			else if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX)
+				vqx->ires_idx = res_idx;
+			else
+				vqx->ires_idx = res_idx++;
+		} else
+			vqx->ires_idx = -1;
+	}
+
+	return (0);
+}
+
+static int
+vtpci_alloc_msi(struct vtpci_softc *sc)
+{
+	device_t dev;
+	int nmsi, cnt;
+
+	dev = sc->vtpci_dev;
+	nmsi = pci_msi_count(dev);
+
+	if (nmsi < 1)
+		return (1);
+
+	cnt = 1;
+	if (pci_alloc_msi(dev, &cnt) == 0 && cnt == 1)
+		return (0);
+
+	return (1);
+}
+
+static int
+vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
+{
+	device_t dev;
+	int nmsix, cnt, required;
+
+	dev = sc->vtpci_dev;
+
+	nmsix = pci_msix_count(dev);
+	if (nmsix < 1)
+		return (1);
+
+	/* An additional vector is needed for the config changes. */
+	required = nvectors + 1;
+	if (nmsix >= required) {
+		cnt = required;
+		if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required)
+			goto out;
+
+		pci_release_msi(dev);
+	}
+
+	/* Attempt shared MSIX configuration. */
+	required = 2;
+	if (nmsix >= required) {
+		cnt = required;
+		if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
+			sc->vtpci_flags |= VIRTIO_PCI_FLAG_SHARED_MSIX;
+			goto out;
+		}
+
+		pci_release_msi(dev);
+	}
+
+	return (1);
+
+out:
+	sc->vtpci_nintr_res = required;
+	sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSIX;
+
+	if (bootverbose) {
+		if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX)
+			device_printf(dev, "using shared virtqueue MSIX\n");
+		else
+			device_printf(dev, "using per virtqueue MSIX\n");
+	}
+
+	return (0);
+}
+
+static int
+vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx)
+{
+	device_t dev;
+	uint16_t vector;
+
+	dev = sc->vtpci_dev;
+
+	if (offset != VIRTIO_MSI_CONFIG_VECTOR &&
+	    offset != VIRTIO_MSI_QUEUE_VECTOR)
+		return (EINVAL);
+
+	if (res_idx != -1) {
+		/* Map from rid to host vector. */
+		vector = sc->vtpci_intr_res[res_idx].rid - 1;
+	} else
+		vector = VIRTIO_MSI_NO_VECTOR;
+
+	/* The first resource is special; make sure it is used correctly. */
+	if (res_idx == 0) {
+		KASSERT(vector == 0, ("unexpected config vector"));
+		KASSERT(offset == VIRTIO_MSI_CONFIG_VECTOR,
+		    ("unexpected config offset"));
+	}
+
+	vtpci_write_config_2(sc, offset, vector);
+
+	if (vtpci_read_config_2(sc, offset) != vector) {
+		device_printf(dev, "insufficient host resources for "
+		    "MSIX interrupts\n");
+		return (ENODEV);
+	}
+
+	return (0);
+}
+
+static void
+vtpci_free_interrupts(struct vtpci_softc *sc)
+{
+	device_t dev;
+	struct vtpci_intr_resource *ires;
+	int i;
+
+	dev = sc->vtpci_dev;
+	sc->vtpci_nintr_res = 0;
+
+	if (sc->vtpci_flags & (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) {
+		pci_release_msi(dev);
+		sc->vtpci_flags &= ~(VIRTIO_PCI_FLAG_MSI |
+		    VIRTIO_PCI_FLAG_MSIX | VIRTIO_PCI_FLAG_SHARED_MSIX);
+	}
+
+	for (i = 0; i < 1 + VIRTIO_MAX_VIRTQUEUES; i++) {
+		ires = &sc->vtpci_intr_res[i];
+
+		if (ires->intrhand != NULL) {
+			bus_teardown_intr(dev, ires->irq, ires->intrhand);
+			ires->intrhand = NULL;
+		}
+
+		if (ires->irq != NULL) {
+			bus_release_resource(dev, SYS_RES_IRQ, ires->rid,
+			    ires->irq);
+			ires->irq = NULL;
+		}
+
+		ires->rid = -1;
+	}
+}
+
+static void
+vtpci_free_virtqueues(struct vtpci_softc *sc)
+{
+	struct vtpci_virtqueue *vqx;
+	int i;
+
+	sc->vtpci_nvqs = 0;
+
+	for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) {
+		vqx = &sc->vtpci_vqx[i];
+
+		if (vqx->vq != NULL) {
+			virtqueue_free(vqx->vq);
+			vqx->vq = NULL;
+		}
+	}
+}
+
+static void
+vtpci_release_child_resources(struct vtpci_softc *sc)
+{
+
+	vtpci_free_interrupts(sc);
+	vtpci_free_virtqueues(sc);
+}
+
+static void
+vtpci_reset(struct vtpci_softc *sc)
+{
+
+	/*
+	 * Setting the status to RESET sets the host device to
+	 * the original, uninitialized state.
+	 */
+	vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
+}
+
+static int
+vtpci_legacy_intr(void *xsc)
+{
+	struct vtpci_softc *sc;
+	struct vtpci_virtqueue *vqx;
+	int i;
+	uint8_t isr;
+
+	sc = xsc;
+	vqx = &sc->vtpci_vqx[0];
+
+	/* Reading the ISR also clears it. */
+	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
+
+	if (isr & VIRTIO_PCI_ISR_CONFIG)
+		vtpci_config_intr(sc);
+
+	if (isr & VIRTIO_PCI_ISR_INTR)
+		for (i = 0; i < sc->vtpci_nvqs; i++, vqx++)
+			virtqueue_intr(vqx->vq);
+
+	return (isr ? FILTER_HANDLED : FILTER_STRAY);
+}
+
+static int
+vtpci_vq_shared_intr(void *xsc)
+{
+	struct vtpci_softc *sc;
+	struct vtpci_virtqueue *vqx;
+	int i, rc;
+
+	rc = 0;
+	sc = xsc;
+	vqx = &sc->vtpci_vqx[0];
+
+	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++)
+		rc |= virtqueue_intr(vqx->vq);
+
+	return (rc ? FILTER_HANDLED : FILTER_STRAY);
+}
+
+static int
+vtpci_vq_intr(void *xvq)
+{
+	struct virtqueue *vq;
+	int rc;
+
+	vq = xvq;
+	rc = virtqueue_intr(vq);
+
+	return (rc ? FILTER_HANDLED : FILTER_STRAY);
+}
+
+static int
+vtpci_config_intr(void *xsc)
+{
+	struct vtpci_softc *sc;
+	device_t child;
+	int rc;
+
+	rc = 0;
+	sc = xsc;
+	child = sc->vtpci_child_dev;
+
+	if (child != NULL)
+		rc = VIRTIO_CONFIG_CHANGE(child);
+
+	return (rc ? FILTER_HANDLED : FILTER_STRAY);
+}
diff --git a/sys/dev/virtio/pci/virtio_pci.h b/sys/dev/virtio/pci/virtio_pci.h
new file mode 100644
index 0000000..6ebfdd5
--- /dev/null
+++ b/sys/dev/virtio/pci/virtio_pci.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright IBM Corp. 2007
+ *
+ * Authors:
+ *  Anthony Liguori  <aliguori@us.ibm.com>
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VIRTIO_PCI_H
+#define _VIRTIO_PCI_H
+
+/* VirtIO PCI vendor/device ID. */
+#define VIRTIO_PCI_VENDORID	0x1AF4
+#define VIRTIO_PCI_DEVICEID_MIN	0x1000
+#define VIRTIO_PCI_DEVICEID_MAX	0x103F
+
+/* VirtIO ABI version, this must match exactly. */
+#define VIRTIO_PCI_ABI_VERSION	0
+
+/*
+ * VirtIO Header, located in BAR 0.
+ */
+#define VIRTIO_PCI_HOST_FEATURES  0  /* host's supported features (32bit, RO)*/
+#define VIRTIO_PCI_GUEST_FEATURES 4  /* guest's supported features (32, RW) */
+#define VIRTIO_PCI_QUEUE_PFN      8  /* physical address of VQ (32, RW) */
+#define VIRTIO_PCI_QUEUE_NUM      12 /* number of ring entries (16, RO) */
+#define VIRTIO_PCI_QUEUE_SEL      14 /* current VQ selection (16, RW) */
+#define VIRTIO_PCI_QUEUE_NOTIFY	  16 /* notify host regarding VQ (16, RW) */
+#define VIRTIO_PCI_STATUS         18 /* device status register (8, RW) */
+#define VIRTIO_PCI_ISR            19 /* interrupt status register, reading
+				      * also clears the register (8, RO) */
+/* Only if MSIX is enabled: */
+#define VIRTIO_MSI_CONFIG_VECTOR  20 /* configuration change vector (16, RW) */
+#define VIRTIO_MSI_QUEUE_VECTOR   22 /* vector for selected VQ notifications
+					(16, RW) */
+
+/* The bit of the ISR which indicates a device has an interrupt. */
+#define VIRTIO_PCI_ISR_INTR	0x1
+/* The bit of the ISR which indicates a device configuration change. */
+#define VIRTIO_PCI_ISR_CONFIG	0x2
+/* Vector value used to disable MSI for queue. */
+#define VIRTIO_MSI_NO_VECTOR	0xFFFF
+
+/*
+ * The remaining space is defined by each driver as the per-driver
+ * configuration space.
+ */
+#define VIRTIO_PCI_CONFIG(sc) \
+    (((sc)->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) ? 24 : 20)
+
+/*
+ * How many bits to shift physical queue address written to QUEUE_PFN.
+ * 12 is historical, and due to x86 page size.
+ */
+#define VIRTIO_PCI_QUEUE_ADDR_SHIFT	12
+
+/* The alignment to use between consumer and producer parts of vring. */
+#define VIRTIO_PCI_VRING_ALIGN	4096
+
+#endif /* _VIRTIO_PCI_H */
diff --git a/sys/dev/virtio/virtio.c b/sys/dev/virtio/virtio.c
new file mode 100644
index 0000000..e385575
--- /dev/null
+++ b/sys/dev/virtio/virtio.c
@@ -0,0 +1,283 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/sbuf.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/_inttypes.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+
+#include "virtio_bus_if.h"
+
+static int virtio_modevent(module_t, int, void *);
+static const char *virtio_feature_name(uint64_t, struct virtio_feature_desc *);
+
+static struct virtio_ident {
+	uint16_t devid;
+	char	*name;
+} virtio_ident_table[] = {
+	{ VIRTIO_ID_NETWORK,	"Network"	},
+	{ VIRTIO_ID_BLOCK,	"Block"		},
+	{ VIRTIO_ID_CONSOLE,	"Console"	},
+	{ VIRTIO_ID_ENTROPY,	"Entropy"	},
+	{ VIRTIO_ID_BALLOON,	"Balloon"	},
+	{ VIRTIO_ID_IOMEMORY,	"IOMemory"	},
+	{ VIRTIO_ID_9P,		"9P Transport"	},
+
+	{ 0, NULL }
+};
+
+/* Device independent features. */
+static struct virtio_feature_desc virtio_common_feature_desc[] = {
+	{ VIRTIO_F_NOTIFY_ON_EMPTY,	"NotifyOnEmpty"	},
+	{ VIRTIO_RING_F_INDIRECT_DESC,	"RingIndirect"	},
+	{ VIRTIO_RING_F_EVENT_IDX,	"EventIdx"	},
+	{ VIRTIO_F_BAD_FEATURE,		"BadFeature"	},
+
+	{ 0, NULL }
+};
+
+const char *
+virtio_device_name(uint16_t devid)
+{
+	struct virtio_ident *ident;
+
+	for (ident = virtio_ident_table; ident->name != NULL; ident++) {
+		if (ident->devid == devid)
+			return (ident->name);
+	}
+
+	return (NULL);
+}
+
+int
+virtio_get_device_type(device_t dev)
+{
+	uintptr_t devtype;
+
+	devtype = -1;
+
+	BUS_READ_IVAR(device_get_parent(dev), dev,
+	    VIRTIO_IVAR_DEVTYPE, &devtype);
+
+	return ((int) devtype);
+}
+
+void
+virtio_set_feature_desc(device_t dev,
+    struct virtio_feature_desc *feature_desc)
+{
+
+	BUS_WRITE_IVAR(device_get_parent(dev), dev,
+	    VIRTIO_IVAR_FEATURE_DESC, (uintptr_t) feature_desc);
+}
+
+void
+virtio_describe(device_t dev, const char *msg,
+    uint64_t features, struct virtio_feature_desc *feature_desc)
+{
+	struct sbuf sb;
+	uint64_t val;
+	char *buf;
+	const char *name;
+	int n;
+
+	if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) {
+		device_printf(dev, "%s features: 0x%"PRIx64"\n", msg,
+		    features);
+		return;
+	}
+
+	sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN);
+	sbuf_printf(&sb, "%s features: 0x%"PRIx64, msg, features);
+
+	for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) {
+		/*
+		 * BAD_FEATURE is used to detect broken Linux clients
+		 * and therefore is not applicable to FreeBSD.
+		 */
+		if (((features & val) == 0) || val == VIRTIO_F_BAD_FEATURE)
+			continue;
+
+		if (n++ == 0)
+			sbuf_cat(&sb, " <");
+		else
+			sbuf_cat(&sb, ",");
+
+		name = NULL;
+		if (feature_desc != NULL)
+			name = virtio_feature_name(val, feature_desc);
+		if (name == NULL)
+			name = virtio_feature_name(val,
+			    virtio_common_feature_desc);
+
+		if (name == NULL)
+			sbuf_printf(&sb, "0x%"PRIx64, val);
+		else
+			sbuf_cat(&sb, name);
+	}
+
+	if (n > 0)
+		sbuf_cat(&sb, ">");
+
+#if __FreeBSD_version < 900020
+	sbuf_finish(&sb);
+	if (sbuf_overflowed(&sb) == 0)
+#else
+	if (sbuf_finish(&sb) == 0)
+#endif
+		device_printf(dev, "%s\n", sbuf_data(&sb));
+
+	sbuf_delete(&sb);
+	free(buf, M_TEMP);
+}
+
+static const char *
+virtio_feature_name(uint64_t val, struct virtio_feature_desc *feature_desc)
+{
+	int i;
+
+	for (i = 0; feature_desc[i].vfd_val != 0; i++)
+		if (val == feature_desc[i].vfd_val)
+			return (feature_desc[i].vfd_str);
+
+	return (NULL);
+}
+
+/*
+ * VirtIO bus method wrappers.
+ */
+
+uint64_t
+virtio_negotiate_features(device_t dev, uint64_t child_features)
+{
+
+	return (VIRTIO_BUS_NEGOTIATE_FEATURES(device_get_parent(dev),
+	    child_features));
+}
+
+int
+virtio_alloc_virtqueues(device_t dev, int flags, int nvqs,
+    struct vq_alloc_info *info)
+{
+
+	return (VIRTIO_BUS_ALLOC_VIRTQUEUES(device_get_parent(dev), flags,
+	    nvqs, info));
+}
+
+int
+virtio_setup_intr(device_t dev, enum intr_type type)
+{
+
+	return (VIRTIO_BUS_SETUP_INTR(device_get_parent(dev), type));
+}
+
+int
+virtio_with_feature(device_t dev, uint64_t feature)
+{
+
+	return (VIRTIO_BUS_WITH_FEATURE(device_get_parent(dev), feature));
+}
+
+void
+virtio_stop(device_t dev)
+{
+
+	VIRTIO_BUS_STOP(device_get_parent(dev));
+}
+
+int
+virtio_reinit(device_t dev, uint64_t features)
+{
+
+	return (VIRTIO_BUS_REINIT(device_get_parent(dev), features));
+}
+
+void
+virtio_reinit_complete(device_t dev)
+{
+
+	VIRTIO_BUS_REINIT_COMPLETE(device_get_parent(dev));
+}
+
+void
+virtio_read_device_config(device_t dev, bus_size_t offset, void *dst, int len)
+{
+
+	VIRTIO_BUS_READ_DEVICE_CONFIG(device_get_parent(dev),
+	    offset, dst, len);
+}
+
+void
+virtio_write_device_config(device_t dev, bus_size_t offset, void *dst, int len)
+{
+
+	VIRTIO_BUS_WRITE_DEVICE_CONFIG(device_get_parent(dev),
+	    offset, dst, len);
+}
+
+static int
+virtio_modevent(module_t mod, int type, void *unused)
+{
+	int error;
+
+	error = 0;
+
+	switch (type) {
+	case MOD_LOAD:
+	case MOD_QUIESCE:
+	case MOD_UNLOAD:
+	case MOD_SHUTDOWN:
+		break;
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+
+	return (error);
+}
+
+static moduledata_t virtio_mod = {
+	"virtio",
+	virtio_modevent,
+	0
+};
+
+DECLARE_MODULE(virtio, virtio_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
+MODULE_VERSION(virtio, 1);
diff --git a/sys/dev/virtio/virtio.h b/sys/dev/virtio/virtio.h
new file mode 100644
index 0000000..ebd3c74
--- /dev/null
+++ b/sys/dev/virtio/virtio.h
@@ -0,0 +1,130 @@
+/*
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VIRTIO_H_
+#define _VIRTIO_H_
+
+#include <sys/types.h>
+
+struct vq_alloc_info;
+
+/* VirtIO device IDs. */
+#define VIRTIO_ID_NETWORK	0x01
+#define VIRTIO_ID_BLOCK		0x02
+#define VIRTIO_ID_CONSOLE	0x03
+#define VIRTIO_ID_ENTROPY	0x04
+#define VIRTIO_ID_BALLOON	0x05
+#define VIRTIO_ID_IOMEMORY	0x06
+#define VIRTIO_ID_9P		0x09
+
+/* Status byte for guest to report progress. */
+#define VIRTIO_CONFIG_STATUS_RESET	0x00
+#define VIRTIO_CONFIG_STATUS_ACK	0x01
+#define VIRTIO_CONFIG_STATUS_DRIVER	0x02
+#define VIRTIO_CONFIG_STATUS_DRIVER_OK	0x04
+#define VIRTIO_CONFIG_STATUS_FAILED	0x80
+
+/*
+ * Generate interrupt when the virtqueue ring is
+ * completely used, even if we've suppressed them.
+ */
+#define VIRTIO_F_NOTIFY_ON_EMPTY (1 << 24)
+
+/*
+ * The guest should never negotiate this feature; it
+ * is used to detect faulty drivers.
+ */
+#define VIRTIO_F_BAD_FEATURE (1 << 30)
+
+/*
+ * Some VirtIO feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. virtio_ring), the
+ * rest are per-device feature bits.
+ */
+#define VIRTIO_TRANSPORT_F_START	28
+#define VIRTIO_TRANSPORT_F_END		32
+
+/*
+ * Maximum number of virtqueues per device.
+ */
+#define VIRTIO_MAX_VIRTQUEUES 8
+
+/*
+ * Each virtqueue indirect descriptor list must be physically contiguous.
+ * To allow us to malloc(9) each list individually, limit the number
+ * supported to what will fit in one page. With 4KB pages, this is a limit
+ * of 256 descriptors. If there is ever a need for more, we can switch to
+ * contigmalloc(9) for the larger allocations, similar to what
+ * bus_dmamem_alloc(9) does.
+ *
+ * Note the sizeof(struct vring_desc) is 16 bytes.
+ */
+#define VIRTIO_MAX_INDIRECT ((int) (PAGE_SIZE / 16))
+
+/*
+ * VirtIO instance variables indices.
+ */
+#define VIRTIO_IVAR_DEVTYPE		1
+#define VIRTIO_IVAR_FEATURE_DESC	2
+
+struct virtio_feature_desc {
+	uint64_t	 vfd_val;
+	char		*vfd_str;
+};
+
+const char *virtio_device_name(uint16_t devid);
+int	 virtio_get_device_type(device_t dev);
+void	 virtio_set_feature_desc(device_t dev,
+	     struct virtio_feature_desc *feature_desc);
+void	 virtio_describe(device_t dev, const char *msg,
+	     uint64_t features, struct virtio_feature_desc *feature_desc);
+
+/*
+ * VirtIO Bus Methods.
+ */
+uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features);
+int	 virtio_alloc_virtqueues(device_t dev, int flags, int nvqs,
+	     struct vq_alloc_info *info);
+int	 virtio_setup_intr(device_t dev, enum intr_type type);
+int	 virtio_with_feature(device_t dev, uint64_t feature);
+void	 virtio_stop(device_t dev);
+int	 virtio_reinit(device_t dev, uint64_t features);
+void	 virtio_reinit_complete(device_t dev);
+
+/*
+ * Read/write a variable amount from the device specific (ie, network)
+ * configuration region. This region is encoded in the same endian as
+ * the guest.
+ */
+void	 virtio_read_device_config(device_t dev, bus_size_t offset,
+	     void *dst, int length);
+void	 virtio_write_device_config(device_t dev, bus_size_t offset,
+	     void *src, int length);
+
+/* Inlined device specific read/write functions for common lengths. */
+#define VIRTIO_RDWR_DEVICE_CONFIG(size, type)				\
+static inline type							\
+__CONCAT(virtio_read_dev_config_,size)(device_t dev,			\
+    bus_size_t offset)							\
+{									\
+	type val;							\
+	virtio_read_device_config(dev, offset, &val, sizeof(type));	\
+	return (val);							\
+}									\
+									\
+static inline void							\
+__CONCAT(virtio_write_dev_config_,size)(device_t dev,			\
+    bus_size_t offset, type val)					\
+{									\
+	virtio_write_device_config(dev, offset, &val, sizeof(type));	\
+}
+
+VIRTIO_RDWR_DEVICE_CONFIG(1, uint8_t);
+VIRTIO_RDWR_DEVICE_CONFIG(2, uint16_t);
+VIRTIO_RDWR_DEVICE_CONFIG(4, uint32_t);
+
+#endif /* _VIRTIO_H_ */
diff --git a/sys/dev/virtio/virtio_bus_if.m b/sys/dev/virtio/virtio_bus_if.m
new file mode 100644
index 0000000..ec2029d
--- /dev/null
+++ b/sys/dev/virtio/virtio_bus_if.m
@@ -0,0 +1,92 @@
+#-
+# Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+#include <sys/bus.h>
+#include <machine/bus.h>
+
+INTERFACE virtio_bus;
+
+HEADER {
+struct vq_alloc_info;
+};
+
+METHOD uint64_t negotiate_features {
+	device_t	dev;
+	uint64_t	child_features;
+};
+
+METHOD int with_feature {
+	device_t	dev;
+	uint64_t	feature;
+};
+
+METHOD int alloc_virtqueues {
+	device_t	dev;
+	int		flags;
+	int		nvqs;
+	struct vq_alloc_info *info;
+};
+HEADER {
+#define VIRTIO_ALLOC_VQS_DISABLE_MSIX 0x1
+};
+
+METHOD int setup_intr {
+	device_t	dev;
+	enum intr_type	type;
+};
+
+METHOD void stop {
+	device_t	dev;
+};
+
+METHOD int reinit {
+	device_t	dev;
+	uint64_t	features;
+};
+
+METHOD void reinit_complete {
+	device_t	dev;
+};
+
+METHOD void notify_vq {
+	device_t	dev;
+	uint16_t	queue;
+};
+
+METHOD void read_device_config {
+	device_t	dev;
+	bus_size_t	offset;
+	void		*dst;
+	int		len;
+};
+
+METHOD void write_device_config {
+	device_t	dev;
+	bus_size_t	offset;
+	void		*src;
+	int		len;
+};
diff --git a/sys/dev/virtio/virtio_if.m b/sys/dev/virtio/virtio_if.m
new file mode 100644
index 0000000..701678c
--- /dev/null
+++ b/sys/dev/virtio/virtio_if.m
@@ -0,0 +1,43 @@
+#-
+# Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+#include <sys/bus.h>
+
+INTERFACE virtio;
+
+CODE {
+	static int
+	virtio_default_config_change(device_t dev)
+	{
+		/* Return that we've handled the change. */
+		return (1);
+	}
+};
+
+METHOD int config_change {
+	device_t	dev;
+} DEFAULT virtio_default_config_change;
diff --git a/sys/dev/virtio/virtio_ring.h b/sys/dev/virtio/virtio_ring.h
new file mode 100644
index 0000000..124cb4d
--- /dev/null
+++ b/sys/dev/virtio/virtio_ring.h
@@ -0,0 +1,119 @@
+/*
+ * This header is BSD licensed so anyone can use the definitions
+ * to implement compatible drivers/servers.
+ *
+ * Copyright Rusty Russell IBM Corporation 2007.
+ */
+/* $FreeBSD$ */
+
+#ifndef VIRTIO_RING_H
+#define	VIRTIO_RING_H
+
+#include <sys/types.h>
+
+/* This marks a buffer as continuing via the next field. */
+#define VRING_DESC_F_NEXT       1
+/* This marks a buffer as write-only (otherwise read-only). */
+#define VRING_DESC_F_WRITE      2
+/* This means the buffer contains a list of buffer descriptors. */
+#define VRING_DESC_F_INDIRECT	4
+
+/* The Host uses this in used->flags to advise the Guest: don't kick me
+ * when you add a buffer.  It's unreliable, so it's simply an
+ * optimization.  Guest will still kick if it's out of buffers. */
+#define VRING_USED_F_NO_NOTIFY  1
+/* The Guest uses this in avail->flags to advise the Host: don't
+ * interrupt me when you consume a buffer.  It's unreliable, so it's
+ * simply an optimization.  */
+#define VRING_AVAIL_F_NO_INTERRUPT      1
+
+/* VirtIO ring descriptors: 16 bytes.
+ * These can chain together via "next". */
+struct vring_desc {
+        /* Address (guest-physical). */
+        uint64_t addr;
+        /* Length. */
+        uint32_t len;
+        /* The flags as indicated above. */
+        uint16_t flags;
+        /* We chain unused descriptors via this, too. */
+        uint16_t next;
+};
+
+struct vring_avail {
+        uint16_t flags;
+        uint16_t idx;
+        uint16_t ring[0];
+};
+
+/* uint32_t is used here for ids for padding reasons. */
+struct vring_used_elem {
+        /* Index of start of used descriptor chain. */
+        uint32_t id;
+        /* Total length of the descriptor chain which was written to. */
+        uint32_t len;
+};
+
+struct vring_used {
+        uint16_t flags;
+        uint16_t idx;
+        struct vring_used_elem ring[0];
+};
+
+struct vring {
+	unsigned int num;
+
+	struct vring_desc *desc;
+	struct vring_avail *avail;
+	struct vring_used *used;
+};
+
+/* The standard layout for the ring is a continuous chunk of memory which
+ * looks like this.  We assume num is a power of 2.
+ *
+ * struct vring {
+ *      // The actual descriptors (16 bytes each)
+ *      struct vring_desc desc[num];
+ *
+ *      // A ring of available descriptor heads with free-running index.
+ *      __u16 avail_flags;
+ *      __u16 avail_idx;
+ *      __u16 available[num];
+ *
+ *      // Padding to the next align boundary.
+ *      char pad[];
+ *
+ *      // A ring of used descriptor heads with free-running index.
+ *      __u16 used_flags;
+ *      __u16 used_idx;
+ *      struct vring_used_elem used[num];
+ * };
+ *
+ * NOTE: for VirtIO PCI, align is 4096.
+ */
+
+static inline int
+vring_size(unsigned int num, unsigned long align)
+{
+	int size;
+
+	size = num * sizeof(struct vring_desc);
+	size += sizeof(struct vring_avail) + (num * sizeof(uint16_t));
+	size = (size + align - 1) & ~(align - 1);
+	size += sizeof(struct vring_used) +
+	    (num * sizeof(struct vring_used_elem));
+	return (size);
+}
+
+static inline void
+vring_init(struct vring *vr, unsigned int num, uint8_t *p,
+    unsigned long align)
+{
+        vr->num = num;
+        vr->desc = (struct vring_desc *) p;
+        vr->avail = (struct vring_avail *) (p +
+	    num * sizeof(struct vring_desc));
+        vr->used = (void *)
+	    (((unsigned long) &vr->avail->ring[num] + align-1) & ~(align-1));
+}
+#endif /* VIRTIO_RING_H */
diff --git a/sys/dev/virtio/virtqueue.c b/sys/dev/virtio/virtqueue.c
new file mode 100644
index 0000000..1fb182e
--- /dev/null
+++ b/sys/dev/virtio/virtqueue.c
@@ -0,0 +1,755 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Implements the virtqueue interface as basically described
+ * in the original VirtIO paper.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/sglist.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/cpu.h>
+#include <machine/bus.h>
+#include <machine/atomic.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+#include <dev/virtio/virtio_ring.h>
+
+#include "virtio_bus_if.h"
+
+struct virtqueue {
+	device_t		 vq_dev;
+	char			 vq_name[VIRTQUEUE_MAX_NAME_SZ];
+	uint16_t		 vq_queue_index;
+	uint16_t		 vq_nentries;
+	uint32_t		 vq_flags;
+#define	VIRTQUEUE_FLAG_INDIRECT	 0x0001
+
+	int			 vq_alignment;
+	int			 vq_ring_size;
+	void			*vq_ring_mem;
+	int			 vq_max_indirect_size;
+	int			 vq_indirect_mem_size;
+	virtqueue_intr_t	*vq_intrhand;
+	void			*vq_intrhand_arg;
+
+	struct vring		 vq_ring;
+	uint16_t		 vq_free_cnt;
+	uint16_t		 vq_queued_cnt;
+	/*
+	 * Head of the free chain in the descriptor table. If
+	 * there are no free descriptors, this will be set to
+	 * VQ_RING_DESC_CHAIN_END.
+	 */
+	uint16_t		 vq_desc_head_idx;
+	/*
+	 * Last consumed descriptor in the used table,
+	 * trails vq_ring.used->idx.
+	 */
+	uint16_t		 vq_used_cons_idx;
+
+	struct vq_desc_extra {
+		void		  *cookie;
+		struct vring_desc *indirect;
+		vm_paddr_t	   indirect_paddr;
+		uint16_t	   ndescs;
+	} vq_descx[0];
+};
+
+/*
+ * The maximum virtqueue size is 2^15. Use that value as the end of
+ * descriptor chain terminator since it will never be a valid index
+ * in the descriptor table. This is used to verify we are correctly
+ * handling vq_free_cnt.
+ */
+#define VQ_RING_DESC_CHAIN_END 32768
+
+#define VQASSERT(_vq, _exp, _msg, ...)				\
+    KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,	\
+	##__VA_ARGS__))
+
+#define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)			\
+    VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,		\
+	"invalid ring index: %d, max: %d", (_idx),		\
+	(_vq)->vq_nentries)
+
+#define VQ_RING_ASSERT_CHAIN_TERM(_vq)				\
+    VQASSERT((_vq), (_vq)->vq_desc_head_idx ==			\
+	VQ_RING_DESC_CHAIN_END,	"full ring terminated "		\
+	"incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
+
+static int	virtqueue_init_indirect(struct virtqueue *vq, int);
+static void	virtqueue_free_indirect(struct virtqueue *vq);
+static void	virtqueue_init_indirect_list(struct virtqueue *,
+		    struct vring_desc *);
+
+static void	vq_ring_init(struct virtqueue *);
+static void	vq_ring_update_avail(struct virtqueue *, uint16_t);
+static uint16_t	vq_ring_enqueue_segments(struct virtqueue *,
+		    struct vring_desc *, uint16_t, struct sglist *, int, int);
+static int	vq_ring_use_indirect(struct virtqueue *, int);
+static void	vq_ring_enqueue_indirect(struct virtqueue *, void *,
+		    struct sglist *, int, int);
+static void	vq_ring_notify_host(struct virtqueue *, int);
+static void	vq_ring_free_chain(struct virtqueue *, uint16_t);
+
+uint64_t
+virtqueue_filter_features(uint64_t features)
+{
+	uint64_t mask;
+
+	mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
+	mask |= VIRTIO_RING_F_INDIRECT_DESC;
+
+	return (features & mask);
+}
+
+int
+virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
+    vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
+{
+	struct virtqueue *vq;
+	int error;
+
+	*vqp = NULL;
+	error = 0;
+
+	if (size == 0) {
+		device_printf(dev,
+		    "virtqueue %d (%s) does not exist (size is zero)\n",
+		    queue, info->vqai_name);
+		return (ENODEV);
+	} else if (!powerof2(size)) {
+		device_printf(dev,
+		    "virtqueue %d (%s) size is not a power of 2: %d\n",
+		    queue, info->vqai_name, size);
+		return (ENXIO);
+	} else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
+		device_printf(dev, "virtqueue %d (%s) requested too many "
+		    "indirect descriptors: %d, max %d\n",
+		    queue, info->vqai_name, info->vqai_maxindirsz,
+		    VIRTIO_MAX_INDIRECT);
+		return (EINVAL);
+	}
+
+	vq = malloc(sizeof(struct virtqueue) +
+	    size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (vq == NULL) {
+		device_printf(dev, "cannot allocate virtqueue\n");
+		return (ENOMEM);
+	}
+
+	vq->vq_dev = dev;
+	strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
+	vq->vq_queue_index = queue;
+	vq->vq_alignment = align;
+	vq->vq_nentries = size;
+	vq->vq_free_cnt = size;
+	vq->vq_intrhand = info->vqai_intr;
+	vq->vq_intrhand_arg = info->vqai_intr_arg;
+
+	if (info->vqai_maxindirsz > 1) {
+		error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
+		if (error)
+			goto fail;
+	}
+
+	vq->vq_ring_size = round_page(vring_size(size, align));
+	vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
+	    M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
+	if (vq->vq_ring_mem == NULL) {
+		device_printf(dev,
+		    "cannot allocate memory for virtqueue ring\n");
+		error = ENOMEM;
+		goto fail;
+	}
+
+	vq_ring_init(vq);
+	virtqueue_disable_intr(vq);
+
+	*vqp = vq;
+
+fail:
+	if (error)
+		virtqueue_free(vq);
+
+	return (error);
+}
+
+static int
+virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
+{
+	device_t dev;
+	struct vq_desc_extra *dxp;
+	int i, size;
+
+	dev = vq->vq_dev;
+
+	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
+		/*
+		 * Indirect descriptors requested by the driver but not
+		 * negotiated. Return zero to keep the initialization
+		 * going: we'll run fine without.
+		 */
+		if (bootverbose)
+			device_printf(dev, "virtqueue %d (%s) requested "
+			    "indirect descriptors but not negotiated\n",
+			    vq->vq_queue_index, vq->vq_name);
+		return (0);
+	}
+
+	size = indirect_size * sizeof(struct vring_desc);
+	vq->vq_max_indirect_size = indirect_size;
+	vq->vq_indirect_mem_size = size;
+	vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
+
+	for (i = 0; i < vq->vq_nentries; i++) {
+		dxp = &vq->vq_descx[i];
+
+		dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
+		if (dxp->indirect == NULL) {
+			device_printf(dev, "cannot allocate indirect list\n");
+			return (ENOMEM);
+		}
+
+		dxp->indirect_paddr = vtophys(dxp->indirect);
+		virtqueue_init_indirect_list(vq, dxp->indirect);
+	}
+
+	return (0);
+}
+
+static void
+virtqueue_free_indirect(struct virtqueue *vq)
+{
+	struct vq_desc_extra *dxp;
+	int i;
+
+	for (i = 0; i < vq->vq_nentries; i++) {
+		dxp = &vq->vq_descx[i];
+
+		if (dxp->indirect == NULL)
+			break;
+
+		free(dxp->indirect, M_DEVBUF);
+		dxp->indirect = NULL;
+		dxp->indirect_paddr = 0;
+	}
+
+	vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
+	vq->vq_indirect_mem_size = 0;
+}
+
+static void
+virtqueue_init_indirect_list(struct virtqueue *vq,
+    struct vring_desc *indirect)
+{
+	int i;
+
+	bzero(indirect, vq->vq_indirect_mem_size);
+
+	for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
+		indirect[i].next = i + 1;
+	indirect[i].next = VQ_RING_DESC_CHAIN_END;
+}
+
+int
+virtqueue_reinit(struct virtqueue *vq, uint16_t size)
+{
+	struct vq_desc_extra *dxp;
+	int i;
+
+	if (vq->vq_nentries != size) {
+		device_printf(vq->vq_dev,
+		    "%s: '%s' changed size; old=%hu, new=%hu\n",
+		    __func__, vq->vq_name, vq->vq_nentries, size);
+		return (EINVAL);
+	}
+
+	/* Warn if the virtqueue was not properly cleaned up. */
+	if (vq->vq_free_cnt != vq->vq_nentries) {
+		device_printf(vq->vq_dev,
+		    "%s: warning, '%s' virtqueue not empty, "
+		    "leaking %d entries\n", __func__, vq->vq_name,
+		    vq->vq_nentries - vq->vq_free_cnt);
+	}
+
+	vq->vq_desc_head_idx = 0;
+	vq->vq_used_cons_idx = 0;
+	vq->vq_queued_cnt = 0;
+	vq->vq_free_cnt = vq->vq_nentries;
+
+	/* To be safe, reset all our allocated memory. */
+	bzero(vq->vq_ring_mem, vq->vq_ring_size);
+	for (i = 0; i < vq->vq_nentries; i++) {
+		dxp = &vq->vq_descx[i];
+		dxp->cookie = NULL;
+		dxp->ndescs = 0;
+		if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
+			virtqueue_init_indirect_list(vq, dxp->indirect);
+	}
+
+	vq_ring_init(vq);
+	virtqueue_disable_intr(vq);
+
+	return (0);
+}
+
+void
+virtqueue_free(struct virtqueue *vq)
+{
+
+	if (vq->vq_free_cnt != vq->vq_nentries) {
+		device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
+		    "leaking %d entries\n", vq->vq_name,
+		    vq->vq_nentries - vq->vq_free_cnt);
+	}
+
+	if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
+		virtqueue_free_indirect(vq);
+
+	if (vq->vq_ring_mem != NULL) {
+		contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
+		vq->vq_ring_size = 0;
+		vq->vq_ring_mem = NULL;
+	}
+
+	free(vq, M_DEVBUF);
+}
+
+vm_paddr_t
+virtqueue_paddr(struct virtqueue *vq)
+{
+
+	return (vtophys(vq->vq_ring_mem));
+}
+
+int
+virtqueue_size(struct virtqueue *vq)
+{
+
+	return (vq->vq_nentries);
+}
+
+int
+virtqueue_empty(struct virtqueue *vq)
+{
+
+	return (vq->vq_nentries == vq->vq_free_cnt);
+}
+
+int
+virtqueue_full(struct virtqueue *vq)
+{
+
+	return (vq->vq_free_cnt == 0);
+}
+
+void
+virtqueue_notify(struct virtqueue *vq)
+{
+
+	vq->vq_queued_cnt = 0;
+	vq_ring_notify_host(vq, 0);
+}
+
+int
+virtqueue_nused(struct virtqueue *vq)
+{
+	uint16_t used_idx, nused;
+
+	used_idx = vq->vq_ring.used->idx;
+	if (used_idx >= vq->vq_used_cons_idx)
+		nused = used_idx - vq->vq_used_cons_idx;
+	else
+		nused = UINT16_MAX - vq->vq_used_cons_idx +
+		    used_idx + 1;
+	VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
+
+	return (nused);
+}
+
+int
+virtqueue_intr(struct virtqueue *vq)
+{
+
+	if (vq->vq_intrhand == NULL ||
+	    vq->vq_used_cons_idx == vq->vq_ring.used->idx)
+		return (0);
+
+	vq->vq_intrhand(vq->vq_intrhand_arg);
+
+	return (1);
+}
+
+int
+virtqueue_enable_intr(struct virtqueue *vq)
+{
+
+	/*
+	 * Enable interrupts, making sure we get the latest
+	 * index of what's already been consumed.
+	 */
+	vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
+
+	mb();
+
+	/*
+	 * Additional items may have been consumed in the time between
+	 * since we last checked and enabled interrupts above. Let our
+	 * caller know so it processes the new entries.
+	 */
+	if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
+		return (1);
+
+	return (0);
+}
+
+void
+virtqueue_disable_intr(struct virtqueue *vq)
+{
+
+	/*
+	 * Note this is only considered a hint to the host.
+	 */
+	vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
+}
+
+int
+virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
+    int readable, int writable)
+{
+	struct vq_desc_extra *dxp;
+	int needed;
+	uint16_t head_idx, idx;
+
+	needed = readable + writable;
+
+	VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
+	VQASSERT(vq, needed == sg->sg_nseg,
+	    "segment count mismatch, %d, %d", needed, sg->sg_nseg);
+	VQASSERT(vq,
+	    needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
+	    "too many segments to enqueue: %d, %d/%d", needed,
+	    vq->vq_nentries, vq->vq_max_indirect_size);
+
+	if (needed < 1)
+		return (EINVAL);
+	if (vq->vq_free_cnt == 0)
+		return (ENOSPC);
+
+	if (vq_ring_use_indirect(vq, needed)) {
+		vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
+		return (0);
+	} else if (vq->vq_free_cnt < needed)
+		return (EMSGSIZE);
+
+	head_idx = vq->vq_desc_head_idx;
+	VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
+	dxp = &vq->vq_descx[head_idx];
+
+	VQASSERT(vq, dxp->cookie == NULL,
+	    "cookie already exists for index %d", head_idx);
+	dxp->cookie = cookie;
+	dxp->ndescs = needed;
+
+	idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
+	    sg, readable, writable);
+
+	vq->vq_desc_head_idx = idx;
+	vq->vq_free_cnt -= needed;
+	if (vq->vq_free_cnt == 0)
+		VQ_RING_ASSERT_CHAIN_TERM(vq);
+	else
+		VQ_RING_ASSERT_VALID_IDX(vq, idx);
+
+	vq_ring_update_avail(vq, head_idx);
+
+	return (0);
+}
+
+void *
+virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
+{
+	struct vring_used_elem *uep;
+	void *cookie;
+	uint16_t used_idx, desc_idx;
+
+	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
+		return (NULL);
+
+	used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
+	uep = &vq->vq_ring.used->ring[used_idx];
+
+	mb();
+	desc_idx = (uint16_t) uep->id;
+	if (len != NULL)
+		*len = uep->len;
+
+	vq_ring_free_chain(vq, desc_idx);
+
+	cookie = vq->vq_descx[desc_idx].cookie;
+	VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
+	vq->vq_descx[desc_idx].cookie = NULL;
+
+	return (cookie);
+}
+
+void *
+virtqueue_poll(struct virtqueue *vq, uint32_t *len)
+{
+	void *cookie;
+
+	while ((cookie = virtqueue_dequeue(vq, len)) == NULL)
+		cpu_spinwait();
+
+	return (cookie);
+}
+
+void *
+virtqueue_drain(struct virtqueue *vq, int *last)
+{
+	void *cookie;
+	int idx;
+
+	cookie = NULL;
+	idx = *last;
+
+	while (idx < vq->vq_nentries && cookie == NULL) {
+		if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
+			vq->vq_descx[idx].cookie = NULL;
+			/* Free chain to keep free count consistent. */
+			vq_ring_free_chain(vq, idx);
+		}
+		idx++;
+	}
+
+	*last = idx;
+
+	return (cookie);
+}
+
+void
+virtqueue_dump(struct virtqueue *vq)
+{
+
+	if (vq == NULL)
+		return;
+
+	printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
+	    "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
+	    "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
+	    vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
+	    virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
+	    vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
+	    vq->vq_ring.used->idx, vq->vq_ring.avail->flags,
+	    vq->vq_ring.used->flags);
+}
+
+static void
+vq_ring_init(struct virtqueue *vq)
+{
+	struct vring *vr;
+	char *ring_mem;
+	int i, size;
+
+	ring_mem = vq->vq_ring_mem;
+	size = vq->vq_nentries;
+	vr = &vq->vq_ring;
+
+	vring_init(vr, size, ring_mem, vq->vq_alignment);
+
+	for (i = 0; i < size - 1; i++)
+		vr->desc[i].next = i + 1;
+	vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
+}
+
+static void
+vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
+{
+	uint16_t avail_idx;
+
+	/*
+	 * Place the head of the descriptor chain into the next slot and make
+	 * it usable to the host. The chain is made available now rather than
+	 * deferring to virtqueue_notify() in the hopes that if the host is
+	 * currently running on another CPU, we can keep it processing the new
+	 * descriptor.
+	 */
+	avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
+	vq->vq_ring.avail->ring[avail_idx] = desc_idx;
+
+	mb();
+	vq->vq_ring.avail->idx++;
+
+	/* Keep pending count until virtqueue_notify() for debugging. */
+	vq->vq_queued_cnt++;
+}
+
+static uint16_t
+vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
+    uint16_t head_idx, struct sglist *sg, int readable, int writable)
+{
+	struct sglist_seg *seg;
+	struct vring_desc *dp;
+	int i, needed;
+	uint16_t idx;
+
+	needed = readable + writable;
+
+	for (i = 0, idx = head_idx, seg = sg->sg_segs;
+	     i < needed;
+	     i++, idx = dp->next, seg++) {
+		VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
+		    "premature end of free desc chain");
+
+		dp = &desc[idx];
+		dp->addr = seg->ss_paddr;
+		dp->len = seg->ss_len;
+		dp->flags = 0;
+
+		if (i < needed - 1)
+			dp->flags |= VRING_DESC_F_NEXT;
+		if (i >= readable)
+			dp->flags |= VRING_DESC_F_WRITE;
+	}
+
+	return (idx);
+}
+
+static int
+vq_ring_use_indirect(struct virtqueue *vq, int needed)
+{
+
+	if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
+		return (0);
+
+	if (vq->vq_max_indirect_size < needed)
+		return (0);
+
+	if (needed < 2)
+		return (0);
+
+	return (1);
+}
+
+static void
+vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
+    struct sglist *sg, int readable, int writable)
+{
+	struct vring_desc *dp;
+	struct vq_desc_extra *dxp;
+	int needed;
+	uint16_t head_idx;
+
+	needed = readable + writable;
+	VQASSERT(vq, needed <= vq->vq_max_indirect_size,
+	    "enqueuing too many indirect descriptors");
+
+	head_idx = vq->vq_desc_head_idx;
+	VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
+	dp = &vq->vq_ring.desc[head_idx];
+	dxp = &vq->vq_descx[head_idx];
+
+	VQASSERT(vq, dxp->cookie == NULL,
+	    "cookie already exists for index %d", head_idx);
+	dxp->cookie = cookie;
+	dxp->ndescs = 1;
+
+	dp->addr = dxp->indirect_paddr;
+	dp->len = needed * sizeof(struct vring_desc);
+	dp->flags = VRING_DESC_F_INDIRECT;
+
+	vq_ring_enqueue_segments(vq, dxp->indirect, 0,
+	    sg, readable, writable);
+
+	vq->vq_desc_head_idx = dp->next;
+	vq->vq_free_cnt--;
+	if (vq->vq_free_cnt == 0)
+		VQ_RING_ASSERT_CHAIN_TERM(vq);
+	else
+		VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
+
+	vq_ring_update_avail(vq, head_idx);
+}
+
+static void
+vq_ring_notify_host(struct virtqueue *vq, int force)
+{
+
+	mb();
+
+	if (force ||
+	    (vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0)
+		VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
+}
+
+static void
+vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
+{
+	struct vring_desc *dp;
+	struct vq_desc_extra *dxp;
+
+	VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
+	dp = &vq->vq_ring.desc[desc_idx];
+	dxp = &vq->vq_descx[desc_idx];
+
+	if (vq->vq_free_cnt == 0)
+		VQ_RING_ASSERT_CHAIN_TERM(vq);
+
+	vq->vq_free_cnt += dxp->ndescs;
+	dxp->ndescs--;
+
+	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
+		while (dp->flags & VRING_DESC_F_NEXT) {
+			VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
+			dp = &vq->vq_ring.desc[dp->next];
+			dxp->ndescs--;
+		}
+	}
+	VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain");
+
+	/*
+	 * We must append the existing free chain, if any, to the end of
+	 * newly freed chain. If the virtqueue was completely used, then
+	 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
+	 */
+	dp->next = vq->vq_desc_head_idx;
+	vq->vq_desc_head_idx = desc_idx;
+}
diff --git a/sys/dev/virtio/virtqueue.h b/sys/dev/virtio/virtqueue.h
new file mode 100644
index 0000000..e790e65
--- /dev/null
+++ b/sys/dev/virtio/virtqueue.h
@@ -0,0 +1,98 @@
+/*-
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VIRTIO_VIRTQUEUE_H
+#define _VIRTIO_VIRTQUEUE_H
+
+#include <sys/types.h>
+
+struct virtqueue;
+struct sglist;
+
+/* Support for indirect buffer descriptors. */
+#define VIRTIO_RING_F_INDIRECT_DESC	(1 << 28)
+
+/* The guest publishes the used index for which it expects an interrupt
+ * at the end of the avail ring. Host should ignore the avail->flags field.
+ * The host publishes the avail index for which it expects a kick
+ * at the end of the used ring. Guest should ignore the used->flags field.
+ */
+#define VIRTIO_RING_F_EVENT_IDX		(1 << 29)
+
+/* Device callback for a virtqueue interrupt. */
+typedef int virtqueue_intr_t(void *);
+
+#define VIRTQUEUE_MAX_NAME_SZ	32
+
+/* One for each virtqueue the device wishes to allocate. */
+struct vq_alloc_info {
+	char		   vqai_name[VIRTQUEUE_MAX_NAME_SZ];
+	int		   vqai_maxindirsz;
+	virtqueue_intr_t  *vqai_intr;
+	void		  *vqai_intr_arg;
+	struct virtqueue **vqai_vq;
+};
+
+#define VQ_ALLOC_INFO_INIT(_i,_nsegs,_intr,_arg,_vqp,_str,...) do {	\
+	snprintf((_i)->vqai_name, VIRTQUEUE_MAX_NAME_SZ, _str,		\
+	    ##__VA_ARGS__);						\
+	(_i)->vqai_maxindirsz = (_nsegs);				\
+	(_i)->vqai_intr = (_intr);					\
+	(_i)->vqai_intr_arg = (_arg);					\
+	(_i)->vqai_vq = (_vqp);						\
+} while (0)
+
+uint64_t virtqueue_filter_features(uint64_t features);
+
+int	 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
+	     int align, vm_paddr_t highaddr, struct vq_alloc_info *info,
+	     struct virtqueue **vqp);
+void	*virtqueue_drain(struct virtqueue *vq, int *last);
+void	 virtqueue_free(struct virtqueue *vq);
+int	 virtqueue_reinit(struct virtqueue *vq, uint16_t size);
+
+int	 virtqueue_intr(struct virtqueue *vq);
+int	 virtqueue_enable_intr(struct virtqueue *vq);
+void	 virtqueue_disable_intr(struct virtqueue *vq);
+
+/* Get physical address of the virtqueue ring. */
+vm_paddr_t virtqueue_paddr(struct virtqueue *vq);
+
+int	 virtqueue_full(struct virtqueue *vq);
+int	 virtqueue_empty(struct virtqueue *vq);
+int	 virtqueue_size(struct virtqueue *vq);
+int	 virtqueue_nused(struct virtqueue *vq);
+void	 virtqueue_notify(struct virtqueue *vq);
+void	 virtqueue_dump(struct virtqueue *vq);
+
+int	 virtqueue_enqueue(struct virtqueue *vq, void *cookie,
+	     struct sglist *sg, int readable, int writable);
+void	*virtqueue_dequeue(struct virtqueue *vq, uint32_t *len);
+void	*virtqueue_poll(struct virtqueue *vq, uint32_t *len);
+
+#endif /* _VIRTIO_VIRTQUEUE_H */
diff --git a/sys/dev/wb/if_wb.c b/sys/dev/wb/if_wb.c
index f924d3e..daac022 100644
--- a/sys/dev/wb/if_wb.c
+++ b/sys/dev/wb/if_wb.c
@@ -113,6 +113,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcivar.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 /* "device miibus" required.  See GENERIC if you get errors here. */
@@ -129,7 +130,7 @@ MODULE_DEPEND(wb, miibus, 1, 1, 1);
 /*
  * Various supported device vendors/types and their names.
  */
-static struct wb_type wb_devs[] = {
+static const struct wb_type const wb_devs[] = {
 	{ WB_VENDORID, WB_DEVICEID_840F,
 		"Winbond W89C840F 10/100BaseTX" },
 	{ CP_VENDORID, CP_DEVICEID_RL100,
@@ -166,10 +167,6 @@ static void wb_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 static void wb_eeprom_putbyte(struct wb_softc *, int);
 static void wb_eeprom_getword(struct wb_softc *, int, u_int16_t *);
 static void wb_read_eeprom(struct wb_softc *, caddr_t, int, int, int);
-static void wb_mii_sync(struct wb_softc *);
-static void wb_mii_send(struct wb_softc *, u_int32_t, int);
-static int wb_mii_readreg(struct wb_softc *, struct wb_mii_frame *);
-static int wb_mii_writereg(struct wb_softc *, struct wb_mii_frame *);
 
 static void wb_setcfg(struct wb_softc *, u_int32_t);
 static void wb_setmulti(struct wb_softc *);
@@ -182,6 +179,24 @@ static int wb_miibus_readreg(device_t, int, int);
 static int wb_miibus_writereg(device_t, int, int, int);
 static void wb_miibus_statchg(device_t);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t wb_mii_bitbang_read(device_t);
+static void wb_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops wb_mii_bitbang_ops = {
+	wb_mii_bitbang_read,
+	wb_mii_bitbang_write,
+	{
+		WB_SIO_MII_DATAOUT,	/* MII_BIT_MDO */
+		WB_SIO_MII_DATAIN,	/* MII_BIT_MDI */
+		WB_SIO_MII_CLK,		/* MII_BIT_MDC */
+		WB_SIO_MII_DIR,		/* MII_BIT_DIR_HOST_PHY */
+		0,			/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 #ifdef WB_USEIOSPACE
 #define WB_RES			SYS_RES_IOPORT
 #define WB_RID			WB_PCI_LOIO
@@ -262,8 +277,6 @@ wb_eeprom_putbyte(sc, addr)
 		SIO_CLR(WB_SIO_EE_CLK);
 		DELAY(100);
 	}
-
-	return;
 }
 
 /*
@@ -304,8 +317,6 @@ wb_eeprom_getword(sc, addr, dest)
 	CSR_WRITE_4(sc, WB_SIO, 0);
 
 	*dest = word;
-
-	return;
 }
 
 /*
@@ -330,194 +341,39 @@ wb_read_eeprom(sc, dest, off, cnt, swap)
 		else
 			*ptr = word;
 	}
-
-	return;
 }
 
 /*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static void
-wb_mii_sync(sc)
-	struct wb_softc		*sc;
+static uint32_t
+wb_mii_bitbang_read(device_t dev)
 {
-	register int		i;
+	struct wb_softc *sc;
+	uint32_t val;
 
-	SIO_SET(WB_SIO_MII_DIR|WB_SIO_MII_DATAIN);
+	sc = device_get_softc(dev);
 
-	for (i = 0; i < 32; i++) {
-		SIO_SET(WB_SIO_MII_CLK);
-		DELAY(1);
-		SIO_CLR(WB_SIO_MII_CLK);
-		DELAY(1);
-	}
+	val = CSR_READ_4(sc, WB_SIO);
+	CSR_BARRIER(sc, WB_SIO, 4,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	return;
+	return (val);
 }
 
 /*
- * Clock a series of bits through the MII.
+ * Write the MII serial port for the MII bit-bang module.
  */
 static void
-wb_mii_send(sc, bits, cnt)
-	struct wb_softc		*sc;
-	u_int32_t		bits;
-	int			cnt;
-{
-	int			i;
-
-	SIO_CLR(WB_SIO_MII_CLK);
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-                if (bits & i) {
-			SIO_SET(WB_SIO_MII_DATAIN);
-                } else {
-			SIO_CLR(WB_SIO_MII_DATAIN);
-                }
-		DELAY(1);
-		SIO_CLR(WB_SIO_MII_CLK);
-		DELAY(1);
-		SIO_SET(WB_SIO_MII_CLK);
-	}
-}
-
-/*
- * Read an PHY register through the MII.
- */
-static int
-wb_mii_readreg(sc, frame)
-	struct wb_softc		*sc;
-	struct wb_mii_frame	*frame;
-	
+wb_mii_bitbang_write(device_t dev, uint32_t val)
 {
-	int			i, ack;
-
-	/*
-	 * Set up frame for RX.
-	 */
-	frame->mii_stdelim = WB_MII_STARTDELIM;
-	frame->mii_opcode = WB_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-	
-	CSR_WRITE_4(sc, WB_SIO, 0);
-
-	/*
- 	 * Turn on data xmit.
-	 */
-	SIO_SET(WB_SIO_MII_DIR);
-
-	wb_mii_sync(sc);
-
-	/*
-	 * Send command/address info.
-	 */
-	wb_mii_send(sc, frame->mii_stdelim, 2);
-	wb_mii_send(sc, frame->mii_opcode, 2);
-	wb_mii_send(sc, frame->mii_phyaddr, 5);
-	wb_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Idle bit */
-	SIO_CLR((WB_SIO_MII_CLK|WB_SIO_MII_DATAIN));
-	DELAY(1);
-	SIO_SET(WB_SIO_MII_CLK);
-	DELAY(1);
-
-	/* Turn off xmit. */
-	SIO_CLR(WB_SIO_MII_DIR);
-	/* Check for ack */
-	SIO_CLR(WB_SIO_MII_CLK);
-	DELAY(1);
-	ack = CSR_READ_4(sc, WB_SIO) & WB_SIO_MII_DATAOUT;
-	SIO_SET(WB_SIO_MII_CLK);
-	DELAY(1);
-	SIO_CLR(WB_SIO_MII_CLK);
-	DELAY(1);
-	SIO_SET(WB_SIO_MII_CLK);
-	DELAY(1);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for(i = 0; i < 16; i++) {
-			SIO_CLR(WB_SIO_MII_CLK);
-			DELAY(1);
-			SIO_SET(WB_SIO_MII_CLK);
-			DELAY(1);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		SIO_CLR(WB_SIO_MII_CLK);
-		DELAY(1);
-		if (!ack) {
-			if (CSR_READ_4(sc, WB_SIO) & WB_SIO_MII_DATAOUT)
-				frame->mii_data |= i;
-			DELAY(1);
-		}
-		SIO_SET(WB_SIO_MII_CLK);
-		DELAY(1);
-	}
-
-fail:
+	struct wb_softc *sc;
 
-	SIO_CLR(WB_SIO_MII_CLK);
-	DELAY(1);
-	SIO_SET(WB_SIO_MII_CLK);
-	DELAY(1);
-
-	if (ack)
-		return(1);
-	return(0);
-}
-
-/*
- * Write to a PHY register through the MII.
- */
-static int
-wb_mii_writereg(sc, frame)
-	struct wb_softc		*sc;
-	struct wb_mii_frame	*frame;
-	
-{
-
-	/*
-	 * Set up frame for TX.
-	 */
-
-	frame->mii_stdelim = WB_MII_STARTDELIM;
-	frame->mii_opcode = WB_MII_WRITEOP;
-	frame->mii_turnaround = WB_MII_TURNAROUND;
-	
-	/*
- 	 * Turn on data output.
-	 */
-	SIO_SET(WB_SIO_MII_DIR);
-
-	wb_mii_sync(sc);
-
-	wb_mii_send(sc, frame->mii_stdelim, 2);
-	wb_mii_send(sc, frame->mii_opcode, 2);
-	wb_mii_send(sc, frame->mii_phyaddr, 5);
-	wb_mii_send(sc, frame->mii_regaddr, 5);
-	wb_mii_send(sc, frame->mii_turnaround, 2);
-	wb_mii_send(sc, frame->mii_data, 16);
-
-	/* Idle bit. */
-	SIO_SET(WB_SIO_MII_CLK);
-	DELAY(1);
-	SIO_CLR(WB_SIO_MII_CLK);
-	DELAY(1);
-
-	/*
-	 * Turn off xmit.
-	 */
-	SIO_CLR(WB_SIO_MII_DIR);
+	sc = device_get_softc(dev);
 
-	return(0);
+	CSR_WRITE_4(sc, WB_SIO, val);
+	CSR_BARRIER(sc, WB_SIO, 4,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
@@ -525,18 +381,8 @@ wb_miibus_readreg(dev, phy, reg)
 	device_t		dev;
 	int			phy, reg;
 {
-	struct wb_softc		*sc;
-	struct wb_mii_frame	frame;
-
-	sc = device_get_softc(dev);
 
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	wb_mii_readreg(sc, &frame);
-
-	return(frame.mii_data);
+	return (mii_bitbang_readreg(dev, &wb_mii_bitbang_ops, phy, reg));
 }
 
 static int
@@ -544,18 +390,8 @@ wb_miibus_writereg(dev, phy, reg, data)
 	device_t		dev;
 	int			phy, reg, data;
 {
-	struct wb_softc		*sc;
-	struct wb_mii_frame	frame;
 
-	sc = device_get_softc(dev);
-
-	bzero((char *)&frame, sizeof(frame));
-
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
-
-	wb_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &wb_mii_bitbang_ops, phy, reg, data);
 
 	return(0);
 }
@@ -570,8 +406,6 @@ wb_miibus_statchg(dev)
 	sc = device_get_softc(dev);
 	mii = device_get_softc(sc->wb_miibus);
 	wb_setcfg(sc, mii->mii_media_active);
-
-	return;
 }
 
 /*
@@ -627,8 +461,6 @@ wb_setmulti(sc)
 	CSR_WRITE_4(sc, WB_MAR0, hashes[0]);
 	CSR_WRITE_4(sc, WB_MAR1, hashes[1]);
 	CSR_WRITE_4(sc, WB_NETCFG, rxfilt);
-
-	return;
 }
 
 /*
@@ -671,8 +503,6 @@ wb_setcfg(sc, media)
 
 	if (restart)
 		WB_SETBIT(sc, WB_NETCFG, WB_NETCFG_TX_ON|WB_NETCFG_RX_ON);
-
-	return;
 }
 
 static void
@@ -742,7 +572,7 @@ static int
 wb_probe(dev)
 	device_t		dev;
 {
-	struct wb_type		*t;
+	const struct wb_type		*t;
 
 	t = wb_devs;
 
@@ -1001,7 +831,7 @@ wb_bfree(buf, args)
 	void			*buf;
 	void			*args;
 {
-	return;
+
 }
 
 /*
@@ -1127,8 +957,6 @@ wb_rxeoc(sc)
 	WB_SETBIT(sc, WB_NETCFG, WB_NETCFG_RX_ON);
 	if (CSR_READ_4(sc, WB_ISR) & WB_RXSTATE_SUSPEND)
 		CSR_WRITE_4(sc, WB_RXSTART, 0xFFFFFFFF);
-
-	return;
 }
 
 /*
@@ -1185,8 +1013,6 @@ wb_txeof(sc)
 
 		sc->wb_cdata.wb_tx_head = cur_tx->wb_nextdesc;
 	}
-
-	return;
 }
 
 /*
@@ -1212,8 +1038,6 @@ wb_txeoc(sc)
 			CSR_WRITE_4(sc, WB_TXSTART, 0xFFFFFFFF);
 		}
 	}
-
-	return;
 }
 
 static void
@@ -1300,8 +1124,6 @@ wb_intr(arg)
 	}
 
 	WB_UNLOCK(sc);
-
-	return;
 }
 
 static void
@@ -1320,8 +1142,6 @@ wb_tick(xsc)
 	if (sc->wb_timer > 0 && --sc->wb_timer == 0)
 		wb_watchdog(sc);
 	callout_reset(&sc->wb_stat_callout, hz, wb_tick, sc);
-
-	return;
 }
 
 /*
@@ -1520,8 +1340,6 @@ wb_start_locked(ifp)
 	 * Set a timeout in case the chip goes out to lunch.
 	 */
 	sc->wb_timer = 5;
-
-	return;
 }
 
 static void
@@ -1647,8 +1465,6 @@ wb_init_locked(sc)
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
 	callout_reset(&sc->wb_stat_callout, hz, wb_tick, sc);
-
-	return;
 }
 
 /*
@@ -1690,8 +1506,6 @@ wb_ifmedia_sts(ifp, ifmr)
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
 	WB_UNLOCK(sc);
-
-	return;
 }
 
 static int
@@ -1757,8 +1571,6 @@ wb_watchdog(sc)
 
 	if (ifp->if_snd.ifq_head != NULL)
 		wb_start_locked(ifp);
-
-	return;
 }
 
 /*
@@ -1809,8 +1621,6 @@ wb_stop(sc)
 		sizeof(sc->wb_ldata->wb_tx_list));
 
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-
-	return;
 }
 
 /*
diff --git a/sys/dev/wb/if_wbreg.h b/sys/dev/wb/if_wbreg.h
index 95d0a8e..16eb8a1 100644
--- a/sys/dev/wb/if_wbreg.h
+++ b/sys/dev/wb/if_wbreg.h
@@ -341,26 +341,9 @@ struct wb_chain_data {
 struct wb_type {
 	u_int16_t		wb_vid;
 	u_int16_t		wb_did;
-	char			*wb_name;
+	const char		*wb_name;
 };
 
-struct wb_mii_frame {
-	u_int8_t		mii_stdelim;
-	u_int8_t		mii_opcode;
-	u_int8_t		mii_phyaddr;
-	u_int8_t		mii_regaddr;
-	u_int8_t		mii_turnaround;
-	u_int16_t		mii_data;
-};
-
-/*
- * MII constants
- */
-#define WB_MII_STARTDELIM	0x01
-#define WB_MII_READOP		0x02
-#define WB_MII_WRITEOP		0x01
-#define WB_MII_TURNAROUND	0x02
-
 struct wb_softc {
 	struct ifnet		*wb_ifp;	/* interface info */
 	device_t		wb_dev;
@@ -395,6 +378,9 @@ struct wb_softc {
 #define CSR_READ_2(sc, reg)		bus_read_2(sc->wb_res, reg)
 #define CSR_READ_1(sc, reg)		bus_read_1(sc->wb_res, reg)
 
+#define	CSR_BARRIER(sc, reg, length, flags)				\
+	bus_barrier(sc->wb_res, reg, length, flags)
+
 #define WB_TIMEOUT		1000
 
 /*
diff --git a/sys/dev/wi/if_wi.c b/sys/dev/wi/if_wi.c
index 69db6a7..c31ad7b 100644
--- a/sys/dev/wi/if_wi.c
+++ b/sys/dev/wi/if_wi.c
@@ -166,7 +166,8 @@ wi_write_val(struct wi_softc *sc, int rid, u_int16_t val)
 	return wi_write_rid(sc, rid, &val, sizeof(val));
 }
 
-SYSCTL_NODE(_hw, OID_AUTO, wi, CTLFLAG_RD, 0, "Wireless driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, wi, CTLFLAG_RD, 0,
+	    "Wireless driver parameters");
 
 static	struct timeval lasttxerror;	/* time of last tx error msg */
 static	int curtxeps;			/* current tx error msgs/sec */
diff --git a/sys/dev/xe/if_xe.c b/sys/dev/xe/if_xe.c
index 77b953a..57411ba 100644
--- a/sys/dev/xe/if_xe.c
+++ b/sys/dev/xe/if_xe.c
@@ -201,7 +201,7 @@ static void	xe_reg_dump(struct xe_softc *scp);
 #ifdef XE_DEBUG
 
 /* sysctl vars */
-SYSCTL_NODE(_hw, OID_AUTO, xe, CTLFLAG_RD, 0, "if_xe parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, xe, CTLFLAG_RD, 0, "if_xe parameters");
 
 int xe_debug = 0;
 SYSCTL_INT(_hw_xe, OID_AUTO, debug, CTLFLAG_RW, &xe_debug, 0,
diff --git a/sys/dev/xen/balloon/balloon.c b/sys/dev/xen/balloon/balloon.c
index 81d1094..6aa8c9b 100644
--- a/sys/dev/xen/balloon/balloon.c
+++ b/sys/dev/xen/balloon/balloon.c
@@ -49,7 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 
-MALLOC_DEFINE(M_BALLOON, "Balloon", "Xen Balloon Driver");
+static MALLOC_DEFINE(M_BALLOON, "Balloon", "Xen Balloon Driver");
 
 struct mtx balloon_mutex;
 
@@ -84,7 +84,7 @@ static struct balloon_stats balloon_stats;
 #define bs balloon_stats
 
 SYSCTL_DECL(_dev_xen);
-SYSCTL_NODE(_dev_xen, OID_AUTO, balloon, CTLFLAG_RD, NULL, "Balloon");
+static SYSCTL_NODE(_dev_xen, OID_AUTO, balloon, CTLFLAG_RD, NULL, "Balloon");
 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, current, CTLFLAG_RD,
     &bs.current_pages, 0, "Current allocation");
 SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, target, CTLFLAG_RD,
@@ -298,8 +298,7 @@ decrease_reservation(unsigned long nr_pages)
 		nr_pages = ARRAY_SIZE(frame_list);
 
 	for (i = 0; i < nr_pages; i++) {
-		int color = 0;
-		if ((page = vm_page_alloc(NULL, color++, 
+		if ((page = vm_page_alloc(NULL, 0, 
 			    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 
 			    VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 			nr_pages = i;
diff --git a/sys/dev/xen/blkback/blkback.c b/sys/dev/xen/blkback/blkback.c
index e52c342..c42bfd9 100644
--- a/sys/dev/xen/blkback/blkback.c
+++ b/sys/dev/xen/blkback/blkback.c
@@ -121,7 +121,7 @@ __FBSDID("$FreeBSD$");
 /**
  * Custom malloc type for all driver allocations.
  */
-MALLOC_DEFINE(M_XENBLOCKBACK, "xbbd", "Xen Block Back Driver Data");
+static MALLOC_DEFINE(M_XENBLOCKBACK, "xbbd", "Xen Block Back Driver Data");
 
 #ifdef XBB_DEBUG
 #define DPRINTF(fmt, args...) \
diff --git a/sys/dev/xen/blkfront/blkfront.c b/sys/dev/xen/blkfront/blkfront.c
index fb530f3..954cf1f 100644
--- a/sys/dev/xen/blkfront/blkfront.c
+++ b/sys/dev/xen/blkfront/blkfront.c
@@ -81,7 +81,7 @@ static int blkif_completion(struct xb_command *);
 static void blkif_free(struct xb_softc *);
 static void blkif_queue_cb(void *, bus_dma_segment_t *, int, int);
 
-MALLOC_DEFINE(M_XENBLOCKFRONT, "xbd", "Xen Block Front driver data");
+static MALLOC_DEFINE(M_XENBLOCKFRONT, "xbd", "Xen Block Front driver data");
 
 #define GRANT_INVALID_REF 0
 
diff --git a/sys/dev/xl/if_xl.c b/sys/dev/xl/if_xl.c
index e09e434..b774e7f 100644
--- a/sys/dev/xl/if_xl.c
+++ b/sys/dev/xl/if_xl.c
@@ -127,6 +127,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -160,7 +161,7 @@ MODULE_DEPEND(xl, miibus, 1, 1, 1);
 /*
  * Various supported device vendors/types and their names.
  */
-static const struct xl_type xl_devs[] = {
+static const struct xl_type const xl_devs[] = {
 	{ TC_VENDORID, TC_DEVICEID_BOOMERANG_10BT,
 		"3Com 3c900-TPO Etherlink XL" },
 	{ TC_VENDORID, TC_DEVICEID_BOOMERANG_10BT_COMBO,
@@ -258,10 +259,6 @@ static void xl_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 
 static int xl_eeprom_wait(struct xl_softc *);
 static int xl_read_eeprom(struct xl_softc *, caddr_t, int, int, int);
-static void xl_mii_sync(struct xl_softc *);
-static void xl_mii_send(struct xl_softc *, u_int32_t, int);
-static int xl_mii_readreg(struct xl_softc *, struct xl_mii_frame *);
-static int xl_mii_writereg(struct xl_softc *, struct xl_mii_frame *);
 
 static void xl_rxfilter(struct xl_softc *);
 static void xl_rxfilter_90x(struct xl_softc *);
@@ -286,6 +283,24 @@ static int xl_miibus_writereg(device_t, int, int, int);
 static void xl_miibus_statchg(device_t);
 static void xl_miibus_mediainit(device_t);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t xl_mii_bitbang_read(device_t);
+static void xl_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops xl_mii_bitbang_ops = {
+	xl_mii_bitbang_read,
+	xl_mii_bitbang_write,
+	{
+		XL_MII_DATA,		/* MII_BIT_MDO */
+		XL_MII_DATA,		/* MII_BIT_MDI */
+		XL_MII_CLK,		/* MII_BIT_MDC */
+		XL_MII_DIR,		/* MII_BIT_DIR_HOST_PHY */
+		0,			/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static device_method_t xl_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		xl_probe),
@@ -359,194 +374,66 @@ xl_wait(struct xl_softc *sc)
  * some chips/CPUs/processor speeds/bus speeds/etc but not
  * with others.
  */
-#define MII_SET(x)					\
-	CSR_WRITE_2(sc, XL_W4_PHY_MGMT,			\
-		CSR_READ_2(sc, XL_W4_PHY_MGMT) | (x))
-
-#define MII_CLR(x)					\
-	CSR_WRITE_2(sc, XL_W4_PHY_MGMT,			\
-		CSR_READ_2(sc, XL_W4_PHY_MGMT) & ~(x))
-
-/*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
- */
-static void
-xl_mii_sync(struct xl_softc *sc)
-{
-	register int		i;
-
-	XL_SEL_WIN(4);
-	MII_SET(XL_MII_DIR|XL_MII_DATA);
-
-	for (i = 0; i < 32; i++) {
-		MII_SET(XL_MII_CLK);
-		MII_SET(XL_MII_DATA);
-		MII_SET(XL_MII_DATA);
-		MII_CLR(XL_MII_CLK);
-		MII_SET(XL_MII_DATA);
-		MII_SET(XL_MII_DATA);
-	}
-}
 
 /*
- * Clock a series of bits through the MII.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static void
-xl_mii_send(struct xl_softc *sc, u_int32_t bits, int cnt)
+static uint32_t
+xl_mii_bitbang_read(device_t dev)
 {
-	int			i;
-
-	XL_SEL_WIN(4);
-	MII_CLR(XL_MII_CLK);
-
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		if (bits & i) {
-			MII_SET(XL_MII_DATA);
-		} else {
-			MII_CLR(XL_MII_DATA);
-		}
-		MII_CLR(XL_MII_CLK);
-		MII_SET(XL_MII_CLK);
-	}
-}
-
-/*
- * Read an PHY register through the MII.
- */
-static int
-xl_mii_readreg(struct xl_softc *sc, struct xl_mii_frame *frame)
-{
-	int			i, ack;
-
-	/* Set up frame for RX. */
-	frame->mii_stdelim = XL_MII_STARTDELIM;
-	frame->mii_opcode = XL_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	/* Select register window 4. */
-	XL_SEL_WIN(4);
-
-	CSR_WRITE_2(sc, XL_W4_PHY_MGMT, 0);
-	/* Turn on data xmit. */
-	MII_SET(XL_MII_DIR);
-
-	xl_mii_sync(sc);
-
-	/* Send command/address info. */
-	xl_mii_send(sc, frame->mii_stdelim, 2);
-	xl_mii_send(sc, frame->mii_opcode, 2);
-	xl_mii_send(sc, frame->mii_phyaddr, 5);
-	xl_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Idle bit */
-	MII_CLR((XL_MII_CLK|XL_MII_DATA));
-	MII_SET(XL_MII_CLK);
-
-	/* Turn off xmit. */
-	MII_CLR(XL_MII_DIR);
-
-	/* Check for ack */
-	MII_CLR(XL_MII_CLK);
-	ack = CSR_READ_2(sc, XL_W4_PHY_MGMT) & XL_MII_DATA;
-	MII_SET(XL_MII_CLK);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for (i = 0; i < 16; i++) {
-			MII_CLR(XL_MII_CLK);
-			MII_SET(XL_MII_CLK);
-		}
-		goto fail;
-	}
+	struct xl_softc		*sc;
+	uint32_t		val;
 
-	for (i = 0x8000; i; i >>= 1) {
-		MII_CLR(XL_MII_CLK);
-		if (!ack) {
-			if (CSR_READ_2(sc, XL_W4_PHY_MGMT) & XL_MII_DATA)
-				frame->mii_data |= i;
-		}
-		MII_SET(XL_MII_CLK);
-	}
+	sc = device_get_softc(dev);
 
-fail:
-	MII_CLR(XL_MII_CLK);
-	MII_SET(XL_MII_CLK);
+	/* We're already in window 4. */
+	val = CSR_READ_2(sc, XL_W4_PHY_MGMT);
+	CSR_BARRIER(sc, XL_W4_PHY_MGMT, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	return (ack ? 1 : 0);
+	return (val);
 }
 
 /*
- * Write to a PHY register through the MII.
+ * Write the MII serial port for the MII bit-bang module.
  */
-static int
-xl_mii_writereg(struct xl_softc *sc, struct xl_mii_frame *frame)
+static void
+xl_mii_bitbang_write(device_t dev, uint32_t val)
 {
+	struct xl_softc		*sc;
 
-	/* Set up frame for TX. */
-	frame->mii_stdelim = XL_MII_STARTDELIM;
-	frame->mii_opcode = XL_MII_WRITEOP;
-	frame->mii_turnaround = XL_MII_TURNAROUND;
-
-	/* Select the window 4. */
-	XL_SEL_WIN(4);
-
-	/* Turn on data output. */
-	MII_SET(XL_MII_DIR);
-
-	xl_mii_sync(sc);
-
-	xl_mii_send(sc, frame->mii_stdelim, 2);
-	xl_mii_send(sc, frame->mii_opcode, 2);
-	xl_mii_send(sc, frame->mii_phyaddr, 5);
-	xl_mii_send(sc, frame->mii_regaddr, 5);
-	xl_mii_send(sc, frame->mii_turnaround, 2);
-	xl_mii_send(sc, frame->mii_data, 16);
-
-	/* Idle bit. */
-	MII_SET(XL_MII_CLK);
-	MII_CLR(XL_MII_CLK);
-
-	/* Turn off xmit. */
-	MII_CLR(XL_MII_DIR);
+	sc = device_get_softc(dev);
 
-	return (0);
+	/* We're already in window 4. */
+	CSR_WRITE_2(sc, XL_W4_PHY_MGMT,	val);
+	CSR_BARRIER(sc, XL_W4_PHY_MGMT, 2,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
 xl_miibus_readreg(device_t dev, int phy, int reg)
 {
 	struct xl_softc		*sc;
-	struct xl_mii_frame	frame;
 
 	sc = device_get_softc(dev);
 
-	bzero((char *)&frame, sizeof(frame));
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-
-	xl_mii_readreg(sc, &frame);
+	/* Select the window 4. */
+	XL_SEL_WIN(4);
 
-	return (frame.mii_data);
+	return (mii_bitbang_readreg(dev, &xl_mii_bitbang_ops, phy, reg));
 }
 
 static int
 xl_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	struct xl_softc		*sc;
-	struct xl_mii_frame	frame;
 
 	sc = device_get_softc(dev);
 
-	bzero((char *)&frame, sizeof(frame));
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
+	/* Select the window 4. */
+	XL_SEL_WIN(4);
 
-	xl_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &xl_mii_bitbang_ops, phy, reg, data);
 
 	return (0);
 }
diff --git a/sys/dev/xl/if_xlreg.h b/sys/dev/xl/if_xlreg.h
index f5494f5..b27e038 100644
--- a/sys/dev/xl/if_xlreg.h
+++ b/sys/dev/xl/if_xlreg.h
@@ -556,27 +556,10 @@ struct xl_chain_data {
 struct xl_type {
 	u_int16_t		xl_vid;
 	u_int16_t		xl_did;
-	char			*xl_name;
-};
-
-struct xl_mii_frame {
-	u_int8_t		mii_stdelim;
-	u_int8_t		mii_opcode;
-	u_int8_t		mii_phyaddr;
-	u_int8_t		mii_regaddr;
-	u_int8_t		mii_turnaround;
-	u_int16_t		mii_data;
+	const char		*xl_name;
 };
 
 /*
- * MII constants
- */
-#define XL_MII_STARTDELIM	0x01
-#define XL_MII_READOP		0x02
-#define XL_MII_WRITEOP		0x01
-#define XL_MII_TURNAROUND	0x02
-
-/*
  * The 3C905B adapters implement a few features that we want to
  * take advantage of, namely the multicast hash filter. With older
  * chips, you only have the option of turning on reception of all
@@ -680,8 +663,17 @@ struct xl_stats {
 #define CSR_READ_1(sc, reg)		\
 	bus_space_read_1(sc->xl_btag, sc->xl_bhandle, reg)
 
-#define XL_SEL_WIN(x)	\
-	CSR_WRITE_2(sc, XL_COMMAND, XL_CMD_WINSEL | x)
+#define CSR_BARRIER(sc, reg, length, flags)				\
+	bus_space_barrier(sc->xl_btag, sc->xl_bhandle, reg, length, flags)
+
+#define XL_SEL_WIN(x) do {						\
+	CSR_BARRIER(sc, XL_COMMAND, 2,					\
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);		\
+	CSR_WRITE_2(sc, XL_COMMAND, XL_CMD_WINSEL | x);			\
+	CSR_BARRIER(sc, XL_COMMAND, 2,					\
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);		\
+} while (0)
+
 #define XL_TIMEOUT		1000
 
 /*
diff --git a/sys/fs/devfs/devfs_rule.c b/sys/fs/devfs/devfs_rule.c
index d26f3504..ea05797 100644
--- a/sys/fs/devfs/devfs_rule.c
+++ b/sys/fs/devfs/devfs_rule.c
@@ -617,7 +617,7 @@ devfs_rule_run(struct devfs_krule *dk, struct devfs_dirent *de, unsigned depth)
 		 * XXX: not work as this is called when devices are created
 		 * XXX: long time after the rules were instantiated.
 		 * XXX: a printf() would probably give too much noise, or
-		 * XXX: DoS the machine.  I guess a a rate-limited message
+		 * XXX: DoS the machine.  I guess a rate-limited message
 		 * XXX: might work.
 		 */
 		if (depth > 0) {
diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c
index 68ab7ce..22908b9 100644
--- a/sys/fs/devfs/devfs_vnops.c
+++ b/sys/fs/devfs/devfs_vnops.c
@@ -261,7 +261,7 @@ devfs_vptocnp(struct vop_vptocnp_args *ap)
 	} else if (vp->v_type == VDIR) {
 		if (dd == dmp->dm_rootdir) {
 			*dvp = vp;
-			vhold(*dvp);
+			vref(*dvp);
 			goto finished;
 		}
 		i -= dd->de_dirent->d_namlen;
@@ -289,6 +289,8 @@ devfs_vptocnp(struct vop_vptocnp_args *ap)
 		mtx_unlock(&devfs_de_interlock);
 		vholdl(*dvp);
 		VI_UNLOCK(*dvp);
+		vref(*dvp);
+		vdrop(*dvp);
 	} else {
 		mtx_unlock(&devfs_de_interlock);
 		error = ENOENT;
@@ -604,6 +606,13 @@ devfs_close_f(struct file *fp, struct thread *td)
 	td->td_fpop = fp;
 	error = vnops.fo_close(fp, td);
 	td->td_fpop = fpop;
+
+	/*
+	 * The f_cdevpriv cannot be assigned non-NULL value while we
+	 * are destroying the file.
+	 */
+	if (fp->f_cdevpriv != NULL)
+		devfs_fpdrop(fp);
 	return (error);
 }
 
@@ -1050,6 +1059,10 @@ devfs_open(struct vop_open_args *ap)
 	dsw = dev_refthread(dev, &ref);
 	if (dsw == NULL)
 		return (ENXIO);
+	if (fp == NULL && dsw->d_fdopen != NULL) {
+		dev_relthread(dev, ref);
+		return (ENXIO);
+	}
 
 	vlocked = VOP_ISLOCKED(vp);
 	VOP_UNLOCK(vp, 0);
diff --git a/sys/fs/ext2fs/ext2_alloc.c b/sys/fs/ext2fs/ext2_alloc.c
index 635a6e2..2f19b87 100644
--- a/sys/fs/ext2fs/ext2_alloc.c
+++ b/sys/fs/ext2fs/ext2_alloc.c
@@ -151,7 +151,7 @@ nospace:
  */
 
 #ifdef FANCY_REALLOC
-SYSCTL_NODE(_vfs, OID_AUTO, ext2fs, CTLFLAG_RW, 0, "EXT2FS filesystem");
+static SYSCTL_NODE(_vfs, OID_AUTO, ext2fs, CTLFLAG_RW, 0, "EXT2FS filesystem");
 
 static int doasyncfree = 1;
 SYSCTL_INT(_vfs_ext2fs, OID_AUTO, doasyncfree, CTLFLAG_RW, &doasyncfree, 0,
diff --git a/sys/fs/msdosfs/msdosfs_conv.c b/sys/fs/msdosfs/msdosfs_conv.c
index 8dc1d07..fde6512 100644
--- a/sys/fs/msdosfs/msdosfs_conv.c
+++ b/sys/fs/msdosfs/msdosfs_conv.c
@@ -61,9 +61,9 @@
 extern struct iconv_functions *msdosfs_iconv;
 
 static int mbsadjpos(const char **, size_t, size_t, int, int, void *handle);
-static u_int16_t dos2unixchr(const u_char **, size_t *, int, struct msdosfsmount *);
+static u_char * dos2unixchr(const u_char **, size_t *, int, struct msdosfsmount *);
 static u_int16_t unix2doschr(const u_char **, size_t *, struct msdosfsmount *);
-static u_int16_t win2unixchr(u_int16_t, struct msdosfsmount *);
+static u_char * win2unixchr(u_int16_t, struct msdosfsmount *);
 static u_int16_t unix2winchr(const u_char **, size_t *, int, struct msdosfsmount *);
 
 /*
@@ -242,7 +242,7 @@ dos2unixfn(dn, un, lower, pmp)
 {
 	size_t i;
 	int thislong = 0;
-	u_int16_t c;
+	u_char *c;
 
 	/*
 	 * If first char of the filename is SLOT_E5 (0x05), then the real
@@ -259,12 +259,10 @@ dos2unixfn(dn, un, lower, pmp)
 	for (i = 8; i > 0 && *dn != ' ';) {
 		c = dos2unixchr((const u_char **)&dn, &i, lower & LCASE_BASE,
 		    pmp);
-		if (c & 0xff00) {
-			*un++ = c >> 8;
+		while (*c != '\0') {
+			*un++ = *c++;
 			thislong++;
 		}
-		*un++ = c;
-		thislong++;
 	}
 	dn += i;
 
@@ -278,12 +276,10 @@ dos2unixfn(dn, un, lower, pmp)
 		for (i = 3; i > 0 && *dn != ' ';) {
 			c = dos2unixchr((const u_char **)&dn, &i,
 			    lower & LCASE_EXT, pmp);
-			if (c & 0xff00) {
-				*un++ = c >> 8;
+			while (*c != '\0') {
+				*un++ = *c++;
 				thislong++;
 			}
-			*un++ = c;
-			thislong++;
 		}
 	}
 	*un++ = 0;
@@ -652,8 +648,9 @@ win2unixfn(nbp, wep, chksum, pmp)
 	int chksum;
 	struct msdosfsmount *pmp;
 {
+	u_char *c;
 	u_int8_t *cp;
-	u_int8_t *np, name[WIN_CHARS * 2 + 1];
+	u_int8_t *np, name[WIN_CHARS * 3 + 1];
 	u_int16_t code;
 	int i;
 
@@ -686,10 +683,9 @@ win2unixfn(nbp, wep, chksum, pmp)
 			*np = '\0';
 			return -1;
 		default:
-			code = win2unixchr(code, pmp);
-			if (code & 0xff00)
-				*np++ = code >> 8;
-			*np++ = code;
+			c = win2unixchr(code, pmp);
+			while (*c != '\0')
+				*np++ = *c++;
 			break;
 		}
 		cp += 2;
@@ -705,10 +701,9 @@ win2unixfn(nbp, wep, chksum, pmp)
 			*np = '\0';
 			return -1;
 		default:
-			code = win2unixchr(code, pmp);
-			if (code & 0xff00)
-				*np++ = code >> 8;
-			*np++ = code;
+			c = win2unixchr(code, pmp);
+			while (*c != '\0')
+				*np++ = *c++;
 			break;
 		}
 		cp += 2;
@@ -724,10 +719,9 @@ win2unixfn(nbp, wep, chksum, pmp)
 			*np = '\0';
 			return -1;
 		default:
-			code = win2unixchr(code, pmp);
-			if (code & 0xff00)
-				*np++ = code >> 8;
-			*np++ = code;
+			c = win2unixchr(code, pmp);
+			while (*c != '\0')
+				*np++ = *c++;
 			break;
 		}
 		cp += 2;
@@ -817,24 +811,22 @@ mbsadjpos(const char **instr, size_t inlen, size_t outlen, int weight, int flag,
 /*
  * Convert DOS char to Local char
  */
-static u_int16_t
+static u_char *
 dos2unixchr(const u_char **instr, size_t *ilen, int lower, struct msdosfsmount *pmp)
 {
-	u_char c;
-	char *outp, outbuf[3];
-	u_int16_t wc;
+	u_char c, *outp, outbuf[5];
 	size_t len, olen;
 
+	outp = outbuf;
 	if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) {
-		olen = len = 2;
-		outp = outbuf;
+		olen = len = 4;
 
 		if (lower & (LCASE_BASE | LCASE_EXT))
 			msdosfs_iconv->convchr_case(pmp->pm_d2u, (const char **)instr,
-						  ilen, &outp, &olen, KICONV_LOWER);
+						  ilen, (char **)&outp, &olen, KICONV_LOWER);
 		else
 			msdosfs_iconv->convchr(pmp->pm_d2u, (const char **)instr,
-					     ilen, &outp, &olen);
+					     ilen, (char **)&outp, &olen);
 		len -= olen;
 
 		/*
@@ -843,21 +835,21 @@ dos2unixchr(const u_char **instr, size_t *ilen, int lower, struct msdosfsmount *
 		if (len == 0) {
 			(*ilen)--;
 			(*instr)++;
-			return ('?');
+			*outp++ = '?';
 		}
-
-		wc = 0;
-		while(len--)
-			wc |= (*(outp - len - 1) & 0xff) << (len << 3);
-		return (wc);
+	} else {
+		(*ilen)--;
+		c = *(*instr)++;
+		c = dos2unix[c];
+		if (lower & (LCASE_BASE | LCASE_EXT))
+			c = u2l[c];
+		*outp++ = c;
+		outbuf[1] = '\0';
 	}
 
-	(*ilen)--;
-	c = *(*instr)++;
-	c = dos2unix[c];
-	if (lower & (LCASE_BASE | LCASE_EXT))
-		c = u2l[c];
-	return ((u_int16_t)c);
+	*outp = '\0';
+	outp = outbuf;
+	return (outp);
 }
 
 /*
@@ -940,23 +932,21 @@ unix2doschr(const u_char **instr, size_t *ilen, struct msdosfsmount *pmp)
 /*
  * Convert Windows char to Local char
  */
-static u_int16_t
+static u_char *
 win2unixchr(u_int16_t wc, struct msdosfsmount *pmp)
 {
-	u_char *inp, *outp, inbuf[3], outbuf[3];
+	u_char *inp, *outp, inbuf[3], outbuf[5];
 	size_t ilen, olen, len;
 
-	if (wc == 0)
-		return (0);
-
+	outp = outbuf;
 	if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) {
 		inbuf[0] = (u_char)(wc>>8);
 		inbuf[1] = (u_char)wc;
 		inbuf[2] = '\0';
 
-		ilen = olen = len = 2;
+		ilen = 2;
+		olen = len = 4;
 		inp = inbuf;
-		outp = outbuf;
 		msdosfs_iconv->convchr(pmp->pm_w2u, (const char **)&inp, &ilen,
 				     (char **)&outp, &olen);
 		len -= olen;
@@ -964,21 +954,15 @@ win2unixchr(u_int16_t wc, struct msdosfsmount *pmp)
 		/*
 		 * return '?' if failed to convert
 		 */
-		if (len == 0) {
-			wc = '?';
-			return (wc);
-		}
-
-		wc = 0;
-		while(len--)
-			wc |= (*(outp - len - 1) & 0xff) << (len << 3);
-		return (wc);
+		if (len == 0)
+			*outp++ = '?';
+	} else {
+		*outp++ = (wc & 0xff00) ? '?' : (u_char)(wc & 0xff);
 	}
 
-	if (wc & 0xff00)
-		wc = '?';
-
-	return (wc);
+	*outp = '\0';
+	outp = outbuf;
+	return (outp);
 }
 
 /*
diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index f095641..d733073 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -418,8 +418,8 @@ int nfsrpc_pathconf(vnode_t, struct nfsv3_pathconf *,
     struct ucred *, NFSPROC_T *, struct nfsvattr *, int *, void *);
 int nfsrpc_renew(struct nfsclclient *, struct ucred *,
     NFSPROC_T *);
-int nfsrpc_rellockown(struct nfsmount *, struct nfscllockowner *,
-    struct ucred *, NFSPROC_T *);
+int nfsrpc_rellockown(struct nfsmount *, struct nfscllockowner *, uint8_t *,
+    int, struct ucred *, NFSPROC_T *);
 int nfsrpc_getdirpath(struct nfsmount *, u_char *, struct ucred *,
     NFSPROC_T *);
 int nfsrpc_delegreturn(struct nfscldeleg *, struct ucred *,
diff --git a/sys/fs/nfs/nfsclstate.h b/sys/fs/nfs/nfsclstate.h
index bb46220..006ef3f 100644
--- a/sys/fs/nfs/nfsclstate.h
+++ b/sys/fs/nfs/nfsclstate.h
@@ -48,7 +48,6 @@ struct nfsclclient {
 	struct nfsclownerhead	nfsc_owner;
 	struct nfscldeleghead	nfsc_deleg;
 	struct nfscldeleghash	nfsc_deleghash[NFSCLDELEGHASHSIZE];
-	struct nfscllockownerhead nfsc_defunctlockowner;
 	struct nfsv4lock nfsc_lock;
 	struct proc	*nfsc_renewthread;
 	struct nfsmount	*nfsc_nmp;
diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c
index b88f7b8..4fa5460 100644
--- a/sys/fs/nfsclient/nfs_clrpcops.c
+++ b/sys/fs/nfsclient/nfs_clrpcops.c
@@ -649,7 +649,8 @@ nfsrpc_doclose(struct nfsmount *nmp, struct nfsclopen *op, NFSPROC_T *p)
 		 * puts on the wire has the file handle for this file appended
 		 * to it, so it can be done now.
 		 */
-		(void)nfsrpc_rellockown(nmp, lp, tcred, p);
+		(void)nfsrpc_rellockown(nmp, lp, lp->nfsl_open->nfso_fh,
+		    lp->nfsl_open->nfso_fhlen, tcred, p);
 	}
 
 	/*
@@ -4027,7 +4028,7 @@ nfsrpc_renew(struct nfsclclient *clp, struct ucred *cred, NFSPROC_T *p)
  */
 APPLESTATIC int
 nfsrpc_rellockown(struct nfsmount *nmp, struct nfscllockowner *lp,
-    struct ucred *cred, NFSPROC_T *p)
+    uint8_t *fh, int fhlen, struct ucred *cred, NFSPROC_T *p)
 {
 	struct nfsrv_descript nfsd, *nd = &nfsd;
 	u_int32_t *tl;
@@ -4039,10 +4040,8 @@ nfsrpc_rellockown(struct nfsmount *nmp, struct nfscllockowner *lp,
 	*tl++ = nmp->nm_clp->nfsc_clientid.lval[0];
 	*tl = nmp->nm_clp->nfsc_clientid.lval[1];
 	NFSBCOPY(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN);
-	NFSBCOPY(lp->nfsl_open->nfso_fh, &own[NFSV4CL_LOCKNAMELEN],
-	    lp->nfsl_open->nfso_fhlen);
-	(void)nfsm_strtom(nd, own, NFSV4CL_LOCKNAMELEN +
-	    lp->nfsl_open->nfso_fhlen);
+	NFSBCOPY(fh, &own[NFSV4CL_LOCKNAMELEN], fhlen);
+	(void)nfsm_strtom(nd, own, NFSV4CL_LOCKNAMELEN + fhlen);
 	nd->nd_flag |= ND_USEGSSNAME;
 	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred,
 	    NFS_PROG, NFS_VER4, NULL, 1, NULL);
diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c
index 7ce2b8e..b18938b 100644
--- a/sys/fs/nfsclient/nfs_clstate.c
+++ b/sys/fs/nfsclient/nfs_clstate.c
@@ -64,8 +64,8 @@ __FBSDID("$FreeBSD$");
  *     the granularity of the OpenOwner, then code must be added to
  *     serialize Ops on the OpenOwner.)
  * - When to get rid of OpenOwners and LockOwners.
- *   - When a process exits, it calls nfscl_cleanup(), which goes
- *     through the client list looking for all Open and Lock Owners.
+ *   - The function nfscl_cleanup_common() is executed after a process exits.
+ *     It goes through the client list looking for all Open and Lock Owners.
  *     When one is found, it is marked "defunct" or in the case of
  *     an OpenOwner without any Opens, freed.
  *     The renew thread scans for defunct Owners and gets rid of them,
@@ -699,7 +699,6 @@ nfscl_getcl(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 {
 	struct nfsclclient *clp;
 	struct nfsclclient *newclp = NULL;
-	struct nfscllockowner *lp, *nlp;
 	struct mount *mp;
 	struct nfsmount *nmp;
 	char uuid[HOSTUUIDLEN];
@@ -744,7 +743,6 @@ nfscl_getcl(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 		TAILQ_INIT(&clp->nfsc_deleg);
 		for (i = 0; i < NFSCLDELEGHASHSIZE; i++)
 			LIST_INIT(&clp->nfsc_deleghash[i]);
-		LIST_INIT(&clp->nfsc_defunctlockowner);
 		clp->nfsc_flags = NFSCLFLAGS_INITED;
 		clp->nfsc_clientidrev = 1;
 		clp->nfsc_cbident = nfscl_nextcbident();
@@ -793,11 +791,6 @@ nfscl_getcl(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 			NFSUNLOCKCLSTATE();
 			return (EACCES);
 		}
-		/* get rid of defunct lockowners */
-		LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list,
-		    nlp) {
-			nfscl_freelockowner(lp, 0);
-		}
 		/*
 		 * If RFC3530 Sec. 14.2.33 is taken literally,
 		 * NFSERR_CLIDINUSE will be returned persistently for the
@@ -1537,13 +1530,6 @@ nfscl_cleanclient(struct nfsclclient *clp)
 {
 	struct nfsclowner *owp, *nowp;
 	struct nfsclopen *op, *nop;
-	struct nfscllockowner *lp, *nlp;
-
-
-	/* get rid of defunct lockowners */
-	LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list, nlp) {
-		nfscl_freelockowner(lp, 0);
-	}
 
 	/* Now, all the OpenOwners, etc. */
 	LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
@@ -1644,33 +1630,9 @@ nfscl_expireclient(struct nfsclclient *clp, struct nfsmount *nmp,
 	}
 }
 
-#ifndef	__FreeBSD__
 /*
- * Called from exit() upon process termination.
- */
-APPLESTATIC void
-nfscl_cleanup(NFSPROC_T *p)
-{
-	struct nfsclclient *clp;
-	u_int8_t own[NFSV4CL_LOCKNAMELEN];
-
-	if (!nfscl_inited)
-		return;
-	nfscl_filllockowner(p->td_proc, own, F_POSIX);
-
-	NFSLOCKCLSTATE();
-	/*
-	 * Loop through all the clientids, looking for the OpenOwners.
-	 */
-	LIST_FOREACH(clp, &nfsclhead, nfsc_list)
-		nfscl_cleanup_common(clp, own);
-	NFSUNLOCKCLSTATE();
-}
-#endif	/* !__FreeBSD__ */
-
-/*
- * Common code used by nfscl_cleanup() and nfscl_cleanupkext().
- * Must be called with CLSTATE lock held.
+ * This function must be called after the process represented by "own" has
+ * exited. Must be called with CLSTATE lock held.
  */
 static void
 nfscl_cleanup_common(struct nfsclclient *clp, u_int8_t *own)
@@ -1717,25 +1679,15 @@ nfscl_cleanup_common(struct nfsclclient *clp, u_int8_t *own)
 		}
 		owp = nowp;
 	}
-
-	/* and check the defunct list */
-	LIST_FOREACH(lp, &clp->nfsc_defunctlockowner, nfsl_list) {
-		if (!NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN))
-		    lp->nfsl_defunct = 1;
-	}
 }
 
-#if defined(APPLEKEXT) || defined(__FreeBSD__)
 /*
- * Simulate the call nfscl_cleanup() by looking for open owners associated
- * with processes that no longer exist, since a call to nfscl_cleanup()
- * can't be patched into exit().
+ * Find open/lock owners for processes that have exited.
  */
 static void
 nfscl_cleanupkext(struct nfsclclient *clp)
 {
 	struct nfsclowner *owp, *nowp;
-	struct nfscllockowner *lp;
 
 	NFSPROCLISTLOCK();
 	NFSLOCKCLSTATE();
@@ -1743,16 +1695,9 @@ nfscl_cleanupkext(struct nfsclclient *clp)
 		if (nfscl_procdoesntexist(owp->nfsow_owner))
 			nfscl_cleanup_common(clp, owp->nfsow_owner);
 	}
-
-	/* and check the defunct list */
-	LIST_FOREACH(lp, &clp->nfsc_defunctlockowner, nfsl_list) {
-		if (nfscl_procdoesntexist(lp->nfsl_owner))
-			lp->nfsl_defunct = 1;
-	}
 	NFSUNLOCKCLSTATE();
 	NFSPROCLISTUNLOCK();
 }
-#endif	/* APPLEKEXT || __FreeBSD__ */
 
 static int	fake_global;	/* Used to force visibility of MNTK_UNMOUNTF */
 /*
@@ -1905,11 +1850,6 @@ nfscl_recover(struct nfsclclient *clp, struct ucred *cred, NFSPROC_T *p)
 	NFSUNLOCKREQ();
 	splx(s);
 
-	/* get rid of defunct lockowners */
-	LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list, nlp) {
-		nfscl_freelockowner(lp, 0);
-	}
-
 	/*
 	 * Now, mark all delegations "need reclaim".
 	 */
@@ -2157,7 +2097,6 @@ nfscl_recover(struct nfsclclient *clp, struct ucred *cred, NFSPROC_T *p)
 APPLESTATIC int
 nfscl_hasexpired(struct nfsclclient *clp, u_int32_t clidrev, NFSPROC_T *p)
 {
-	struct nfscllockowner *lp, *nlp;
 	struct nfsmount *nmp;
 	struct ucred *cred;
 	int igotlock = 0, error, trycnt;
@@ -2207,12 +2146,6 @@ nfscl_hasexpired(struct nfsclclient *clp, u_int32_t clidrev, NFSPROC_T *p)
 		clp->nfsc_flags &= ~(NFSCLFLAGS_HASCLIENTID |
 		    NFSCLFLAGS_RECOVER);
 	} else {
-		/* get rid of defunct lockowners */
-		LIST_FOREACH_SAFE(lp, &clp->nfsc_defunctlockowner, nfsl_list,
-		    nlp) {
-			nfscl_freelockowner(lp, 0);
-		}
-
 		/*
 		 * Expire the state for the client.
 		 */
@@ -2406,6 +2339,8 @@ nfscl_renewthread(struct nfsclclient *clp, NFSPROC_T *p)
 	u_int32_t clidrev;
 	int error, cbpathdown, islept, igotlock, ret, clearok;
 	uint32_t recover_done_time = 0;
+	struct timespec mytime;
+	static time_t prevsec = 0;
 
 	cred = newnfs_getcred();
 	NFSLOCKCLSTATE();
@@ -2486,25 +2421,6 @@ nfscl_renewthread(struct nfsclclient *clp, NFSPROC_T *p)
 		    owp = nowp;
 		}
 
-		/* also search the defunct list */
-		lp = LIST_FIRST(&clp->nfsc_defunctlockowner);
-		while (lp != NULL) {
-		    nlp = LIST_NEXT(lp, nfsl_list);
-		    if (lp->nfsl_defunct) {
-			LIST_FOREACH(olp, &lh, nfsl_list) {
-			    if (!NFSBCMP(olp->nfsl_owner, lp->nfsl_owner,
-				NFSV4CL_LOCKNAMELEN))
-				break;
-			}
-			if (olp == NULL) {
-			    LIST_REMOVE(lp, nfsl_list);
-			    LIST_INSERT_HEAD(&lh, lp, nfsl_list);
-			} else {
-			    nfscl_freelockowner(lp, 0);
-			}
-		    }
-		    lp = nlp;
-		}
 		/* and release defunct lock owners */
 		LIST_FOREACH_SAFE(lp, &lh, nfsl_list, nlp) {
 		    nfscl_freelockowner(lp, 0);
@@ -2612,22 +2528,15 @@ tryagain:
 			FREE((caddr_t)dp, M_NFSCLDELEG);
 		}
 
-#if defined(APPLEKEXT) || defined(__FreeBSD__)
 		/*
-		 * Simulate the calls to nfscl_cleanup() when a process
-		 * exits, since the call can't be patched into exit().
+		 * Call nfscl_cleanupkext() once per second to check for
+		 * open/lock owners where the process has exited.
 		 */
-		{
-			struct timespec mytime;
-			static time_t prevsec = 0;
-
-			NFSGETNANOTIME(&mytime);
-			if (prevsec != mytime.tv_sec) {
-				prevsec = mytime.tv_sec;
-				nfscl_cleanupkext(clp);
-			}
+		NFSGETNANOTIME(&mytime);
+		if (prevsec != mytime.tv_sec) {
+			prevsec = mytime.tv_sec;
+			nfscl_cleanupkext(clp);
 		}
-#endif	/* APPLEKEXT || __FreeBSD__ */
 
 		NFSLOCKCLSTATE();
 		if ((clp->nfsc_flags & NFSCLFLAGS_RECOVER) == 0)
diff --git a/sys/fs/nfsclient/nfs_clvfsops.c b/sys/fs/nfsclient/nfs_clvfsops.c
index 3339428..461980e 100644
--- a/sys/fs/nfsclient/nfs_clvfsops.c
+++ b/sys/fs/nfsclient/nfs_clvfsops.c
@@ -715,7 +715,7 @@ static const char *nfs_opts[] = { "from", "nfs_args",
     "retrans", "acregmin", "acregmax", "acdirmin", "acdirmax", "resvport",
     "readahead", "hostname", "timeout", "addr", "fh", "nfsv3", "sec",
     "principal", "nfsv4", "gssname", "allgssname", "dirpath",
-    "negnametimeo", "nocto",
+    "negnametimeo", "nocto", "wcommitsize",
     NULL };
 
 /*
@@ -949,6 +949,15 @@ nfs_mount(struct mount *mp)
 		}
 		args.flags |= NFSMNT_ACDIRMAX;
 	}
+	if (vfs_getopt(mp->mnt_optnew, "wcommitsize", (void **)&opt, NULL) == 0) {
+		ret = sscanf(opt, "%d", &args.wcommitsize);
+		if (ret != 1 || args.wcommitsize < 0) {
+			vfs_mount_error(mp, "illegal wcommitsize: %s", opt);
+			error = EINVAL;
+			goto out;
+		}
+		args.flags |= NFSMNT_WCOMMITSIZE;
+	}
 	if (vfs_getopt(mp->mnt_optnew, "timeout", (void **)&opt, NULL) == 0) {
 		ret = sscanf(opt, "%d", &args.timeo);
 		if (ret != 1 || args.timeo <= 0) {
@@ -1212,7 +1221,20 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
 	vfs_getnewfsid(mp);
 	nmp->nm_mountp = mp;
 	mtx_init(&nmp->nm_mtx, "NFSmount lock", NULL, MTX_DEF | MTX_DUPOK);			
+
+	/*
+	 * Since nfs_decode_args() might optionally set them, these need to
+	 * set to defaults before the call, so that the optional settings
+	 * aren't overwritten.
+	 */
 	nmp->nm_negnametimeo = negnametimeo;
+	nmp->nm_timeo = NFS_TIMEO;
+	nmp->nm_retry = NFS_RETRANS;
+	nmp->nm_readahead = NFS_DEFRAHEAD;
+	if (desiredvnodes >= 11000)
+		nmp->nm_wcommitsize = hibufspace / (desiredvnodes / 1000);
+	else
+		nmp->nm_wcommitsize = hibufspace / 10;
 
 	nfs_decode_args(mp, nmp, argp, hst, cred, td);
 
@@ -1223,24 +1245,18 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
 	 *
 	 * For V3, ncl_fsinfo will adjust this as necessary.  Assume maximum
 	 * that we can handle until we find out otherwise.
-	 * XXX Our "safe" limit on the client is what we can store in our
-	 * buffer cache using signed(!) block numbers.
 	 */
 	if ((argp->flags & (NFSMNT_NFSV3 | NFSMNT_NFSV4)) == 0)
 		nmp->nm_maxfilesize = 0xffffffffLL;
 	else
 		nmp->nm_maxfilesize = OFF_MAX;
 
-	nmp->nm_timeo = NFS_TIMEO;
-	nmp->nm_retry = NFS_RETRANS;
 	if ((argp->flags & (NFSMNT_NFSV3 | NFSMNT_NFSV4)) == 0) {
 		nmp->nm_wsize = NFS_WSIZE;
 		nmp->nm_rsize = NFS_RSIZE;
 		nmp->nm_readdirsize = NFS_READDIRSIZE;
 	}
-	nmp->nm_wcommitsize = hibufspace / (desiredvnodes / 1000);
 	nmp->nm_numgrps = NFS_MAXGRPS;
-	nmp->nm_readahead = NFS_DEFRAHEAD;
 	nmp->nm_tprintf_delay = nfs_tprintf_delay;
 	if (nmp->nm_tprintf_delay < 0)
 		nmp->nm_tprintf_delay = 0;
diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c
index 7e87b6e..8bc0be9 100644
--- a/sys/fs/nfsclient/nfs_clvnops.c
+++ b/sys/fs/nfsclient/nfs_clvnops.c
@@ -2580,6 +2580,16 @@ nfs_strategy(struct vop_strategy_args *ap)
 static int
 nfs_fsync(struct vop_fsync_args *ap)
 {
+
+	if (ap->a_vp->v_type != VREG) {
+		/*
+		 * For NFS, metadata is changed synchronously on the server,
+		 * so there is nothing to flush. Also, ncl_flush() clears
+		 * the NMODIFIED flag and that shouldn't be done here for
+		 * directories.
+		 */
+		return (0);
+	}
 	return (ncl_flush(ap->a_vp, ap->a_waitfor, NULL, ap->a_td, 1, 0));
 }
 
@@ -2941,6 +2951,8 @@ nfs_advlock(struct vop_advlock_args *ap)
 	u_quad_t size;
 	
 	if (NFS_ISV4(vp) && (ap->a_flags & (F_POSIX | F_FLOCK)) != 0) {
+		if (vp->v_type != VREG)
+			return (EINVAL);
 		if ((ap->a_flags & F_POSIX) != 0)
 			cred = p->p_ucred;
 		else
diff --git a/sys/fs/ntfs/ntfs_subr.c b/sys/fs/ntfs/ntfs_subr.c
index a6c3b85..f5071d7 100644
--- a/sys/fs/ntfs/ntfs_subr.c
+++ b/sys/fs/ntfs/ntfs_subr.c
@@ -51,10 +51,12 @@
 #include <fs/ntfs/ntfs_compr.h>
 #include <fs/ntfs/ntfs_ihash.h>
 
-MALLOC_DEFINE(M_NTFSNTVATTR, "ntfs_vattr", "NTFS file attribute information");
-MALLOC_DEFINE(M_NTFSRDATA, "ntfsd_resdata", "NTFS resident data");
-MALLOC_DEFINE(M_NTFSRUN, "ntfs_vrun", "NTFS vrun storage");
-MALLOC_DEFINE(M_NTFSDECOMP, "ntfs_decomp", "NTFS decompression temporary");
+static MALLOC_DEFINE(M_NTFSNTVATTR, "ntfs_vattr",
+    "NTFS file attribute information");
+static MALLOC_DEFINE(M_NTFSRDATA, "ntfsd_resdata", "NTFS resident data");
+static MALLOC_DEFINE(M_NTFSRUN, "ntfs_vrun", "NTFS vrun storage");
+static MALLOC_DEFINE(M_NTFSDECOMP, "ntfs_decomp",
+    "NTFS decompression temporary");
 
 static int ntfs_ntlookupattr(struct ntfsmount *, const char *, int, int *, char **);
 static int ntfs_findvattr(struct ntfsmount *, struct ntnode *, struct ntvattr **, struct ntvattr **, u_int32_t, const char *, size_t, cn_t);
diff --git a/sys/fs/nullfs/null_subr.c b/sys/fs/nullfs/null_subr.c
index 5717a01..a45ccf7 100644
--- a/sys/fs/nullfs/null_subr.c
+++ b/sys/fs/nullfs/null_subr.c
@@ -289,22 +289,12 @@ null_checkvp(vp, fil, lno)
 #endif
 	if (a->null_lowervp == NULLVP) {
 		/* Should never happen */
-		int i; u_long *p;
-		printf("vp = %p, ZERO ptr\n", (void *)vp);
-		for (p = (u_long *) a, i = 0; i < 8; i++)
-			printf(" %lx", p[i]);
-		printf("\n");
-		panic("null_checkvp");
+		panic("null_checkvp %p", vp);
 	}
 	VI_LOCK_FLAGS(a->null_lowervp, MTX_DUPOK);
-	if (a->null_lowervp->v_usecount < 1) {
-		int i; u_long *p;
-		printf("vp = %p, unref'ed lowervp\n", (void *)vp);
-		for (p = (u_long *) a, i = 0; i < 8; i++)
-			printf(" %lx", p[i]);
-		printf("\n");
-		panic ("null with unref'ed lowervp");
-	}
+	if (a->null_lowervp->v_usecount < 1)
+		panic ("null with unref'ed lowervp, vp %p lvp %p",
+		    vp, a->null_lowervp);
 	VI_UNLOCK(a->null_lowervp);
 #ifdef notyet
 	printf("null %x/%d -> %x/%d [%s, %d]\n",
diff --git a/sys/fs/nullfs/null_vnops.c b/sys/fs/nullfs/null_vnops.c
index 30a38da..bcf8750 100644
--- a/sys/fs/nullfs/null_vnops.c
+++ b/sys/fs/nullfs/null_vnops.c
@@ -729,7 +729,7 @@ null_print(struct vop_print_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 
-	printf("\tvp=%p, lowervp=%p\n", vp, NULLVPTOLOWERVP(vp));
+	printf("\tvp=%p, lowervp=%p\n", vp, VTONULL(vp)->null_lowervp);
 	return (0);
 }
 
@@ -784,6 +784,7 @@ null_vptocnp(struct vop_vptocnp_args *ap)
 	vhold(lvp);
 	VOP_UNLOCK(vp, 0); /* vp is held by vn_vptocnp_locked that called us */
 	ldvp = lvp;
+	vref(lvp);
 	error = vn_vptocnp(&ldvp, cred, ap->a_buf, ap->a_buflen);
 	vdrop(lvp);
 	if (error != 0) {
@@ -797,19 +798,17 @@ null_vptocnp(struct vop_vptocnp_args *ap)
 	 */
 	error = vn_lock(ldvp, LK_EXCLUSIVE);
 	if (error != 0) {
+		vrele(ldvp);
 		vn_lock(vp, locked | LK_RETRY);
-		vdrop(ldvp);
 		return (ENOENT);
 	}
 	vref(ldvp);
-	vdrop(ldvp);
 	error = null_nodeget(vp->v_mount, ldvp, dvp);
 	if (error == 0) {
 #ifdef DIAGNOSTIC
 		NULLVPTOLOWERVP(*dvp);
 #endif
-		vhold(*dvp);
-		vput(*dvp);
+		VOP_UNLOCK(*dvp, 0); /* keep reference on *dvp */
 	} else
 		vput(ldvp);
 
diff --git a/sys/fs/nwfs/nwfs_mount.h b/sys/fs/nwfs/nwfs_mount.h
index a1670942..92c11af 100644
--- a/sys/fs/nwfs/nwfs_mount.h
+++ b/sys/fs/nwfs/nwfs_mount.h
@@ -65,11 +65,4 @@ struct nwfs_args {
 	int		tz;
 };
 
-#ifdef _KERNEL
-
-#ifdef MALLOC_DECLARE
-MALLOC_DECLARE(M_NWFSMNT);
-#endif
-
-#endif
 #endif /* !_NWFS_MOUNT_H_ */
diff --git a/sys/fs/procfs/procfs_status.c b/sys/fs/procfs/procfs_status.c
index 7850504..4bfa06f 100644
--- a/sys/fs/procfs/procfs_status.c
+++ b/sys/fs/procfs/procfs_status.c
@@ -113,12 +113,14 @@ procfs_doprocstatus(PFS_FILL_ARGS)
 	}
 
 	tdfirst = FIRST_THREAD_IN_PROC(p);
+	thread_lock(tdfirst);
 	if (tdfirst->td_wchan != NULL) {
 		KASSERT(tdfirst->td_wmesg != NULL,
 		    ("wchan %p has no wmesg", tdfirst->td_wchan));
 		wmesg = tdfirst->td_wmesg;
 	} else
 		wmesg = "nochan";
+	thread_unlock(tdfirst);
 
 	if (p->p_flag & P_INMEM) {
 		struct timeval start, ut, st;
@@ -175,7 +177,7 @@ procfs_doproccmdline(PFS_FILL_ARGS)
 	/*
 	 * If we are using the ps/cmdline caching, use that.  Otherwise
 	 * revert back to the old way which only implements full cmdline
-	 * for the currept process and just p->p_comm for all other
+	 * for the current process and just p->p_comm for all other
 	 * processes.
 	 * Note that if the argv is no longer available, we deliberately
 	 * don't fall back on p->p_comm or return an error: the authentic
diff --git a/sys/fs/pseudofs/pseudofs_vncache.c b/sys/fs/pseudofs/pseudofs_vncache.c
index 19cc70d..41a9c35 100644
--- a/sys/fs/pseudofs/pseudofs_vncache.c
+++ b/sys/fs/pseudofs/pseudofs_vncache.c
@@ -52,7 +52,7 @@ static struct pfs_vdata *pfs_vncache;
 static eventhandler_tag pfs_exit_tag;
 static void pfs_exit(void *arg, struct proc *p);
 
-SYSCTL_NODE(_vfs_pfs, OID_AUTO, vncache, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_vfs_pfs, OID_AUTO, vncache, CTLFLAG_RW, 0,
     "pseudofs vnode cache");
 
 static int pfs_vncache_entries;
diff --git a/sys/fs/pseudofs/pseudofs_vnops.c b/sys/fs/pseudofs/pseudofs_vnops.c
index f8343a9..3f0c00e 100644
--- a/sys/fs/pseudofs/pseudofs_vnops.c
+++ b/sys/fs/pseudofs/pseudofs_vnops.c
@@ -410,8 +410,7 @@ pfs_vptocnp(struct vop_vptocnp_args *ap)
 	}
 
 	*buflen = i;
-	vhold(*dvp);
-	vput(*dvp);
+	VOP_UNLOCK(*dvp, 0);
 	vn_lock(vp, locked | LK_RETRY);
 	vfs_unbusy(mp);
 
@@ -632,8 +631,6 @@ pfs_read(struct vop_read_args *va)
 	    (offset = uio->uio_offset) != uio->uio_offset ||
 	    (resid = uio->uio_resid) != uio->uio_resid ||
 	    (buflen = offset + resid + 1) < offset || buflen > INT_MAX) {
-		if (proc != NULL)
-			PRELE(proc);
 		error = EINVAL;
 		goto ret;
 	}
@@ -713,6 +710,13 @@ pfs_iterate(struct thread *td, struct proc *proc, struct pfs_node *pd,
 	return (0);
 }
 
+/* Directory entry list */
+struct pfsentry {
+	STAILQ_ENTRY(pfsentry)	link;
+	struct dirent		entry;
+};
+STAILQ_HEAD(pfsdirentlist, pfsentry);
+
 /*
  * Return directory entries.
  */
@@ -725,12 +729,14 @@ pfs_readdir(struct vop_readdir_args *va)
 	pid_t pid = pvd->pvd_pid;
 	struct proc *p, *proc;
 	struct pfs_node *pn;
-	struct dirent *entry;
 	struct uio *uio;
+	struct pfsentry *pfsent, *pfsent2;
+	struct pfsdirentlist lst;
 	off_t offset;
 	int error, i, resid;
-	char *buf, *ent;
 
+	STAILQ_INIT(&lst);
+	error = 0;
 	KASSERT(pd->pn_info == vn->v_mount->mnt_data,
 	    ("%s(): pn_info does not match mountpoint", __func__));
 	PFS_TRACE(("%s pid %lu", pd->pn_name, (unsigned long)pid));
@@ -750,8 +756,6 @@ pfs_readdir(struct vop_readdir_args *va)
 	if (resid == 0)
 		PFS_RETURN (0);
 
-	/* can't do this while holding the proc lock... */
-	buf = malloc(resid, M_IOV, M_WAITOK | M_ZERO);
 	sx_slock(&allproc_lock);
 	pfs_lock(pd);
 
@@ -759,7 +763,6 @@ pfs_readdir(struct vop_readdir_args *va)
         if (!pfs_visible(curthread, pd, pid, &proc)) {
 		sx_sunlock(&allproc_lock);
 		pfs_unlock(pd);
-		free(buf, M_IOV);
                 PFS_RETURN (ENOENT);
 	}
 	KASSERT(pid == NO_PID || proc != NULL,
@@ -773,57 +776,64 @@ pfs_readdir(struct vop_readdir_args *va)
 				PROC_UNLOCK(proc);
 			pfs_unlock(pd);
 			sx_sunlock(&allproc_lock);
-			free(buf, M_IOV);
 			PFS_RETURN (0);
 		}
 	}
 
 	/* fill in entries */
-	ent = buf;
 	while (pfs_iterate(curthread, proc, pd, &pn, &p) != -1 &&
 	    resid >= PFS_DELEN) {
-		entry = (struct dirent *)ent;
-		entry->d_reclen = PFS_DELEN;
-		entry->d_fileno = pn_fileno(pn, pid);
+		if ((pfsent = malloc(sizeof(struct pfsentry), M_IOV,
+		    M_NOWAIT | M_ZERO)) == NULL) {
+			error = ENOMEM;
+			break;
+		}
+		pfsent->entry.d_reclen = PFS_DELEN;
+		pfsent->entry.d_fileno = pn_fileno(pn, pid);
 		/* PFS_DELEN was picked to fit PFS_NAMLEN */
 		for (i = 0; i < PFS_NAMELEN - 1 && pn->pn_name[i] != '\0'; ++i)
-			entry->d_name[i] = pn->pn_name[i];
-		entry->d_name[i] = 0;
-		entry->d_namlen = i;
+			pfsent->entry.d_name[i] = pn->pn_name[i];
+		pfsent->entry.d_name[i] = 0;
+		pfsent->entry.d_namlen = i;
 		switch (pn->pn_type) {
 		case pfstype_procdir:
 			KASSERT(p != NULL,
 			    ("reached procdir node with p == NULL"));
-			entry->d_namlen = snprintf(entry->d_name,
+			pfsent->entry.d_namlen = snprintf(pfsent->entry.d_name,
 			    PFS_NAMELEN, "%d", p->p_pid);
 			/* fall through */
 		case pfstype_root:
 		case pfstype_dir:
 		case pfstype_this:
 		case pfstype_parent:
-			entry->d_type = DT_DIR;
+			pfsent->entry.d_type = DT_DIR;
 			break;
 		case pfstype_file:
-			entry->d_type = DT_REG;
+			pfsent->entry.d_type = DT_REG;
 			break;
 		case pfstype_symlink:
-			entry->d_type = DT_LNK;
+			pfsent->entry.d_type = DT_LNK;
 			break;
 		default:
 			panic("%s has unexpected node type: %d", pn->pn_name, pn->pn_type);
 		}
-		PFS_TRACE(("%s", entry->d_name));
+		PFS_TRACE(("%s", pfsent->entry.d_name));
+		STAILQ_INSERT_TAIL(&lst, pfsent, link);
 		offset += PFS_DELEN;
 		resid -= PFS_DELEN;
-		ent += PFS_DELEN;
 	}
 	if (proc != NULL)
 		PROC_UNLOCK(proc);
 	pfs_unlock(pd);
 	sx_sunlock(&allproc_lock);
-	PFS_TRACE(("%zd bytes", ent - buf));
-	error = uiomove(buf, ent - buf, uio);
-	free(buf, M_IOV);
+	i = 0;
+	STAILQ_FOREACH_SAFE(pfsent, &lst, link, pfsent2) {
+		if (error == 0)
+			error = uiomove(&pfsent->entry, PFS_DELEN, uio);
+		free(pfsent, M_IOV);
+		i++;
+	}
+	PFS_TRACE(("%d bytes", i * PFS_DELEN));
 	PFS_RETURN (error);
 }
 
diff --git a/sys/fs/smbfs/smbfs.h b/sys/fs/smbfs/smbfs.h
index 84c79a1..486959b 100644
--- a/sys/fs/smbfs/smbfs.h
+++ b/sys/fs/smbfs/smbfs.h
@@ -61,10 +61,6 @@ struct smbfs_args {
 
 #include <sys/_sx.h>
 
-#ifdef MALLOC_DECLARE
-MALLOC_DECLARE(M_SMBFSMNT);
-#endif
-
 struct smbnode;
 struct smb_share;
 struct u_cred;
diff --git a/sys/fs/smbfs/smbfs_node.c b/sys/fs/smbfs/smbfs_node.c
index 661579c..9de8eb5 100644
--- a/sys/fs/smbfs/smbfs_node.c
+++ b/sys/fs/smbfs/smbfs_node.c
@@ -58,7 +58,7 @@
 
 extern struct vop_vector smbfs_vnodeops;	/* XXX -> .h file */
 
-MALLOC_DEFINE(M_SMBNODE, "smbufs_node", "SMBFS vnode private part");
+static MALLOC_DEFINE(M_SMBNODE, "smbufs_node", "SMBFS vnode private part");
 static MALLOC_DEFINE(M_SMBNODENAME, "smbufs_nname", "SMBFS node name");
 
 int smbfs_hashprint(struct mount *mp);
diff --git a/sys/fs/smbfs/smbfs_smb.c b/sys/fs/smbfs/smbfs_smb.c
index bebc8ee..05162d5 100644
--- a/sys/fs/smbfs/smbfs_smb.c
+++ b/sys/fs/smbfs/smbfs_smb.c
@@ -34,6 +34,7 @@
 #include <sys/vnode.h>
 #include <sys/mbuf.h>
 #include <sys/mount.h>
+#include <sys/endian.h>
 
 #ifdef USE_MD5_HASH
 #include <sys/md5.h>
@@ -393,6 +394,10 @@ smbfs_smb_setpattr(struct smbnode *np, u_int16_t attr, struct timespec *mtime,
 		if (error)
 			break;
 		mb_put_uint8(mbp, SMB_DT_ASCII);
+		if (SMB_UNICODE_STRINGS(SSTOVC(ssp))) {
+			mb_put_padbyte(mbp);
+			mb_put_uint8(mbp, 0);	/* 1st byte of NULL Unicode char */
+		}
 		mb_put_uint8(mbp, 0);
 		smb_rq_bend(rqp);
 		error = smb_rq_simple(rqp);
@@ -909,6 +914,10 @@ smbfs_smb_search(struct smbfs_fctx *ctx)
 		mb_put_uint16le(mbp, 0);	/* context length */
 		ctx->f_flags &= ~SMBFS_RDD_FINDFIRST;
 	} else {
+		if (SMB_UNICODE_STRINGS(vcp)) {
+			mb_put_padbyte(mbp);
+			mb_put_uint8(mbp, 0);
+		}
 		mb_put_uint8(mbp, 0);	/* file name length */
 		mb_put_uint8(mbp, SMB_DT_VARIABLE);
 		mb_put_uint16le(mbp, SMB_SKEYLEN);
@@ -1069,7 +1078,7 @@ smbfs_smb_trans2find2(struct smbfs_fctx *ctx)
 		mb_put_uint32le(mbp, 0);		/* resume key */
 		mb_put_uint16le(mbp, flags);
 		if (ctx->f_rname)
-			mb_put_mem(mbp, ctx->f_rname, strlen(ctx->f_rname) + 1, MB_MSYSTEM);
+			mb_put_mem(mbp, ctx->f_rname, ctx->f_rnamelen + 1, MB_MSYSTEM);
 		else
 			mb_put_uint8(mbp, 0);	/* resume file name */
 #if 0
@@ -1152,7 +1161,10 @@ static int
 smbfs_findopenLM2(struct smbfs_fctx *ctx, struct smbnode *dnp,
 	const char *wildcard, int wclen, int attr, struct smb_cred *scred)
 {
-	ctx->f_name = malloc(SMB_MAXFNAMELEN, M_SMBFSDATA, M_WAITOK);
+	if (SMB_UNICODE_STRINGS(SSTOVC(ctx->f_ssp))) {
+		ctx->f_name = malloc(SMB_MAXFNAMELEN * 2, M_SMBFSDATA, M_WAITOK);
+	} else
+		ctx->f_name = malloc(SMB_MAXFNAMELEN, M_SMBFSDATA, M_WAITOK);
 	if (ctx->f_name == NULL)
 		return ENOMEM;
 	ctx->f_infolevel = SMB_DIALECT(SSTOVC(ctx->f_ssp)) < SMB_DIALECT_NTLM0_12 ?
@@ -1231,7 +1243,10 @@ smbfs_findnextLM2(struct smbfs_fctx *ctx, int limit)
 		SMBERROR("unexpected info level %d\n", ctx->f_infolevel);
 		return EINVAL;
 	}
-	nmlen = min(size, SMB_MAXFNAMELEN);
+	if (SMB_UNICODE_STRINGS(SSTOVC(ctx->f_ssp))) {
+		nmlen = min(size, SMB_MAXFNAMELEN * 2);
+	} else
+		nmlen = min(size, SMB_MAXFNAMELEN);
 	cp = ctx->f_name;
 	error = md_get_mem(mbp, cp, nmlen, MB_MSYSTEM);
 	if (error)
@@ -1245,8 +1260,12 @@ smbfs_findnextLM2(struct smbfs_fctx *ctx, int limit)
 			return EBADRPC;
 		}
 	}
-	if (nmlen && cp[nmlen - 1] == 0)
-		nmlen--;
+	if (SMB_UNICODE_STRINGS(SSTOVC(ctx->f_ssp))) {
+		if (nmlen > 1 && cp[nmlen - 1] == 0 && cp[nmlen - 2] == 0)
+			nmlen -= 2;
+	} else
+		if (nmlen && cp[nmlen - 1] == 0)
+			nmlen--;
 	if (nmlen == 0)
 		return EBADRPC;
 
@@ -1330,10 +1349,17 @@ smbfs_findnext(struct smbfs_fctx *ctx, int limit, struct smb_cred *scred)
 			error = smbfs_findnextLM2(ctx, limit);
 		if (error)
 			return error;
-		if ((ctx->f_nmlen == 1 && ctx->f_name[0] == '.') ||
-		    (ctx->f_nmlen == 2 && ctx->f_name[0] == '.' &&
-		     ctx->f_name[1] == '.'))
-			continue;
+		if (SMB_UNICODE_STRINGS(SSTOVC(ctx->f_ssp))) {
+			if ((ctx->f_nmlen == 2 &&
+			     *(u_int16_t *)ctx->f_name == htole16(0x002e)) ||
+			    (ctx->f_nmlen == 4 &&
+			     *(u_int32_t *)ctx->f_name == htole32(0x002e002e)))
+				continue;
+		} else
+			if ((ctx->f_nmlen == 1 && ctx->f_name[0] == '.') ||
+			    (ctx->f_nmlen == 2 && ctx->f_name[0] == '.' &&
+			     ctx->f_name[1] == '.'))
+				continue;
 		break;
 	}
 	smbfs_fname_tolocal(SSTOVC(ctx->f_ssp), ctx->f_name, &ctx->f_nmlen,
diff --git a/sys/fs/smbfs/smbfs_subr.c b/sys/fs/smbfs/smbfs_subr.c
index b775dff..51ee4d5 100644
--- a/sys/fs/smbfs/smbfs_subr.c
+++ b/sys/fs/smbfs/smbfs_subr.c
@@ -130,7 +130,10 @@ smb_fphelp(struct mbchain *mbp, struct smb_vc *vcp, struct smbnode *np,
 		return smb_put_dmem(mbp, vcp, "\\", 2, caseopt);*/
 	while (i--) {
 		np = *--npp;
-		error = mb_put_uint8(mbp, '\\');
+		if (SMB_UNICODE_STRINGS(vcp))
+			error = mb_put_uint16le(mbp, '\\');
+		else
+			error = mb_put_uint8(mbp, '\\');
 		if (error)
 			break;
 		error = smb_put_dmem(mbp, vcp, np->n_name, np->n_nmlen, caseopt);
@@ -148,6 +151,11 @@ smbfs_fullpath(struct mbchain *mbp, struct smb_vc *vcp, struct smbnode *dnp,
 	int caseopt = SMB_CS_NONE;
 	int error;
 
+	if (SMB_UNICODE_STRINGS(vcp)) {
+		error = mb_put_padbyte(mbp);
+		if (error)
+			return error;
+	}
 	if (SMB_DIALECT(vcp) < SMB_DIALECT_LANMAN1_0)
 		caseopt |= SMB_CS_UPPER;
 	if (dnp != NULL) {
@@ -156,7 +164,10 @@ smbfs_fullpath(struct mbchain *mbp, struct smb_vc *vcp, struct smbnode *dnp,
 			return error;
 	}
 	if (name) {
-		error = mb_put_uint8(mbp, '\\');
+		if (SMB_UNICODE_STRINGS(vcp))
+			error = mb_put_uint16le(mbp, '\\');
+		else
+			error = mb_put_uint8(mbp, '\\');
 		if (error)
 			return error;
 		error = smb_put_dmem(mbp, vcp, name, nmlen, caseopt);
@@ -164,6 +175,8 @@ smbfs_fullpath(struct mbchain *mbp, struct smb_vc *vcp, struct smbnode *dnp,
 			return error;
 	}
 	error = mb_put_uint8(mbp, 0);
+	if (SMB_UNICODE_STRINGS(vcp) && error == 0)
+		error = mb_put_uint8(mbp, 0);
 	return error;
 }
 
@@ -191,6 +204,17 @@ smbfs_fname_tolocal(struct smb_vc *vcp, char *name, int *nmlen, int caseopt)
 
 		error = iconv_conv_case
 			(vcp->vc_tolocal, (const char **)&ibuf, &ilen, &obuf, &olen, copt);
+		if (error && SMB_UNICODE_STRINGS(vcp)) {
+			/*
+			 * If using unicode, leaving a file name as it was when
+			 * convert fails will cause a problem because the file name
+			 * will contain NULL.
+			 * Here, put '?' and give converted file name.
+			 */
+			*obuf = '?';
+			olen--;
+			error = 0;
+		}
 		if (!error) {
 			*nmlen = sizeof(outbuf) - olen;
 			memcpy(name, outbuf, *nmlen);
diff --git a/sys/fs/tmpfs/tmpfs.h b/sys/fs/tmpfs/tmpfs.h
index e95e3c8..fe00fea 100644
--- a/sys/fs/tmpfs/tmpfs.h
+++ b/sys/fs/tmpfs/tmpfs.h
@@ -502,11 +502,8 @@ int	tmpfs_truncate(struct vnode *, off_t);
 static __inline size_t
 tmpfs_mem_info(void)
 {
-	size_t size;
 
-	size = swap_pager_avail + cnt.v_free_count + cnt.v_inactive_count;
-	size -= size > cnt.v_wire_count ? cnt.v_wire_count : size;
-	return size;
+	return (swap_pager_avail + cnt.v_free_count + cnt.v_cache_count);
 }
 
 /* Returns the maximum size allowed for a tmpfs file system.  This macro
diff --git a/sys/fs/tmpfs/tmpfs_vfsops.c b/sys/fs/tmpfs/tmpfs_vfsops.c
index 9d23bd2..74aeba1 100644
--- a/sys/fs/tmpfs/tmpfs_vfsops.c
+++ b/sys/fs/tmpfs/tmpfs_vfsops.c
@@ -156,9 +156,6 @@ tmpfs_mount(struct mount *mp)
 		return EOPNOTSUPP;
 	}
 
-	printf("WARNING: TMPFS is considered to be a highly experimental "
-	    "feature in FreeBSD.\n");
-
 	vn_lock(mp->mnt_vnodecovered, LK_SHARED | LK_RETRY);
 	error = VOP_GETATTR(mp->mnt_vnodecovered, &va, mp->mnt_cred);
 	VOP_UNLOCK(mp->mnt_vnodecovered, 0);
diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c
index 74aba04..6f52491 100644
--- a/sys/fs/tmpfs/tmpfs_vnops.c
+++ b/sys/fs/tmpfs/tmpfs_vnops.c
@@ -1138,6 +1138,7 @@ tmpfs_rename(struct vop_rename_args *v)
 		 * really reclaimed. */
 		tmpfs_free_dirent(VFS_TO_TMPFS(tvp->v_mount), de, TRUE);
 	}
+	cache_purge(fvp);
 
 	error = 0;
 
diff --git a/sys/fs/udf/udf_vnops.c b/sys/fs/udf/udf_vnops.c
index 1c9e0aa..b8e39b3 100644
--- a/sys/fs/udf/udf_vnops.c
+++ b/sys/fs/udf/udf_vnops.c
@@ -107,8 +107,8 @@ struct vop_vector udf_fifoops = {
 	.vop_vptofh =		udf_vptofh,
 };
 
-MALLOC_DEFINE(M_UDFFID, "udf_fid", "UDF FileId structure");
-MALLOC_DEFINE(M_UDFDS, "udf_ds", "UDF Dirstream structure");
+static MALLOC_DEFINE(M_UDFFID, "udf_fid", "UDF FileId structure");
+static MALLOC_DEFINE(M_UDFDS, "udf_ds", "UDF Dirstream structure");
 
 #define UDF_INVALID_BMAP	-1
 
diff --git a/sys/geom/cache/g_cache.c b/sys/geom/cache/g_cache.c
index fc3490f..378c2f2 100644
--- a/sys/geom/cache/g_cache.c
+++ b/sys/geom/cache/g_cache.c
@@ -48,7 +48,8 @@ FEATURE(geom_cache, "GEOM cache module");
 static MALLOC_DEFINE(M_GCACHE, "gcache_data", "GEOM_CACHE Data");
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, cache, CTLFLAG_RW, 0, "GEOM_CACHE stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, cache, CTLFLAG_RW, 0,
+    "GEOM_CACHE stuff");
 static u_int g_cache_debug = 0;
 SYSCTL_UINT(_kern_geom_cache, OID_AUTO, debug, CTLFLAG_RW, &g_cache_debug, 0,
     "Debug level");
diff --git a/sys/geom/concat/g_concat.c b/sys/geom/concat/g_concat.c
index b2196b7..9143166 100644
--- a/sys/geom/concat/g_concat.c
+++ b/sys/geom/concat/g_concat.c
@@ -45,7 +45,8 @@ FEATURE(geom_concat, "GEOM concatenation support");
 static MALLOC_DEFINE(M_CONCAT, "concat_data", "GEOM_CONCAT Data");
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, concat, CTLFLAG_RW, 0, "GEOM_CONCAT stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, concat, CTLFLAG_RW, 0,
+    "GEOM_CONCAT stuff");
 static u_int g_concat_debug = 0;
 TUNABLE_INT("kern.geom.concat.debug", &g_concat_debug);
 SYSCTL_UINT(_kern_geom_concat, OID_AUTO, debug, CTLFLAG_RW, &g_concat_debug, 0,
@@ -117,25 +118,33 @@ g_concat_remove_disk(struct g_concat_disk *disk)
 	struct g_consumer *cp;
 	struct g_concat_softc *sc;
 
+	g_topology_assert();
 	KASSERT(disk->d_consumer != NULL, ("Non-valid disk in %s.", __func__));
 	sc = disk->d_softc;
 	cp = disk->d_consumer;
 
-	G_CONCAT_DEBUG(0, "Disk %s removed from %s.", cp->provider->name,
-	    sc->sc_name);
+	if (!disk->d_removed) {
+		G_CONCAT_DEBUG(0, "Disk %s removed from %s.",
+		    cp->provider->name, sc->sc_name);
+		disk->d_removed = 1;
+	}
 
-	disk->d_consumer = NULL;
 	if (sc->sc_provider != NULL) {
 		sc->sc_provider->flags |= G_PF_WITHER;
+		G_CONCAT_DEBUG(0, "Device %s deactivated.",
+		    sc->sc_provider->name);
 		g_orphan_provider(sc->sc_provider, ENXIO);
 		sc->sc_provider = NULL;
-		G_CONCAT_DEBUG(0, "Device %s removed.", sc->sc_name);
 	}
 
 	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
-		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+		return;
+	disk->d_consumer = NULL;
 	g_detach(cp);
 	g_destroy_consumer(cp);
+	/* If there are no valid disks anymore, remove device. */
+	if (LIST_EMPTY(&sc->sc_geom->consumer))
+		g_concat_destroy(sc, 1);
 }
 
 static void
@@ -155,37 +164,18 @@ g_concat_orphan(struct g_consumer *cp)
 	if (disk == NULL)	/* Possible? */
 		return;
 	g_concat_remove_disk(disk);
-
-	/* If there are no valid disks anymore, remove device. */
-	if (g_concat_nvalid(sc) == 0)
-		g_concat_destroy(sc, 1);
 }
 
 static int
 g_concat_access(struct g_provider *pp, int dr, int dw, int de)
 {
-	struct g_consumer *cp1, *cp2;
-	struct g_concat_softc *sc;
+	struct g_consumer *cp1, *cp2, *tmp;
+	struct g_concat_disk *disk;
 	struct g_geom *gp;
 	int error;
 
+	g_topology_assert();
 	gp = pp->geom;
-	sc = gp->softc;
-
-	if (sc == NULL) {
-		/*
-		 * It looks like geom is being withered.
-		 * In that case we allow only negative requests.
-		 */
-		KASSERT(dr <= 0 && dw <= 0 && de <= 0,
-		    ("Positive access request (device=%s).", pp->name));
-		if ((pp->acr + dr) == 0 && (pp->acw + dw) == 0 &&
-		    (pp->ace + de) == 0) {
-			G_CONCAT_DEBUG(0, "Device %s definitely destroyed.",
-			    gp->name);
-		}
-		return (0);
-	}
 
 	/* On first open, grab an extra "exclusive" bit */
 	if (pp->acr == 0 && pp->acw == 0 && pp->ace == 0)
@@ -194,22 +184,24 @@ g_concat_access(struct g_provider *pp, int dr, int dw, int de)
 	if ((pp->acr + dr) == 0 && (pp->acw + dw) == 0 && (pp->ace + de) == 0)
 		de--;
 
-	error = ENXIO;
-	LIST_FOREACH(cp1, &gp->consumer, consumer) {
+	LIST_FOREACH_SAFE(cp1, &gp->consumer, consumer, tmp) {
 		error = g_access(cp1, dr, dw, de);
-		if (error == 0)
-			continue;
-		/*
-		 * If we fail here, backout all previous changes.
-		 */
-		LIST_FOREACH(cp2, &gp->consumer, consumer) {
-			if (cp1 == cp2)
-				return (error);
-			g_access(cp2, -dr, -dw, -de);
+		if (error != 0)
+			goto fail;
+		disk = cp1->private;
+		if (cp1->acr == 0 && cp1->acw == 0 && cp1->ace == 0 &&
+		    disk->d_removed) {
+			g_concat_remove_disk(disk); /* May destroy geom. */
 		}
-		/* NOTREACHED */
 	}
+	return (0);
 
+fail:
+	LIST_FOREACH(cp2, &gp->consumer, consumer) {
+		if (cp1 == cp2)
+			break;
+		g_access(cp2, -dr, -dw, -de);
+	}
 	return (error);
 }
 
@@ -391,6 +383,7 @@ g_concat_check_and_run(struct g_concat_softc *sc)
 	u_int no, sectorsize = 0;
 	off_t start;
 
+	g_topology_assert();
 	if (g_concat_nvalid(sc) != sc->sc_ndisks)
 		return;
 
@@ -419,7 +412,7 @@ g_concat_check_and_run(struct g_concat_softc *sc)
 	sc->sc_provider = pp;
 	g_error_provider(pp, 0);
 
-	G_CONCAT_DEBUG(0, "Device %s activated.", sc->sc_name);
+	G_CONCAT_DEBUG(0, "Device %s activated.", sc->sc_provider->name);
 }
 
 static int
@@ -461,6 +454,7 @@ g_concat_add_disk(struct g_concat_softc *sc, struct g_provider *pp, u_int no)
 	struct g_geom *gp;
 	int error;
 
+	g_topology_assert();
 	/* Metadata corrupted? */
 	if (no >= sc->sc_ndisks)
 		return (EINVAL);
@@ -509,6 +503,7 @@ g_concat_add_disk(struct g_concat_softc *sc, struct g_provider *pp, u_int no)
 	disk->d_softc = sc;
 	disk->d_start = 0;	/* not yet */
 	disk->d_end = 0;	/* not yet */
+	disk->d_removed = 0;
 
 	G_CONCAT_DEBUG(0, "Disk %s attached to %s.", pp->name, sc->sc_name);
 
@@ -576,8 +571,8 @@ static int
 g_concat_destroy(struct g_concat_softc *sc, boolean_t force)
 {
 	struct g_provider *pp;
+	struct g_consumer *cp, *cp1;
 	struct g_geom *gp;
-	u_int no;
 
 	g_topology_assert();
 
@@ -597,24 +592,23 @@ g_concat_destroy(struct g_concat_softc *sc, boolean_t force)
 		}
 	}
 
-	for (no = 0; no < sc->sc_ndisks; no++) {
-		if (sc->sc_disks[no].d_consumer != NULL)
-			g_concat_remove_disk(&sc->sc_disks[no]);
+	gp = sc->sc_geom;
+	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
+		g_concat_remove_disk(cp->private);
+		if (cp1 == NULL)
+			return (0);	/* Recursion happened. */
 	}
+	if (!LIST_EMPTY(&gp->consumer))
+		return (EINPROGRESS);
 
-	gp = sc->sc_geom;
 	gp->softc = NULL;
 	KASSERT(sc->sc_provider == NULL, ("Provider still exists? (device=%s)",
 	    gp->name));
 	free(sc->sc_disks, M_CONCAT);
 	free(sc, M_CONCAT);
 
-	pp = LIST_FIRST(&gp->provider);
-	if (pp == NULL || (pp->acr == 0 && pp->acw == 0 && pp->ace == 0))
-		G_CONCAT_DEBUG(0, "Device %s destroyed.", gp->name);
-
+	G_CONCAT_DEBUG(0, "Device %s destroyed.", gp->name);
 	g_wither_geom(gp, ENXIO);
-
 	return (0);
 }
 
diff --git a/sys/geom/concat/g_concat.h b/sys/geom/concat/g_concat.h
index def0a98..1c1e6f5 100644
--- a/sys/geom/concat/g_concat.h
+++ b/sys/geom/concat/g_concat.h
@@ -72,6 +72,7 @@ struct g_concat_disk {
 	struct g_concat_softc	*d_softc;
 	off_t			 d_start;
 	off_t			 d_end;
+	int			 d_removed;
 };
 
 struct g_concat_softc {
diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index b5e4f7b..874aa26 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -443,16 +443,15 @@ g_eli_worker(void *arg)
 	sc = wr->w_softc;
 #ifdef SMP
 	/* Before sched_bind() to a CPU, wait for all CPUs to go on-line. */
-	if (mp_ncpus > 1 && sc->sc_crypto == G_ELI_CRYPTO_SW &&
-	    g_eli_threads == 0) {
+	if (sc->sc_cpubind) {
 		while (!smp_started)
 			tsleep(wr, 0, "geli:smp", hz / 4);
 	}
 #endif
 	thread_lock(curthread);
 	sched_prio(curthread, PUSER);
-	if (sc->sc_crypto == G_ELI_CRYPTO_SW && g_eli_threads == 0)
-		sched_bind(curthread, wr->w_number);
+	if (sc->sc_cpubind)
+		sched_bind(curthread, wr->w_number % mp_ncpus);
 	thread_unlock(curthread);
 
 	G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
@@ -813,11 +812,7 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 	threads = g_eli_threads;
 	if (threads == 0)
 		threads = mp_ncpus;
-	else if (threads > mp_ncpus) {
-		/* There is really no need for too many worker threads. */
-		threads = mp_ncpus;
-		G_ELI_DEBUG(0, "Reducing number of threads to %u.", threads);
-	}
+	sc->sc_cpubind = (mp_ncpus > 1 && threads == mp_ncpus);
 	for (i = 0; i < threads; i++) {
 		if (g_eli_cpu_is_disabled(i)) {
 			G_ELI_DEBUG(1, "%s: CPU %u disabled, skipping.",
@@ -857,9 +852,6 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 			goto failed;
 		}
 		LIST_INSERT_HEAD(&sc->sc_workers, wr, w_next);
-		/* If we have hardware support, one thread is enough. */
-		if (sc->sc_crypto == G_ELI_CRYPTO_HW)
-			break;
 	}
 
 	/*
diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h
index 0b48db2..a5aad8f 100644
--- a/sys/geom/eli/g_eli.h
+++ b/sys/geom/eli/g_eli.h
@@ -192,6 +192,7 @@ struct g_eli_softc {
 	size_t		 sc_sectorsize;
 	u_int		 sc_bytes_per_sector;
 	u_int		 sc_data_per_sector;
+	boolean_t	 sc_cpubind;
 
 	/* Only for software cryptography. */
 	struct bio_queue_head sc_queue;
diff --git a/sys/geom/gate/g_gate.c b/sys/geom/gate/g_gate.c
index 886b993..52df914 100644
--- a/sys/geom/gate/g_gate.c
+++ b/sys/geom/gate/g_gate.c
@@ -59,7 +59,8 @@ FEATURE(geom_gate, "GEOM Gate module");
 static MALLOC_DEFINE(M_GATE, "gg_data", "GEOM Gate Data");
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, gate, CTLFLAG_RW, 0, "GEOM_GATE stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, gate, CTLFLAG_RW, 0,
+    "GEOM_GATE stuff");
 static int g_gate_debug = 0;
 TUNABLE_INT("kern.geom.gate.debug", &g_gate_debug);
 SYSCTL_INT(_kern_geom_gate, OID_AUTO, debug, CTLFLAG_RW, &g_gate_debug, 0,
diff --git a/sys/geom/geom_disk.c b/sys/geom/geom_disk.c
index 6b6c5ae..8eb486a 100644
--- a/sys/geom/geom_disk.c
+++ b/sys/geom/geom_disk.c
@@ -88,7 +88,8 @@ static struct g_class g_disk_class = {
 };
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, disk, CTLFLAG_RW, 0, "GEOM_DISK stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, disk, CTLFLAG_RW, 0,
+    "GEOM_DISK stuff");
 
 static void
 g_disk_init(struct g_class *mp __unused)
diff --git a/sys/geom/geom_event.c b/sys/geom/geom_event.c
index 1e2fc8d..1c92ccb 100644
--- a/sys/geom/geom_event.c
+++ b/sys/geom/geom_event.c
@@ -294,7 +294,7 @@ g_run_events()
 		} else {
 			g_topology_unlock();
 			msleep(&g_wait_event, &g_eventlock, PRIBIO | PDROP,
-			    "-", 0);
+			    "-", TAILQ_EMPTY(&g_doorstep) ? 0 : hz / 10);
 		}
 	}
 	/* NOTREACHED */
diff --git a/sys/geom/geom_vfs.c b/sys/geom/geom_vfs.c
index f90321d..a3e5138 100644
--- a/sys/geom/geom_vfs.c
+++ b/sys/geom/geom_vfs.c
@@ -31,7 +31,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bio.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
+#include <sys/mutex.h>
 #include <sys/vnode.h>
 #include <sys/mount.h>	/* XXX Temporary for VFS_LOCK_GIANT */
 
@@ -45,6 +47,13 @@ __FBSDID("$FreeBSD$");
  */
 #include <sys/buf.h>
 
+struct g_vfs_softc {
+	struct mtx	 sc_mtx;
+	struct bufobj	*sc_bo;
+	int		 sc_active;
+	int		 sc_orphaned;
+};
+
 static struct buf_ops __g_vfs_bufops = {
 	.bop_name =	"GEOM_VFS",
 	.bop_write =	bufwrite,
@@ -66,21 +75,29 @@ static struct g_class g_vfs_class = {
 DECLARE_GEOM_CLASS(g_vfs_class, g_vfs);
 
 static void
+g_vfs_destroy(void *arg, int flags __unused)
+{
+	struct g_consumer *cp;
+
+	g_topology_assert();
+	cp = arg;
+	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
+		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+	g_detach(cp);
+	if (cp->geom->softc == NULL)
+		g_wither_geom(cp->geom, ENXIO);
+}
+
+static void
 g_vfs_done(struct bio *bip)
 {
+	struct g_consumer *cp;
+	struct g_vfs_softc *sc;
 	struct buf *bp;
-	int vfslocked;
-
-	/*
-	 * Provider ('bio_to') could have withered away sometime
-	 * between incrementing the 'nend' in g_io_deliver() and now,
-	 * making 'bio_to' a dangling pointer.  We cannot do that
-	 * in g_wither_geom(), as it would require going over
-	 * the 'g_bio_run_up' list, resetting the pointer.
-	 */
-	if (bip->bio_from->provider == NULL)
-		bip->bio_to = NULL;
+	int vfslocked, destroy;
 
+	cp = bip->bio_from;
+	sc = cp->geom->softc;
 	if (bip->bio_error) {
 		printf("g_vfs_done():");
 		g_print_bio(bip);
@@ -93,6 +110,13 @@ g_vfs_done(struct bio *bip)
 		bp->b_ioflags |= BIO_ERROR;
 	bp->b_resid = bp->b_bcount - bip->bio_completed;
 	g_destroy_bio(bip);
+
+	mtx_lock(&sc->sc_mtx);
+	destroy = ((--sc->sc_active) == 0 && sc->sc_orphaned);
+	mtx_unlock(&sc->sc_mtx);
+	if (destroy)
+		g_post_event(g_vfs_destroy, cp, M_WAITOK, NULL);
+
 	vfslocked = VFS_LOCK_GIANT(((struct mount *)NULL));
 	bufdone(bp);
 	VFS_UNLOCK_GIANT(vfslocked);
@@ -101,17 +125,20 @@ g_vfs_done(struct bio *bip)
 void
 g_vfs_strategy(struct bufobj *bo, struct buf *bp)
 {
+	struct g_vfs_softc *sc;
 	struct g_consumer *cp;
 	struct bio *bip;
 	int vfslocked;
 
 	cp = bo->bo_private;
-	/* G_VALID_CONSUMER(cp); We likely lack topology lock */
+	sc = cp->geom->softc;
 
 	/*
 	 * If the provider has orphaned us, just return EXIO.
 	 */
-	if (cp->provider == NULL) {
+	mtx_lock(&sc->sc_mtx);
+	if (sc->sc_orphaned) {
+		mtx_unlock(&sc->sc_mtx);
 		bp->b_error = ENXIO;
 		bp->b_ioflags |= BIO_ERROR;
 		vfslocked = VFS_LOCK_GIANT(((struct mount *)NULL));
@@ -119,6 +146,8 @@ g_vfs_strategy(struct bufobj *bo, struct buf *bp)
 		VFS_UNLOCK_GIANT(vfslocked);
 		return;
 	}
+	sc->sc_active++;
+	mtx_unlock(&sc->sc_mtx);
 
 	bip = g_alloc_bio();
 	bip->bio_cmd = bp->b_iocmd;
@@ -134,14 +163,20 @@ static void
 g_vfs_orphan(struct g_consumer *cp)
 {
 	struct g_geom *gp;
+	struct g_vfs_softc *sc;
+	int destroy;
 
 	g_topology_assert();
 
 	gp = cp->geom;
+	sc = gp->softc;
 	g_trace(G_T_TOPOLOGY, "g_vfs_orphan(%p(%s))", cp, gp->name);
-	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
-		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
-	g_detach(cp);
+	mtx_lock(&sc->sc_mtx);
+	sc->sc_orphaned = 1;
+	destroy = (sc->sc_active == 0);
+	mtx_unlock(&sc->sc_mtx);
+	if (destroy)
+		g_vfs_destroy(cp, 0);
 
 	/*
 	 * Do not destroy the geom.  Filesystem will do that during unmount.
@@ -154,6 +189,7 @@ g_vfs_open(struct vnode *vp, struct g_consumer **cpp, const char *fsname, int wr
 	struct g_geom *gp;
 	struct g_provider *pp;
 	struct g_consumer *cp;
+	struct g_vfs_softc *sc;
 	struct bufobj *bo;
 	int vfslocked;
 	int error;
@@ -169,6 +205,10 @@ g_vfs_open(struct vnode *vp, struct g_consumer **cpp, const char *fsname, int wr
 	if (pp == NULL)
 		return (ENOENT);
 	gp = g_new_geomf(&g_vfs_class, "%s.%s", fsname, pp->name);
+	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
+	mtx_init(&sc->sc_mtx, "g_vfs", NULL, MTX_DEF);
+	sc->sc_bo = bo;
+	gp->softc = sc;
 	cp = g_new_consumer(gp);
 	g_attach(cp, pp);
 	error = g_access(cp, 1, wr, wr);
@@ -184,7 +224,6 @@ g_vfs_open(struct vnode *vp, struct g_consumer **cpp, const char *fsname, int wr
 	bo->bo_ops = g_vfs_bufops;
 	bo->bo_private = cp;
 	bo->bo_bsize = pp->sectorsize;
-	gp->softc = bo;
 
 	return (error);
 }
@@ -193,13 +232,17 @@ void
 g_vfs_close(struct g_consumer *cp)
 {
 	struct g_geom *gp;
-	struct bufobj *bo;
+	struct g_vfs_softc *sc;
 
 	g_topology_assert();
 
 	gp = cp->geom;
-	bo = gp->softc;
-	bufobj_invalbuf(bo, V_SAVE, 0, 0);
-	bo->bo_private = cp->private;
-	g_wither_geom_close(gp, ENXIO);
+	sc = gp->softc;
+	bufobj_invalbuf(sc->sc_bo, V_SAVE, 0, 0);
+	sc->sc_bo->bo_private = cp->private;
+	gp->softc = NULL;
+	mtx_destroy(&sc->sc_mtx);
+	if (!sc->sc_orphaned || cp->provider == NULL)
+		g_wither_geom_close(gp, ENXIO);
+	g_free(sc);
 }
diff --git a/sys/geom/journal/g_journal.c b/sys/geom/journal/g_journal.c
index 254ba92..706d6d4 100644
--- a/sys/geom/journal/g_journal.c
+++ b/sys/geom/journal/g_journal.c
@@ -93,7 +93,8 @@ static u_int g_journal_accept_immediately = 64;
 static u_int g_journal_record_entries = GJ_RECORD_HEADER_NENTRIES;
 static u_int g_journal_do_optimize = 1;
 
-SYSCTL_NODE(_kern_geom, OID_AUTO, journal, CTLFLAG_RW, 0, "GEOM_JOURNAL stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, journal, CTLFLAG_RW, 0,
+    "GEOM_JOURNAL stuff");
 SYSCTL_INT(_kern_geom_journal, OID_AUTO, debug, CTLFLAG_RW, &g_journal_debug, 0,
     "Debug level");
 SYSCTL_UINT(_kern_geom_journal, OID_AUTO, switch_time, CTLFLAG_RW,
@@ -140,7 +141,7 @@ static u_int g_journal_cache_misses = 0;
 static u_int g_journal_cache_alloc_failures = 0;
 static u_int g_journal_cache_low = 0;
 
-SYSCTL_NODE(_kern_geom_journal, OID_AUTO, cache, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom_journal, OID_AUTO, cache, CTLFLAG_RW, 0,
     "GEOM_JOURNAL cache");
 SYSCTL_UINT(_kern_geom_journal_cache, OID_AUTO, used, CTLFLAG_RD,
     &g_journal_cache_used, 0, "Number of allocated bytes");
@@ -195,7 +196,7 @@ static u_long g_journal_stats_wait_for_copy = 0;
 static u_long g_journal_stats_journal_full = 0;
 static u_long g_journal_stats_low_mem = 0;
 
-SYSCTL_NODE(_kern_geom_journal, OID_AUTO, stats, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom_journal, OID_AUTO, stats, CTLFLAG_RW, 0,
     "GEOM_JOURNAL statistics");
 SYSCTL_ULONG(_kern_geom_journal_stats, OID_AUTO, skipped_bytes, CTLFLAG_RW,
     &g_journal_stats_bytes_skipped, 0, "Number of skipped bytes");
diff --git a/sys/geom/mirror/g_mirror.c b/sys/geom/mirror/g_mirror.c
index 6688864..1eb46de 100644
--- a/sys/geom/mirror/g_mirror.c
+++ b/sys/geom/mirror/g_mirror.c
@@ -51,7 +51,8 @@ FEATURE(geom_mirror, "GEOM mirroring support");
 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0, "GEOM_MIRROR stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
+    "GEOM_MIRROR stuff");
 u_int g_mirror_debug = 0;
 TUNABLE_INT("kern.geom.mirror.debug", &g_mirror_debug);
 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RW, &g_mirror_debug, 0,
diff --git a/sys/geom/mountver/g_mountver.c b/sys/geom/mountver/g_mountver.c
index b2e1a63..30af511 100644
--- a/sys/geom/mountver/g_mountver.c
+++ b/sys/geom/mountver/g_mountver.c
@@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$");
 
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, mountver, CTLFLAG_RW,
+static SYSCTL_NODE(_kern_geom, OID_AUTO, mountver, CTLFLAG_RW,
     0, "GEOM_MOUNTVER stuff");
 static u_int g_mountver_debug = 0;
 static u_int g_mountver_check_ident = 1;
diff --git a/sys/geom/multipath/g_multipath.c b/sys/geom/multipath/g_multipath.c
index 947cce0..db1ff2e 100644
--- a/sys/geom/multipath/g_multipath.c
+++ b/sys/geom/multipath/g_multipath.c
@@ -1,4 +1,5 @@
 /*-
+ * Copyright (c) 2011 Alexander Motin <mav@FreeBSD.org>
  * Copyright (c) 2006-2007 Matthew Jacob <mjacob@FreeBSD.org>
  * All rights reserved.
  *
@@ -48,11 +49,14 @@ __FBSDID("$FreeBSD$");
 FEATURE(geom_multipath, "GEOM multipath support");
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, multipath, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom, OID_AUTO, multipath, CTLFLAG_RW, 0,
     "GEOM_MULTIPATH tunables");
 static u_int g_multipath_debug = 0;
 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW,
     &g_multipath_debug, 0, "Debug level");
+static u_int g_multipath_exclusive = 1;
+SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW,
+    &g_multipath_exclusive, 0, "Exclusively open providers");
 
 static enum {
 	GKT_NIL,
@@ -79,6 +83,7 @@ static g_taste_t g_multipath_taste;
 static g_ctl_req_t g_multipath_config;
 static g_init_t g_multipath_init;
 static g_fini_t g_multipath_fini;
+static g_dumpconf_t g_multipath_dumpconf;
 
 struct g_class g_multipath_class = {
 	.name		= G_MULTIPATH_CLASS_NAME,
@@ -90,35 +95,144 @@ struct g_class g_multipath_class = {
 	.fini		= g_multipath_fini
 };
 
-#define	MP_BAD		0x1
-#define	MP_POSTED	0x2
+#define	MP_FAIL		0x00000001
+#define	MP_LOST		0x00000002
+#define	MP_NEW		0x00000004
+#define	MP_POSTED	0x00000008
+#define	MP_BAD		(MP_FAIL | MP_LOST | MP_NEW)
+#define MP_IDLE		0x00000010
+#define MP_IDLE_MASK	0xfffffff0
+
+static int
+g_multipath_good(struct g_geom *gp)
+{
+	struct g_consumer *cp;
+	int n = 0;
+
+	LIST_FOREACH(cp, &gp->consumer, consumer) {
+		if ((cp->index & MP_BAD) == 0)
+			n++;
+	}
+	return (n);
+}
+
+static void
+g_multipath_fault(struct g_consumer *cp, int cause)
+{
+	struct g_multipath_softc *sc;
+	struct g_consumer *lcp;
+	struct g_geom *gp;
+
+	gp = cp->geom;
+	sc = gp->softc;
+	cp->index |= cause;
+	if (g_multipath_good(gp) == 0 && sc->sc_ndisks > 0) {
+		LIST_FOREACH(lcp, &gp->consumer, consumer) {
+			if (lcp->provider == NULL ||
+			    (lcp->index & (MP_LOST | MP_NEW)))
+				continue;
+			if (sc->sc_ndisks > 1 && lcp == cp)
+				continue;
+			printf("GEOM_MULTIPATH: "
+			    "all paths in %s were marked FAIL, restore %s\n",
+			    sc->sc_name, lcp->provider->name);
+			lcp->index &= ~MP_FAIL;
+		}
+	}
+	if (cp != sc->sc_active)
+		return;
+	sc->sc_active = NULL;
+	LIST_FOREACH(lcp, &gp->consumer, consumer) {
+		if ((lcp->index & MP_BAD) == 0) {
+			sc->sc_active = lcp;
+			break;
+		}
+	}
+	if (sc->sc_active == NULL) {
+		printf("GEOM_MULTIPATH: out of providers for %s\n",
+		    sc->sc_name);
+	} else if (!sc->sc_active_active) {
+		printf("GEOM_MULTIPATH: %s is now active path in %s\n",
+		    sc->sc_active->provider->name, sc->sc_name);
+	}
+}
+
+static struct g_consumer *
+g_multipath_choose(struct g_geom *gp)
+{
+	struct g_multipath_softc *sc;
+	struct g_consumer *best, *cp;
+
+	sc = gp->softc;
+	if (!sc->sc_active_active)
+		return (sc->sc_active);
+	best = NULL;
+	LIST_FOREACH(cp, &gp->consumer, consumer) {
+		if (cp->index & MP_BAD)
+			continue;
+		cp->index += MP_IDLE;
+		if (best == NULL || cp->private < best->private ||
+		    (cp->private == best->private && cp->index > best->index))
+			best = cp;
+	}
+	if (best != NULL)
+		best->index &= ~MP_IDLE_MASK;
+	return (best);
+}
 
 static void
 g_mpd(void *arg, int flags __unused)
 {
+	struct g_geom *gp;
+	struct g_multipath_softc *sc;
 	struct g_consumer *cp;
+	int w;
 
 	g_topology_assert();
 	cp = arg;
-	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
+	gp = cp->geom;
+	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0) {
+		w = cp->acw;
 		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+		if (w > 0 && cp->provider != NULL &&
+		    (cp->provider->geom->flags & G_GEOM_WITHER) == 0) {
+			g_post_event(g_mpd, cp, M_WAITOK, NULL);
+			return;
+		}
+	}
+	sc = gp->softc;
+	mtx_lock(&sc->sc_mtx);
 	if (cp->provider) {
 		printf("GEOM_MULTIPATH: %s removed from %s\n",
-		    cp->provider->name, cp->geom->name);
+		    cp->provider->name, gp->name);
 		g_detach(cp);
 	}
 	g_destroy_consumer(cp);
+	mtx_unlock(&sc->sc_mtx);
+	if (LIST_EMPTY(&gp->consumer))
+		g_multipath_destroy(gp);
 }
 
 static void
 g_multipath_orphan(struct g_consumer *cp)
 {
-	if ((cp->index & MP_POSTED) == 0) {
+	struct g_multipath_softc *sc;
+	uintptr_t *cnt;
+
+	g_topology_assert();
+	printf("GEOM_MULTIPATH: %s in %s was disconnected\n",
+	    cp->provider->name, cp->geom->name);
+	sc = cp->geom->softc;
+	cnt = (uintptr_t *)&cp->private;
+	mtx_lock(&sc->sc_mtx);
+	sc->sc_ndisks--;
+	g_multipath_fault(cp, MP_LOST);
+	if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
 		cp->index |= MP_POSTED;
-		printf("GEOM_MULTIPATH: %s orphaned in %s\n",
-		    cp->provider->name, cp->geom->name);
+		mtx_unlock(&sc->sc_mtx);
 		g_mpd(cp, 0);
-	}
+	} else
+		mtx_unlock(&sc->sc_mtx);
 }
 
 static void
@@ -128,20 +242,29 @@ g_multipath_start(struct bio *bp)
 	struct g_geom *gp;
 	struct g_consumer *cp;
 	struct bio *cbp;
+	uintptr_t *cnt;
 
 	gp = bp->bio_to->geom;
 	sc = gp->softc;
 	KASSERT(sc != NULL, ("NULL sc"));
-	cp = sc->cp_active;
-	if (cp == NULL) {
-		g_io_deliver(bp, ENXIO);
-		return;
-	}
 	cbp = g_clone_bio(bp);
 	if (cbp == NULL) {
 		g_io_deliver(bp, ENOMEM);
 		return;
 	}
+	mtx_lock(&sc->sc_mtx);
+	cp = g_multipath_choose(gp);
+	if (cp == NULL) {
+		mtx_unlock(&sc->sc_mtx);
+		g_destroy_bio(cbp);
+		g_io_deliver(bp, ENXIO);
+		return;
+	}
+	if ((uintptr_t)bp->bio_driver1 < sc->sc_ndisks)
+		bp->bio_driver1 = (void *)(uintptr_t)sc->sc_ndisks;
+	cnt = (uintptr_t *)&cp->private;
+	(*cnt)++;
+	mtx_unlock(&sc->sc_mtx);
 	cbp->bio_done = g_multipath_done;
 	g_io_request(cbp, cp);
 }
@@ -149,12 +272,27 @@ g_multipath_start(struct bio *bp)
 static void
 g_multipath_done(struct bio *bp)
 {
+	struct g_multipath_softc *sc;
+	struct g_consumer *cp;
+	uintptr_t *cnt;
+
 	if (bp->bio_error == ENXIO || bp->bio_error == EIO) {
 		mtx_lock(&gmtbq_mtx);
 		bioq_insert_tail(&gmtbq, bp);
-		wakeup(&g_multipath_kt_state);
 		mtx_unlock(&gmtbq_mtx);
+		wakeup(&g_multipath_kt_state);
 	} else {
+		cp = bp->bio_from;
+		sc = cp->geom->softc;
+		cnt = (uintptr_t *)&cp->private;
+		mtx_lock(&sc->sc_mtx);
+		(*cnt)--;
+		if (*cnt == 0 && (cp->index & MP_LOST)) {
+			cp->index |= MP_POSTED;
+			mtx_unlock(&sc->sc_mtx);
+			g_post_event(g_mpd, cp, M_WAITOK, NULL);
+		} else
+			mtx_unlock(&sc->sc_mtx);
 		g_std_done(bp);
 	}
 }
@@ -167,6 +305,7 @@ g_multipath_done_error(struct bio *bp)
 	struct g_multipath_softc *sc;
 	struct g_consumer *cp;
 	struct g_provider *pp;
+	uintptr_t *cnt;
 
 	/*
 	 * If we had a failure, we have to check first to see
@@ -176,47 +315,31 @@ g_multipath_done_error(struct bio *bp)
 	 * to the next available consumer.
 	 */
 
-	g_topology_lock();
 	pbp = bp->bio_parent;
 	gp = pbp->bio_to->geom;
 	sc = gp->softc;
 	cp = bp->bio_from;
 	pp = cp->provider;
-
-	cp->index |= MP_BAD;
-	if (cp->nend == cp->nstart && pp->nend == pp->nstart) {
+	cnt = (uintptr_t *)&cp->private;
+
+	mtx_lock(&sc->sc_mtx);
+	printf("GEOM_MULTIPATH: Error %d, %s in %s marked FAIL\n",
+	    bp->bio_error, pp->name, sc->sc_name);
+	g_multipath_fault(cp, MP_FAIL);
+	(*cnt)--;
+	if (*cnt == 0 && (cp->index & (MP_LOST | MP_POSTED)) == MP_LOST) {
 		cp->index |= MP_POSTED;
-		g_post_event(g_mpd, cp, M_NOWAIT, NULL);
-	}
-	if (cp == sc->cp_active) {
-		struct g_consumer *lcp;
-		printf("GEOM_MULTIPATH: %s failed in %s\n",
-		    pp->name, sc->sc_name);
-		sc->cp_active = NULL;
-		LIST_FOREACH(lcp, &gp->consumer, consumer) {
-			if ((lcp->index & MP_BAD) == 0) {
-				sc->cp_active = lcp;
-				break;
-			}
-		}
-		if (sc->cp_active == NULL || sc->cp_active->provider == NULL) {
-			printf("GEOM_MULTIPATH: out of providers for %s\n",
-			    sc->sc_name);
-			g_topology_unlock();
-			return;
-		} else {
-			printf("GEOM_MULTIPATH: %s now active path in %s\n",
-			    sc->cp_active->provider->name, sc->sc_name);
-		}
-	}
-	g_topology_unlock();
+		mtx_unlock(&sc->sc_mtx);
+		g_post_event(g_mpd, cp, M_WAITOK, NULL);
+	} else
+		mtx_unlock(&sc->sc_mtx);
 
 	/*
 	 * If we can fruitfully restart the I/O, do so.
 	 */
-	if (sc->cp_active) {
+	if (pbp->bio_children < (uintptr_t)pbp->bio_driver1) {
+		pbp->bio_inbed++;
 		g_destroy_bio(bp);
-		pbp->bio_children--;
 		g_multipath_start(pbp);
 	} else {
 		g_std_done(bp);
@@ -254,6 +377,7 @@ g_multipath_access(struct g_provider *pp, int dr, int dw, int de)
 {
 	struct g_geom *gp;
 	struct g_consumer *cp, *badcp = NULL;
+	struct g_multipath_softc *sc;
 	int error;
 
 	gp = pp->geom;
@@ -265,6 +389,10 @@ g_multipath_access(struct g_provider *pp, int dr, int dw, int de)
 			goto fail;
 		}
 	}
+	sc = gp->softc;
+	sc->sc_opened += dr + dw + de;
+	if (sc->sc_stopping && sc->sc_opened == 0)
+		g_multipath_destroy(gp);
 	return (0);
 
 fail:
@@ -286,6 +414,9 @@ g_multipath_create(struct g_class *mp, struct g_multipath_metadata *md)
 	g_topology_assert();
 
 	LIST_FOREACH(gp, &mp->geom, geom) {
+		sc = gp->softc;
+		if (sc == NULL || sc->sc_stopping)
+			continue;
 		if (strcmp(gp->name, md->md_name) == 0) {
 			printf("GEOM_MULTIPATH: name %s already exists\n",
 			    md->md_name);
@@ -295,19 +426,25 @@ g_multipath_create(struct g_class *mp, struct g_multipath_metadata *md)
 
 	gp = g_new_geomf(mp, md->md_name);
 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
+	mtx_init(&sc->sc_mtx, "multipath", NULL, MTX_DEF);
+	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
+	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
+	sc->sc_active_active = md->md_active_active;
 	gp->softc = sc;
 	gp->start = g_multipath_start;
 	gp->orphan = g_multipath_orphan;
 	gp->access = g_multipath_access;
-	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
-	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
+	gp->dumpconf = g_multipath_dumpconf;
 
 	pp = g_new_providerf(gp, "multipath/%s", md->md_name);
-	/* limit the provider to not have it stomp on metadata */
-	pp->mediasize = md->md_size - md->md_sectorsize;
-	pp->sectorsize = md->md_sectorsize;
-	sc->pp = pp;
+	if (md->md_size != 0) {
+		pp->mediasize = md->md_size -
+		    ((md->md_uuid[0] != 0) ? md->md_sectorsize : 0);
+		pp->sectorsize = md->md_sectorsize;
+	}
+	sc->sc_pp = pp;
 	g_error_provider(pp, 0);
+	printf("GEOM_MULTIPATH: %s created\n", gp->name);
 	return (gp);
 }
 
@@ -316,7 +453,7 @@ g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
 {
 	struct g_multipath_softc *sc;
 	struct g_consumer *cp, *nxtcp;
-	int error;
+	int error, acr, acw, ace;
 
 	g_topology_assert();
 
@@ -337,6 +474,8 @@ g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
 	}
 	nxtcp = LIST_FIRST(&gp->consumer);
 	cp = g_new_consumer(gp);
+	cp->private = NULL;
+	cp->index = MP_NEW;
 	error = g_attach(cp, pp);
 	if (error != 0) {
 		printf("GEOM_MULTIPATH: cannot attach %s to %s",
@@ -344,29 +483,51 @@ g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
 		g_destroy_consumer(cp);
 		return (error);
 	}
-	cp->private = sc;
-	cp->index = 0;
 
 	/*
 	 * Set access permissions on new consumer to match other consumers
 	 */
-	if (nxtcp && (nxtcp->acr + nxtcp->acw +  nxtcp->ace)) {
-		error = g_access(cp, nxtcp->acr, nxtcp->acw, nxtcp->ace);
-		if (error) {
-			printf("GEOM_MULTIPATH: cannot set access in "
-			    "attaching %s to %s/%s (%d)\n",
-			    pp->name, sc->sc_name, sc->sc_uuid, error);
-			g_detach(cp);
-			g_destroy_consumer(cp);
-			return (error);
-		}
+	if (sc->sc_pp) {
+		acr = sc->sc_pp->acr;
+		acw = sc->sc_pp->acw;
+		ace = sc->sc_pp->ace;
+	} else
+		acr = acw = ace = 0;
+	if (g_multipath_exclusive) {
+		acr++;
+		acw++;
+		ace++;
 	}
-	printf("GEOM_MULTIPATH: adding %s to %s/%s\n",
-	    pp->name, sc->sc_name, sc->sc_uuid);
-	if (sc->cp_active == NULL) {
-		sc->cp_active = cp;
-		printf("GEOM_MULTIPATH: %s now active path in %s\n",
-		    pp->name, sc->sc_name);
+	error = g_access(cp, acr, acw, ace);
+	if (error) {
+		printf("GEOM_MULTIPATH: cannot set access in "
+		    "attaching %s to %s (%d)\n",
+		    pp->name, sc->sc_name, error);
+		g_detach(cp);
+		g_destroy_consumer(cp);
+		return (error);
+	}
+	if (sc->sc_pp != NULL && sc->sc_pp->mediasize == 0) {
+		sc->sc_pp->mediasize = pp->mediasize -
+		    ((sc->sc_uuid[0] != 0) ? pp->sectorsize : 0);
+		sc->sc_pp->sectorsize = pp->sectorsize;
+	}
+	if (sc->sc_pp != NULL &&
+	    sc->sc_pp->stripesize == 0 && sc->sc_pp->stripeoffset == 0) {
+		sc->sc_pp->stripesize = pp->stripesize;
+		sc->sc_pp->stripeoffset = pp->stripeoffset;
+	}
+	mtx_lock(&sc->sc_mtx);
+	cp->index = 0;
+	sc->sc_ndisks++;
+	mtx_unlock(&sc->sc_mtx);
+	printf("GEOM_MULTIPATH: %s added to %s\n",
+	    pp->name, sc->sc_name);
+	if (sc->sc_active == NULL) {
+		sc->sc_active = cp;
+		if (!sc->sc_active_active)
+			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
+			    pp->name, sc->sc_name);
 	}
 	return (0);
 }
@@ -374,17 +535,41 @@ g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
 static int
 g_multipath_destroy(struct g_geom *gp)
 {
-	struct g_provider *pp;
+	struct g_multipath_softc *sc;
+	struct g_consumer *cp, *cp1;
 
 	g_topology_assert();
 	if (gp->softc == NULL)
 		return (ENXIO);
-	pp = LIST_FIRST(&gp->provider);
-	if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0))
-		return (EBUSY);
-	printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
+	sc = gp->softc;
+	if (!sc->sc_stopping) {
+		printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
+		sc->sc_stopping = 1;
+	}
+	if (sc->sc_opened != 0) {
+		if (sc->sc_pp != NULL) {
+			g_wither_provider(sc->sc_pp, ENXIO);
+			sc->sc_pp = NULL;
+		}
+		return (EINPROGRESS);
+	}
+	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
+		mtx_lock(&sc->sc_mtx);
+		if ((cp->index & MP_POSTED) == 0) {
+			cp->index |= MP_POSTED;
+			mtx_unlock(&sc->sc_mtx);
+			g_mpd(cp, 0);
+			if (cp1 == NULL)
+				return(0);	/* Recursion happened. */
+		} else
+			mtx_unlock(&sc->sc_mtx);
+	}
+	if (!LIST_EMPTY(&gp->consumer))
+		return (EINPROGRESS);
+	mtx_destroy(&sc->sc_mtx);
 	g_free(gp->softc);
 	gp->softc = NULL;
+	printf("GEOM_MULTIPATH: %s destroyed\n", gp->name);
 	g_wither_geom(gp, ENXIO);
 	return (0);
 }
@@ -408,15 +593,15 @@ g_multipath_rotate(struct g_geom *gp)
 		return (ENXIO);
 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
 		if ((lcp->index & MP_BAD) == 0) {
-			if (sc->cp_active != lcp) {
+			if (sc->sc_active != lcp)
 				break;
-			}
 		}
 	}
 	if (lcp) {
-		sc->cp_active = lcp;
-		printf("GEOM_MULTIPATH: %s now active path in %s\n",
-		    lcp->provider->name, sc->sc_name);
+		sc->sc_active = lcp;
+		if (!sc->sc_active_active)
+			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
+			    lcp->provider->name, sc->sc_name);
 	}
 	return (0);
 }
@@ -504,6 +689,10 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		    G_MULTIPATH_VERSION);
 		return (NULL);
 	}
+	if (md.md_size != 0 && md.md_size != pp->mediasize)
+		return (NULL);
+	if (md.md_sectorsize != 0 && md.md_sectorsize != pp->sectorsize)
+		return (NULL);
 	if (g_multipath_debug)
 		printf("MULTIPATH: %s/%s\n", md.md_name, md.md_uuid);
 
@@ -521,7 +710,7 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 	sc = NULL;
 	LIST_FOREACH(gp, &mp->geom, geom) {
 		sc = gp->softc;
-		if (sc == NULL)
+		if (sc == NULL || sc->sc_stopping)
 			continue;
 		if (strncmp(md.md_uuid, sc->sc_uuid, sizeof(md.md_uuid)) == 0)
 			break;
@@ -531,7 +720,7 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		if (gp1 == gp)
 			continue;
 		sc = gp1->softc;
-		if (sc == NULL)
+		if (sc == NULL || sc->sc_stopping)
 			continue;
 		if (strncmp(md.md_name, sc->sc_name, sizeof(md.md_name)) == 0)
 			break;
@@ -591,12 +780,14 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 }
 
 static void
-g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
+g_multipath_ctl_add_name(struct gctl_req *req, struct g_class *mp,
+    const char *name)
 {
+	struct g_multipath_softc *sc;
 	struct g_geom *gp;
 	struct g_consumer *cp;
-	struct g_provider *pp, *pp0;
-	const char *name, *mpname;
+	struct g_provider *pp;
+	const char *mpname;
 	static const char devpf[6] = "/dev/";
 
 	g_topology_assert();
@@ -611,12 +802,8 @@ g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "Device %s is invalid", mpname);
 		return;
 	}
+	sc = gp->softc;
 
-	name = gctl_get_asciiparam(req, "arg1");
-	if (name == NULL) {
-		gctl_error(req, "No 'arg1' argument");
-		return;
-	}
 	if (strncmp(name, devpf, 5) == 0)
 		name += 5;
 	pp = g_provider_by_name(name);
@@ -626,33 +813,30 @@ g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
 	}
 
 	/*
-	 * Check to make sure parameters match, if we already have one.
+	 * Check to make sure parameters match.
 	 */
-	cp = LIST_FIRST(&gp->consumer);
-	if (cp) {
-		pp0 = cp->provider;
-	} else {
-		pp0 = NULL;
-	}
-	if (pp0) {
-		if (pp0 == pp) {
-			gctl_error(req, "providers %s and %s are the same",
-			    pp0->name, pp->name);
-			return;
-		}
-		if (pp0->mediasize != pp->mediasize) {
-			gctl_error(req, "Provider %s is %jd; Provider %s is %jd",
-			    pp0->name, (intmax_t) pp0->mediasize,
-			    pp->name, (intmax_t) pp->mediasize);
-			return;
-		}
-		if (pp0->sectorsize != pp->sectorsize) {
-			gctl_error(req, "Provider %s has sectorsize %u; Provider %s "
-			    "has sectorsize %u", pp0->name, pp0->sectorsize,
-			    pp->name, pp->sectorsize);
+	LIST_FOREACH(cp, &gp->consumer, consumer) {
+		if (cp->provider == pp) {
+			gctl_error(req, "provider %s is already there",
+			    pp->name);
 			return;
 		}
 	}
+	if (sc->sc_pp != NULL && sc->sc_pp->mediasize != 0 &&
+	    sc->sc_pp->mediasize + (sc->sc_uuid[0] != 0 ? pp->sectorsize : 0)
+	     != pp->mediasize) {
+		gctl_error(req, "Providers size mismatch %jd != %jd",
+		    (intmax_t) sc->sc_pp->mediasize +
+			(sc->sc_uuid[0] != 0 ? pp->sectorsize : 0),
+		    (intmax_t) pp->mediasize);
+		return;
+	}
+	if (sc->sc_pp != NULL && sc->sc_pp->sectorsize != 0 &&
+	    sc->sc_pp->sectorsize != pp->sectorsize) {
+		gctl_error(req, "Providers sectorsize mismatch %u != %u",
+		    sc->sc_pp->sectorsize, pp->sectorsize);
+		return;
+	}
 
 	/*
 	 * Now add....
@@ -660,24 +844,244 @@ g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
 	(void) g_multipath_add_disk(gp, pp);
 }
 
+static void
+g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_multipath_softc *sc;
+	struct g_geom *gp;
+	const char *mpname, *name;
+
+	mpname = gctl_get_asciiparam(req, "arg0");
+        if (mpname == NULL) {
+                gctl_error(req, "No 'arg0' argument");
+                return;
+        }
+	gp = g_multipath_find_geom(mp, mpname);
+	if (gp == NULL) {
+		gctl_error(req, "Device %s not found", mpname);
+		return;
+	}
+	sc = gp->softc;
+
+	name = gctl_get_asciiparam(req, "arg1");
+	if (name == NULL) {
+		gctl_error(req, "No 'arg1' argument");
+		return;
+	}
+	g_multipath_ctl_add_name(req, mp, name);
+}
+
+static void
+g_multipath_ctl_create(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_multipath_metadata md;
+	struct g_multipath_softc *sc;
+	struct g_geom *gp;
+	const char *mpname, *name;
+	char param[16];
+	int *nargs, i, *active_active;
+
+	g_topology_assert();
+
+	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
+	if (*nargs < 2) {
+		gctl_error(req, "wrong number of arguments.");
+		return;
+	}
+
+	mpname = gctl_get_asciiparam(req, "arg0");
+        if (mpname == NULL) {
+                gctl_error(req, "No 'arg0' argument");
+                return;
+        }
+	gp = g_multipath_find_geom(mp, mpname);
+	if (gp != NULL) {
+		gctl_error(req, "Device %s already exist", mpname);
+		return;
+	}
+	sc = gp->softc;
+
+	memset(&md, 0, sizeof(md));
+	strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
+	md.md_version = G_MULTIPATH_VERSION;
+	strlcpy(md.md_name, mpname, sizeof(md.md_name));
+	md.md_size = 0;
+	md.md_sectorsize = 0;
+	md.md_uuid[0] = 0;
+	active_active = gctl_get_paraml(req, "active_active",
+	    sizeof(*active_active));
+	md.md_active_active =
+	    (active_active == NULL || *active_active == 0) ? 0 : 1;
+	gp = g_multipath_create(mp, &md);
+	if (gp == NULL) {
+		gctl_error(req, "GEOM_MULTIPATH: cannot create geom %s/%s\n",
+		    md.md_name, md.md_uuid);
+		return;
+	}
+	sc = gp->softc;
+
+	for (i = 1; i < *nargs; i++) {
+		snprintf(param, sizeof(param), "arg%d", i);
+		name = gctl_get_asciiparam(req, param);
+		g_multipath_ctl_add_name(req, mp, name);
+	}
+
+	if (sc->sc_ndisks != (*nargs - 1))
+		g_multipath_destroy(gp);
+}
+
+static void
+g_multipath_ctl_fail(struct gctl_req *req, struct g_class *mp, int fail)
+{
+	struct g_multipath_softc *sc;
+	struct g_geom *gp;
+	struct g_consumer *cp;
+	const char *mpname, *name;
+	int found;
+
+	mpname = gctl_get_asciiparam(req, "arg0");
+        if (mpname == NULL) {
+                gctl_error(req, "No 'arg0' argument");
+                return;
+        }
+	gp = g_multipath_find_geom(mp, mpname);
+	if (gp == NULL) {
+		gctl_error(req, "Device %s not found", mpname);
+		return;
+	}
+	sc = gp->softc;
+
+	name = gctl_get_asciiparam(req, "arg1");
+	if (name == NULL) {
+		gctl_error(req, "No 'arg1' argument");
+		return;
+	}
+
+	found = 0;
+	mtx_lock(&sc->sc_mtx);
+	LIST_FOREACH(cp, &gp->consumer, consumer) {
+		if (cp->provider != NULL &&
+		    strcmp(cp->provider->name, name) == 0 &&
+		    (cp->index & MP_LOST) == 0) {
+			found = 1;
+			printf("GEOM_MULTIPATH: %s in %s is marked %s.\n",
+				name, sc->sc_name, fail ? "FAIL" : "OK");
+			if (fail) {
+				g_multipath_fault(cp, MP_FAIL);
+			} else {
+				cp->index &= ~MP_FAIL;
+			}
+		}
+	}
+	mtx_unlock(&sc->sc_mtx);
+	if (found == 0)
+		gctl_error(req, "Provider %s not found", name);
+}
+
+static void
+g_multipath_ctl_remove(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_multipath_softc *sc;
+	struct g_geom *gp;
+	struct g_consumer *cp, *cp1;
+	const char *mpname, *name;
+	uintptr_t *cnt;
+	int found;
+
+	mpname = gctl_get_asciiparam(req, "arg0");
+        if (mpname == NULL) {
+                gctl_error(req, "No 'arg0' argument");
+                return;
+        }
+	gp = g_multipath_find_geom(mp, mpname);
+	if (gp == NULL) {
+		gctl_error(req, "Device %s not found", mpname);
+		return;
+	}
+	sc = gp->softc;
+
+	name = gctl_get_asciiparam(req, "arg1");
+	if (name == NULL) {
+		gctl_error(req, "No 'arg1' argument");
+		return;
+	}
+
+	found = 0;
+	mtx_lock(&sc->sc_mtx);
+	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
+		if (cp->provider != NULL &&
+		    strcmp(cp->provider->name, name) == 0 &&
+		    (cp->index & MP_LOST) == 0) {
+			found = 1;
+			printf("GEOM_MULTIPATH: removing %s from %s\n",
+			    cp->provider->name, cp->geom->name);
+			sc->sc_ndisks--;
+			g_multipath_fault(cp, MP_LOST);
+			cnt = (uintptr_t *)&cp->private;
+			if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
+				cp->index |= MP_POSTED;
+				mtx_unlock(&sc->sc_mtx);
+				g_mpd(cp, 0);
+				if (cp1 == NULL)
+					return;	/* Recursion happened. */
+				mtx_lock(&sc->sc_mtx);
+			}
+		}
+	}
+	mtx_unlock(&sc->sc_mtx);
+	if (found == 0)
+		gctl_error(req, "Provider %s not found", name);
+}
+
 static struct g_geom *
 g_multipath_find_geom(struct g_class *mp, const char *name)
 {
 	struct g_geom *gp;
+	struct g_multipath_softc *sc;
 
 	LIST_FOREACH(gp, &mp->geom, geom) {
-		if (strcmp(gp->name, name) == 0) {
+		sc = gp->softc;
+		if (sc == NULL || sc->sc_stopping)
+			continue;
+		if (strcmp(gp->name, name) == 0)
 			return (gp);
-		}
 	}
 	return (NULL);
 }
 
 static void
+g_multipath_ctl_stop(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_geom *gp;
+	const char *name;
+	int error;
+
+	g_topology_assert();
+
+	name = gctl_get_asciiparam(req, "arg0");
+        if (name == NULL) {
+                gctl_error(req, "No 'arg0' argument");
+                return;
+        }
+	gp = g_multipath_find_geom(mp, name);
+	if (gp == NULL) {
+		gctl_error(req, "Device %s is invalid", name);
+		return;
+	}
+	error = g_multipath_destroy(gp);
+	if (error != 0 && error != EINPROGRESS)
+		gctl_error(req, "failed to stop %s (err=%d)", name, error);
+}
+
+static void
 g_multipath_ctl_destroy(struct gctl_req *req, struct g_class *mp)
 {
 	struct g_geom *gp;
+	struct g_multipath_softc *sc;
+	struct g_consumer *cp;
+	struct g_provider *pp;
 	const char *name;
+	uint8_t *buf;
 	int error;
 
 	g_topology_assert();
@@ -692,10 +1096,31 @@ g_multipath_ctl_destroy(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "Device %s is invalid", name);
 		return;
 	}
+	sc = gp->softc;
+
+	if (sc->sc_uuid[0] != 0 && sc->sc_active != NULL) {
+		cp = sc->sc_active;
+		pp = cp->provider;
+		error = g_access(cp, 1, 1, 1);
+		if (error != 0) {
+			gctl_error(req, "Can't open %s (%d)", pp->name, error);
+			goto destroy;
+		}
+		g_topology_unlock();
+		buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
+		error = g_write_data(cp, pp->mediasize - pp->sectorsize,
+		    buf, pp->sectorsize);
+		g_topology_lock();
+		g_access(cp, -1, -1, -1);
+		if (error != 0)
+			gctl_error(req, "Can't erase metadata on %s (%d)",
+			    pp->name, error);
+	}
+
+destroy:
 	error = g_multipath_destroy(gp);
-	if (error != 0) {
+	if (error != 0 && error != EINPROGRESS)
 		gctl_error(req, "failed to destroy %s (err=%d)", name, error);
-	}
 }
 
 static void
@@ -729,7 +1154,9 @@ g_multipath_ctl_getactive(struct gctl_req *req, struct g_class *mp)
 	struct sbuf *sb;
 	struct g_geom *gp;
 	struct g_multipath_softc *sc;
+	struct g_consumer *cp;
 	const char *name;
+	int empty;
 
 	sb = sbuf_new_auto();
 
@@ -745,8 +1172,21 @@ g_multipath_ctl_getactive(struct gctl_req *req, struct g_class *mp)
 		return;
 	}
 	sc = gp->softc;
-	if (sc->cp_active && sc->cp_active->provider) {
-		sbuf_printf(sb, "%s\n", sc->cp_active->provider->name);
+	if (sc->sc_active_active) {
+		empty = 1;
+		LIST_FOREACH(cp, &gp->consumer, consumer) {
+			if (cp->index & MP_BAD)
+				continue;
+			if (!empty)
+				sbuf_cat(sb, " ");
+			sbuf_cat(sb, cp->provider->name);
+			empty = 0;
+		}
+		if (empty)
+			sbuf_cat(sb, "none");
+		sbuf_cat(sb, "\n");
+	} else if (sc->sc_active && sc->sc_active->provider) {
+		sbuf_printf(sb, "%s\n", sc->sc_active->provider->name);
 	} else {
 		sbuf_printf(sb, "none\n");
 	}
@@ -767,8 +1207,18 @@ g_multipath_config(struct gctl_req *req, struct g_class *mp, const char *verb)
 		gctl_error(req, "Userland and kernel parts are out of sync");
 	} else if (strcmp(verb, "add") == 0) {
 		g_multipath_ctl_add(req, mp);
+	} else if (strcmp(verb, "create") == 0) {
+		g_multipath_ctl_create(req, mp);
+	} else if (strcmp(verb, "stop") == 0) {
+		g_multipath_ctl_stop(req, mp);
 	} else if (strcmp(verb, "destroy") == 0) {
 		g_multipath_ctl_destroy(req, mp);
+	} else if (strcmp(verb, "fail") == 0) {
+		g_multipath_ctl_fail(req, mp, 1);
+	} else if (strcmp(verb, "restore") == 0) {
+		g_multipath_ctl_fail(req, mp, 0);
+	} else if (strcmp(verb, "remove") == 0) {
+		g_multipath_ctl_remove(req, mp);
 	} else if (strcmp(verb, "rotate") == 0) {
 		g_multipath_ctl_rotate(req, mp);
 	} else if (strcmp(verb, "getactive") == 0) {
@@ -777,4 +1227,40 @@ g_multipath_config(struct gctl_req *req, struct g_class *mp, const char *verb)
 		gctl_error(req, "Unknown verb %s", verb);
 	}
 }
+
+static void
+g_multipath_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
+    struct g_consumer *cp, struct g_provider *pp)
+{
+	struct g_multipath_softc *sc;
+	int good;
+
+	g_topology_assert();
+
+	sc = gp->softc;
+	if (sc == NULL)
+		return;
+	if (cp != NULL) {
+		sbuf_printf(sb, "%s<State>%s</State>", indent,
+		    (cp->index & MP_NEW) ? "NEW" :
+		    (cp->index & MP_LOST) ? "LOST" :
+		    (cp->index & MP_FAIL) ? "FAIL" :
+		    (sc->sc_active_active || sc->sc_active == cp) ?
+		     "ACTIVE" : "PASSIVE");
+	} else {
+		good = g_multipath_good(gp);
+		sbuf_printf(sb, "%s<State>%s</State>", indent,
+		    good == 0 ? "BROKEN" :
+		    (good != sc->sc_ndisks || sc->sc_ndisks == 1) ?
+		    "DEGRADED" : "OPTIMAL");
+	}
+	if (cp == NULL && pp == NULL) {
+		sbuf_printf(sb, "%s<UUID>%s</UUID>", indent, sc->sc_uuid);
+		sbuf_printf(sb, "%s<Mode>Active/%s</Mode>", indent,
+		    sc->sc_active_active ? "Active" : "Passive");
+		sbuf_printf(sb, "%s<Type>%s</Type>", indent,
+		    sc->sc_uuid[0] == 0 ? "MANUAL" : "AUTOMATIC");
+	}
+}
+
 DECLARE_GEOM_CLASS(g_multipath_class, g_multipath);
diff --git a/sys/geom/multipath/g_multipath.h b/sys/geom/multipath/g_multipath.h
index 22d6157..33c44f1 100644
--- a/sys/geom/multipath/g_multipath.h
+++ b/sys/geom/multipath/g_multipath.h
@@ -43,10 +43,15 @@
 #ifdef	_KERNEL
 
 struct g_multipath_softc {
-	struct g_provider *	pp;
-	struct g_consumer *	cp_active;
+	struct g_provider *	sc_pp;
+	struct g_consumer *	sc_active;
+	struct mtx		sc_mtx;
 	char			sc_name[16];
 	char			sc_uuid[40];
+	int			sc_opened;
+	int			sc_stopping;
+	int			sc_ndisks;
+	int			sc_active_active; /* Active/Active mode */
 };
 #endif	/* _KERNEL */
 
@@ -57,6 +62,7 @@ struct g_multipath_metadata {
 	uint32_t	md_version;	/* version */
 	uint32_t	md_sectorsize;	/* sectorsize of provider */
 	uint64_t	md_size;	/* absolute size of provider */
+	uint8_t		md_active_active; /* Active/Active mode */
 };
 
 static __inline void
@@ -79,6 +85,8 @@ multipath_metadata_encode(const struct g_multipath_metadata *md, u_char *data)
 	le32enc(data, md->md_sectorsize);
 	data += sizeof(md->md_sectorsize);
 	le64enc(data, md->md_size);
+	data += sizeof(md->md_size);
+	*data = md->md_active_active;
 }
 
 static __inline void
@@ -95,5 +103,7 @@ multipath_metadata_decode(u_char *data, struct g_multipath_metadata *md)
 	md->md_sectorsize = le32dec(data);
 	data += sizeof(md->md_sectorsize);
 	md->md_size = le64dec(data);
+	data += sizeof(md->md_size);
+	md->md_active_active = *data;
 }
 #endif	/* _G_MULTIPATH_H_ */
diff --git a/sys/geom/nop/g_nop.c b/sys/geom/nop/g_nop.c
index 675f06e..c28a77f 100644
--- a/sys/geom/nop/g_nop.c
+++ b/sys/geom/nop/g_nop.c
@@ -42,7 +42,7 @@ __FBSDID("$FreeBSD$");
 
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, nop, CTLFLAG_RW, 0, "GEOM_NOP stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, nop, CTLFLAG_RW, 0, "GEOM_NOP stuff");
 static u_int g_nop_debug = 0;
 SYSCTL_UINT(_kern_geom_nop, OID_AUTO, debug, CTLFLAG_RW, &g_nop_debug, 0,
     "Debug level");
diff --git a/sys/geom/part/g_part.c b/sys/geom/part/g_part.c
index f24e7b5..4cc1ed5 100644
--- a/sys/geom/part/g_part.c
+++ b/sys/geom/part/g_part.c
@@ -106,7 +106,8 @@ struct g_part_alias_list {
 };
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, part, CTLFLAG_RW, 0, "GEOM_PART stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, part, CTLFLAG_RW, 0,
+    "GEOM_PART stuff");
 static u_int check_integrity = 1;
 TUNABLE_INT("kern.geom.part.check_integrity", &check_integrity);
 SYSCTL_UINT(_kern_geom_part, OID_AUTO, check_integrity, CTLFLAG_RW,
@@ -451,6 +452,10 @@ g_part_parm_geom(struct gctl_req *req, const char *name, struct g_geom **v)
 		gctl_error(req, "%d %s '%s'", EINVAL, name, gname);
 		return (EINVAL);
 	}
+	if ((gp->flags & G_GEOM_WITHER) != 0) {
+		gctl_error(req, "%d %s", ENXIO, gname);
+		return (ENXIO);
+	}
 	*v = gp;
 	return (0);
 }
diff --git a/sys/geom/raid/tr_raid1.c b/sys/geom/raid/tr_raid1.c
index 8e5eea2..36b7f7b 100644
--- a/sys/geom/raid/tr_raid1.c
+++ b/sys/geom/raid/tr_raid1.c
@@ -43,7 +43,7 @@ __FBSDID("$FreeBSD$");
 #include "g_raid_tr_if.h"
 
 SYSCTL_DECL(_kern_geom_raid);
-SYSCTL_NODE(_kern_geom_raid, OID_AUTO, raid1, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom_raid, OID_AUTO, raid1, CTLFLAG_RW, 0,
     "RAID1 parameters");
 
 #define RAID1_REBUILD_SLAB	(1 << 20) /* One transation in a rebuild */
diff --git a/sys/geom/raid/tr_raid1e.c b/sys/geom/raid/tr_raid1e.c
index 23e8ba9..c827117 100644
--- a/sys/geom/raid/tr_raid1e.c
+++ b/sys/geom/raid/tr_raid1e.c
@@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$");
 #define N	2
 
 SYSCTL_DECL(_kern_geom_raid);
-SYSCTL_NODE(_kern_geom_raid, OID_AUTO, raid1e, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom_raid, OID_AUTO, raid1e, CTLFLAG_RW, 0,
     "RAID1E parameters");
 
 #define RAID1E_REBUILD_SLAB	(1 << 20) /* One transation in a rebuild */
diff --git a/sys/geom/raid3/g_raid3.c b/sys/geom/raid3/g_raid3.c
index eef7c10..029d255 100644
--- a/sys/geom/raid3/g_raid3.c
+++ b/sys/geom/raid3/g_raid3.c
@@ -51,7 +51,8 @@ FEATURE(geom_raid3, "GEOM RAID-3 functionality");
 static MALLOC_DEFINE(M_RAID3, "raid3_data", "GEOM_RAID3 Data");
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, raid3, CTLFLAG_RW, 0, "GEOM_RAID3 stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, raid3, CTLFLAG_RW, 0,
+    "GEOM_RAID3 stuff");
 u_int g_raid3_debug = 0;
 TUNABLE_INT("kern.geom.raid3.debug", &g_raid3_debug);
 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, debug, CTLFLAG_RW, &g_raid3_debug, 0,
@@ -91,7 +92,7 @@ TUNABLE_INT("kern.geom.raid3.n4k", &g_raid3_n4k);
 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n4k, CTLFLAG_RD, &g_raid3_n4k, 0,
     "Maximum number of 4kB allocations");
 
-SYSCTL_NODE(_kern_geom_raid3, OID_AUTO, stat, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom_raid3, OID_AUTO, stat, CTLFLAG_RW, 0,
     "GEOM_RAID3 statistics");
 static u_int g_raid3_parity_mismatch = 0;
 SYSCTL_UINT(_kern_geom_raid3_stat, OID_AUTO, parity_mismatch, CTLFLAG_RD,
diff --git a/sys/geom/sched/gs_rr.c b/sys/geom/sched/gs_rr.c
index 1fc8cda..6f26879 100644
--- a/sys/geom/sched/gs_rr.c
+++ b/sys/geom/sched/gs_rr.c
@@ -198,7 +198,7 @@ static struct g_rr_params me = {
 struct g_rr_params *gs_rr_me = &me;
 
 SYSCTL_DECL(_kern_geom_sched);
-SYSCTL_NODE(_kern_geom_sched, OID_AUTO, rr, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern_geom_sched, OID_AUTO, rr, CTLFLAG_RW, 0,
     "GEOM_SCHED ROUND ROBIN stuff");
 SYSCTL_INT(_kern_geom_sched_rr, OID_AUTO, units, CTLFLAG_RD,
     &me.units, 0, "Scheduler instances");
diff --git a/sys/geom/shsec/g_shsec.c b/sys/geom/shsec/g_shsec.c
index abf0317..9f8723f 100644
--- a/sys/geom/shsec/g_shsec.c
+++ b/sys/geom/shsec/g_shsec.c
@@ -68,7 +68,8 @@ struct g_class g_shsec_class = {
 };
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, shsec, CTLFLAG_RW, 0, "GEOM_SHSEC stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, shsec, CTLFLAG_RW, 0,
+    "GEOM_SHSEC stuff");
 static u_int g_shsec_debug = 0;
 TUNABLE_INT("kern.geom.shsec.debug", &g_shsec_debug);
 SYSCTL_UINT(_kern_geom_shsec, OID_AUTO, debug, CTLFLAG_RW, &g_shsec_debug, 0,
diff --git a/sys/geom/stripe/g_stripe.c b/sys/geom/stripe/g_stripe.c
index bb94631..575ec5f 100644
--- a/sys/geom/stripe/g_stripe.c
+++ b/sys/geom/stripe/g_stripe.c
@@ -68,7 +68,8 @@ struct g_class g_stripe_class = {
 };
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, stripe, CTLFLAG_RW, 0, "GEOM_STRIPE stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, stripe, CTLFLAG_RW, 0,
+    "GEOM_STRIPE stuff");
 static u_int g_stripe_debug = 0;
 TUNABLE_INT("kern.geom.stripe.debug", &g_stripe_debug);
 SYSCTL_UINT(_kern_geom_stripe, OID_AUTO, debug, CTLFLAG_RW, &g_stripe_debug, 0,
@@ -160,28 +161,35 @@ static void
 g_stripe_remove_disk(struct g_consumer *cp)
 {
 	struct g_stripe_softc *sc;
-	u_int no;
 
+	g_topology_assert();
 	KASSERT(cp != NULL, ("Non-valid disk in %s.", __func__));
-	sc = (struct g_stripe_softc *)cp->private;
+	sc = (struct g_stripe_softc *)cp->geom->softc;
 	KASSERT(sc != NULL, ("NULL sc in %s.", __func__));
-	no = cp->index;
 
-	G_STRIPE_DEBUG(0, "Disk %s removed from %s.", cp->provider->name,
-	    sc->sc_name);
+	if (cp->private == NULL) {
+		G_STRIPE_DEBUG(0, "Disk %s removed from %s.",
+		    cp->provider->name, sc->sc_name);
+		cp->private = (void *)(uintptr_t)-1;
+	}
 
-	sc->sc_disks[no] = NULL;
 	if (sc->sc_provider != NULL) {
 		sc->sc_provider->flags |= G_PF_WITHER;
+		G_STRIPE_DEBUG(0, "Device %s deactivated.",
+		    sc->sc_provider->name);
 		g_orphan_provider(sc->sc_provider, ENXIO);
 		sc->sc_provider = NULL;
-		G_STRIPE_DEBUG(0, "Device %s removed.", sc->sc_name);
 	}
 
 	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
-		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+		return;
+	sc->sc_disks[cp->index] = NULL;
+	cp->index = 0;
 	g_detach(cp);
 	g_destroy_consumer(cp);
+	/* If there are no valid disks anymore, remove device. */
+	if (LIST_EMPTY(&sc->sc_geom->consumer))
+		g_stripe_destroy(sc, 1);
 }
 
 static void
@@ -197,36 +205,20 @@ g_stripe_orphan(struct g_consumer *cp)
 		return;
 
 	g_stripe_remove_disk(cp);
-	/* If there are no valid disks anymore, remove device. */
-	if (g_stripe_nvalid(sc) == 0)
-		g_stripe_destroy(sc, 1);
 }
 
 static int
 g_stripe_access(struct g_provider *pp, int dr, int dw, int de)
 {
-	struct g_consumer *cp1, *cp2;
+	struct g_consumer *cp1, *cp2, *tmp;
 	struct g_stripe_softc *sc;
 	struct g_geom *gp;
 	int error;
 
+	g_topology_assert();
 	gp = pp->geom;
 	sc = gp->softc;
-
-	if (sc == NULL) {
-		/*
-		 * It looks like geom is being withered.
-		 * In that case we allow only negative requests.
-		 */
-		KASSERT(dr <= 0 && dw <= 0 && de <= 0,
-		    ("Positive access request (device=%s).", pp->name));
-		if ((pp->acr + dr) == 0 && (pp->acw + dw) == 0 &&
-		    (pp->ace + de) == 0) {
-			G_STRIPE_DEBUG(0, "Device %s definitely destroyed.",
-			    gp->name);
-		}
-		return (0);
-	}
+	KASSERT(sc != NULL, ("NULL sc in %s.", __func__));
 
 	/* On first open, grab an extra "exclusive" bit */
 	if (pp->acr == 0 && pp->acw == 0 && pp->ace == 0)
@@ -235,22 +227,23 @@ g_stripe_access(struct g_provider *pp, int dr, int dw, int de)
 	if ((pp->acr + dr) == 0 && (pp->acw + dw) == 0 && (pp->ace + de) == 0)
 		de--;
 
-	error = ENXIO;
-	LIST_FOREACH(cp1, &gp->consumer, consumer) {
+	LIST_FOREACH_SAFE(cp1, &gp->consumer, consumer, tmp) {
 		error = g_access(cp1, dr, dw, de);
-		if (error == 0)
-			continue;
-		/*
-		 * If we fail here, backout all previous changes.
-		 */
-		LIST_FOREACH(cp2, &gp->consumer, consumer) {
-			if (cp1 == cp2)
-				return (error);
-			g_access(cp2, -dr, -dw, -de);
+		if (error != 0)
+			goto fail;
+		if (cp1->acr == 0 && cp1->acw == 0 && cp1->ace == 0 &&
+		    cp1->private != NULL) {
+			g_stripe_remove_disk(cp1); /* May destroy geom. */
 		}
-		/* NOTREACHED */
 	}
+	return (0);
 
+fail:
+	LIST_FOREACH(cp2, &gp->consumer, consumer) {
+		if (cp1 == cp2)
+			break;
+		g_access(cp2, -dr, -dw, -de);
+	}
 	return (error);
 }
 
@@ -652,6 +645,7 @@ g_stripe_check_and_run(struct g_stripe_softc *sc)
 	off_t mediasize, ms;
 	u_int no, sectorsize = 0;
 
+	g_topology_assert();
 	if (g_stripe_nvalid(sc) != sc->sc_ndisks)
 		return;
 
@@ -681,7 +675,7 @@ g_stripe_check_and_run(struct g_stripe_softc *sc)
 	sc->sc_provider->stripeoffset = 0;
 	g_error_provider(sc->sc_provider, 0);
 
-	G_STRIPE_DEBUG(0, "Device %s activated.", sc->sc_name);
+	G_STRIPE_DEBUG(0, "Device %s activated.", sc->sc_provider->name);
 }
 
 static int
@@ -722,6 +716,7 @@ g_stripe_add_disk(struct g_stripe_softc *sc, struct g_provider *pp, u_int no)
 	struct g_geom *gp;
 	int error;
 
+	g_topology_assert();
 	/* Metadata corrupted? */
 	if (no >= sc->sc_ndisks)
 		return (EINVAL);
@@ -734,6 +729,8 @@ g_stripe_add_disk(struct g_stripe_softc *sc, struct g_provider *pp, u_int no)
 	fcp = LIST_FIRST(&gp->consumer);
 
 	cp = g_new_consumer(gp);
+	cp->private = NULL;
+	cp->index = no;
 	error = g_attach(cp, pp);
 	if (error != 0) {
 		g_destroy_consumer(cp);
@@ -764,12 +761,8 @@ g_stripe_add_disk(struct g_stripe_softc *sc, struct g_provider *pp, u_int no)
 		}
 	}
 
-	cp->private = sc;
-	cp->index = no;
 	sc->sc_disks[no] = cp;
-
 	G_STRIPE_DEBUG(0, "Disk %s attached to %s.", pp->name, sc->sc_name);
-
 	g_stripe_check_and_run(sc);
 
 	return (0);
@@ -789,6 +782,7 @@ g_stripe_create(struct g_class *mp, const struct g_stripe_metadata *md,
 	struct g_geom *gp;
 	u_int no;
 
+	g_topology_assert();
 	G_STRIPE_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
 	    md->md_id);
 
@@ -850,8 +844,8 @@ static int
 g_stripe_destroy(struct g_stripe_softc *sc, boolean_t force)
 {
 	struct g_provider *pp;
+	struct g_consumer *cp, *cp1;
 	struct g_geom *gp;
-	u_int no;
 
 	g_topology_assert();
 
@@ -871,24 +865,22 @@ g_stripe_destroy(struct g_stripe_softc *sc, boolean_t force)
 		}
 	}
 
-	for (no = 0; no < sc->sc_ndisks; no++) {
-		if (sc->sc_disks[no] != NULL)
-			g_stripe_remove_disk(sc->sc_disks[no]);
+	gp = sc->sc_geom;
+	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
+		g_stripe_remove_disk(cp);
+		if (cp1 == NULL)
+			return (0);	/* Recursion happened. */
 	}
+	if (!LIST_EMPTY(&gp->consumer))
+		return (EINPROGRESS);
 
-	gp = sc->sc_geom;
 	gp->softc = NULL;
 	KASSERT(sc->sc_provider == NULL, ("Provider still exists? (device=%s)",
 	    gp->name));
 	free(sc->sc_disks, M_STRIPE);
 	free(sc, M_STRIPE);
-
-	pp = LIST_FIRST(&gp->provider);
-	if (pp == NULL || (pp->acr == 0 && pp->acw == 0 && pp->ace == 0))
-		G_STRIPE_DEBUG(0, "Device %s destroyed.", gp->name);
-
+	G_STRIPE_DEBUG(0, "Device %s destroyed.", gp->name);
 	g_wither_geom(gp, ENXIO);
-
 	return (0);
 }
 
diff --git a/sys/geom/uzip/g_uzip.c b/sys/geom/uzip/g_uzip.c
index c8b0dfc..f19e14c 100644
--- a/sys/geom/uzip/g_uzip.c
+++ b/sys/geom/uzip/g_uzip.c
@@ -50,7 +50,7 @@ FEATURE(geom_uzip, "GEOM uzip read-only compressed disks support");
 #define DPRINTF(a)
 #endif
 
-MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures");
+static MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures");
 
 #define UZIP_CLASS_NAME	"UZIP"
 
diff --git a/sys/geom/vinum/geom_vinum.c b/sys/geom/vinum/geom_vinum.c
index ab4d1a4..15a62a1 100644
--- a/sys/geom/vinum/geom_vinum.c
+++ b/sys/geom/vinum/geom_vinum.c
@@ -47,7 +47,8 @@ __FBSDID("$FreeBSD$");
 #include <geom/vinum/geom_vinum_raid5.h>
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, vinum, CTLFLAG_RW, 0, "GEOM_VINUM stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, vinum, CTLFLAG_RW, 0,
+    "GEOM_VINUM stuff");
 u_int g_vinum_debug = 0;
 TUNABLE_INT("kern.geom.vinum.debug", &g_vinum_debug);
 SYSCTL_UINT(_kern_geom_vinum, OID_AUTO, debug, CTLFLAG_RW, &g_vinum_debug, 0,
diff --git a/sys/geom/virstor/g_virstor.c b/sys/geom/virstor/g_virstor.c
index e9bee5c..a180ef2 100644
--- a/sys/geom/virstor/g_virstor.c
+++ b/sys/geom/virstor/g_virstor.c
@@ -80,7 +80,8 @@ struct g_class g_virstor_class = {
 
 /* Declare sysctl's and loader tunables */
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, virstor, CTLFLAG_RW, 0, "GEOM_GVIRSTOR information");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, virstor, CTLFLAG_RW, 0,
+    "GEOM_GVIRSTOR information");
 
 static u_int g_virstor_debug = 2; /* XXX: lower to 2 when released to public */
 TUNABLE_INT("kern.geom.virstor.debug", &g_virstor_debug);
diff --git a/sys/geom/zero/g_zero.c b/sys/geom/zero/g_zero.c
index b8899e7..1c2c54f 100644
--- a/sys/geom/zero/g_zero.c
+++ b/sys/geom/zero/g_zero.c
@@ -42,7 +42,8 @@ __FBSDID("$FreeBSD$");
 #define	G_ZERO_CLASS_NAME	"ZERO"
 
 SYSCTL_DECL(_kern_geom);
-SYSCTL_NODE(_kern_geom, OID_AUTO, zero, CTLFLAG_RW, 0, "GEOM_ZERO stuff");
+static SYSCTL_NODE(_kern_geom, OID_AUTO, zero, CTLFLAG_RW, 0,
+    "GEOM_ZERO stuff");
 static int g_zero_clear = 1;
 SYSCTL_INT(_kern_geom_zero, OID_AUTO, clear, CTLFLAG_RW, &g_zero_clear, 0,
     "Clear read data buffer");
diff --git a/sys/gnu/fs/xfs/FreeBSD/support/debug.c b/sys/gnu/fs/xfs/FreeBSD/support/debug.c
index 72345f0..cf2ef94 100644
--- a/sys/gnu/fs/xfs/FreeBSD/support/debug.c
+++ b/sys/gnu/fs/xfs/FreeBSD/support/debug.c
@@ -40,7 +40,7 @@
 
 #include <support/debug.h>
 
-SYSCTL_NODE(_debug, OID_AUTO, xfs, CTLFLAG_RD, 0, "XFS debug options");
+static SYSCTL_NODE(_debug, OID_AUTO, xfs, CTLFLAG_RD, 0, "XFS debug options");
 
 static int verbosity = 10;
 SYSCTL_INT(_debug_xfs, OID_AUTO, verbosity, CTLFLAG_RW, &verbosity, 0, "");
diff --git a/sys/gnu/fs/xfs/FreeBSD/xfs_mountops.c b/sys/gnu/fs/xfs/FreeBSD/xfs_mountops.c
index e9ccfc9..413a524 100644
--- a/sys/gnu/fs/xfs/FreeBSD/xfs_mountops.c
+++ b/sys/gnu/fs/xfs/FreeBSD/xfs_mountops.c
@@ -71,7 +71,7 @@
 
 #include <xfs_mountops.h>
 
-MALLOC_DEFINE(M_XFSNODE, "XFS node", "XFS vnode private part");
+static MALLOC_DEFINE(M_XFSNODE, "XFS node", "XFS vnode private part");
 
 static vfs_mount_t	_xfs_mount;
 static vfs_unmount_t	_xfs_unmount;
diff --git a/sys/i386/conf/DEFAULTS b/sys/i386/conf/DEFAULTS
index 78d807c..1c5cd96 100644
--- a/sys/i386/conf/DEFAULTS
+++ b/sys/i386/conf/DEFAULTS
@@ -30,3 +30,6 @@ options 	NATIVE
 device		atpic
 
 options 	NEW_PCIB
+
+# Allow mounting non-MPSAFE filesystems
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC
index 12bcda4..c9b6a18 100644
--- a/sys/i386/conf/GENERIC
+++ b/sys/i386/conf/GENERIC
@@ -65,8 +65,11 @@ options 	MAC			# TrustedBSD MAC Framework
 #options 	KDTRACE_HOOKS		# Kernel DTrace hooks
 options 	INCLUDE_CONFIG_FILE     # Include this file in kernel
 
-# Debugging for use in -current
+# Debugging support.  Always need this:
 options 	KDB			# Enable kernel debugger support.
+# For minimum debugger support (stable branch) use:
+#options 	KDB_TRACE		# Print a stack trace for a panic.
+# For full debugger support use this instead:
 options 	DDB			# Support DDB.
 options 	GDB			# Support remote GDB.
 options 	DEADLKRES		# Enable the deadlock resolver
@@ -107,7 +110,7 @@ options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
 device		ahd		# AHA39320/29320 and onboard AIC79xx devices
 options 	AHD_REG_PRETTY_PRINT	# Print register bitfields in debug
 					# output.  Adds ~215k to driver.
-device		amd		# AMD 53C974 (Tekram DC-390(T))
+device		esp		# AMD Am53C974 (Tekram DC-390(T))
 device		hptiop		# Highpoint RocketRaid 3xxx series
 device		isp		# Qlogic family
 #device		ispfw		# Firmware for QLogic HBAs- normally a module
@@ -358,3 +361,8 @@ device		snd_hda		# Intel High Definition Audio
 device		snd_ich		# Intel, NVidia and other ICH AC'97 Audio
 device		snd_uaudio	# USB Audio
 device		snd_via8233	# VIA VT8233x Audio
+
+# MMC/SD
+device		mmc		# MMC/SD bus
+device		mmcsd		# MMC/SD memory card
+device		sdhci		# Generic PCI SD Host Controller
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index 839dd0d..c7352d9 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -3299,7 +3299,8 @@ int
 fill_fpregs(struct thread *td, struct fpreg *fpregs)
 {
 
-	KASSERT(td == curthread || TD_IS_SUSPENDED(td),
+	KASSERT(td == curthread || TD_IS_SUSPENDED(td) ||
+	    P_SHOULDSTOP(td->td_proc),
 	    ("not suspended thread %p", td));
 #ifdef DEV_NPX
 	npxgetregs(td);
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 0900901..11d23ba 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -1152,7 +1152,7 @@ start_ap(int apic_id)
 u_int xhits_gbl[MAXCPU];
 u_int xhits_pg[MAXCPU];
 u_int xhits_rng[MAXCPU];
-SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
+static SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
     sizeof(xhits_gbl), "IU", "");
 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index db5f995..3c9d83b 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -218,7 +218,7 @@ pt_entry_t pg_nx;
 static uma_zone_t pdptzone;
 #endif
 
-SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
+static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
 
 static int pat_works = 1;
 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1,
@@ -795,7 +795,7 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0,
 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0,
 	"Page share factor per proc");
 
-SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
     "2/4MB page mapping counters");
 
 static u_long pmap_pde_demotions;
@@ -1720,7 +1720,6 @@ pmap_pinit(pmap_t pmap)
 {
 	vm_page_t m, ptdpg[NPGPTD];
 	vm_paddr_t pa;
-	static int color;
 	int i;
 
 	PMAP_LOCK_INIT(pmap);
@@ -1754,9 +1753,8 @@ pmap_pinit(pmap_t pmap)
 	 * allocate the page directory page(s)
 	 */
 	for (i = 0; i < NPGPTD;) {
-		m = vm_page_alloc(NULL, color++,
-		    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
-		    VM_ALLOC_ZERO);
+		m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
+		    VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 		if (m == NULL)
 			VM_WAIT;
 		else {
@@ -2274,7 +2272,6 @@ get_pv_entry(pmap_t pmap, int try)
 {
 	static const struct timeval printinterval = { 60, 0 };
 	static struct timeval lastprint;
-	static vm_pindex_t colour;
 	struct vpgqueues *pq;
 	int bit, field;
 	pv_entry_t pv;
@@ -2320,7 +2317,7 @@ retry:
 	 * queues lock.  If "pv_vafree" is currently non-empty, it will
 	 * remain non-empty until pmap_ptelist_alloc() completes.
 	 */
-	if (pv_vafree == 0 || (m = vm_page_alloc(NULL, colour, (pq ==
+	if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, (pq ==
 	    &vm_page_queues[PQ_ACTIVE] ? VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL) |
 	    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 		if (try) {
@@ -2346,7 +2343,6 @@ retry:
 	}
 	PV_STAT(pc_chunk_count++);
 	PV_STAT(pc_chunk_allocs++);
-	colour++;
 	pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
 	pmap_qenter((vm_offset_t)pc, &m, 1);
 	pc->pc_pmap = pmap;
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index 5006f41..6e7f1d9 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -136,7 +136,7 @@ void dblfault_handler(void);
 
 extern inthand_t IDTVEC(lcall_syscall);
 
-#define MAX_TRAP_MSG		30
+#define MAX_TRAP_MSG		33
 static char *trap_msg[] = {
 	"",					/*  0 unused */
 	"privileged instruction fault",		/*  1 T_PRIVINFLT */
@@ -169,6 +169,10 @@ static char *trap_msg[] = {
 	"machine check trap",			/* 28 T_MCHK */
 	"SIMD floating-point exception",	/* 29 T_XMMFLT */
 	"reserved (unknown) fault",		/* 30 T_RESERVED */
+	"",					/* 31 unused (reserved) */
+	"DTrace pid return trap",               /* 32 T_DTRACE_RET */
+	"DTrace fasttrap probe trap",           /* 33 T_DTRACE_PROBE */
+
 };
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
@@ -265,10 +269,6 @@ trap(struct trapframe *frame)
 	 * handled the trap and modified the trap frame so that this
 	 * function can return normally.
 	 */
-	if ((type == T_PROTFLT || type == T_PAGEFLT) &&
-	    dtrace_trap_func != NULL)
-		if ((*dtrace_trap_func)(frame, type))
-			goto out;
 	if (type == T_DTRACE_PROBE || type == T_DTRACE_RET ||
 	    type == T_BPTFLT) {
 		struct reg regs;
@@ -287,6 +287,9 @@ trap(struct trapframe *frame)
 		    dtrace_return_probe_ptr(&regs) == 0)
 			goto out;
 	}
+	if ((type == T_PROTFLT || type == T_PAGEFLT) &&
+	    dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type))
+		goto out;
 #endif
 
 	if ((frame->tf_eflags & PSL_I) == 0) {
@@ -1062,9 +1065,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
 #include "../../kern/subr_syscall.c"
 
 /*
- *	syscall -	system call request C handler
- *
- *	A system call is essentially treated as a trap.
+ * syscall - system call request C handler.  A system call is
+ * essentially treated as a trap by reusing the frame layout.
  */
 void
 syscall(struct trapframe *frame)
diff --git a/sys/i386/ibcs2/ibcs2_fcntl.c b/sys/i386/ibcs2/ibcs2_fcntl.c
index fddfcb5..2902da7 100644
--- a/sys/i386/ibcs2/ibcs2_fcntl.c
+++ b/sys/i386/ibcs2/ibcs2_fcntl.c
@@ -243,7 +243,7 @@ ibcs2_access(td, uap)
 	int error;
 
         CHECKALTEXIST(td, uap->path, &path);
-	error = kern_access(td, path, UIO_SYSSPACE, uap->flags);
+	error = kern_access(td, path, UIO_SYSSPACE, uap->amode);
 	free(path, M_TEMP);
 	return (error);
 }
diff --git a/sys/i386/ibcs2/ibcs2_proto.h b/sys/i386/ibcs2/ibcs2_proto.h
index 4172565..bc5aeba 100644
--- a/sys/i386/ibcs2/ibcs2_proto.h
+++ b/sys/i386/ibcs2/ibcs2_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/ibcs2/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/i386/ibcs2/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #ifndef _IBCS2_SYSPROTO_H_
@@ -120,7 +120,7 @@ struct ibcs2_utime_args {
 };
 struct ibcs2_access_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char amode_l_[PADL_(int)]; int amode; char amode_r_[PADR_(int)];
 };
 struct ibcs2_nice_args {
 	char incr_l_[PADL_(int)]; int incr; char incr_r_[PADR_(int)];
diff --git a/sys/i386/ibcs2/ibcs2_syscall.h b/sys/i386/ibcs2/ibcs2_syscall.h
index 64dca57..2a1d9c3 100644
--- a/sys/i386/ibcs2/ibcs2_syscall.h
+++ b/sys/i386/ibcs2/ibcs2_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/ibcs2/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/i386/ibcs2/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #define	IBCS2_SYS_syscall	0
diff --git a/sys/i386/ibcs2/ibcs2_sysent.c b/sys/i386/ibcs2/ibcs2_sysent.c
index 1fb4905..5c11b8d 100644
--- a/sys/i386/ibcs2/ibcs2_sysent.c
+++ b/sys/i386/ibcs2/ibcs2_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/ibcs2/syscalls.master 203660 2010-02-08 10:02:01Z ed 
+ * created from FreeBSD: head/sys/i386/ibcs2/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #include <sys/param.h>
diff --git a/sys/i386/ibcs2/syscalls.master b/sys/i386/ibcs2/syscalls.master
index 9d0eda6..fd06d1e 100644
--- a/sys/i386/ibcs2/syscalls.master
+++ b/sys/i386/ibcs2/syscalls.master
@@ -80,7 +80,7 @@
 				    struct ibcs2_utimbuf *buf); }
 31	AUE_NULL	UNIMPL	ibcs2_stty
 32	AUE_NULL	UNIMPL	ibcs2_gtty
-33	AUE_ACCESS	STD	{ int ibcs2_access(char *path, int flags); }
+33	AUE_ACCESS	STD	{ int ibcs2_access(char *path, int amode); }
 34	AUE_NICE	STD	{ int ibcs2_nice(int incr); }
 35	AUE_STATFS	STD	{ int ibcs2_statfs(char *path, \
 				    struct ibcs2_statfs *buf, int len, \
diff --git a/sys/i386/include/segments.h b/sys/i386/include/segments.h
index 1c1ddee..9e65c3c 100644
--- a/sys/i386/include/segments.h
+++ b/sys/i386/include/segments.h
@@ -207,7 +207,7 @@ struct region_descriptor {
 #define	IDT_XF		19	/* #XF: SIMD Floating-Point Exception */
 #define	IDT_IO_INTS	NRSVIDT	/* Base of IDT entries for I/O interrupts. */
 #define	IDT_SYSCALL	0x80	/* System Call Interrupt Vector */
-#define	IDT_DTRACE_RET	0x92	/* DTrace pid provider Interrupt Vector */
+#define	IDT_DTRACE_RET	0x20	/* DTrace pid provider Interrupt Vector */
 
 /*
  * Entries in the Global Descriptor Table (GDT)
diff --git a/sys/i386/include/signal.h b/sys/i386/include/signal.h
index 7a5f876..c636c2c 100644
--- a/sys/i386/include/signal.h
+++ b/sys/i386/include/signal.h
@@ -46,16 +46,17 @@ typedef int sig_atomic_t;
 #include <machine/trap.h>	/* codes for SIGILL, SIGFPE */
 
 /*
- * Only the kernel should need these old type definitions.
- */
-#if defined(_KERNEL) && defined(COMPAT_43)
-/*
  * Information pushed on stack when a signal is delivered.
  * This is used by the kernel to restore state following
  * execution of the signal handler.  It is also made available
  * to the handler to allow it to restore state properly if
  * a non-standard exit is performed.
  */
+
+#if defined(_KERNEL) && defined(COMPAT_43)
+/*
+ * Only the kernel should need these old type definitions.
+ */
 struct osigcontext {
 	int	sc_onstack;		/* sigstack state to restore */
 	osigset_t sc_mask;		/* signal mask to restore */
@@ -83,7 +84,7 @@ struct osigcontext {
 
 /*
  * The sequence of the fields/registers in struct sigcontext should match
- * those in mcontext_t.
+ * those in mcontext_t and struct trapframe.
  */
 struct sigcontext {
 	struct __sigset sc_mask;	/* signal mask to restore */
@@ -109,8 +110,8 @@ struct sigcontext {
 	int	sc_ss;
 	int	sc_len;			/* sizeof(mcontext_t) */
 	/*
-	 * XXX - See <machine/ucontext.h> and <machine/npx.h> for
-	 *       the following fields.
+	 * See <machine/ucontext.h> and <machine/npx.h> for
+	 * the following fields.
 	 */
 	int	sc_fpformat;
 	int	sc_ownedfp;
diff --git a/sys/i386/include/trap.h b/sys/i386/include/trap.h
index d8e36b5..a395d62 100644
--- a/sys/i386/include/trap.h
+++ b/sys/i386/include/trap.h
@@ -62,8 +62,8 @@
 #define	T_MCHK		28	/* machine check trap */
 #define	T_XMMFLT	29	/* SIMD floating-point exception */
 #define	T_RESERVED	30	/* reserved (unknown) */
-#define	T_DTRACE_RET	31	/* DTrace pid return */
-#define	T_DTRACE_PROBE	32	/* DTrace fasttrap probe */
+#define	T_DTRACE_RET	32	/* DTrace pid return */
+#define	T_DTRACE_PROBE	33	/* DTrace fasttrap probe */
 
 /* XXX most of the following codes aren't used, but could be. */
 
diff --git a/sys/i386/include/ucontext.h b/sys/i386/include/ucontext.h
index d8657d3..d9f8344 100644
--- a/sys/i386/include/ucontext.h
+++ b/sys/i386/include/ucontext.h
@@ -33,9 +33,9 @@
 
 typedef struct __mcontext {
 	/*
-	 * The first 20 fields must match the definition of
-	 * sigcontext. So that we can support sigcontext
-	 * and ucontext_t at the same time.
+	 * The definition of mcontext_t shall match the layout of
+	 * struct sigcontext after the sc_mask member.  So that we can
+	 * support sigcontext and ucontext_t at the same time.
 	 */
 	__register_t	mc_onstack;	/* XXX - sigcontext compat. */
 	__register_t	mc_gs;		/* machine state (struct trapframe) */
diff --git a/sys/i386/linux/linux_proto.h b/sys/i386/linux/linux_proto.h
index 6f51bb2..d480012 100644
--- a/sys/i386/linux/linux_proto.h
+++ b/sys/i386/linux/linux_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/linux/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/i386/linux/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 #ifndef _LINUX_SYSPROTO_H_
@@ -12,6 +12,7 @@
 #include <sys/signal.h>
 #include <sys/acl.h>
 #include <sys/cpuset.h>
+#include <sys/_ffcounter.h>
 #include <sys/_semaphore.h>
 #include <sys/ucontext.h>
 
@@ -135,7 +136,7 @@ struct linux_utime_args {
 };
 struct linux_access_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char flags_l_[PADL_(l_int)]; l_int flags; char flags_r_[PADR_(l_int)];
+	char amode_l_[PADL_(l_int)]; l_int amode; char amode_r_[PADR_(l_int)];
 };
 struct linux_nice_args {
 	char inc_l_[PADL_(l_int)]; l_int inc; char inc_r_[PADR_(l_int)];
@@ -961,7 +962,7 @@ struct linux_linkat_args {
 	char oldname_l_[PADL_(const char *)]; const char * oldname; char oldname_r_[PADR_(const char *)];
 	char newdfd_l_[PADL_(l_int)]; l_int newdfd; char newdfd_r_[PADR_(l_int)];
 	char newname_l_[PADL_(const char *)]; const char * newname; char newname_r_[PADR_(const char *)];
-	char flags_l_[PADL_(l_int)]; l_int flags; char flags_r_[PADR_(l_int)];
+	char flag_l_[PADL_(l_int)]; l_int flag; char flag_r_[PADR_(l_int)];
 };
 struct linux_symlinkat_args {
 	char oldname_l_[PADL_(const char *)]; const char * oldname; char oldname_r_[PADR_(const char *)];
@@ -982,7 +983,8 @@ struct linux_fchmodat_args {
 struct linux_faccessat_args {
 	char dfd_l_[PADL_(l_int)]; l_int dfd; char dfd_r_[PADR_(l_int)];
 	char filename_l_[PADL_(const char *)]; const char * filename; char filename_r_[PADR_(const char *)];
-	char mode_l_[PADL_(l_int)]; l_int mode; char mode_r_[PADR_(l_int)];
+	char amode_l_[PADL_(l_int)]; l_int amode; char amode_r_[PADR_(l_int)];
+	char flag_l_[PADL_(l_int)]; l_int flag; char flag_r_[PADR_(l_int)];
 };
 struct linux_pselect6_args {
 	register_t dummy;
diff --git a/sys/i386/linux/linux_syscall.h b/sys/i386/linux/linux_syscall.h
index a00963c..367dbf7 100644
--- a/sys/i386/linux/linux_syscall.h
+++ b/sys/i386/linux/linux_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/linux/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/i386/linux/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 #define	LINUX_SYS_exit	1
diff --git a/sys/i386/linux/linux_syscalls.c b/sys/i386/linux/linux_syscalls.c
index 6f10d5d..5ebe4f9 100644
--- a/sys/i386/linux/linux_syscalls.c
+++ b/sys/i386/linux/linux_syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/linux/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/i386/linux/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 const char *linux_syscallnames[] = {
diff --git a/sys/i386/linux/linux_sysent.c b/sys/i386/linux/linux_sysent.c
index 491ead7..27b2d3b 100644
--- a/sys/i386/linux/linux_sysent.c
+++ b/sys/i386/linux/linux_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/i386/linux/syscalls.master 220028 2011-03-26 10:51:56Z avg 
+ * created from FreeBSD: head/sys/i386/linux/syscalls.master 227693 2011-11-19 07:19:37Z ed 
  */
 
 #include <sys/param.h>
diff --git a/sys/i386/linux/linux_systrace_args.c b/sys/i386/linux/linux_systrace_args.c
index ce45424..4c06a7f 100644
--- a/sys/i386/linux/linux_systrace_args.c
+++ b/sys/i386/linux/linux_systrace_args.c
@@ -238,7 +238,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 33: {
 		struct linux_access_args *p = params;
 		uarg[0] = (intptr_t) p->path; /* char * */
-		iarg[1] = p->flags; /* l_int */
+		iarg[1] = p->amode; /* l_int */
 		*n_args = 2;
 		break;
 	}
@@ -2087,7 +2087,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		uarg[1] = (intptr_t) p->oldname; /* const char * */
 		iarg[2] = p->newdfd; /* l_int */
 		uarg[3] = (intptr_t) p->newname; /* const char * */
-		iarg[4] = p->flags; /* l_int */
+		iarg[4] = p->flag; /* l_int */
 		*n_args = 5;
 		break;
 	}
@@ -2124,8 +2124,9 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		struct linux_faccessat_args *p = params;
 		iarg[0] = p->dfd; /* l_int */
 		uarg[1] = (intptr_t) p->filename; /* const char * */
-		iarg[2] = p->mode; /* l_int */
-		*n_args = 3;
+		iarg[2] = p->amode; /* l_int */
+		iarg[3] = p->flag; /* l_int */
+		*n_args = 4;
 		break;
 	}
 	/* linux_pselect6 */
@@ -2186,7 +2187,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	};
 }
 static void
-systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 {
 	const char *p = NULL;
 	switch (sysnum) {
@@ -5360,6 +5361,9 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 		case 2:
 			p = "l_int";
 			break;
+		case 3:
+			p = "l_int";
+			break;
 		default:
 			break;
 		};
@@ -5420,3 +5424,1230 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 	if (p != NULL)
 		strlcpy(desc, p, descsz);
 }
+static void
+systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+{
+	const char *p = NULL;
+	switch (sysnum) {
+#define	nosys	linux_nosys
+	/* sys_exit */
+	case 1:
+		if (ndx == 0 || ndx == 1)
+			p = "void";
+		break;
+	/* linux_fork */
+	case 2:
+	/* read */
+	case 3:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* write */
+	case 4:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_open */
+	case 5:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* close */
+	case 6:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_waitpid */
+	case 7:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_creat */
+	case 8:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_link */
+	case 9:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_unlink */
+	case 10:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_execve */
+	case 11:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chdir */
+	case 12:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_time */
+	case 13:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mknod */
+	case 14:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chmod */
+	case 15:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lchown16 */
+	case 16:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_stat */
+	case 18:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lseek */
+	case 19:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getpid */
+	case 20:
+	/* linux_mount */
+	case 21:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_oldumount */
+	case 22:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setuid16 */
+	case 23:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getuid16 */
+	case 24:
+	/* linux_stime */
+	case 25:
+	/* linux_ptrace */
+	case 26:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_alarm */
+	case 27:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstat */
+	case 28:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pause */
+	case 29:
+	/* linux_utime */
+	case 30:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_access */
+	case 33:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_nice */
+	case 34:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sync */
+	case 36:
+	/* linux_kill */
+	case 37:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rename */
+	case 38:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mkdir */
+	case 39:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rmdir */
+	case 40:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* dup */
+	case 41:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pipe */
+	case 42:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_times */
+	case 43:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_brk */
+	case 45:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setgid16 */
+	case 46:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getgid16 */
+	case 47:
+	/* linux_signal */
+	case 48:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_geteuid16 */
+	case 49:
+	/* linux_getegid16 */
+	case 50:
+	/* acct */
+	case 51:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_umount */
+	case 52:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ioctl */
+	case 54:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fcntl */
+	case 55:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setpgid */
+	case 57:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_olduname */
+	case 59:
+	/* umask */
+	case 60:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chroot */
+	case 61:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ustat */
+	case 62:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* dup2 */
+	case 63:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getppid */
+	case 64:
+	/* getpgrp */
+	case 65:
+	/* setsid */
+	case 66:
+	/* linux_sigaction */
+	case 67:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sgetmask */
+	case 68:
+	/* linux_ssetmask */
+	case 69:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setreuid16 */
+	case 70:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setregid16 */
+	case 71:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigsuspend */
+	case 72:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigpending */
+	case 73:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sethostname */
+	case 74:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setrlimit */
+	case 75:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_old_getrlimit */
+	case 76:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getrusage */
+	case 77:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* gettimeofday */
+	case 78:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* settimeofday */
+	case 79:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getgroups16 */
+	case 80:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setgroups16 */
+	case 81:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_old_select */
+	case 82:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_symlink */
+	case 83:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lstat */
+	case 84:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readlink */
+	case 85:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_uselib */
+	case 86:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* swapon */
+	case 87:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_reboot */
+	case 88:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readdir */
+	case 89:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mmap */
+	case 90:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munmap */
+	case 91:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_truncate */
+	case 92:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ftruncate */
+	case 93:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchmod */
+	case 94:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchown */
+	case 95:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getpriority */
+	case 96:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setpriority */
+	case 97:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_statfs */
+	case 99:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstatfs */
+	case 100:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ioperm */
+	case 101:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_socketcall */
+	case 102:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_syslog */
+	case 103:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setitimer */
+	case 104:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getitimer */
+	case 105:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newstat */
+	case 106:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newlstat */
+	case 107:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newfstat */
+	case 108:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_uname */
+	case 109:
+	/* linux_iopl */
+	case 110:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_vhangup */
+	case 111:
+	/* linux_vm86old */
+	case 113:
+	/* linux_wait4 */
+	case 114:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_swapoff */
+	case 115:
+	/* linux_sysinfo */
+	case 116:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ipc */
+	case 117:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fsync */
+	case 118:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigreturn */
+	case 119:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clone */
+	case 120:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setdomainname */
+	case 121:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_newuname */
+	case 122:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_modify_ldt */
+	case 123:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_adjtimex */
+	case 124:
+	/* linux_mprotect */
+	case 125:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigprocmask */
+	case 126:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_create_module */
+	case 127:
+	/* linux_init_module */
+	case 128:
+	/* linux_delete_module */
+	case 129:
+	/* linux_get_kernel_syms */
+	case 130:
+	/* linux_quotactl */
+	case 131:
+	/* getpgid */
+	case 132:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchdir */
+	case 133:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_bdflush */
+	case 134:
+	/* linux_sysfs */
+	case 135:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_personality */
+	case 136:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsuid16 */
+	case 138:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsgid16 */
+	case 139:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_llseek */
+	case 140:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getdents */
+	case 141:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_select */
+	case 142:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* flock */
+	case 143:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_msync */
+	case 144:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* readv */
+	case 145:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* writev */
+	case 146:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getsid */
+	case 147:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fdatasync */
+	case 148:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sysctl */
+	case 149:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mlock */
+	case 150:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlock */
+	case 151:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mlockall */
+	case 152:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlockall */
+	case 153:
+	/* sched_setparam */
+	case 154:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_getparam */
+	case 155:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_setscheduler */
+	case 156:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_getscheduler */
+	case 157:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_yield */
+	case 158:
+	/* linux_sched_get_priority_max */
+	case 159:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_get_priority_min */
+	case 160:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_rr_get_interval */
+	case 161:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_nanosleep */
+	case 162:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mremap */
+	case 163:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setresuid16 */
+	case 164:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getresuid16 */
+	case 165:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_vm86 */
+	case 166:
+	/* linux_query_module */
+	case 167:
+	/* poll */
+	case 168:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_nfsservctl */
+	case 169:
+	/* linux_setresgid16 */
+	case 170:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getresgid16 */
+	case 171:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_prctl */
+	case 172:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigreturn */
+	case 173:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigaction */
+	case 174:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigprocmask */
+	case 175:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigpending */
+	case 176:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigtimedwait */
+	case 177:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_rt_sigqueueinfo */
+	case 178:
+	/* linux_rt_sigsuspend */
+	case 179:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pread */
+	case 180:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pwrite */
+	case 181:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chown16 */
+	case 182:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getcwd */
+	case 183:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_capget */
+	case 184:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_capset */
+	case 185:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sigaltstack */
+	case 186:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sendfile */
+	case 187:
+	/* linux_vfork */
+	case 190:
+	/* linux_getrlimit */
+	case 191:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mmap2 */
+	case 192:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_truncate64 */
+	case 193:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_ftruncate64 */
+	case 194:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_stat64 */
+	case 195:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lstat64 */
+	case 196:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstat64 */
+	case 197:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lchown */
+	case 198:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getuid */
+	case 199:
+	/* linux_getgid */
+	case 200:
+	/* geteuid */
+	case 201:
+	/* getegid */
+	case 202:
+	/* setreuid */
+	case 203:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setregid */
+	case 204:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getgroups */
+	case 205:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setgroups */
+	case 206:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchown */
+	case 207:
+	/* setresuid */
+	case 208:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresuid */
+	case 209:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setresgid */
+	case 210:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresgid */
+	case 211:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_chown */
+	case 212:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setuid */
+	case 213:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setgid */
+	case 214:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsuid */
+	case 215:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_setfsgid */
+	case 216:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pivot_root */
+	case 217:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mincore */
+	case 218:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* madvise */
+	case 219:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_getdents64 */
+	case 220:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fcntl64 */
+	case 221:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_gettid */
+	case 224:
+	/* linux_setxattr */
+	case 226:
+	/* linux_lsetxattr */
+	case 227:
+	/* linux_fsetxattr */
+	case 228:
+	/* linux_getxattr */
+	case 229:
+	/* linux_lgetxattr */
+	case 230:
+	/* linux_fgetxattr */
+	case 231:
+	/* linux_listxattr */
+	case 232:
+	/* linux_llistxattr */
+	case 233:
+	/* linux_flistxattr */
+	case 234:
+	/* linux_removexattr */
+	case 235:
+	/* linux_lremovexattr */
+	case 236:
+	/* linux_fremovexattr */
+	case 237:
+	/* linux_tkill */
+	case 238:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sys_futex */
+	case 240:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_setaffinity */
+	case 241:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_sched_getaffinity */
+	case 242:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_set_thread_area */
+	case 243:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_get_thread_area */
+	case 244:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fadvise64 */
+	case 250:
+	/* linux_exit_group */
+	case 252:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_lookup_dcookie */
+	case 253:
+	/* linux_epoll_create */
+	case 254:
+	/* linux_epoll_ctl */
+	case 255:
+	/* linux_epoll_wait */
+	case 256:
+	/* linux_remap_file_pages */
+	case 257:
+	/* linux_set_tid_address */
+	case 258:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_timer_create */
+	case 259:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_timer_settime */
+	case 260:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_timer_gettime */
+	case 261:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_timer_getoverrun */
+	case 262:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_timer_delete */
+	case 263:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_settime */
+	case 264:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_gettime */
+	case 265:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_getres */
+	case 266:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_clock_nanosleep */
+	case 267:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_statfs64 */
+	case 268:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstatfs64 */
+	case 269:
+	/* linux_tgkill */
+	case 270:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_utimes */
+	case 271:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fadvise64_64 */
+	case 272:
+	/* linux_mbind */
+	case 274:
+	/* linux_get_mempolicy */
+	case 275:
+	/* linux_set_mempolicy */
+	case 276:
+	/* linux_mq_open */
+	case 277:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mq_unlink */
+	case 278:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mq_timedsend */
+	case 279:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mq_timedreceive */
+	case 280:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mq_notify */
+	case 281:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mq_getsetattr */
+	case 282:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_kexec_load */
+	case 283:
+	/* linux_waitid */
+	case 284:
+	/* linux_add_key */
+	case 286:
+	/* linux_request_key */
+	case 287:
+	/* linux_keyctl */
+	case 288:
+	/* linux_ioprio_set */
+	case 289:
+	/* linux_ioprio_get */
+	case 290:
+	/* linux_inotify_init */
+	case 291:
+	/* linux_inotify_add_watch */
+	case 292:
+	/* linux_inotify_rm_watch */
+	case 293:
+	/* linux_migrate_pages */
+	case 294:
+	/* linux_openat */
+	case 295:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mkdirat */
+	case 296:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_mknodat */
+	case 297:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fchownat */
+	case 298:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_futimesat */
+	case 299:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fstatat64 */
+	case 300:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_unlinkat */
+	case 301:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_renameat */
+	case 302:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_linkat */
+	case 303:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_symlinkat */
+	case 304:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_readlinkat */
+	case 305:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_fchmodat */
+	case 306:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_faccessat */
+	case 307:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_pselect6 */
+	case 308:
+	/* linux_ppoll */
+	case 309:
+	/* linux_unshare */
+	case 310:
+	/* linux_set_robust_list */
+	case 311:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_get_robust_list */
+	case 312:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linux_splice */
+	case 313:
+	/* linux_sync_file_range */
+	case 314:
+	/* linux_tee */
+	case 315:
+	/* linux_vmsplice */
+	case 316:
+	default:
+		break;
+	};
+	if (p != NULL)
+		strlcpy(desc, p, descsz);
+}
diff --git a/sys/i386/linux/syscalls.master b/sys/i386/linux/syscalls.master
index 27ddef6..7e7965f 100644
--- a/sys/i386/linux/syscalls.master
+++ b/sys/i386/linux/syscalls.master
@@ -86,7 +86,7 @@
 				    struct l_utimbuf *times); }
 31	AUE_NULL	UNIMPL	stty
 32	AUE_NULL	UNIMPL	gtty
-33	AUE_ACCESS	STD	{ int linux_access(char *path, l_int flags); }
+33	AUE_ACCESS	STD	{ int linux_access(char *path, l_int amode); }
 34	AUE_NICE	STD	{ int linux_nice(l_int inc); }
 35	AUE_NULL	UNIMPL	ftime
 36	AUE_SYNC	NOPROTO	{ int sync(void); }
@@ -493,14 +493,14 @@
 302	AUE_RENAMEAT	STD	{ int linux_renameat(l_int olddfd, const char *oldname, \
 					l_int newdfd, const char *newname); }
 303	AUE_LINKAT	STD	{ int linux_linkat(l_int olddfd, const char *oldname, \
-					l_int newdfd, const char *newname, l_int flags); }
+					l_int newdfd, const char *newname, l_int flag); }
 304	AUE_SYMLINKAT	STD	{ int linux_symlinkat(const char *oldname, l_int newdfd, \
 					const char *newname); }
 305	AUE_READLINKAT	STD	{ int linux_readlinkat(l_int dfd, const char *path, \
 					char *buf, l_int bufsiz); }
 306	AUE_FCHMODAT	STD	{ int linux_fchmodat(l_int dfd, const char *filename, \
 					l_mode_t mode); }
-307	AUE_FACCESSAT	STD	{ int linux_faccessat(l_int dfd, const char *filename, l_int mode); }
+307	AUE_FACCESSAT	STD	{ int linux_faccessat(l_int dfd, const char *filename, l_int amode, l_int flag); }
 308	AUE_NULL	STD	{ int linux_pselect6(void); }
 309	AUE_NULL	STD	{ int linux_ppoll(void); }
 310	AUE_NULL	STD	{ int linux_unshare(void); }
diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c
index b19f75c..c81834b 100644
--- a/sys/i386/xen/pmap.c
+++ b/sys/i386/xen/pmap.c
@@ -277,7 +277,7 @@ SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD,
 	   "Number of times pmap_pte_quick didn't change PMAP1");
 static struct mtx PMAP2mutex;
 
-SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
+static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
 static int pg_ps_enabled;
 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0,
     "Are large page mappings enabled?");
@@ -286,7 +286,7 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0,
 	"Max number of PV entries");
 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0,
 	"Page share factor per proc");
-SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
     "2/4MB page mapping counters");
 
 static u_long pmap_pde_mappings;
@@ -1475,7 +1475,6 @@ pmap_pinit(pmap_t pmap)
 {
 	vm_page_t m, ptdpg[NPGPTD + 1];
 	int npgptd = NPGPTD + 1;
-	static int color;
 	int i;
 
 #ifdef HAMFISTED_LOCKING
@@ -1507,9 +1506,8 @@ pmap_pinit(pmap_t pmap)
 	 * allocate the page directory page(s)
 	 */
 	for (i = 0; i < npgptd;) {
-		m = vm_page_alloc(NULL, color++,
-		    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
-		    VM_ALLOC_ZERO);
+		m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
+		    VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 		if (m == NULL)
 			VM_WAIT;
 		else {
@@ -2103,7 +2101,6 @@ get_pv_entry(pmap_t pmap, int try)
 {
 	static const struct timeval printinterval = { 60, 0 };
 	static struct timeval lastprint;
-	static vm_pindex_t colour;
 	struct vpgqueues *pq;
 	int bit, field;
 	pv_entry_t pv;
@@ -2149,7 +2146,7 @@ retry:
 	 * queues lock.  If "pv_vafree" is currently non-empty, it will
 	 * remain non-empty until pmap_ptelist_alloc() completes.
 	 */
-	if (pv_vafree == 0 || (m = vm_page_alloc(NULL, colour, (pq ==
+	if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, (pq ==
 	    &vm_page_queues[PQ_ACTIVE] ? VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL) |
 	    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 		if (try) {
@@ -2175,7 +2172,6 @@ retry:
 	}
 	PV_STAT(pc_chunk_count++);
 	PV_STAT(pc_chunk_allocs++);
-	colour++;
 	pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
 	pmap_qenter((vm_offset_t)pc, &m, 1);
 	if ((m->flags & PG_ZERO) == 0)
diff --git a/sys/ia64/conf/DEFAULTS b/sys/ia64/conf/DEFAULTS
index a3e10d6..e31e26b 100644
--- a/sys/ia64/conf/DEFAULTS
+++ b/sys/ia64/conf/DEFAULTS
@@ -20,3 +20,6 @@ options 	GEOM_PART_GPT
 options 	GEOM_PART_MBR
 
 options 	NEW_PCIB
+
+# Allow mounting non-MPSAFE filesystems
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/ia64/conf/GENERIC b/sys/ia64/conf/GENERIC
index a9dd567..542b832 100644
--- a/sys/ia64/conf/GENERIC
+++ b/sys/ia64/conf/GENERIC
@@ -60,8 +60,11 @@ options 	UFS_DIRHASH	# Hash-based directory lookup scheme
 options 	UFS_GJOURNAL	# Enable gjournal-based UFS journaling
 options 	_KPOSIX_PRIORITY_SCHEDULING	# Posix P1003_1B RT extensions
 
-# Debugging for use in -current
-options 	KDB		# Enable kernel debugger support
+# Debugging support.  Always need this:
+options 	KDB		# Enable kernel debugger support.
+# For minimum debugger support (stable branch) use:
+#options 	KDB_TRACE	# Print a stack trace for a panic.
+# For full debugger support use this instead:
 options 	DDB		# Support DDB
 options 	GDB		# Support remote GDB
 options 	DEADLKRES	# Enable the deadlock resolver
diff --git a/sys/ia64/ia64/busdma_machdep.c b/sys/ia64/ia64/busdma_machdep.c
index d7152df..c78ba71 100644
--- a/sys/ia64/ia64/busdma_machdep.c
+++ b/sys/ia64/ia64/busdma_machdep.c
@@ -88,7 +88,7 @@ static int total_bpages;
 static int total_bounced;
 static int total_deferred;
 
-SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
 SYSCTL_INT(_hw_busdma, OID_AUTO, free_bpages, CTLFLAG_RD, &free_bpages, 0,
     "Free bounce pages");
 SYSCTL_INT(_hw_busdma, OID_AUTO, reserved_bpages, CTLFLAG_RD, &reserved_bpages,
diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index 2e14a2e..349800d 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -98,8 +98,8 @@ __FBSDID("$FreeBSD$");
 #include <machine/unwind.h>
 #include <machine/vmparam.h>
 
-SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, "");
-SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RD, 0, "");
+static SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, "");
+static SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RD, 0, "");
 
 static u_int bus_freq;
 SYSCTL_UINT(_hw_freq, OID_AUTO, bus, CTLFLAG_RD, &bus_freq, 0,
diff --git a/sys/ia64/ia64/mca.c b/sys/ia64/ia64/mca.c
index 8484d5a..7b882f6 100644
--- a/sys/ia64/ia64/mca.c
+++ b/sys/ia64/ia64/mca.c
@@ -43,7 +43,7 @@
 #include <machine/sal.h>
 #include <machine/smp.h>
 
-MALLOC_DEFINE(M_MCA, "MCA", "Machine Check Architecture");
+static MALLOC_DEFINE(M_MCA, "MCA", "Machine Check Architecture");
 
 struct mca_info {
 	STAILQ_ENTRY(mca_info) mi_link;
@@ -59,7 +59,7 @@ static int64_t		mca_info_size[SAL_INFO_TYPES];
 static vm_offset_t	mca_info_block;
 static struct mtx	mca_info_block_lock;
 
-SYSCTL_NODE(_hw, OID_AUTO, mca, CTLFLAG_RW, NULL, "MCA container");
+static SYSCTL_NODE(_hw, OID_AUTO, mca, CTLFLAG_RW, NULL, "MCA container");
 
 static int mca_count;		/* Number of records stored. */
 static int mca_first;		/* First (lowest) record ID. */
diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c
index 4c289df..5ba5856 100644
--- a/sys/ia64/ia64/pmap.c
+++ b/sys/ia64/ia64/pmap.c
@@ -207,7 +207,7 @@ static uma_zone_t ptezone;
  * Virtual Hash Page Table (VHPT) data.
  */
 /* SYSCTL_DECL(_machdep); */
-SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
+static SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
 
 struct ia64_bucket *pmap_vhpt_bucket;
 
diff --git a/sys/ia64/ia64/sapic.c b/sys/ia64/ia64/sapic.c
index 5aa1449..58c3426 100644
--- a/sys/ia64/ia64/sapic.c
+++ b/sys/ia64/ia64/sapic.c
@@ -99,7 +99,7 @@ struct sapic_rte {
 	uint64_t	rte_destination_id	:8;
 };
 
-MALLOC_DEFINE(M_SAPIC, "sapic", "I/O SAPIC devices");
+static MALLOC_DEFINE(M_SAPIC, "sapic", "I/O SAPIC devices");
 
 struct sapic *ia64_sapics[16];		/* XXX make this resizable */
 int ia64_sapic_count;
diff --git a/sys/ia64/ia64/sscdisk.c b/sys/ia64/ia64/sscdisk.c
index f9abc0a..b38a73a 100644
--- a/sys/ia64/ia64/sscdisk.c
+++ b/sys/ia64/ia64/sscdisk.c
@@ -72,7 +72,7 @@ ssc(u_int64_t in0, u_int64_t in1, u_int64_t in2, u_int64_t in3, int which)
 #define SSC_NSECT 409600
 #endif
 
-MALLOC_DEFINE(M_SSC, "ssc_disk", "Simulator Disk");
+static MALLOC_DEFINE(M_SSC, "ssc_disk", "Simulator Disk");
 
 static d_strategy_t sscstrategy;
 
diff --git a/sys/ia64/ia64/unwind.c b/sys/ia64/ia64/unwind.c
index 6343ed8..7b4c6d2 100644
--- a/sys/ia64/ia64/unwind.c
+++ b/sys/ia64/ia64/unwind.c
@@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
 
 #include <uwx.h>
 
-MALLOC_DEFINE(M_UNWIND, "Unwind", "Unwind information");
+static MALLOC_DEFINE(M_UNWIND, "Unwind", "Unwind information");
 
 struct unw_entry {
 	uint64_t	ue_start;	/* procedure start */
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 2c094f8..61bf595 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -641,7 +641,7 @@ static char init_path[MAXPATHLEN] =
 #ifdef	INIT_PATH
     __XSTRING(INIT_PATH);
 #else
-    "/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init:/stand/sysinstall";
+    "/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init";
 #endif
 SYSCTL_STRING(_kern, OID_AUTO, init_path, CTLFLAG_RD, init_path, 0,
 	"Path used to search the init process");
diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c
index 0854cec..7ace326 100644
--- a/sys/kern/init_sysent.c
+++ b/sys/kern/init_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 224987 2011-08-18 22:51:30Z jonathan 
+ * created from FreeBSD: head/sys/kern/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #include "opt_compat.h"
@@ -275,9 +275,9 @@ struct sysent sysent[] = {
 	{ AS(ktimer_gettime_args), (sy_call_t *)sys_ktimer_gettime, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 238 = ktimer_gettime */
 	{ AS(ktimer_getoverrun_args), (sy_call_t *)sys_ktimer_getoverrun, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 239 = ktimer_getoverrun */
 	{ AS(nanosleep_args), (sy_call_t *)sys_nanosleep, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 240 = nanosleep */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 241 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 242 = nosys */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 243 = nosys */
+	{ AS(ffclock_getcounter_args), (sy_call_t *)sys_ffclock_getcounter, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 241 = ffclock_getcounter */
+	{ AS(ffclock_setestimate_args), (sy_call_t *)sys_ffclock_setestimate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 242 = ffclock_setestimate */
+	{ AS(ffclock_getestimate_args), (sy_call_t *)sys_ffclock_getestimate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 243 = ffclock_getestimate */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 244 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 245 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 246 = nosys */
@@ -565,5 +565,5 @@ struct sysent sysent[] = {
 	{ AS(rctl_add_rule_args), (sy_call_t *)sys_rctl_add_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 528 = rctl_add_rule */
 	{ AS(rctl_remove_rule_args), (sy_call_t *)sys_rctl_remove_rule, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 529 = rctl_remove_rule */
 	{ AS(posix_fallocate_args), (sy_call_t *)sys_posix_fallocate, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 530 = posix_fallocate */
-	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },			/* 531 = posix_fadvise */
+	{ AS(posix_fadvise_args), (sy_call_t *)sys_posix_fadvise, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 531 = posix_fadvise */
 };
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 4f78073..cf9d6a8 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -304,7 +304,8 @@ static struct kthread_desc deadlkres_kd = {
 
 SYSINIT(deadlkres, SI_SUB_CLOCKS, SI_ORDER_ANY, kthread_start, &deadlkres_kd);
 
-SYSCTL_NODE(_debug, OID_AUTO, deadlkres, CTLFLAG_RW, 0, "Deadlock resolver");
+static SYSCTL_NODE(_debug, OID_AUTO, deadlkres, CTLFLAG_RW, 0,
+    "Deadlock resolver");
 SYSCTL_INT(_debug_deadlkres, OID_AUTO, slptime_threshold, CTLFLAG_RW,
     &slptime_threshold, 0,
     "Number of seconds within is valid to sleep on a sleepqueue");
diff --git a/sys/kern/kern_conf.c b/sys/kern/kern_conf.c
index 8da16d2..267a207 100644
--- a/sys/kern/kern_conf.c
+++ b/sys/kern/kern_conf.c
@@ -1009,14 +1009,13 @@ make_dev_physpath_alias(int flags, struct cdev **cdev, struct cdev *pdev,
 	}
 
 	sprintf(devfspath, "%s/%s", physpath, pdev->si_name);
-	if (old_alias != NULL
-	 && strcmp(old_alias->si_name, devfspath) == 0) {
+	if (old_alias != NULL && strcmp(old_alias->si_name, devfspath) == 0) {
 		/* Retain the existing alias. */
 		*cdev = old_alias;
 		old_alias = NULL;
 		ret = 0;
 	} else {
-		ret = make_dev_alias_p(flags, cdev, pdev, devfspath);
+		ret = make_dev_alias_p(flags, cdev, pdev, "%s", devfspath);
 	}
 out:
 	if (old_alias != NULL)	
diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c
index 3429a0f..c32a63c 100644
--- a/sys/kern/kern_cpu.c
+++ b/sys/kern/kern_cpu.c
@@ -135,7 +135,8 @@ static int		cf_lowest_freq;
 static int		cf_verbose;
 TUNABLE_INT("debug.cpufreq.lowest", &cf_lowest_freq);
 TUNABLE_INT("debug.cpufreq.verbose", &cf_verbose);
-SYSCTL_NODE(_debug, OID_AUTO, cpufreq, CTLFLAG_RD, NULL, "cpufreq debugging");
+static SYSCTL_NODE(_debug, OID_AUTO, cpufreq, CTLFLAG_RD, NULL,
+    "cpufreq debugging");
 SYSCTL_INT(_debug_cpufreq, OID_AUTO, lowest, CTLFLAG_RW, &cf_lowest_freq, 1,
     "Don't provide levels below this frequency.");
 SYSCTL_INT(_debug_cpufreq, OID_AUTO, verbose, CTLFLAG_RW, &cf_verbose, 1,
diff --git a/sys/kern/kern_ctf.c b/sys/kern/kern_ctf.c
index bdff96e..2737860 100644
--- a/sys/kern/kern_ctf.c
+++ b/sys/kern/kern_ctf.c
@@ -90,7 +90,7 @@ link_elf_ctf_get(linker_file_t lf, linker_ctf_t *lc)
 	 * ctfcnt to -1. See below.
 	 */
 	if (ef->ctfcnt < 0)
-		return (0);
+		return (EFTYPE);
 
 	/* Now check if we've already loaded the CTF data.. */
 	if (ef->ctfcnt > 0) {
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 8b3e20e..2ef36d1 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -838,12 +838,12 @@ do_dup(struct thread *td, int flags, int old, int new,
 	if (flags & DUP_FIXED) {
 		if (new >= fdp->fd_nfiles) {
 			/*
-			 * The resource limits are here instead of e.g. fdalloc(),
-			 * because the file descriptor table may be shared between
-			 * processes, so we can't really use racct_add()/racct_sub().
-			 * Instead of counting the number of actually allocated
-			 * descriptors, just put the limit on the size of the file
-			 * descriptor table.
+			 * The resource limits are here instead of e.g.
+			 * fdalloc(), because the file descriptor table may be
+			 * shared between processes, so we can't really use
+			 * racct_add()/racct_sub().  Instead of counting the
+			 * number of actually allocated descriptors, just put
+			 * the limit on the size of the file descriptor table.
 			 */
 #ifdef RACCT
 			PROC_LOCK(p);
@@ -1516,7 +1516,7 @@ fdalloc(struct thread *td, int minfd, int *result)
 	FILEDESC_XLOCK_ASSERT(fdp);
 
 	if (fdp->fd_freefile > minfd)
-		minfd = fdp->fd_freefile;	   
+		minfd = fdp->fd_freefile;
 
 	PROC_LOCK(p);
 	maxfd = min((int)lim_cur(p, RLIMIT_NOFILE), maxfilesperproc);
@@ -2248,7 +2248,7 @@ closef(struct file *fp, struct thread *td)
 
 /*
  * Initialize the file pointer with the specified properties.
- * 
+ *
  * The ops are set with release semantics to be certain that the flags, type,
  * and data are visible when ops is.  This is to prevent ops methods from being
  * called with bad data.
@@ -2575,12 +2575,6 @@ _fdrop(struct file *fp, struct thread *td)
 		panic("fdrop: count %d", fp->f_count);
 	if (fp->f_ops != &badfileops)
 		error = fo_close(fp, td);
-	/*
-	 * The f_cdevpriv cannot be assigned non-NULL value while we
-	 * are destroying the file.
-	 */
-	if (fp->f_cdevpriv != NULL)
-		devfs_fpdrop(fp);
 	atomic_subtract_int(&openfiles, 1);
 	crfree(fp->f_cred);
 	uma_zfree(file_zone, fp);
@@ -3764,28 +3758,32 @@ SYSINIT(select, SI_SUB_LOCK, SI_ORDER_FIRST, filelistinit, NULL);
 /*-------------------------------------------------------------------*/
 
 static int
-badfo_readwrite(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td)
+badfo_readwrite(struct file *fp, struct uio *uio, struct ucred *active_cred,
+    int flags, struct thread *td)
 {
 
 	return (EBADF);
 }
 
 static int
-badfo_truncate(struct file *fp, off_t length, struct ucred *active_cred, struct thread *td)
+badfo_truncate(struct file *fp, off_t length, struct ucred *active_cred,
+    struct thread *td)
 {
 
 	return (EINVAL);
 }
 
 static int
-badfo_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred, struct thread *td)
+badfo_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
+    struct thread *td)
 {
 
 	return (EBADF);
 }
 
 static int
-badfo_poll(struct file *fp, int events, struct ucred *active_cred, struct thread *td)
+badfo_poll(struct file *fp, int events, struct ucred *active_cred,
+    struct thread *td)
 {
 
 	return (0);
@@ -3799,7 +3797,8 @@ badfo_kqfilter(struct file *fp, struct knote *kn)
 }
 
 static int
-badfo_stat(struct file *fp, struct stat *sb, struct ucred *active_cred, struct thread *td)
+badfo_stat(struct file *fp, struct stat *sb, struct ucred *active_cred,
+    struct thread *td)
 {
 
 	return (EBADF);
diff --git a/sys/kern/kern_dtrace.c b/sys/kern/kern_dtrace.c
index 2082fed..5582fb9 100644
--- a/sys/kern/kern_dtrace.c
+++ b/sys/kern/kern_dtrace.c
@@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$");
 FEATURE(kdtrace_hooks,
     "Kernel DTrace hooks which are required to load DTrace kernel modules");
 
-MALLOC_DEFINE(M_KDTRACE, "kdtrace", "DTrace hooks");
+static MALLOC_DEFINE(M_KDTRACE, "kdtrace", "DTrace hooks");
 
 /* Return the DTrace process data size compiled in the kernel hooks. */
 size_t
diff --git a/sys/kern/kern_et.c b/sys/kern/kern_et.c
index 6cf5860..3156c81 100644
--- a/sys/kern/kern_et.c
+++ b/sys/kern/kern_et.c
@@ -41,7 +41,7 @@ struct mtx	et_eventtimers_mtx;
 MTX_SYSINIT(et_eventtimers_init, &et_eventtimers_mtx, "et_mtx", MTX_DEF);
 
 SYSCTL_NODE(_kern, OID_AUTO, eventtimer, CTLFLAG_RW, 0, "Event timers");
-SYSCTL_NODE(_kern_eventtimer, OID_AUTO, et, CTLFLAG_RW, 0, "");
+static SYSCTL_NODE(_kern_eventtimer, OID_AUTO, et, CTLFLAG_RW, 0, "");
 
 /*
  * Register a new event timer hardware.
diff --git a/sys/kern/kern_fail.c b/sys/kern/kern_fail.c
index f192471..85d81e8 100644
--- a/sys/kern/kern_fail.c
+++ b/sys/kern/kern_fail.c
@@ -69,7 +69,7 @@ __FBSDID("$FreeBSD$");
 ILOG_DEFINE_FOR_FILE(L_ISI_FAIL_POINT, L_ILOG, fail_point);
 #endif
 
-MALLOC_DEFINE(M_FAIL_POINT, "Fail Points", "fail points system");
+static MALLOC_DEFINE(M_FAIL_POINT, "Fail Points", "fail points system");
 #define fp_free(ptr) free(ptr, M_FAIL_POINT)
 #define fp_malloc(size, flags) malloc((size), M_FAIL_POINT, (flags))
 
diff --git a/sys/kern/kern_ffclock.c b/sys/kern/kern_ffclock.c
new file mode 100644
index 0000000..62b91f4
--- /dev/null
+++ b/sys/kern/kern_ffclock.c
@@ -0,0 +1,473 @@
+/*-
+ * Copyright (c) 2011 The University of Melbourne
+ * All rights reserved.
+ *
+ * This software was developed by Julien Ridoux at the University of Melbourne
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_ffclock.h"
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/priv.h>
+#include <sys/proc.h>
+#include <sys/sbuf.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/timeffc.h>
+
+#ifdef FFCLOCK
+
+extern struct ffclock_estimate ffclock_estimate;
+extern struct bintime ffclock_boottime;
+extern int8_t ffclock_updated;
+extern struct mtx ffclock_mtx;
+
+/*
+ * Feed-forward clock absolute time. This should be the preferred way to read
+ * the feed-forward clock for "wall-clock" type time. The flags allow to compose
+ * various flavours of absolute time (e.g. with or without leap seconds taken
+ * into account). If valid pointers are provided, the ffcounter value and an
+ * upper bound on clock error associated with the bintime are provided.
+ * NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value
+ * read earlier.
+ */
+void
+ffclock_abstime(ffcounter *ffcount, struct bintime *bt,
+    struct bintime *error_bound, uint32_t flags)
+{
+	struct ffclock_estimate cest;
+	ffcounter ffc;
+	ffcounter update_ffcount;
+	ffcounter ffdelta_error;
+
+	/* Get counter and corresponding time. */
+	if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST)
+		ffclock_last_tick(&ffc, bt, flags);
+	else {
+		ffclock_read_counter(&ffc);
+		ffclock_convert_abs(ffc, bt, flags);
+	}
+
+	/* Current ffclock estimate, use update_ffcount as generation number. */
+	do {
+		update_ffcount = ffclock_estimate.update_ffcount;
+		bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
+	} while (update_ffcount != ffclock_estimate.update_ffcount);
+
+	/*
+	 * Leap second adjustment. Total as seen by synchronisation algorithm
+	 * since it started. cest.leapsec_next is the ffcounter prediction of
+	 * when the next leapsecond occurs.
+	 */
+	if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) {
+		bt->sec -= cest.leapsec_total;
+		if (ffc > cest.leapsec_next)
+			bt->sec -= cest.leapsec;
+	}
+
+	/* Boot time adjustment, for uptime/monotonic clocks. */
+	if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) {
+		bintime_sub(bt, &ffclock_boottime);
+	}
+
+	/* Compute error bound if a valid pointer has been passed. */
+	if (error_bound) {
+		ffdelta_error = ffc - cest.update_ffcount;
+		ffclock_convert_diff(ffdelta_error, error_bound);
+		/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
+		bintime_mul(error_bound, cest.errb_rate *
+		    (uint64_t)18446744073709LL);
+		/* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */
+		bintime_addx(error_bound, cest.errb_abs *
+		    (uint64_t)18446744073LL);
+	}
+
+	if (ffcount)
+		*ffcount = ffc;
+}
+
+/*
+ * Feed-forward difference clock. This should be the preferred way to convert a
+ * time interval in ffcounter values into a time interval in seconds. If a valid
+ * pointer is passed, an upper bound on the error in computing the time interval
+ * in seconds is provided.
+ */
+void
+ffclock_difftime(ffcounter ffdelta, struct bintime *bt,
+    struct bintime *error_bound)
+{
+	ffcounter update_ffcount;
+	uint32_t err_rate;
+
+	ffclock_convert_diff(ffdelta, bt);
+
+	if (error_bound) {
+		do {
+			update_ffcount = ffclock_estimate.update_ffcount;
+			err_rate = ffclock_estimate.errb_rate;
+		} while (update_ffcount != ffclock_estimate.update_ffcount);
+
+		ffclock_convert_diff(ffdelta, error_bound);
+		/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
+		bintime_mul(error_bound, err_rate * (uint64_t)18446744073709LL);
+	}
+}
+
+/*
+ * Sysctl for the Feed-Forward Clock.
+ */
+
+static int ffclock_version = 2;
+SYSCTL_NODE(_kern, OID_AUTO, ffclock, CTLFLAG_RW, 0,
+    "Feed-Forward Clock Support");
+SYSCTL_INT(_kern_ffclock, OID_AUTO, version, CTLFLAG_RD, &ffclock_version, 0,
+    "Version of Feed-Forward Clock Support");
+
+/*
+ * Sysctl to select which clock is read when calling any of the
+ * [get]{bin,nano,micro}[up]time() functions.
+ */
+char *sysclocks[] = {"feedback", "feed-forward"};
+
+#define	NUM_SYSCLOCKS (sizeof(sysclocks) / sizeof(*sysclocks))
+
+/* Report or change the active timecounter hardware. */
+static int
+sysctl_kern_ffclock_choice(SYSCTL_HANDLER_ARGS)
+{
+	struct sbuf *s;
+	int clk, error;
+
+	s = sbuf_new_for_sysctl(NULL, NULL, 16 * NUM_SYSCLOCKS, req);
+	if (s == NULL)
+		return (ENOMEM);
+
+	for (clk = 0; clk < NUM_SYSCLOCKS; clk++) {
+		sbuf_cat(s, sysclocks[clk]);
+		if (clk + 1 < NUM_SYSCLOCKS)
+			sbuf_cat(s, " ");
+	}
+	error = sbuf_finish(s);
+	sbuf_delete(s);
+
+	return (error);
+}
+
+SYSCTL_PROC(_kern_ffclock, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
+    0, 0, sysctl_kern_ffclock_choice, "A", "Clock paradigms available");
+
+extern int sysclock_active;
+
+static int
+sysctl_kern_ffclock_active(SYSCTL_HANDLER_ARGS)
+{
+	char newclock[32];
+	int error;
+
+	switch (sysclock_active) {
+	case SYSCLOCK_FBCK:
+		strlcpy(newclock, sysclocks[SYSCLOCK_FBCK], sizeof(newclock));
+		break;
+	case SYSCLOCK_FFWD:
+		strlcpy(newclock, sysclocks[SYSCLOCK_FFWD], sizeof(newclock));
+		break;
+	}
+
+	error = sysctl_handle_string(oidp, &newclock[0], sizeof(newclock), req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	if (strncmp(newclock, sysclocks[SYSCLOCK_FBCK],
+	    sizeof(sysclocks[SYSCLOCK_FBCK])) == 0)
+		sysclock_active = SYSCLOCK_FBCK;
+	else if (strncmp(newclock, sysclocks[SYSCLOCK_FFWD],
+	    sizeof(sysclocks[SYSCLOCK_FFWD])) == 0)
+		sysclock_active = SYSCLOCK_FFWD;
+	else
+		return (EINVAL);
+
+	return (error);
+}
+
+SYSCTL_PROC(_kern_ffclock, OID_AUTO, active, CTLTYPE_STRING | CTLFLAG_RW,
+    0, 0, sysctl_kern_ffclock_active, "A", "Kernel clock selected");
+
+int sysctl_kern_ffclock_ffcounter_bypass = 0;
+
+SYSCTL_INT(_kern_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW,
+    &sysctl_kern_ffclock_ffcounter_bypass, 0,
+    "Use reliable hardware timecounter as the Feed-Forward Counter");
+
+/*
+ * High level functions to access the Feed-Forward Clock.
+ */
+void
+ffclock_bintime(struct bintime *bt)
+{
+
+	ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
+}
+
+void
+ffclock_nanotime(struct timespec *tsp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
+	bintime2timespec(&bt, tsp);
+}
+
+void
+ffclock_microtime(struct timeval *tvp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
+	bintime2timeval(&bt, tvp);
+}
+
+void
+ffclock_getbintime(struct bintime *bt)
+{
+
+	ffclock_abstime(NULL, bt, NULL,
+	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
+}
+
+void
+ffclock_getnanotime(struct timespec *tsp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL,
+	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
+	bintime2timespec(&bt, tsp);
+}
+
+void
+ffclock_getmicrotime(struct timeval *tvp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL,
+	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
+	bintime2timeval(&bt, tvp);
+}
+
+void
+ffclock_binuptime(struct bintime *bt)
+{
+
+	ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
+}
+
+void
+ffclock_nanouptime(struct timespec *tsp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
+	bintime2timespec(&bt, tsp);
+}
+
+void
+ffclock_microuptime(struct timeval *tvp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
+	bintime2timeval(&bt, tvp);
+}
+
+void
+ffclock_getbinuptime(struct bintime *bt)
+{
+
+	ffclock_abstime(NULL, bt, NULL,
+	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
+}
+
+void
+ffclock_getnanouptime(struct timespec *tsp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL,
+	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
+	bintime2timespec(&bt, tsp);
+}
+
+void
+ffclock_getmicrouptime(struct timeval *tvp)
+{
+	struct bintime bt;
+
+	ffclock_abstime(NULL, &bt, NULL,
+	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
+	bintime2timeval(&bt, tvp);
+}
+
+void
+ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt)
+{
+
+	ffclock_difftime(ffdelta, bt, NULL);
+}
+
+void
+ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp)
+{
+	struct bintime bt;
+
+	ffclock_difftime(ffdelta, &bt, NULL);
+	bintime2timespec(&bt, tsp);
+}
+
+void
+ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp)
+{
+	struct bintime bt;
+
+	ffclock_difftime(ffdelta, &bt, NULL);
+	bintime2timeval(&bt, tvp);
+}
+
+/*
+ * System call allowing userland applications to retrieve the current value of
+ * the Feed-Forward Clock counter.
+ */
+#ifndef _SYS_SYSPROTO_H_
+struct ffclock_getcounter_args {
+	ffcounter *ffcount;
+};
+#endif
+/* ARGSUSED */
+int
+sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap)
+{
+	ffcounter ffcount;
+	int error;
+
+	ffcount = 0;
+	ffclock_read_counter(&ffcount);
+	if (ffcount == 0)
+		return (EAGAIN);
+	error = copyout(&ffcount, uap->ffcount, sizeof(ffcounter));
+
+	return (error);
+}
+
+/*
+ * System call allowing the synchronisation daemon to push new feed-foward clock
+ * estimates to the kernel. Acquire ffclock_mtx to prevent concurrent updates
+ * and ensure data consistency.
+ * NOTE: ffclock_updated signals the fftimehands that new estimates are
+ * available. The updated estimates are picked up by the fftimehands on next
+ * tick, which could take as long as 1/hz seconds (if ticks are not missed).
+ */
+#ifndef _SYS_SYSPROTO_H_
+struct ffclock_setestimate_args {
+	struct ffclock_estimate *cest;
+};
+#endif
+/* ARGSUSED */
+int
+sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap)
+{
+	struct ffclock_estimate cest;
+	int error;
+
+	/* Reuse of PRIV_CLOCK_SETTIME. */
+	if ((error = priv_check(td, PRIV_CLOCK_SETTIME)) != 0)
+		return (error);
+
+	if ((error = copyin(uap->cest, &cest, sizeof(struct ffclock_estimate)))
+	    != 0)
+		return (error);
+
+	mtx_lock(&ffclock_mtx);
+	memcpy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
+	ffclock_updated++;
+	mtx_unlock(&ffclock_mtx);
+	return (error);
+}
+
+/*
+ * System call allowing userland applications to retrieve the clock estimates
+ * stored within the kernel. It is useful to kickstart the synchronisation
+ * daemon with the kernel's knowledge of hardware timecounter.
+ */
+#ifndef _SYS_SYSPROTO_H_
+struct ffclock_getestimate_args {
+	struct ffclock_estimate *cest;
+};
+#endif
+/* ARGSUSED */
+int
+sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap)
+{
+	struct ffclock_estimate cest;
+	int error;
+
+	mtx_lock(&ffclock_mtx);
+	memcpy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
+	mtx_unlock(&ffclock_mtx);
+	error = copyout(&cest, uap->cest, sizeof(struct ffclock_estimate));
+	return (error);
+}
+
+#else /* !FFCLOCK */
+
+int
+sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap)
+{
+
+	return (ENOSYS);
+}
+
+int
+sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap)
+{
+
+	return (ENOSYS);
+}
+
+int
+sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap)
+{
+
+	return (ENOSYS);
+}
+
+#endif /* FFCLOCK */
diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
index 37a2c59..84f5575 100644
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -78,7 +78,7 @@ __FBSDID("$FreeBSD$");
 #define	DEFAULT_HOSTUUID	"00000000-0000-0000-0000-000000000000"
 
 MALLOC_DEFINE(M_PRISON, "prison", "Prison structures");
-MALLOC_DEFINE(M_PRISON_RACCT, "prison_racct", "Prison racct structures");
+static MALLOC_DEFINE(M_PRISON_RACCT, "prison_racct", "Prison racct structures");
 
 /* Keep struct prison prison0 and some code in kern_jail_set() readable. */
 #ifdef INET
@@ -3944,7 +3944,7 @@ prison_path(struct prison *pr1, struct prison *pr2)
 /*
  * Jail-related sysctls.
  */
-SYSCTL_NODE(_security, OID_AUTO, jail, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security, OID_AUTO, jail, CTLFLAG_RW, 0,
     "Jails");
 
 static int
diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c
index eff3d5b..004a19e 100644
--- a/sys/kern/kern_ktr.c
+++ b/sys/kern/kern_ktr.c
@@ -80,7 +80,7 @@ __FBSDID("$FreeBSD$");
 
 FEATURE(ktr, "Kernel support for KTR kernel tracing facility");
 
-SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options");
+static SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options");
 
 int	ktr_mask = KTR_MASK;
 TUNABLE_INT("debug.ktr.mask", &ktr_mask);
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index 5c5bfbd..d7e5ade 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -28,6 +28,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
+#include "opt_kld.h"
 #include "opt_hwpmc_hooks.h"
 
 #include <sys/param.h>
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 7311872..459811e 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -131,15 +131,16 @@ CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
 #define	lockmgr_xlocked(lk)						\
 	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
 
-static void	 assert_lockmgr(struct lock_object *lock, int how);
+static void	assert_lockmgr(const struct lock_object *lock, int how);
 #ifdef DDB
-static void	 db_show_lockmgr(struct lock_object *lock);
+static void	db_show_lockmgr(const struct lock_object *lock);
 #endif
-static void	 lock_lockmgr(struct lock_object *lock, int how);
+static void	lock_lockmgr(struct lock_object *lock, int how);
 #ifdef KDTRACE_HOOKS
-static int	 owner_lockmgr(struct lock_object *lock, struct thread **owner);
+static int	owner_lockmgr(const struct lock_object *lock,
+		    struct thread **owner);
 #endif
-static int	 unlock_lockmgr(struct lock_object *lock);
+static int	unlock_lockmgr(struct lock_object *lock);
 
 struct lock_class lock_class_lockmgr = {
 	.lc_name = "lockmgr",
@@ -158,13 +159,14 @@ struct lock_class lock_class_lockmgr = {
 #ifdef ADAPTIVE_LOCKMGRS
 static u_int alk_retries = 10;
 static u_int alk_loops = 10000;
-SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging");
+static SYSCTL_NODE(_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL,
+    "lockmgr debugging");
 SYSCTL_UINT(_debug_lockmgr, OID_AUTO, retries, CTLFLAG_RW, &alk_retries, 0, "");
 SYSCTL_UINT(_debug_lockmgr, OID_AUTO, loops, CTLFLAG_RW, &alk_loops, 0, "");
 #endif
 
 static __inline struct thread *
-lockmgr_xholder(struct lock *lk)
+lockmgr_xholder(const struct lock *lk)
 {
 	uintptr_t x;
 
@@ -334,7 +336,7 @@ wakeupshlk(struct lock *lk, const char *file, int line)
 }
 
 static void
-assert_lockmgr(struct lock_object *lock, int what)
+assert_lockmgr(const struct lock_object *lock, int what)
 {
 
 	panic("lockmgr locks do not support assertions");
@@ -356,7 +358,7 @@ unlock_lockmgr(struct lock_object *lock)
 
 #ifdef KDTRACE_HOOKS
 static int
-owner_lockmgr(struct lock_object *lock, struct thread **owner)
+owner_lockmgr(const struct lock_object *lock, struct thread **owner)
 {
 
 	panic("lockmgr locks do not support owner inquiring");
@@ -1259,7 +1261,7 @@ _lockmgr_disown(struct lock *lk, const char *file, int line)
 }
 
 void
-lockmgr_printinfo(struct lock *lk)
+lockmgr_printinfo(const struct lock *lk)
 {
 	struct thread *td;
 	uintptr_t x;
@@ -1288,7 +1290,7 @@ lockmgr_printinfo(struct lock *lk)
 }
 
 int
-lockstatus(struct lock *lk)
+lockstatus(const struct lock *lk)
 {
 	uintptr_t v, x;
 	int ret;
@@ -1318,7 +1320,7 @@ FEATURE(invariant_support,
 #endif
 
 void
-_lockmgr_assert(struct lock *lk, int what, const char *file, int line)
+_lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
 {
 	int slocked = 0;
 
@@ -1411,12 +1413,12 @@ lockmgr_chain(struct thread *td, struct thread **ownerp)
 }
 
 static void
-db_show_lockmgr(struct lock_object *lock)
+db_show_lockmgr(const struct lock_object *lock)
 {
 	struct thread *td;
-	struct lock *lk;
+	const struct lock *lk;
 
-	lk = (struct lock *)lock;
+	lk = (const struct lock *)lock;
 
 	db_printf(" state: ");
 	if (lk->lk_lock == LK_UNLOCKED)
diff --git a/sys/kern/kern_lockf.c b/sys/kern/kern_lockf.c
index ddf6846..6d6dc51 100644
--- a/sys/kern/kern_lockf.c
+++ b/sys/kern/kern_lockf.c
@@ -90,7 +90,7 @@ static int	lockf_debug = 0; /* control debug output */
 SYSCTL_INT(_debug, OID_AUTO, lockf_debug, CTLFLAG_RW, &lockf_debug, 0, "");
 #endif
 
-MALLOC_DEFINE(M_LOCKF, "lockf", "Byte-range locking structures");
+static MALLOC_DEFINE(M_LOCKF, "lockf", "Byte-range locking structures");
 
 struct owner_edge;
 struct owner_vertex;
diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index ecc45db..9b61ecd 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -231,7 +231,7 @@ static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
 static time_t t_malloc_fail;
 
 #if defined(MALLOC_MAKE_FAILURES) || (MALLOC_DEBUG_MAXZONES > 1)
-SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
     "Kernel malloc debugging options");
 #endif
 
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 041b480..eddb0d3 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -85,14 +85,15 @@ __FBSDID("$FreeBSD$");
 
 #define	mtx_owner(m)	((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK))
 
-static void	assert_mtx(struct lock_object *lock, int what);
+static void	assert_mtx(const struct lock_object *lock, int what);
 #ifdef DDB
-static void	db_show_mtx(struct lock_object *lock);
+static void	db_show_mtx(const struct lock_object *lock);
 #endif
 static void	lock_mtx(struct lock_object *lock, int how);
 static void	lock_spin(struct lock_object *lock, int how);
 #ifdef KDTRACE_HOOKS
-static int	owner_mtx(struct lock_object *lock, struct thread **owner);
+static int	owner_mtx(const struct lock_object *lock,
+		    struct thread **owner);
 #endif
 static int	unlock_mtx(struct lock_object *lock);
 static int	unlock_spin(struct lock_object *lock);
@@ -134,10 +135,10 @@ struct mtx blocked_lock;
 struct mtx Giant;
 
 void
-assert_mtx(struct lock_object *lock, int what)
+assert_mtx(const struct lock_object *lock, int what)
 {
 
-	mtx_assert((struct mtx *)lock, what);
+	mtx_assert((const struct mtx *)lock, what);
 }
 
 void
@@ -174,9 +175,9 @@ unlock_spin(struct lock_object *lock)
 
 #ifdef KDTRACE_HOOKS
 int
-owner_mtx(struct lock_object *lock, struct thread **owner)
+owner_mtx(const struct lock_object *lock, struct thread **owner)
 {
-	struct mtx *m = (struct mtx *)lock;
+	const struct mtx *m = (const struct mtx *)lock;
 
 	*owner = mtx_owner(m);
 	return (mtx_unowned(m) == 0);
@@ -273,7 +274,7 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
  * is already owned, it will recursively acquire the lock.
  */
 int
-_mtx_trylock(struct mtx *m, int opts, const char *file, int line)
+mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line)
 {
 #ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
@@ -539,7 +540,7 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
 #endif /* SMP */
 
 void
-_thread_lock_flags(struct thread *td, int opts, const char *file, int line)
+thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
 {
 	struct mtx *m;
 	uintptr_t tid;
@@ -693,7 +694,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
  */
 #ifdef INVARIANT_SUPPORT
 void
-_mtx_assert(struct mtx *m, int what, const char *file, int line)
+_mtx_assert(const struct mtx *m, int what, const char *file, int line)
 {
 
 	if (panicstr != NULL || dumping)
@@ -871,12 +872,12 @@ mutex_init(void)
 
 #ifdef DDB
 void
-db_show_mtx(struct lock_object *lock)
+db_show_mtx(const struct lock_object *lock)
 {
 	struct thread *td;
-	struct mtx *m;
+	const struct mtx *m;
 
-	m = (struct mtx *)lock;
+	m = (const struct mtx *)lock;
 
 	db_printf(" flags: {");
 	if (LOCK_CLASS(lock) == &lock_class_mtx_spin)
diff --git a/sys/kern/kern_poll.c b/sys/kern/kern_poll.c
index 8441126..0010452 100644
--- a/sys/kern/kern_poll.c
+++ b/sys/kern/kern_poll.c
@@ -98,7 +98,7 @@ static uint32_t poll_burst = 5;
 static uint32_t poll_burst_max = 150;	/* good for 100Mbit net and HZ=1000 */
 static uint32_t poll_each_burst = 5;
 
-SYSCTL_NODE(_kern, OID_AUTO, polling, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern, OID_AUTO, polling, CTLFLAG_RW, 0,
 	"Device polling parameters");
 
 SYSCTL_UINT(_kern_polling, OID_AUTO, burst, CTLFLAG_RD,
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 998e7ca..1fcedd2 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/loginclass.h>
 #include <sys/malloc.h>
+#include <sys/mman.h>
 #include <sys/mount.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
@@ -75,6 +76,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
+#include <vm/vm_page.h>
 #include <vm/uma.h>
 
 #ifdef COMPAT_FREEBSD32
@@ -1526,7 +1528,7 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS)
 	}
 	kve = malloc(sizeof(*kve), M_TEMP, M_WAITOK);
 
-	map = &p->p_vmspace->vm_map;	/* XXXRW: More locking required? */
+	map = &vm->vm_map;
 	vm_map_lock_read(map);
 	for (entry = map->header.next; entry != &map->header;
 	    entry = entry->next) {
@@ -1704,13 +1706,14 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS)
 	}
 	kve = malloc(sizeof(*kve), M_TEMP, M_WAITOK);
 
-	map = &vm->vm_map;	/* XXXRW: More locking required? */
+	map = &vm->vm_map;
 	vm_map_lock_read(map);
 	for (entry = map->header.next; entry != &map->header;
 	    entry = entry->next) {
 		vm_object_t obj, tobj, lobj;
 		vm_offset_t addr;
-		int vfslocked;
+		vm_paddr_t locked_pa;
+		int vfslocked, mincoreinfo;
 
 		if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 			continue;
@@ -1728,8 +1731,14 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS)
 		kve->kve_resident = 0;
 		addr = entry->start;
 		while (addr < entry->end) {
-			if (pmap_extract(map->pmap, addr))
+			locked_pa = 0;
+			mincoreinfo = pmap_mincore(map->pmap, addr, &locked_pa);
+			if (locked_pa != 0)
+				vm_page_unlock(PHYS_TO_VM_PAGE(locked_pa));
+			if (mincoreinfo & MINCORE_INCORE)
 				kve->kve_resident++;
+			if (mincoreinfo & MINCORE_SUPER)
+				kve->kve_flags |= KVME_FLAG_SUPER;
 			addr += PAGE_SIZE;
 		}
 
diff --git a/sys/kern/kern_rctl.c b/sys/kern/kern_rctl.c
index ad31456..aaea7d5 100644
--- a/sys/kern/kern_rctl.c
+++ b/sys/kern/kern_rctl.c
@@ -169,7 +169,7 @@ RW_SYSINIT(rctl_lock, &rctl_lock, "RCTL lock");
 static int rctl_rule_fully_specified(const struct rctl_rule *rule);
 static void rctl_rule_to_sbuf(struct sbuf *sb, const struct rctl_rule *rule);
 
-MALLOC_DEFINE(M_RCTL, "rctl", "Resource Limits");
+static MALLOC_DEFINE(M_RCTL, "rctl", "Resource Limits");
 
 static const char *
 rctl_subject_type_name(int subject)
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index e6e8021..0a36772 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -1118,6 +1118,10 @@ lim_hold(limp)
 void
 lim_fork(struct proc *p1, struct proc *p2)
 {
+
+	PROC_LOCK_ASSERT(p1, MA_OWNED);
+	PROC_LOCK_ASSERT(p2, MA_OWNED);
+
 	p2->p_limit = lim_hold(p1->p_limit);
 	callout_init_mtx(&p2->p_limco, &p2->p_mtx, 0);
 	if (p1->p_cpulimit != RLIM_INFINITY)
diff --git a/sys/kern/kern_rmlock.c b/sys/kern/kern_rmlock.c
index 1c7337d..181825a 100644
--- a/sys/kern/kern_rmlock.c
+++ b/sys/kern/kern_rmlock.c
@@ -69,10 +69,10 @@ static __inline void compiler_memory_barrier(void) {
 	__asm __volatile("":::"memory");
 }
 
-static void	assert_rm(struct lock_object *lock, int what);
+static void	assert_rm(const struct lock_object *lock, int what);
 static void	lock_rm(struct lock_object *lock, int how);
 #ifdef KDTRACE_HOOKS
-static int	owner_rm(struct lock_object *lock, struct thread **owner);
+static int	owner_rm(const struct lock_object *lock, struct thread **owner);
 #endif
 static int	unlock_rm(struct lock_object *lock);
 
@@ -93,7 +93,7 @@ struct lock_class lock_class_rm = {
 };
 
 static void
-assert_rm(struct lock_object *lock, int what)
+assert_rm(const struct lock_object *lock, int what)
 {
 
 	panic("assert_rm called");
@@ -115,7 +115,7 @@ unlock_rm(struct lock_object *lock)
 
 #ifdef KDTRACE_HOOKS
 static int
-owner_rm(struct lock_object *lock, struct thread **owner)
+owner_rm(const struct lock_object *lock, struct thread **owner)
 {
 
 	panic("owner_rm called");
@@ -227,7 +227,7 @@ rm_destroy(struct rmlock *rm)
 }
 
 int
-rm_wowned(struct rmlock *rm)
+rm_wowned(const struct rmlock *rm)
 {
 
 	if (rm->lock_object.lo_flags & RM_SLEEPABLE)
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index 81b4c5f..a4d3e96 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -58,7 +58,8 @@ __FBSDID("$FreeBSD$");
 #ifdef ADAPTIVE_RWLOCKS
 static int rowner_retries = 10;
 static int rowner_loops = 10000;
-SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, "rwlock debugging");
+static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL,
+    "rwlock debugging");
 SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, "");
 SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
 #endif
@@ -66,12 +67,12 @@ SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
 #ifdef DDB
 #include <ddb/ddb.h>
 
-static void	db_show_rwlock(struct lock_object *lock);
+static void	db_show_rwlock(const struct lock_object *lock);
 #endif
-static void	assert_rw(struct lock_object *lock, int what);
+static void	assert_rw(const struct lock_object *lock, int what);
 static void	lock_rw(struct lock_object *lock, int how);
 #ifdef KDTRACE_HOOKS
-static int	owner_rw(struct lock_object *lock, struct thread **owner);
+static int	owner_rw(const struct lock_object *lock, struct thread **owner);
 #endif
 static int	unlock_rw(struct lock_object *lock);
 
@@ -120,10 +121,10 @@ struct lock_class lock_class_rw = {
 #endif
 
 void
-assert_rw(struct lock_object *lock, int what)
+assert_rw(const struct lock_object *lock, int what)
 {
 
-	rw_assert((struct rwlock *)lock, what);
+	rw_assert((const struct rwlock *)lock, what);
 }
 
 void
@@ -156,9 +157,9 @@ unlock_rw(struct lock_object *lock)
 
 #ifdef KDTRACE_HOOKS
 int
-owner_rw(struct lock_object *lock, struct thread **owner)
+owner_rw(const struct lock_object *lock, struct thread **owner)
 {
-	struct rwlock *rw = (struct rwlock *)lock;
+	const struct rwlock *rw = (const struct rwlock *)lock;
 	uintptr_t x = rw->rw_lock;
 
 	*owner = rw_wowner(rw);
@@ -222,7 +223,7 @@ rw_sysinit_flags(void *arg)
 }
 
 int
-rw_wowned(struct rwlock *rw)
+rw_wowned(const struct rwlock *rw)
 {
 
 	return (rw_wowner(rw) == curthread);
@@ -1010,7 +1011,7 @@ out:
  * thread owns an rlock.
  */
 void
-_rw_assert(struct rwlock *rw, int what, const char *file, int line)
+_rw_assert(const struct rwlock *rw, int what, const char *file, int line)
 {
 
 	if (panicstr != NULL)
@@ -1083,12 +1084,12 @@ _rw_assert(struct rwlock *rw, int what, const char *file, int line)
 
 #ifdef DDB
 void
-db_show_rwlock(struct lock_object *lock)
+db_show_rwlock(const struct lock_object *lock)
 {
-	struct rwlock *rw;
+	const struct rwlock *rw;
 	struct thread *td;
 
-	rw = (struct rwlock *)lock;
+	rw = (const struct rwlock *)lock;
 
 	db_printf(" state: ");
 	if (rw->rw_lock == RW_UNLOCKED)
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index 039ca7e..2b5a5ae 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -121,7 +121,8 @@ SYSCTL_INT(_kern, OID_AUTO, sync_on_panic, CTLFLAG_RW | CTLFLAG_TUN,
 	&sync_on_panic, 0, "Do a sync before rebooting from a panic");
 TUNABLE_INT("kern.sync_on_panic", &sync_on_panic);
 
-SYSCTL_NODE(_kern, OID_AUTO, shutdown, CTLFLAG_RW, 0, "Shutdown environment");
+static SYSCTL_NODE(_kern, OID_AUTO, shutdown, CTLFLAG_RW, 0,
+    "Shutdown environment");
 
 #ifndef DIAGNOSTIC
 static int show_busybufs;
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 188bf83..e73f78c 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -136,7 +136,8 @@ static int	kern_forcesigexit = 1;
 SYSCTL_INT(_kern, OID_AUTO, forcesigexit, CTLFLAG_RW,
     &kern_forcesigexit, 0, "Force trap signal to be handled");
 
-SYSCTL_NODE(_kern, OID_AUTO, sigqueue, CTLFLAG_RW, 0, "POSIX real time signal");
+static SYSCTL_NODE(_kern, OID_AUTO, sigqueue, CTLFLAG_RW, 0,
+    "POSIX real time signal");
 
 static int	max_pending_per_proc = 128;
 SYSCTL_INT(_kern_sigqueue, OID_AUTO, max_pending_per_proc, CTLFLAG_RW,
@@ -2058,7 +2059,7 @@ tdsendsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
 	 */
 	mtx_lock(&ps->ps_mtx);
 	if (SIGISMEMBER(ps->ps_sigignore, sig)) {
-		SDT_PROBE(proc, kernel, , signal_discard, ps, td, sig, 0, 0 );
+		SDT_PROBE(proc, kernel, , signal_discard, td, p, sig, 0, 0 );
 
 		mtx_unlock(&ps->ps_mtx);
 		if (ksi && (ksi->ksi_flags & KSI_INS))
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index b824eac..8121d00 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -105,13 +105,13 @@ CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE);
 #define	sx_recurse		lock_object.lo_data
 #define	sx_recursed(sx)		((sx)->sx_recurse != 0)
 
-static void	assert_sx(struct lock_object *lock, int what);
+static void	assert_sx(const struct lock_object *lock, int what);
 #ifdef DDB
-static void	db_show_sx(struct lock_object *lock);
+static void	db_show_sx(const struct lock_object *lock);
 #endif
 static void	lock_sx(struct lock_object *lock, int how);
 #ifdef KDTRACE_HOOKS
-static int	owner_sx(struct lock_object *lock, struct thread **owner);
+static int	owner_sx(const struct lock_object *lock, struct thread **owner);
 #endif
 static int	unlock_sx(struct lock_object *lock);
 
@@ -136,16 +136,16 @@ struct lock_class lock_class_sx = {
 #ifdef ADAPTIVE_SX
 static u_int asx_retries = 10;
 static u_int asx_loops = 10000;
-SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
+static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
 SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, "");
 SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, "");
 #endif
 
 void
-assert_sx(struct lock_object *lock, int what)
+assert_sx(const struct lock_object *lock, int what)
 {
 
-	sx_assert((struct sx *)lock, what);
+	sx_assert((const struct sx *)lock, what);
 }
 
 void
@@ -178,9 +178,9 @@ unlock_sx(struct lock_object *lock)
 
 #ifdef KDTRACE_HOOKS
 int
-owner_sx(struct lock_object *lock, struct thread **owner)
+owner_sx(const struct lock_object *lock, struct thread **owner)
 {
-        struct sx *sx = (struct sx *)lock;
+        const struct sx *sx = (const struct sx *)lock;
 	uintptr_t x = sx->sx_lock;
 
         *owner = (struct thread *)SX_OWNER(x);
@@ -256,7 +256,7 @@ _sx_slock(struct sx *sx, int opts, const char *file, int line)
 }
 
 int
-_sx_try_slock(struct sx *sx, const char *file, int line)
+sx_try_slock_(struct sx *sx, const char *file, int line)
 {
 	uintptr_t x;
 
@@ -300,7 +300,7 @@ _sx_xlock(struct sx *sx, int opts, const char *file, int line)
 }
 
 int
-_sx_try_xlock(struct sx *sx, const char *file, int line)
+sx_try_xlock_(struct sx *sx, const char *file, int line)
 {
 	int rval;
 
@@ -364,7 +364,7 @@ _sx_xunlock(struct sx *sx, const char *file, int line)
  * Return 1 if if the upgrade succeed, 0 otherwise.
  */
 int
-_sx_try_upgrade(struct sx *sx, const char *file, int line)
+sx_try_upgrade_(struct sx *sx, const char *file, int line)
 {
 	uintptr_t x;
 	int success;
@@ -394,7 +394,7 @@ _sx_try_upgrade(struct sx *sx, const char *file, int line)
  * Downgrade an unrecursed exclusive lock into a single shared lock.
  */
 void
-_sx_downgrade(struct sx *sx, const char *file, int line)
+sx_downgrade_(struct sx *sx, const char *file, int line)
 {
 	uintptr_t x;
 	int wakeup_swapper;
@@ -1005,7 +1005,7 @@ _sx_sunlock_hard(struct sx *sx, const char *file, int line)
  * thread owns an slock.
  */
 void
-_sx_assert(struct sx *sx, int what, const char *file, int line)
+_sx_assert(const struct sx *sx, int what, const char *file, int line)
 {
 #ifndef WITNESS
 	int slocked = 0;
@@ -1088,12 +1088,12 @@ _sx_assert(struct sx *sx, int what, const char *file, int line)
 
 #ifdef DDB
 static void
-db_show_sx(struct lock_object *lock)
+db_show_sx(const struct lock_object *lock)
 {
 	struct thread *td;
-	struct sx *sx;
+	const struct sx *sx;
 
-	sx = (struct sx *)lock;
+	sx = (const struct sx *)lock;
 
 	db_printf(" state: ");
 	if (sx->sx_lock == SX_LOCK_UNLOCKED)
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index a2c26ae..bab9178 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -325,16 +325,33 @@ msleep_spin(void *ident, struct mtx *mtx, const char *wmesg, int timo)
 }
 
 /*
- * pause() is like tsleep() except that the intention is to not be
- * explicitly woken up by another thread.  Instead, the current thread
- * simply wishes to sleep until the timeout expires.  It is
- * implemented using a dummy wait channel.
+ * pause() delays the calling thread by the given number of system ticks.
+ * During cold bootup, pause() uses the DELAY() function instead of
+ * the tsleep() function to do the waiting. The "timo" argument must be
+ * greater than zero.
  */
 int
 pause(const char *wmesg, int timo)
 {
+	KASSERT(timo >= 0, ("pause: timo must be >= 0"));
 
-	KASSERT(timo != 0, ("pause: timeout required"));
+	/* silently convert invalid timeouts */
+	if (timo < 1)
+		timo = 1;
+
+	if (cold) {
+		/*
+		 * We delay one HZ at a time to avoid overflowing the
+		 * system specific DELAY() function(s):
+		 */
+		while (timo >= hz) {
+			DELAY(1000000);
+			timo -= hz;
+		}
+		if (timo > 0)
+			DELAY(timo * tick);
+		return (0);
+	}
 	return (tsleep(&pause_wchan, 0, wmesg, timo));
 }
 
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index 0c52071..e5db8d9 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -5,18 +5,32 @@
  * can do whatever you want with this stuff. If we meet some day, and you think
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
+ *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Julien Ridoux at the University
+ * of Melbourne under sponsorship from the FreeBSD Foundation.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_ntp.h"
+#include "opt_ffclock.h"
 
 #include <sys/param.h>
 #include <sys/kernel.h>
+#ifdef FFCLOCK
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#endif
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 #include <sys/systm.h>
+#ifdef FFCLOCK
+#include <sys/timeffc.h>
+#endif
 #include <sys/timepps.h>
 #include <sys/timetc.h>
 #include <sys/timex.h>
@@ -99,7 +113,7 @@ SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD,
     NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime");
 
 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, "");
-SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
+static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
 
 static int timestepwarnings;
 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
@@ -163,6 +177,144 @@ tc_delta(struct timehands *th)
  * the comment in <sys/time.h> for a description of these 12 functions.
  */
 
+#ifdef FFCLOCK
+static void
+fbclock_binuptime(struct bintime *bt)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		*bt = th->th_offset;
+		bintime_addx(bt, th->th_scale * tc_delta(th));
+	} while (gen == 0 || gen != th->th_generation);
+}
+
+static void
+fbclock_nanouptime(struct timespec *tsp)
+{
+	struct bintime bt;
+
+	binuptime(&bt);
+	bintime2timespec(&bt, tsp);
+}
+
+static void
+fbclock_microuptime(struct timeval *tvp)
+{
+	struct bintime bt;
+
+	binuptime(&bt);
+	bintime2timeval(&bt, tvp);
+}
+
+static void
+fbclock_bintime(struct bintime *bt)
+{
+
+	binuptime(bt);
+	bintime_add(bt, &boottimebin);
+}
+
+static void
+fbclock_nanotime(struct timespec *tsp)
+{
+	struct bintime bt;
+
+	bintime(&bt);
+	bintime2timespec(&bt, tsp);
+}
+
+static void
+fbclock_microtime(struct timeval *tvp)
+{
+	struct bintime bt;
+
+	bintime(&bt);
+	bintime2timeval(&bt, tvp);
+}
+
+static void
+fbclock_getbinuptime(struct bintime *bt)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		*bt = th->th_offset;
+	} while (gen == 0 || gen != th->th_generation);
+}
+
+static void
+fbclock_getnanouptime(struct timespec *tsp)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		bintime2timespec(&th->th_offset, tsp);
+	} while (gen == 0 || gen != th->th_generation);
+}
+
+static void
+fbclock_getmicrouptime(struct timeval *tvp)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		bintime2timeval(&th->th_offset, tvp);
+	} while (gen == 0 || gen != th->th_generation);
+}
+
+static void
+fbclock_getbintime(struct bintime *bt)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		*bt = th->th_offset;
+	} while (gen == 0 || gen != th->th_generation);
+	bintime_add(bt, &boottimebin);
+}
+
+static void
+fbclock_getnanotime(struct timespec *tsp)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		*tsp = th->th_nanotime;
+	} while (gen == 0 || gen != th->th_generation);
+}
+
+static void
+fbclock_getmicrotime(struct timeval *tvp)
+{
+	struct timehands *th;
+	unsigned int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		*tvp = th->th_microtime;
+	} while (gen == 0 || gen != th->th_generation);
+}
+#else /* !FFCLOCK */
 void
 binuptime(struct bintime *bt)
 {
@@ -299,6 +451,584 @@ getmicrotime(struct timeval *tvp)
 		*tvp = th->th_microtime;
 	} while (gen == 0 || gen != th->th_generation);
 }
+#endif /* FFCLOCK */
+
+#ifdef FFCLOCK
+/*
+ * Support for feed-forward synchronization algorithms. This is heavily inspired
+ * by the timehands mechanism but kept independent from it. *_windup() functions
+ * have some connection to avoid accessing the timecounter hardware more than
+ * necessary.
+ */
+
+int sysclock_active = SYSCLOCK_FBCK;
+
+/* Feed-forward clock estimates kept updated by the synchronization daemon. */
+struct ffclock_estimate ffclock_estimate;
+struct bintime ffclock_boottime;	/* Feed-forward boot time estimate. */
+uint32_t ffclock_status;		/* Feed-forward clock status. */
+int8_t ffclock_updated;			/* New estimates are available. */
+struct mtx ffclock_mtx;			/* Mutex on ffclock_estimate. */
+
+struct sysclock_ops {
+	int active;
+	void (*binuptime) (struct bintime *bt);
+	void (*nanouptime) (struct timespec *tsp);
+	void (*microuptime) (struct timeval *tvp);
+	void (*bintime) (struct bintime *bt);
+	void (*nanotime) (struct timespec *tsp);
+	void (*microtime) (struct timeval *tvp);
+	void (*getbinuptime) (struct bintime *bt);
+	void (*getnanouptime) (struct timespec *tsp);
+	void (*getmicrouptime) (struct timeval *tvp);
+	void (*getbintime) (struct bintime *bt);
+	void (*getnanotime) (struct timespec *tsp);
+	void (*getmicrotime) (struct timeval *tvp);
+};
+
+static struct sysclock_ops sysclock = {
+	.active = SYSCLOCK_FBCK,
+	.binuptime = fbclock_binuptime,
+	.nanouptime = fbclock_nanouptime,
+	.microuptime = fbclock_microuptime,
+	.bintime = fbclock_bintime,
+	.nanotime = fbclock_nanotime,
+	.microtime = fbclock_microtime,
+	.getbinuptime = fbclock_getbinuptime,
+	.getnanouptime = fbclock_getnanouptime,
+	.getmicrouptime = fbclock_getmicrouptime,
+	.getbintime = fbclock_getbintime,
+	.getnanotime = fbclock_getnanotime,
+	.getmicrotime = fbclock_getmicrotime
+};
+
+struct fftimehands {
+	struct ffclock_estimate	cest;
+	struct bintime		tick_time;
+	struct bintime		tick_time_lerp;
+	ffcounter		tick_ffcount;
+	uint64_t		period_lerp;
+	volatile uint8_t	gen;
+	struct fftimehands	*next;
+};
+
+#define	NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
+
+static struct fftimehands ffth[10];
+static struct fftimehands *volatile fftimehands = ffth;
+
+static void
+ffclock_init(void)
+{
+	struct fftimehands *cur;
+	struct fftimehands *last;
+
+	memset(ffth, 0, sizeof(ffth));
+
+	last = ffth + NUM_ELEMENTS(ffth) - 1;
+	for (cur = ffth; cur < last; cur++)
+		cur->next = cur + 1;
+	last->next = ffth;
+
+	ffclock_updated = 0;
+	ffclock_status = FFCLOCK_STA_UNSYNC;
+	mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
+}
+
+/*
+ * Reset the feed-forward clock estimates. Called from inittodr() to get things
+ * kick started and uses the timecounter nominal frequency as a first period
+ * estimate. Note: this function may be called several time just after boot.
+ * Note: this is the only function that sets the value of boot time for the
+ * monotonic (i.e. uptime) version of the feed-forward clock.
+ */
+void
+ffclock_reset_clock(struct timespec *ts)
+{
+	struct timecounter *tc;
+	struct ffclock_estimate cest;
+
+	tc = timehands->th_counter;
+	memset(&cest, 0, sizeof(struct ffclock_estimate));
+
+	timespec2bintime(ts, &ffclock_boottime);
+	timespec2bintime(ts, &(cest.update_time));
+	ffclock_read_counter(&cest.update_ffcount);
+	cest.leapsec_next = 0;
+	cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
+	cest.errb_abs = 0;
+	cest.errb_rate = 0;
+	cest.status = FFCLOCK_STA_UNSYNC;
+	cest.leapsec_total = 0;
+	cest.leapsec = 0;
+
+	mtx_lock(&ffclock_mtx);
+	bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
+	ffclock_updated = INT8_MAX;
+	mtx_unlock(&ffclock_mtx);
+
+	printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
+	    (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
+	    (unsigned long)ts->tv_nsec);
+}
+
+/*
+ * Sub-routine to convert a time interval measured in RAW counter units to time
+ * in seconds stored in bintime format.
+ * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
+ * larger than the max value of u_int (on 32 bit architecture). Loop to consume
+ * extra cycles.
+ */
+static void
+ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
+{
+	struct bintime bt2;
+	ffcounter delta, delta_max;
+
+	delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
+	bintime_clear(bt);
+	do {
+		if (ffdelta > delta_max)
+			delta = delta_max;
+		else
+			delta = ffdelta;
+		bt2.sec = 0;
+		bt2.frac = period;
+		bintime_mul(&bt2, (unsigned int)delta);
+		bintime_add(bt, &bt2);
+		ffdelta -= delta;
+	} while (ffdelta > 0);
+}
+
+/*
+ * Update the fftimehands.
+ * Push the tick ffcount and time(s) forward based on current clock estimate.
+ * The conversion from ffcounter to bintime relies on the difference clock
+ * principle, whose accuracy relies on computing small time intervals. If a new
+ * clock estimate has been passed by the synchronisation daemon, make it
+ * current, and compute the linear interpolation for monotonic time if needed.
+ */
+static void
+ffclock_windup(unsigned int delta)
+{
+	struct ffclock_estimate *cest;
+	struct fftimehands *ffth;
+	struct bintime bt, gap_lerp;
+	ffcounter ffdelta;
+	uint64_t frac;
+	unsigned int polling;
+	uint8_t forward_jump, ogen;
+
+	/*
+	 * Pick the next timehand, copy current ffclock estimates and move tick
+	 * times and counter forward.
+	 */
+	forward_jump = 0;
+	ffth = fftimehands->next;
+	ogen = ffth->gen;
+	ffth->gen = 0;
+	cest = &ffth->cest;
+	bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
+	ffdelta = (ffcounter)delta;
+	ffth->period_lerp = fftimehands->period_lerp;
+
+	ffth->tick_time = fftimehands->tick_time;
+	ffclock_convert_delta(ffdelta, cest->period, &bt);
+	bintime_add(&ffth->tick_time, &bt);
+
+	ffth->tick_time_lerp = fftimehands->tick_time_lerp;
+	ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
+	bintime_add(&ffth->tick_time_lerp, &bt);
+
+	ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
+
+	/*
+	 * Assess the status of the clock, if the last update is too old, it is
+	 * likely the synchronisation daemon is dead and the clock is free
+	 * running.
+	 */
+	if (ffclock_updated == 0) {
+		ffdelta = ffth->tick_ffcount - cest->update_ffcount;
+		ffclock_convert_delta(ffdelta, cest->period, &bt);
+		if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
+			ffclock_status |= FFCLOCK_STA_UNSYNC;
+	}
+
+	/*
+	 * If available, grab updated clock estimates and make them current.
+	 * Recompute time at this tick using the updated estimates. The clock
+	 * estimates passed the feed-forward synchronisation daemon may result
+	 * in time conversion that is not monotonically increasing (just after
+	 * the update). time_lerp is a particular linear interpolation over the
+	 * synchronisation algo polling period that ensures monotonicity for the
+	 * clock ids requesting it.
+	 */
+	if (ffclock_updated > 0) {
+		bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
+		ffdelta = ffth->tick_ffcount - cest->update_ffcount;
+		ffth->tick_time = cest->update_time;
+		ffclock_convert_delta(ffdelta, cest->period, &bt);
+		bintime_add(&ffth->tick_time, &bt);
+
+		/* ffclock_reset sets ffclock_updated to INT8_MAX */
+		if (ffclock_updated == INT8_MAX)
+			ffth->tick_time_lerp = ffth->tick_time;
+
+		if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
+			forward_jump = 1;
+		else
+			forward_jump = 0;
+
+		bintime_clear(&gap_lerp);
+		if (forward_jump) {
+			gap_lerp = ffth->tick_time;
+			bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
+		} else {
+			gap_lerp = ffth->tick_time_lerp;
+			bintime_sub(&gap_lerp, &ffth->tick_time);
+		}
+
+		/*
+		 * The reset from the RTC clock may be far from accurate, and
+		 * reducing the gap between real time and interpolated time
+		 * could take a very long time if the interpolated clock insists
+		 * on strict monotonicity. The clock is reset under very strict
+		 * conditions (kernel time is known to be wrong and
+		 * synchronization daemon has been restarted recently.
+		 * ffclock_boottime absorbs the jump to ensure boot time is
+		 * correct and uptime functions stay consistent.
+		 */
+		if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
+		    ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
+		    ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
+			if (forward_jump)
+				bintime_add(&ffclock_boottime, &gap_lerp);
+			else
+				bintime_sub(&ffclock_boottime, &gap_lerp);
+			ffth->tick_time_lerp = ffth->tick_time;
+			bintime_clear(&gap_lerp);
+		}
+
+		ffclock_status = cest->status;
+		ffth->period_lerp = cest->period;
+
+		/*
+		 * Compute corrected period used for the linear interpolation of
+		 * time. The rate of linear interpolation is capped to 5000PPM
+		 * (5ms/s).
+		 */
+		if (bintime_isset(&gap_lerp)) {
+			ffdelta = cest->update_ffcount;
+			ffdelta -= fftimehands->cest.update_ffcount;
+			ffclock_convert_delta(ffdelta, cest->period, &bt);
+			polling = bt.sec;
+			bt.sec = 0;
+			bt.frac = 5000000 * (uint64_t)18446744073LL;
+			bintime_mul(&bt, polling);
+			if (bintime_cmp(&gap_lerp, &bt, >))
+				gap_lerp = bt;
+
+			/* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
+			frac = 0;
+			if (gap_lerp.sec > 0) {
+				frac -= 1;
+				frac /= ffdelta / gap_lerp.sec;
+			}
+			frac += gap_lerp.frac / ffdelta;
+
+			if (forward_jump)
+				ffth->period_lerp += frac;
+			else
+				ffth->period_lerp -= frac;
+		}
+
+		ffclock_updated = 0;
+	}
+	if (++ogen == 0)
+		ogen = 1;
+	ffth->gen = ogen;
+	fftimehands = ffth;
+}
+
+/*
+ * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
+ * the old and new hardware counter cannot be read simultaneously. tc_windup()
+ * does read the two counters 'back to back', but a few cycles are effectively
+ * lost, and not accumulated in tick_ffcount. This is a fairly radical
+ * operation for a feed-forward synchronization daemon, and it is its job to not
+ * pushing irrelevant data to the kernel. Because there is no locking here,
+ * simply force to ignore pending or next update to give daemon a chance to
+ * realize the counter has changed.
+ */
+static void
+ffclock_change_tc(struct timehands *th)
+{
+	struct fftimehands *ffth;
+	struct ffclock_estimate *cest;
+	struct timecounter *tc;
+	uint8_t ogen;
+
+	tc = th->th_counter;
+	ffth = fftimehands->next;
+	ogen = ffth->gen;
+	ffth->gen = 0;
+
+	cest = &ffth->cest;
+	bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
+	cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
+	cest->errb_abs = 0;
+	cest->errb_rate = 0;
+	cest->status |= FFCLOCK_STA_UNSYNC;
+
+	ffth->tick_ffcount = fftimehands->tick_ffcount;
+	ffth->tick_time_lerp = fftimehands->tick_time_lerp;
+	ffth->tick_time = fftimehands->tick_time;
+	ffth->period_lerp = cest->period;
+
+	/* Do not lock but ignore next update from synchronization daemon. */
+	ffclock_updated--;
+
+	if (++ogen == 0)
+		ogen = 1;
+	ffth->gen = ogen;
+	fftimehands = ffth;
+}
+
+static void
+change_sysclock(int new_sysclock)
+{
+
+	sysclock.active = new_sysclock;
+
+	switch (sysclock.active) {
+	case SYSCLOCK_FBCK:
+		sysclock.binuptime = fbclock_binuptime;
+		sysclock.nanouptime = fbclock_nanouptime;
+		sysclock.microuptime = fbclock_microuptime;
+		sysclock.bintime = fbclock_bintime;
+		sysclock.nanotime = fbclock_nanotime;
+		sysclock.microtime = fbclock_microtime;
+		sysclock.getbinuptime = fbclock_getbinuptime;
+		sysclock.getnanouptime = fbclock_getnanouptime;
+		sysclock.getmicrouptime = fbclock_getmicrouptime;
+		sysclock.getbintime = fbclock_getbintime;
+		sysclock.getnanotime = fbclock_getnanotime;
+		sysclock.getmicrotime = fbclock_getmicrotime;
+		break;
+	case SYSCLOCK_FFWD:
+		sysclock.binuptime = ffclock_binuptime;
+		sysclock.nanouptime = ffclock_nanouptime;
+		sysclock.microuptime = ffclock_microuptime;
+		sysclock.bintime = ffclock_bintime;
+		sysclock.nanotime = ffclock_nanotime;
+		sysclock.microtime = ffclock_microtime;
+		sysclock.getbinuptime = ffclock_getbinuptime;
+		sysclock.getnanouptime = ffclock_getnanouptime;
+		sysclock.getmicrouptime = ffclock_getmicrouptime;
+		sysclock.getbintime = ffclock_getbintime;
+		sysclock.getnanotime = ffclock_getnanotime;
+		sysclock.getmicrotime = ffclock_getmicrotime;
+		break;
+	default:
+		break;
+	}
+}
+
+/*
+ * Retrieve feed-forward counter and time of last kernel tick.
+ */
+void
+ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
+{
+	struct fftimehands *ffth;
+	uint8_t gen;
+
+	/*
+	 * No locking but check generation has not changed. Also need to make
+	 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
+	 */
+	do {
+		ffth = fftimehands;
+		gen = ffth->gen;
+		if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
+			*bt = ffth->tick_time_lerp;
+		else
+			*bt = ffth->tick_time;
+		*ffcount = ffth->tick_ffcount;
+	} while (gen == 0 || gen != ffth->gen);
+}
+
+/*
+ * Absolute clock conversion. Low level function to convert ffcounter to
+ * bintime. The ffcounter is converted using the current ffclock period estimate
+ * or the "interpolated period" to ensure monotonicity.
+ * NOTE: this conversion may have been deferred, and the clock updated since the
+ * hardware counter has been read.
+ */
+void
+ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
+{
+	struct fftimehands *ffth;
+	struct bintime bt2;
+	ffcounter ffdelta;
+	uint8_t gen;
+
+	/*
+	 * No locking but check generation has not changed. Also need to make
+	 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
+	 */
+	do {
+		ffth = fftimehands;
+		gen = ffth->gen;
+		if (ffcount > ffth->tick_ffcount)
+			ffdelta = ffcount - ffth->tick_ffcount;
+		else
+			ffdelta = ffth->tick_ffcount - ffcount;
+
+		if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
+			*bt = ffth->tick_time_lerp;
+			ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
+		} else {
+			*bt = ffth->tick_time;
+			ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
+		}
+
+		if (ffcount > ffth->tick_ffcount)
+			bintime_add(bt, &bt2);
+		else
+			bintime_sub(bt, &bt2);
+	} while (gen == 0 || gen != ffth->gen);
+}
+
+/*
+ * Difference clock conversion.
+ * Low level function to Convert a time interval measured in RAW counter units
+ * into bintime. The difference clock allows measuring small intervals much more
+ * reliably than the absolute clock.
+ */
+void
+ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
+{
+	struct fftimehands *ffth;
+	uint8_t gen;
+
+	/* No locking but check generation has not changed. */
+	do {
+		ffth = fftimehands;
+		gen = ffth->gen;
+		ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
+	} while (gen == 0 || gen != ffth->gen);
+}
+
+/*
+ * Access to current ffcounter value.
+ */
+void
+ffclock_read_counter(ffcounter *ffcount)
+{
+	struct timehands *th;
+	struct fftimehands *ffth;
+	unsigned int gen, delta;
+
+	/*
+	 * ffclock_windup() called from tc_windup(), safe to rely on
+	 * th->th_generation only, for correct delta and ffcounter.
+	 */
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		ffth = fftimehands;
+		delta = tc_delta(th);
+		*ffcount = ffth->tick_ffcount;
+	} while (gen == 0 || gen != th->th_generation);
+
+	*ffcount += delta;
+}
+
+void
+binuptime(struct bintime *bt)
+{
+
+	sysclock.binuptime(bt);
+}
+
+void
+nanouptime(struct timespec *tsp)
+{
+
+	sysclock.nanouptime(tsp);
+}
+
+void
+microuptime(struct timeval *tvp)
+{
+
+	sysclock.microuptime(tvp);
+}
+
+void
+bintime(struct bintime *bt)
+{
+
+	sysclock.bintime(bt);
+}
+
+void
+nanotime(struct timespec *tsp)
+{
+
+	sysclock.nanotime(tsp);
+}
+
+void
+microtime(struct timeval *tvp)
+{
+
+	sysclock.microtime(tvp);
+}
+
+void
+getbinuptime(struct bintime *bt)
+{
+
+	sysclock.getbinuptime(bt);
+}
+
+void
+getnanouptime(struct timespec *tsp)
+{
+
+	sysclock.getnanouptime(tsp);
+}
+
+void
+getmicrouptime(struct timeval *tvp)
+{
+
+	sysclock.getmicrouptime(tvp);
+}
+
+void
+getbintime(struct bintime *bt)
+{
+
+	sysclock.getbintime(bt);
+}
+
+void
+getnanotime(struct timespec *tsp)
+{
+
+	sysclock.getnanotime(tsp);
+}
+
+void
+getmicrotime(struct timeval *tvp)
+{
+
+	sysclock.getmicrouptime(tvp);
+}
+#endif /* FFCLOCK */
 
 /*
  * Initialize a new timecounter and possibly use it.
@@ -440,6 +1170,9 @@ tc_windup(void)
 		ncount = timecounter->tc_get_timecount(timecounter);
 	else
 		ncount = 0;
+#ifdef FFCLOCK
+	ffclock_windup(delta);
+#endif
 	th->th_offset_count += delta;
 	th->th_offset_count &= th->th_counter->tc_counter_mask;
 	while (delta > th->th_counter->tc_frequency) {
@@ -502,6 +1235,9 @@ tc_windup(void)
 		th->th_offset_count = ncount;
 		tc_min_ticktock_freq = max(1, timecounter->tc_frequency /
 		    (((uint64_t)timecounter->tc_counter_mask + 1) / 3));
+#ifdef FFCLOCK
+		ffclock_change_tc(th);
+#endif
 	}
 
 	/*-
@@ -532,6 +1268,11 @@ tc_windup(void)
 	scale /= th->th_counter->tc_frequency;
 	th->th_scale = scale * 2;
 
+#ifdef FFCLOCK
+	if (sysclock_active != sysclock.active)
+		change_sysclock(sysclock_active);
+#endif
+
 	/*
 	 * Now that the struct timehands is again consistent, set the new
 	 * generation number, making sure to not make it zero.
@@ -541,8 +1282,21 @@ tc_windup(void)
 	th->th_generation = ogen;
 
 	/* Go live with the new struct timehands. */
-	time_second = th->th_microtime.tv_sec;
-	time_uptime = th->th_offset.sec;
+#ifdef FFCLOCK
+	switch (sysclock_active) {
+	case SYSCLOCK_FBCK:
+#endif
+		time_second = th->th_microtime.tv_sec;
+		time_uptime = th->th_offset.sec;
+#ifdef FFCLOCK
+		break;
+	case SYSCLOCK_FFWD:
+		time_second = fftimehands->tick_time_lerp.sec;
+		time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
+		break;
+	}
+#endif
+
 	timehands = th;
 }
 
@@ -611,6 +1365,9 @@ pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
 {
 	pps_params_t *app;
 	struct pps_fetch_args *fapi;
+#ifdef FFCLOCK
+	struct pps_fetch_ffc_args *fapi_ffc;
+#endif
 #ifdef PPS_SYNC
 	struct pps_kcbind_args *kapi;
 #endif
@@ -625,6 +1382,11 @@ pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
 		app = (pps_params_t *)data;
 		if (app->mode & ~pps->ppscap)
 			return (EINVAL);
+#ifdef FFCLOCK
+		/* Ensure only a single clock is selected for ffc timestamp. */
+		if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
+			return (EINVAL);
+#endif
 		pps->ppsparam = *app;
 		return (0);
 	case PPS_IOC_GETPARAMS:
@@ -644,6 +1406,31 @@ pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
 		pps->ppsinfo.current_mode = pps->ppsparam.mode;
 		fapi->pps_info_buf = pps->ppsinfo;
 		return (0);
+#ifdef FFCLOCK
+	case PPS_IOC_FETCH_FFCOUNTER:
+		fapi_ffc = (struct pps_fetch_ffc_args *)data;
+		if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
+		    PPS_TSFMT_TSPEC)
+			return (EINVAL);
+		if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
+			return (EOPNOTSUPP);
+		pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
+		fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
+		/* Overwrite timestamps if feedback clock selected. */
+		switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
+		case PPS_TSCLK_FBCK:
+			fapi_ffc->pps_info_buf_ffc.assert_timestamp =
+			    pps->ppsinfo.assert_timestamp;
+			fapi_ffc->pps_info_buf_ffc.clear_timestamp =
+			    pps->ppsinfo.clear_timestamp;
+			break;
+		case PPS_TSCLK_FFWD:
+			break;
+		default:
+			break;
+		}
+		return (0);
+#endif /* FFCLOCK */
 	case PPS_IOC_KCBIND:
 #ifdef PPS_SYNC
 		kapi = (struct pps_kcbind_args *)data;
@@ -672,6 +1459,9 @@ pps_init(struct pps_state *pps)
 		pps->ppscap |= PPS_OFFSETASSERT;
 	if (pps->ppscap & PPS_CAPTURECLEAR)
 		pps->ppscap |= PPS_OFFSETCLEAR;
+#ifdef FFCLOCK
+	pps->ppscap |= PPS_TSCLK_MASK;
+#endif
 }
 
 void
@@ -683,6 +1473,9 @@ pps_capture(struct pps_state *pps)
 	th = timehands;
 	pps->capgen = th->th_generation;
 	pps->capth = th;
+#ifdef FFCLOCK
+	pps->capffth = fftimehands;
+#endif
 	pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
 	if (pps->capgen != th->th_generation)
 		pps->capgen = 0;
@@ -696,6 +1489,11 @@ pps_event(struct pps_state *pps, int event)
 	u_int tcount, *pcount;
 	int foff, fhard;
 	pps_seq_t *pseq;
+#ifdef FFCLOCK
+	struct timespec *tsp_ffc;
+	pps_seq_t *pseq_ffc;
+	ffcounter *ffcount;
+#endif
 
 	KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
 	/* If the timecounter was wound up underneath us, bail out. */
@@ -710,6 +1508,11 @@ pps_event(struct pps_state *pps, int event)
 		fhard = pps->kcmode & PPS_CAPTUREASSERT;
 		pcount = &pps->ppscount[0];
 		pseq = &pps->ppsinfo.assert_sequence;
+#ifdef FFCLOCK
+		ffcount = &pps->ppsinfo_ffc.assert_ffcount;
+		tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
+		pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
+#endif
 	} else {
 		tsp = &pps->ppsinfo.clear_timestamp;
 		osp = &pps->ppsparam.clear_offset;
@@ -717,6 +1520,11 @@ pps_event(struct pps_state *pps, int event)
 		fhard = pps->kcmode & PPS_CAPTURECLEAR;
 		pcount = &pps->ppscount[1];
 		pseq = &pps->ppsinfo.clear_sequence;
+#ifdef FFCLOCK
+		ffcount = &pps->ppsinfo_ffc.clear_ffcount;
+		tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
+		pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
+#endif
 	}
 
 	/*
@@ -753,6 +1561,17 @@ pps_event(struct pps_state *pps, int event)
 			tsp->tv_sec -= 1;
 		}
 	}
+
+#ifdef FFCLOCK
+	*ffcount = pps->capffth->tick_ffcount + tcount;
+	bt = pps->capffth->tick_time;
+	ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
+	bintime_add(&bt, &pps->capffth->tick_time);
+	bintime2timespec(&bt, &ts);
+	(*pseq_ffc)++;
+	*tsp_ffc = ts;
+#endif
+
 #ifdef PPS_SYNC
 	if (fhard) {
 		uint64_t scale;
@@ -820,6 +1639,10 @@ inittimecounter(void *dummy)
 	p = (tc_tick * 1000000) / hz;
 	printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
 
+#ifdef FFCLOCK
+	ffclock_init();
+	change_sysclock(sysclock_active);
+#endif
 	/* warm up new timecounter (again) and get rolling. */
 	(void)timecounter->tc_get_timecount(timecounter);
 	(void)timecounter->tc_get_timecount(timecounter);
diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c
index e997e48..881fd55 100644
--- a/sys/kern/kern_thr.c
+++ b/sys/kern/kern_thr.c
@@ -58,7 +58,8 @@ __FBSDID("$FreeBSD$");
 
 #include <security/audit/audit.h>
 
-SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation");
+static SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0,
+    "thread allocation");
 
 static int max_threads_per_proc = 1500;
 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
@@ -473,7 +474,8 @@ kern_thr_suspend(struct thread *td, struct timespec *tsp)
 	}
 
 	if (tsp != NULL) {
-		if (tsp->tv_nsec < 0 || tsp->tv_nsec > 1000000000)
+		if (tsp->tv_sec < 0 || tsp->tv_nsec < 0 ||
+		    tsp->tv_nsec > 1000000000)
 			return (EINVAL);
 		if (tsp->tv_sec == 0 && tsp->tv_nsec == 0)
 			error = EWOULDBLOCK;
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 29bdaa2..d9df5c3 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -566,6 +566,8 @@ calc_remaining(struct proc *p, int mode)
 {
 	int remaining;
 
+	PROC_LOCK_ASSERT(p, MA_OWNED);
+	PROC_SLOCK_ASSERT(p, MA_OWNED);
 	if (mode == SINGLE_EXIT)
 		remaining = p->p_numthreads;
 	else if (mode == SINGLE_BOUNDARY)
@@ -819,8 +821,11 @@ thread_suspend_check(int return_instead)
 			td->td_flags &= ~TDF_BOUNDARY;
 		thread_unlock(td);
 		PROC_LOCK(p);
-		if (return_instead == 0)
+		if (return_instead == 0) {
+			PROC_SLOCK(p);
 			p->p_boundary_count--;
+			PROC_SUNLOCK(p);
+		}
 	}
 	return (0);
 }
diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c
index b820bd9..5e1d4f3 100644
--- a/sys/kern/kern_timeout.c
+++ b/sys/kern/kern_timeout.c
@@ -156,7 +156,7 @@ struct callout_cpu cc_cpu;
 static int timeout_cpu;
 void (*callout_new_inserted)(int cpu, int ticks) = NULL;
 
-MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
+static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
 
 /**
  * Locked by cc_lock:
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index dfd6b12..80e2c73 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -186,7 +186,7 @@ static struct umtxq_chain	umtxq_chains[2][UMTX_CHAINS];
 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
 static int			umtx_pi_allocated;
 
-SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
+static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
     &umtx_pi_allocated, 0, "Allocated umtx_pi");
 
diff --git a/sys/kern/makesyscalls.sh b/sys/kern/makesyscalls.sh
index d1162b5..21e6046 100644
--- a/sys/kern/makesyscalls.sh
+++ b/sys/kern/makesyscalls.sh
@@ -38,6 +38,7 @@ sysinc="sysinc.switch.$$"
 sysarg="sysarg.switch.$$"
 sysprotoend="sysprotoend.$$"
 systracetmp="systrace.$$"
+systraceret="systraceret.$$"
 
 if [ -r capabilities.conf ]; then
 	capenabled=`cat capabilities.conf | grep -v "^#" | grep -v "^$"`
@@ -46,9 +47,9 @@ else
 	capenabled=""
 fi
 
-trap "rm $sysaue $sysdcl $syscompat $syscompatdcl $syscompat4 $syscompat4dcl $syscompat6 $syscompat6dcl $syscompat7 $syscompat7dcl $sysent $sysinc $sysarg $sysprotoend $systracetmp" 0
+trap "rm $sysaue $sysdcl $syscompat $syscompatdcl $syscompat4 $syscompat4dcl $syscompat6 $syscompat6dcl $syscompat7 $syscompat7dcl $sysent $sysinc $sysarg $sysprotoend $systracetmp $systraceret" 0
 
-touch $sysaue $sysdcl $syscompat $syscompatdcl $syscompat4 $syscompat4dcl $syscompat6 $syscompat6dcl $syscompat7 $syscompat7dcl $sysent $sysinc $sysarg $sysprotoend $systracetmp
+touch $sysaue $sysdcl $syscompat $syscompatdcl $syscompat4 $syscompat4dcl $syscompat6 $syscompat6dcl $syscompat7 $syscompat7dcl $sysent $sysinc $sysarg $sysprotoend $systracetmp $systraceret
 
 case $# in
     0)	echo "usage: $0 input-file <config-file>" 1>&2
@@ -96,6 +97,7 @@ s/\$//g
 		sysmk = \"$sysmk\"
 		systrace = \"$systrace\"
 		systracetmp = \"$systracetmp\"
+		systraceret = \"$systraceret\"
 		compat = \"$compat\"
 		compat4 = \"$compat4\"
 		compat6 = \"$compat6\"
@@ -152,6 +154,7 @@ s/\$//g
 		printf "#include <sys/signal.h>\n" > sysarg
 		printf "#include <sys/acl.h>\n" > sysarg
 		printf "#include <sys/cpuset.h>\n" > sysarg
+		printf "#include <sys/_ffcounter.h>\n" > sysarg
 		printf "#include <sys/_semaphore.h>\n" > sysarg
 		printf "#include <sys/ucontext.h>\n\n" > sysarg
 		printf "#include <bsm/audit_kevents.h>\n\n" > sysarg
@@ -179,9 +182,12 @@ s/\$//g
 		printf "\tint64_t *iarg  = (int64_t *) uarg;\n" > systrace
 		printf "\tswitch (sysnum) {\n" > systrace
 
-		printf "static void\nsystrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systracetmp
+		printf "static void\nsystrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systracetmp
 		printf "\tswitch (sysnum) {\n" > systracetmp
 
+		printf "static void\nsystrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systraceret
+		printf "\tswitch (sysnum) {\n" > systraceret
+
 		next
 	}
 	NF == 0 || $1 ~ /^;/ {
@@ -202,6 +208,7 @@ s/\$//g
 		print > sysnames
 		print > systrace
 		print > systracetmp
+		print > systraceret
 		savesyscall = syscall
 		next
 	}
@@ -216,6 +223,7 @@ s/\$//g
 		print > sysnames
 		print > systrace
 		print > systracetmp
+		print > systraceret
 		syscall = savesyscall
 		next
 	}
@@ -230,6 +238,7 @@ s/\$//g
 		print > sysnames
 		print > systrace
 		print > systracetmp
+		print > systraceret
 		next
 	}
 	syscall != $1 {
@@ -303,7 +312,8 @@ s/\$//g
 			parserr($end, ")")
 		end--
 
-		f++	#function return type
+		syscallret=$f
+		f++
 
 		funcname=$f
 
@@ -387,6 +397,7 @@ s/\$//g
 		parseline()
 		printf("\t/* %s */\n\tcase %d: {\n", funcname, syscall) > systrace
 		printf("\t/* %s */\n\tcase %d:\n", funcname, syscall) > systracetmp
+		printf("\t/* %s */\n\tcase %d:\n", funcname, syscall) > systraceret
 		if (argc > 0) {
 			printf("\t\tswitch(ndx) {\n") > systracetmp
 			printf("\t\tstruct %s *p = params;\n", argalias) > systrace
@@ -406,6 +417,10 @@ s/\$//g
 					     argname[i], argtype[i]) > systrace
 			}
 			printf("\t\tdefault:\n\t\t\tbreak;\n\t\t};\n") > systracetmp
+
+			printf("\t\tif (ndx == 0 || ndx == 1)\n") > systraceret
+			printf("\t\t\tp = \"%s\";\n", syscallret) > systraceret
+			printf("\t\tbreak;\n") > systraceret
 		}
 		printf("\t\t*n_args = %d;\n\t\tbreak;\n\t}\n", argc) > systrace
 		printf("\t\tbreak;\n") > systracetmp
@@ -623,6 +638,7 @@ s/\$//g
 		    > syshdr
 		printf "\tdefault:\n\t\t*n_args = 0;\n\t\tbreak;\n\t};\n}\n" > systrace
 		printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systracetmp
+		printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systraceret
 	} '
 
 cat $sysinc $sysent >> $syssw
@@ -633,4 +649,5 @@ cat $sysarg $sysdcl \
 	$syscompat7 $syscompat7dcl \
 	$sysaue $sysprotoend > $sysproto
 cat $systracetmp >> $systrace
+cat $systraceret >> $systrace
 
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 136080c..0fd6481 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -205,7 +205,8 @@ SYSCTL_PROC(_kern_sched, OID_AUTO, quantum, CTLTYPE_INT | CTLFLAG_RW,
 
 #ifdef SMP
 /* Enable forwarding of wakeups to all other cpus */
-SYSCTL_NODE(_kern_sched, OID_AUTO, ipiwakeup, CTLFLAG_RD, NULL, "Kernel SMP");
+static SYSCTL_NODE(_kern_sched, OID_AUTO, ipiwakeup, CTLFLAG_RD, NULL,
+    "Kernel SMP");
 
 static int runq_fuzz = 1;
 SYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, "");
diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c
index f46660f..c2551d7 100644
--- a/sys/kern/subr_bus.c
+++ b/sys/kern/subr_bus.c
@@ -1881,6 +1881,39 @@ device_delete_child(device_t dev, device_t child)
 }
 
 /**
+ * @brief Delete all children devices of the given device, if any.
+ *
+ * This function deletes all children devices of the given device, if
+ * any, using the device_delete_child() function for each device it
+ * finds. If a child device cannot be deleted, this function will
+ * return an error code.
+ * 
+ * @param dev		the parent device
+ *
+ * @retval 0		success
+ * @retval non-zero	a device would not detach
+ */
+int
+device_delete_all_children(device_t dev)
+{
+	device_t child;
+	int error;
+
+	PDEBUG(("Deleting all children of %s", DEVICENAME(dev)));
+
+	error = 0;
+
+	while ( (child = TAILQ_FIRST(&dev->children)) ) {
+		error = device_delete_child(dev, child);
+		if (error) {
+			PDEBUG(("Failed deleting %s", DEVICENAME(child)));
+			break;
+		}
+	}
+	return (error);
+}
+
+/**
  * @brief Find a device given a unit number
  *
  * This is similar to devclass_get_devices() but only searches for
diff --git a/sys/kern/subr_devstat.c b/sys/kern/subr_devstat.c
index eaf6427..bbbfdbf 100644
--- a/sys/kern/subr_devstat.c
+++ b/sys/kern/subr_devstat.c
@@ -402,7 +402,8 @@ sysctl_devstat(SYSCTL_HANDLER_ARGS)
  * Sysctl entries for devstat.  The first one is a node that all the rest
  * hang off of. 
  */
-SYSCTL_NODE(_kern, OID_AUTO, devstat, CTLFLAG_RD, NULL, "Device Statistics");
+static SYSCTL_NODE(_kern, OID_AUTO, devstat, CTLFLAG_RD, NULL,
+    "Device Statistics");
 
 SYSCTL_PROC(_kern_devstat, OID_AUTO, all, CTLFLAG_RD|CTLTYPE_OPAQUE,
     NULL, 0, sysctl_devstat, "S,devstat", "All devices in the devstat list");
diff --git a/sys/kern/subr_firmware.c b/sys/kern/subr_firmware.c
index 209f3a7..9a1ec3f 100644
--- a/sys/kern/subr_firmware.c
+++ b/sys/kern/subr_firmware.c
@@ -198,10 +198,8 @@ firmware_register(const char *imagename, const void *data, size_t datasize,
 	frp->fw.data = data;
 	frp->fw.datasize = datasize;
 	frp->fw.version = version;
-	if (parent != NULL) {
+	if (parent != NULL)
 		frp->parent = PRIV_FW(parent);
-		frp->parent->refcnt++;
-	}
 	mtx_unlock(&firmware_mtx);
 	if (bootverbose)
 		printf("firmware: '%s' version %u: %zu bytes loaded at %p\n",
@@ -235,8 +233,6 @@ firmware_unregister(const char *imagename)
 	}  else {
 		linker_file_t x = fp->file;	/* save value */
 
-		if (fp->parent != NULL)	/* release parent reference */
-			fp->parent->refcnt--;
 		/*
 		 * Clear the whole entry with bzero to make sure we
 		 * do not forget anything. Then restore 'file' which is
@@ -341,6 +337,8 @@ firmware_get(const char *imagename)
 		return NULL;
 	}
 found:				/* common exit point on success */
+	if (fp->refcnt == 0 && fp->parent != NULL)
+		fp->parent->refcnt++;
 	fp->refcnt++;
 	mtx_unlock(&firmware_mtx);
 	return &fp->fw;
@@ -363,6 +361,8 @@ firmware_put(const struct firmware *p, int flags)
 	mtx_lock(&firmware_mtx);
 	fp->refcnt--;
 	if (fp->refcnt == 0) {
+		if (fp->parent != NULL)
+			fp->parent->refcnt--;
 		if (flags & FIRMWARE_UNLOAD)
 			fp->flags |= FW_UNLOAD;
 		if (fp->file)
diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c
index 1d23f21..42e188b 100644
--- a/sys/kern/subr_kdb.c
+++ b/sys/kern/subr_kdb.c
@@ -82,7 +82,7 @@ static int kdb_sysctl_panic(SYSCTL_HANDLER_ARGS);
 static int kdb_sysctl_trap(SYSCTL_HANDLER_ARGS);
 static int kdb_sysctl_trap_code(SYSCTL_HANDLER_ARGS);
 
-SYSCTL_NODE(_debug, OID_AUTO, kdb, CTLFLAG_RW, NULL, "KDB nodes");
+static SYSCTL_NODE(_debug, OID_AUTO, kdb, CTLFLAG_RW, NULL, "KDB nodes");
 
 SYSCTL_PROC(_debug_kdb, OID_AUTO, available, CTLTYPE_STRING | CTLFLAG_RD, NULL,
     0, kdb_sysctl_available, "A", "list of available KDB backends");
diff --git a/sys/kern/subr_kobj.c b/sys/kern/subr_kobj.c
index 4ee7182..5be746a 100644
--- a/sys/kern/subr_kobj.c
+++ b/sys/kern/subr_kobj.c
@@ -60,18 +60,9 @@ static struct mtx kobj_mtx;
 static int kobj_mutex_inited;
 static int kobj_next_id = 1;
 
-/*
- * In the event that kobj_mtx has not been initialized yet,
- * we will ignore it, and run without locks in order to support
- * use of KOBJ before mutexes are available. This early in the boot
- * process, everything is single threaded and so races should not
- * happen. This is used to provide the PMAP layer on PowerPC, as well
- * as board support.
- */
-
-#define KOBJ_LOCK()	if (kobj_mutex_inited) mtx_lock(&kobj_mtx);
-#define KOBJ_UNLOCK()	if (kobj_mutex_inited) mtx_unlock(&kobj_mtx);
-#define KOBJ_ASSERT(what) if (kobj_mutex_inited) mtx_assert(&kobj_mtx,what);
+#define	KOBJ_LOCK()		mtx_lock(&kobj_mtx)
+#define	KOBJ_UNLOCK()		mtx_unlock(&kobj_mtx)
+#define	KOBJ_ASSERT(what)	mtx_assert(&kobj_mtx, what);
 
 SYSCTL_INT(_kern, OID_AUTO, kobj_methodcount, CTLFLAG_RD,
 	   &kobj_next_id, 0, "");
@@ -92,7 +83,7 @@ SYSINIT(kobj, SI_SUB_LOCK, SI_ORDER_ANY, kobj_init_mutex, NULL);
  * desc pointer is NULL, it is guaranteed never to match any read
  * descriptors.
  */
-static struct kobj_method null_method = {
+static const struct kobj_method null_method = {
 	0, 0,
 };
 
@@ -104,28 +95,11 @@ kobj_error_method(void)
 }
 
 static void
-kobj_register_method(struct kobjop_desc *desc)
-{
-	KOBJ_ASSERT(MA_OWNED);
-
-	if (desc->id == 0) {
-		desc->id = kobj_next_id++;
-	}
-}
-
-static void
-kobj_unregister_method(struct kobjop_desc *desc)
-{
-}
-
-static void
 kobj_class_compile_common(kobj_class_t cls, kobj_ops_t ops)
 {
 	kobj_method_t *m;
 	int i;
 
-	KOBJ_ASSERT(MA_OWNED);
-
 	/*
 	 * Don't do anything if we are already compiled.
 	 */
@@ -135,8 +109,10 @@ kobj_class_compile_common(kobj_class_t cls, kobj_ops_t ops)
 	/*
 	 * First register any methods which need it.
 	 */
-	for (i = 0, m = cls->methods; m->desc; i++, m++)
-		kobj_register_method(m->desc);
+	for (i = 0, m = cls->methods; m->desc; i++, m++) {
+		if (m->desc->id == 0)
+			m->desc->id = kobj_next_id++;
+	}
 
 	/*
 	 * Then initialise the ops table.
@@ -159,7 +135,7 @@ kobj_class_compile(kobj_class_t cls)
 	 */
 	ops = malloc(sizeof(struct kobj_ops), M_KOBJ, M_NOWAIT);
 	if (!ops)
-		panic("kobj_compile_methods: out of memory");
+		panic("%s: out of memory", __func__);
 
 	KOBJ_LOCK();
 	
@@ -182,17 +158,14 @@ void
 kobj_class_compile_static(kobj_class_t cls, kobj_ops_t ops)
 {
 
-	KOBJ_ASSERT(MA_NOTOWNED);
+	KASSERT(kobj_mutex_inited == 0,
+	    ("%s: only supported during early cycles", __func__));
 
 	/*
 	 * Increment refs to make sure that the ops table is not freed.
 	 */
-	KOBJ_LOCK();
-
 	cls->refs++;
 	kobj_class_compile_common(cls, ops);
-
-	KOBJ_UNLOCK();
 }
 
 static kobj_method_t*
@@ -251,7 +224,7 @@ kobj_lookup_method(kobj_class_t cls,
 
 	ce = kobj_lookup_method_mi(cls, desc);
 	if (!ce)
-		ce = desc->deflt;
+		ce = &desc->deflt;
 	*cep = ce;
 	return ce;
 }
@@ -259,8 +232,6 @@ kobj_lookup_method(kobj_class_t cls,
 void
 kobj_class_free(kobj_class_t cls)
 {
-	int i;
-	kobj_method_t *m;
 	void* ops = NULL;
 
 	KOBJ_ASSERT(MA_NOTOWNED);
@@ -272,10 +243,9 @@ kobj_class_free(kobj_class_t cls)
 	 */
 	if (cls->refs == 0) {
 		/*
-		 * Unregister any methods which are no longer used.
+		 * For now we don't do anything to unregister any methods
+		 * which are no longer used.
 		 */
-		for (i = 0, m = cls->methods; m->desc; i++, m++)
-			kobj_unregister_method(m->desc);
 
 		/*
 		 * Free memory and clean up.
@@ -308,6 +278,14 @@ kobj_create(kobj_class_t cls,
 	return obj;
 }
 
+static void
+kobj_init_common(kobj_t obj, kobj_class_t cls)
+{
+
+	obj->ops = cls->ops;
+	cls->refs++;
+}
+
 void
 kobj_init(kobj_t obj, kobj_class_t cls)
 {
@@ -329,13 +307,22 @@ kobj_init(kobj_t obj, kobj_class_t cls)
 		goto retry;
 	}
 
-	obj->ops = cls->ops;
-	cls->refs++;
+	kobj_init_common(obj, cls);
 
 	KOBJ_UNLOCK();
 }
 
 void
+kobj_init_static(kobj_t obj, kobj_class_t cls)
+{
+
+	KASSERT(kobj_mutex_inited == 0,
+	    ("%s: only supported during early cycles", __func__));
+
+	kobj_init_common(obj, cls);
+}
+
+void
 kobj_delete(kobj_t obj, struct malloc_type *mtype)
 {
 	kobj_class_t cls = obj->ops->cls;
diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c
index be9fa31..1c18f65 100644
--- a/sys/kern/subr_lock.c
+++ b/sys/kern/subr_lock.c
@@ -645,8 +645,9 @@ out:
 	critical_exit();
 }
 
-SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
-SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, "lock profiling");
+static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
+static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL,
+    "lock profiling");
 SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,
     &lock_prof_skipspin, 0, "Skip profiling on spinlocks.");
 SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW,
diff --git a/sys/kern/subr_mchain.c b/sys/kern/subr_mchain.c
index e40e5ff..3fd265b 100644
--- a/sys/kern/subr_mchain.c
+++ b/sys/kern/subr_mchain.c
@@ -128,6 +128,21 @@ mb_reserve(struct mbchain *mbp, int size)
 }
 
 int
+mb_put_padbyte(struct mbchain *mbp)
+{
+	caddr_t dst;
+	char x = 0;
+
+	dst = mtod(mbp->mb_cur, caddr_t) + mbp->mb_cur->m_len;
+
+	/* only add padding if address is odd */
+	if ((unsigned long)dst & 1)
+		return mb_put_mem(mbp, (caddr_t)&x, 1, MB_MSYSTEM);
+	else
+	return 0;
+}
+
+int
 mb_put_uint8(struct mbchain *mbp, uint8_t x)
 {
 	return mb_put_mem(mbp, (caddr_t)&x, sizeof(x), MB_MSYSTEM);
diff --git a/sys/kern/subr_pcpu.c b/sys/kern/subr_pcpu.c
index ba76bb2..505a4df 100644
--- a/sys/kern/subr_pcpu.c
+++ b/sys/kern/subr_pcpu.c
@@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sx.h>
 #include <ddb/ddb.h>
 
-MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting.");
+static MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting.");
 
 struct dpcpu_free {
 	uintptr_t	df_start;
diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c
index 7ff5414..c5b6b08 100644
--- a/sys/kern/subr_prof.c
+++ b/sys/kern/subr_prof.c
@@ -385,7 +385,7 @@ sysctl_kern_prof(SYSCTL_HANDLER_ARGS)
 	/* NOTREACHED */
 }
 
-SYSCTL_NODE(_kern, KERN_PROF, prof, CTLFLAG_RW, sysctl_kern_prof, "");
+static SYSCTL_NODE(_kern, KERN_PROF, prof, CTLFLAG_RW, sysctl_kern_prof, "");
 #endif /* GPROF */
 
 /*
diff --git a/sys/kern/subr_rman.c b/sys/kern/subr_rman.c
index abd72c0..db8eb2f 100644
--- a/sys/kern/subr_rman.c
+++ b/sys/kern/subr_rman.c
@@ -1085,7 +1085,7 @@ found:
 	return (error);
 }
 
-SYSCTL_NODE(_hw_bus, OID_AUTO, rman, CTLFLAG_RD, sysctl_rman,
+static SYSCTL_NODE(_hw_bus, OID_AUTO, rman, CTLFLAG_RD, sysctl_rman,
     "kernel resource manager");
 
 #ifdef DDB
diff --git a/sys/kern/subr_rtc.c b/sys/kern/subr_rtc.c
index 120ad5f..ed2befc 100644
--- a/sys/kern/subr_rtc.c
+++ b/sys/kern/subr_rtc.c
@@ -1,12 +1,17 @@
 /*-
  * Copyright (c) 1988 University of Utah.
  * Copyright (c) 1982, 1990, 1993
- *	The Regents of the University of California.  All rights reserved.
+ *	The Regents of the University of California.
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
  *
  * This code is derived from software contributed to Berkeley by
  * the Systems Programming Group of the University of Utah Computer
  * Science Department.
  *
+ * Portions of this software were developed by Julien Ridoux at the University
+ * of Melbourne under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -50,12 +55,17 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_ffclock.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/clock.h>
 #include <sys/sysctl.h>
+#ifdef FFCLOCK
+#include <sys/timeffc.h>
+#endif
 #include <sys/timetc.h>
 
 #include "clock_if.h"
@@ -133,6 +143,9 @@ inittodr(time_t base)
 	ts.tv_sec += utc_offset();
 	timespecadd(&ts, &clock_adj);
 	tc_setclock(&ts);
+#ifdef FFCLOCK
+	ffclock_reset_clock(&ts);
+#endif
 	return;
 
 wrong_time:
diff --git a/sys/kern/subr_sleepqueue.c b/sys/kern/subr_sleepqueue.c
index 4ef7c48..bbf79db 100644
--- a/sys/kern/subr_sleepqueue.c
+++ b/sys/kern/subr_sleepqueue.c
@@ -139,8 +139,8 @@ struct sleepqueue_chain {
 
 #ifdef SLEEPQUEUE_PROFILING
 u_int sleepq_max_depth;
-SYSCTL_NODE(_debug, OID_AUTO, sleepq, CTLFLAG_RD, 0, "sleepq profiling");
-SYSCTL_NODE(_debug_sleepq, OID_AUTO, chains, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_debug, OID_AUTO, sleepq, CTLFLAG_RD, 0, "sleepq profiling");
+static SYSCTL_NODE(_debug_sleepq, OID_AUTO, chains, CTLFLAG_RD, 0,
     "sleepq chain stats");
 SYSCTL_UINT(_debug_sleepq, OID_AUTO, max_depth, CTLFLAG_RD, &sleepq_max_depth,
     0, "maxmimum depth achieved of a single chain");
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 87d536c..5dd0c55 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -70,7 +70,8 @@ int mp_maxcpus = MAXCPU;
 volatile int smp_started;
 u_int mp_maxid;
 
-SYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD|CTLFLAG_CAPRD, NULL, "Kernel SMP");
+static SYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD|CTLFLAG_CAPRD, NULL,
+    "Kernel SMP");
 
 SYSCTL_INT(_kern_smp, OID_AUTO, maxid, CTLFLAG_RD|CTLFLAG_CAPRD, &mp_maxid, 0,
     "Max CPU ID.");
@@ -415,13 +416,16 @@ smp_rendezvous_cpus(cpuset_t map,
 {
 	int curcpumap, i, ncpus = 0;
 
+	/* Look comments in the !SMP case. */
 	if (!smp_started) {
+		spinlock_enter();
 		if (setup_func != NULL)
 			setup_func(arg);
 		if (action_func != NULL)
 			action_func(arg);
 		if (teardown_func != NULL)
 			teardown_func(arg);
+		spinlock_exit();
 		return;
 	}
 
@@ -666,12 +670,18 @@ smp_rendezvous_cpus(cpuset_t map,
 	void (*teardown_func)(void *),
 	void *arg)
 {
+	/*
+	 * In the !SMP case we just need to ensure the same initial conditions
+	 * as the SMP case.
+	 */
+	spinlock_enter();
 	if (setup_func != NULL)
 		setup_func(arg);
 	if (action_func != NULL)
 		action_func(arg);
 	if (teardown_func != NULL)
 		teardown_func(arg);
+	spinlock_exit();
 }
 
 void
@@ -681,12 +691,15 @@ smp_rendezvous(void (*setup_func)(void *),
 	       void *arg)
 {
 
+	/* Look comments in the smp_rendezvous_cpus() case. */
+	spinlock_enter();
 	if (setup_func != NULL)
 		setup_func(arg);
 	if (action_func != NULL)
 		action_func(arg);
 	if (teardown_func != NULL)
 		teardown_func(arg);
+	spinlock_exit();
 }
 
 /*
diff --git a/sys/kern/subr_stack.c b/sys/kern/subr_stack.c
index c26dad8..6408aec 100644
--- a/sys/kern/subr_stack.c
+++ b/sys/kern/subr_stack.c
@@ -77,7 +77,7 @@ stack_put(struct stack *st, vm_offset_t pc)
 }
 
 void
-stack_copy(struct stack *src, struct stack *dst)
+stack_copy(const struct stack *src, struct stack *dst)
 {
 
 	*dst = *src;
@@ -91,7 +91,7 @@ stack_zero(struct stack *st)
 }
 
 void
-stack_print(struct stack *st)
+stack_print(const struct stack *st)
 {
 	char namebuf[64];
 	long offset;
@@ -107,7 +107,7 @@ stack_print(struct stack *st)
 }
 
 void
-stack_print_short(struct stack *st)
+stack_print_short(const struct stack *st)
 {
 	char namebuf[64];
 	long offset;
@@ -127,7 +127,7 @@ stack_print_short(struct stack *st)
 }
 
 void
-stack_print_ddb(struct stack *st)
+stack_print_ddb(const struct stack *st)
 {
 	const char *name;
 	long offset;
@@ -143,7 +143,7 @@ stack_print_ddb(struct stack *st)
 
 #ifdef DDB
 void
-stack_print_short_ddb(struct stack *st)
+stack_print_short_ddb(const struct stack *st)
 {
 	const char *name;
 	long offset;
@@ -167,7 +167,7 @@ stack_print_short_ddb(struct stack *st)
  * other for use in the live kernel.
  */
 void
-stack_sbuf_print(struct sbuf *sb, struct stack *st)
+stack_sbuf_print(struct sbuf *sb, const struct stack *st)
 {
 	char namebuf[64];
 	long offset;
@@ -184,7 +184,7 @@ stack_sbuf_print(struct sbuf *sb, struct stack *st)
 
 #ifdef DDB
 void
-stack_sbuf_print_ddb(struct sbuf *sb, struct stack *st)
+stack_sbuf_print_ddb(struct sbuf *sb, const struct stack *st)
 {
 	const char *name;
 	long offset;
@@ -201,8 +201,8 @@ stack_sbuf_print_ddb(struct sbuf *sb, struct stack *st)
 
 #ifdef KTR
 void
-stack_ktr(u_int mask, const char *file, int line, struct stack *st, u_int depth,
-    int cheap)
+stack_ktr(u_int mask, const char *file, int line, const struct stack *st,
+    u_int depth, int cheap)
 {
 #ifdef DDB
 	const char *name;
diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c
index 1d32d66..008dada 100644
--- a/sys/kern/subr_turnstile.c
+++ b/sys/kern/subr_turnstile.c
@@ -138,8 +138,9 @@ struct turnstile_chain {
 
 #ifdef TURNSTILE_PROFILING
 u_int turnstile_max_depth;
-SYSCTL_NODE(_debug, OID_AUTO, turnstile, CTLFLAG_RD, 0, "turnstile profiling");
-SYSCTL_NODE(_debug_turnstile, OID_AUTO, chains, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_debug, OID_AUTO, turnstile, CTLFLAG_RD, 0,
+    "turnstile profiling");
+static SYSCTL_NODE(_debug_turnstile, OID_AUTO, chains, CTLFLAG_RD, 0,
     "turnstile chain stats");
 SYSCTL_UINT(_debug_turnstile, OID_AUTO, max_depth, CTLFLAG_RD,
     &turnstile_max_depth, 0, "maxmimum depth achieved of a single chain");
diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c
index 6e81328..d49e3c3 100644
--- a/sys/kern/subr_uio.c
+++ b/sys/kern/subr_uio.c
@@ -437,7 +437,7 @@ copyinstrfrom(const void * __restrict src, void * __restrict dst, size_t len,
 }
 
 int
-copyiniov(struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
+copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
 {
 	u_int iovlen;
 
@@ -455,7 +455,7 @@ copyiniov(struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
 }
 
 int
-copyinuio(struct iovec *iovp, u_int iovcnt, struct uio **uiop)
+copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
 {
 	struct iovec *iov;
 	struct uio *uio;
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c
index a4cf5a2..bbf9a97 100644
--- a/sys/kern/subr_witness.c
+++ b/sys/kern/subr_witness.c
@@ -184,7 +184,7 @@ __FBSDID("$FreeBSD$");
 #define	WITNESS_INDEX_ASSERT(i)						\
 	MPASS((i) > 0 && (i) <= w_max_used_index && (i) < WITNESS_COUNT)
 
-MALLOC_DEFINE(M_WITNESS, "Witness", "Witness");
+static MALLOC_DEFINE(M_WITNESS, "Witness", "Witness");
 
 /*
  * Lock instances.  A lock instance is the data associated with a lock while
@@ -332,7 +332,7 @@ static void	depart(struct witness *w);
 static struct witness	*enroll(const char *description,
 			    struct lock_class *lock_class);
 static struct lock_instance	*find_instance(struct lock_list_entry *list,
-				    struct lock_object *lock);
+				    const struct lock_object *lock);
 static int	isitmychild(struct witness *parent, struct witness *child);
 static int	isitmydescendant(struct witness *parent, struct witness *child);
 static void	itismychild(struct witness *parent, struct witness *child);
@@ -376,7 +376,8 @@ static void	witness_setflag(struct lock_object *lock, int flag, int set);
 #define	witness_debugger(c)
 #endif
 
-SYSCTL_NODE(_debug, OID_AUTO, witness, CTLFLAG_RW, NULL, "Witness Locking");
+static SYSCTL_NODE(_debug, OID_AUTO, witness, CTLFLAG_RW, NULL,
+    "Witness Locking");
 
 /*
  * If set to 0, lock order checking is disabled.  If set to -1,
@@ -2062,7 +2063,7 @@ witness_lock_list_free(struct lock_list_entry *lle)
 }
 
 static struct lock_instance *
-find_instance(struct lock_list_entry *list, struct lock_object *lock)
+find_instance(struct lock_list_entry *list, const struct lock_object *lock)
 {
 	struct lock_list_entry *lle;
 	struct lock_instance *instance;
@@ -2209,7 +2210,8 @@ witness_restore(struct lock_object *lock, const char *file, int line)
 }
 
 void
-witness_assert(struct lock_object *lock, int flags, const char *file, int line)
+witness_assert(const struct lock_object *lock, int flags, const char *file,
+    int line)
 {
 #ifdef INVARIANT_SUPPORT
 	struct lock_instance *instance;
diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c
index 8d4ba09..3be2689 100644
--- a/sys/kern/sys_generic.c
+++ b/sys/kern/sys_generic.c
@@ -831,6 +831,54 @@ sys_select(struct thread *td, struct select_args *uap)
 	    NFDBITS));
 }
 
+/*
+ * In the unlikely case when user specified n greater then the last
+ * open file descriptor, check that no bits are set after the last
+ * valid fd.  We must return EBADF if any is set.
+ *
+ * There are applications that rely on the behaviour.
+ *
+ * nd is fd_lastfile + 1.
+ */
+static int
+select_check_badfd(fd_set *fd_in, int nd, int ndu, int abi_nfdbits)
+{
+	char *addr, *oaddr;
+	int b, i, res;
+	uint8_t bits;
+
+	if (nd >= ndu || fd_in == NULL)
+		return (0);
+
+	oaddr = NULL;
+	bits = 0; /* silence gcc */
+	for (i = nd; i < ndu; i++) {
+		b = i / NBBY;
+#if BYTE_ORDER == LITTLE_ENDIAN
+		addr = (char *)fd_in + b;
+#else
+		addr = (char *)fd_in;
+		if (abi_nfdbits == NFDBITS) {
+			addr += rounddown(b, sizeof(fd_mask)) +
+			    sizeof(fd_mask) - 1 - b % sizeof(fd_mask);
+		} else {
+			addr += rounddown(b, sizeof(uint32_t)) +
+			    sizeof(uint32_t) - 1 - b % sizeof(uint32_t);
+		}
+#endif
+		if (addr != oaddr) {
+			res = fubyte(addr);
+			if (res == -1)
+				return (EFAULT);
+			oaddr = addr;
+			bits = res;
+		}
+		if ((bits & (1 << (i % NBBY))) != 0)
+			return (EBADF);
+	}
+	return (0);
+}
+
 int
 kern_select(struct thread *td, int nd, fd_set *fd_in, fd_set *fd_ou,
     fd_set *fd_ex, struct timeval *tvp, int abi_nfdbits)
@@ -845,14 +893,26 @@ kern_select(struct thread *td, int nd, fd_set *fd_in, fd_set *fd_ou,
 	fd_mask s_selbits[howmany(2048, NFDBITS)];
 	fd_mask *ibits[3], *obits[3], *selbits, *sbp;
 	struct timeval atv, rtv, ttv;
-	int error, timo;
+	int error, lf, ndu, timo;
 	u_int nbufbytes, ncpbytes, ncpubytes, nfdbits;
 
 	if (nd < 0)
 		return (EINVAL);
 	fdp = td->td_proc->p_fd;
-	if (nd > fdp->fd_lastfile + 1)
-		nd = fdp->fd_lastfile + 1;
+	ndu = nd;
+	lf = fdp->fd_lastfile;
+	if (nd > lf + 1)
+		nd = lf + 1;
+
+	error = select_check_badfd(fd_in, nd, ndu, abi_nfdbits);
+	if (error != 0)
+		return (error);
+	error = select_check_badfd(fd_ou, nd, ndu, abi_nfdbits);
+	if (error != 0)
+		return (error);
+	error = select_check_badfd(fd_ex, nd, ndu, abi_nfdbits);
+	if (error != 0)
+		return (error);
 
 	/*
 	 * Allocate just enough bits for the non-null fd_sets.  Use the
diff --git a/sys/kern/syscalls.c b/sys/kern/syscalls.c
index e353188..dd82db7 100644
--- a/sys/kern/syscalls.c
+++ b/sys/kern/syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 224987 2011-08-18 22:51:30Z jonathan 
+ * created from FreeBSD: head/sys/kern/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 const char *syscallnames[] = {
@@ -248,9 +248,9 @@ const char *syscallnames[] = {
 	"ktimer_gettime",			/* 238 = ktimer_gettime */
 	"ktimer_getoverrun",			/* 239 = ktimer_getoverrun */
 	"nanosleep",			/* 240 = nanosleep */
-	"#241",			/* 241 = nosys */
-	"#242",			/* 242 = nosys */
-	"#243",			/* 243 = nosys */
+	"ffclock_getcounter",			/* 241 = ffclock_getcounter */
+	"ffclock_setestimate",			/* 242 = ffclock_setestimate */
+	"ffclock_getestimate",			/* 243 = ffclock_getestimate */
 	"#244",			/* 244 = nosys */
 	"#245",			/* 245 = nosys */
 	"#246",			/* 246 = nosys */
@@ -538,5 +538,5 @@ const char *syscallnames[] = {
 	"rctl_add_rule",			/* 528 = rctl_add_rule */
 	"rctl_remove_rule",			/* 529 = rctl_remove_rule */
 	"posix_fallocate",			/* 530 = posix_fallocate */
-	"#531",			/* 531 = posix_fadvise */
+	"posix_fadvise",			/* 531 = posix_fadvise */
 };
diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master
index b79c6c7..e10af89 100644
--- a/sys/kern/syscalls.master
+++ b/sys/kern/syscalls.master
@@ -115,7 +115,7 @@
 32	AUE_GETSOCKNAME	STD	{ int getsockname(int fdes, \
 				    struct sockaddr * __restrict asa, \
 				    __socklen_t * __restrict alen); }
-33	AUE_ACCESS	STD	{ int access(char *path, int flags); }
+33	AUE_ACCESS	STD	{ int access(char *path, int amode); }
 34	AUE_CHFLAGS	STD	{ int chflags(char *path, int flags); }
 35	AUE_FCHFLAGS	STD	{ int fchflags(int fd, int flags); }
 36	AUE_SYNC	STD	{ int sync(void); }
@@ -456,9 +456,11 @@
 239	AUE_NULL	STD	{ int ktimer_getoverrun(int timerid); }
 240	AUE_NULL	STD	{ int nanosleep(const struct timespec *rqtp, \
 				    struct timespec *rmtp); }
-241	AUE_NULL	UNIMPL	nosys
-242	AUE_NULL	UNIMPL	nosys
-243	AUE_NULL	UNIMPL	nosys
+241	AUE_NULL	STD	{ int ffclock_getcounter(ffcounter *ffcount); }
+242	AUE_NULL	STD	{ int ffclock_setestimate( \
+				    struct ffclock_estimate *cest); }
+243	AUE_NULL	STD	{ int ffclock_getestimate( \
+				    struct ffclock_estimate *cest); }
 244	AUE_NULL	UNIMPL	nosys
 245	AUE_NULL	UNIMPL	nosys
 246	AUE_NULL	UNIMPL	nosys
@@ -671,7 +673,7 @@
 				    const char *attrname); }
 374	AUE_NULL	STD	{ int __setugid(int flag); }
 375	AUE_NULL	UNIMPL	nfsclnt
-376	AUE_EACCESS	STD	{ int eaccess(char *path, int flags); }
+376	AUE_EACCESS	STD	{ int eaccess(char *path, int amode); }
 377	AUE_NULL	NOSTD|NOTSTATIC	{ int afs3_syscall(long syscall, \
 				    long parm1, long parm2, long parm3, \
 				    long parm4, long parm5, long parm6); }
@@ -870,7 +872,7 @@
 488	AUE_NULL	STD	{ int cpuset_setaffinity(cpulevel_t level, \
 				    cpuwhich_t which, id_t id, size_t cpusetsize, \
 				    const cpuset_t *mask); }
-489	AUE_FACCESSAT	STD	{ int faccessat(int fd, char *path, int mode, \
+489	AUE_FACCESSAT	STD	{ int faccessat(int fd, char *path, int amode, \
 				    int flag); }
 490	AUE_FCHMODAT	STD	{ int fchmodat(int fd, char *path, mode_t mode, \
 				    int flag); }
@@ -947,6 +949,7 @@
 				    size_t outbuflen); }
 530	AUE_NULL	STD	{ int posix_fallocate(int fd, \
 				    off_t offset, off_t len); }
-531	AUE_NULL	UNIMPL	posix_fadvise
+531	AUE_NULL	STD	{ int posix_fadvise(int fd, off_t offset, \
+				    off_t len, int advice); }
 ; Please copy any additions and changes to the following compatability tables:
 ; sys/compat/freebsd32/syscalls.master
diff --git a/sys/kern/systrace_args.c b/sys/kern/systrace_args.c
index 9343613..4a5d72a 100644
--- a/sys/kern/systrace_args.c
+++ b/sys/kern/systrace_args.c
@@ -245,7 +245,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 33: {
 		struct access_args *p = params;
 		uarg[0] = (intptr_t) p->path; /* char * */
-		iarg[1] = p->flags; /* int */
+		iarg[1] = p->amode; /* int */
 		*n_args = 2;
 		break;
 	}
@@ -1337,6 +1337,27 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 2;
 		break;
 	}
+	/* ffclock_getcounter */
+	case 241: {
+		struct ffclock_getcounter_args *p = params;
+		uarg[0] = (intptr_t) p->ffcount; /* ffcounter * */
+		*n_args = 1;
+		break;
+	}
+	/* ffclock_setestimate */
+	case 242: {
+		struct ffclock_setestimate_args *p = params;
+		uarg[0] = (intptr_t) p->cest; /* struct ffclock_estimate * */
+		*n_args = 1;
+		break;
+	}
+	/* ffclock_getestimate */
+	case 243: {
+		struct ffclock_getestimate_args *p = params;
+		uarg[0] = (intptr_t) p->cest; /* struct ffclock_estimate * */
+		*n_args = 1;
+		break;
+	}
 	/* ntp_gettime */
 	case 248: {
 		struct ntp_gettime_args *p = params;
@@ -2018,7 +2039,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 	case 376: {
 		struct eaccess_args *p = params;
 		uarg[0] = (intptr_t) p->path; /* char * */
-		iarg[1] = p->flags; /* int */
+		iarg[1] = p->amode; /* int */
 		*n_args = 2;
 		break;
 	}
@@ -2873,7 +2894,7 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		struct faccessat_args *p = params;
 		iarg[0] = p->fd; /* int */
 		uarg[1] = (intptr_t) p->path; /* char * */
-		iarg[2] = p->mode; /* int */
+		iarg[2] = p->amode; /* int */
 		iarg[3] = p->flag; /* int */
 		*n_args = 4;
 		break;
@@ -3234,13 +3255,23 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 3;
 		break;
 	}
+	/* posix_fadvise */
+	case 531: {
+		struct posix_fadvise_args *p = params;
+		iarg[0] = p->fd; /* int */
+		iarg[1] = p->offset; /* off_t */
+		iarg[2] = p->len; /* off_t */
+		iarg[3] = p->advice; /* int */
+		*n_args = 4;
+		break;
+	}
 	default:
 		*n_args = 0;
 		break;
 	};
 }
 static void
-systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 {
 	const char *p = NULL;
 	switch (sysnum) {
@@ -5371,6 +5402,36 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
+	/* ffclock_getcounter */
+	case 241:
+		switch(ndx) {
+		case 0:
+			p = "ffcounter *";
+			break;
+		default:
+			break;
+		};
+		break;
+	/* ffclock_setestimate */
+	case 242:
+		switch(ndx) {
+		case 0:
+			p = "struct ffclock_estimate *";
+			break;
+		default:
+			break;
+		};
+		break;
+	/* ffclock_getestimate */
+	case 243:
+		switch(ndx) {
+		case 0:
+			p = "struct ffclock_estimate *";
+			break;
+		default:
+			break;
+		};
+		break;
 	/* ntp_gettime */
 	case 248:
 		switch(ndx) {
@@ -8603,6 +8664,1908 @@ systrace_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
+	/* posix_fadvise */
+	case 531:
+		switch(ndx) {
+		case 0:
+			p = "int";
+			break;
+		case 1:
+			p = "off_t";
+			break;
+		case 2:
+			p = "off_t";
+			break;
+		case 3:
+			p = "int";
+			break;
+		default:
+			break;
+		};
+		break;
+	default:
+		break;
+	};
+	if (p != NULL)
+		strlcpy(desc, p, descsz);
+}
+static void
+systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
+{
+	const char *p = NULL;
+	switch (sysnum) {
+	/* nosys */
+	case 0:
+	/* sys_exit */
+	case 1:
+		if (ndx == 0 || ndx == 1)
+			p = "void";
+		break;
+	/* fork */
+	case 2:
+	/* read */
+	case 3:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* write */
+	case 4:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* open */
+	case 5:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* close */
+	case 6:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* wait4 */
+	case 7:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* link */
+	case 9:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* unlink */
+	case 10:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chdir */
+	case 12:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchdir */
+	case 13:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mknod */
+	case 14:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chmod */
+	case 15:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chown */
+	case 16:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* obreak */
+	case 17:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpid */
+	case 20:
+	/* mount */
+	case 21:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* unmount */
+	case 22:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setuid */
+	case 23:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getuid */
+	case 24:
+	/* geteuid */
+	case 25:
+	/* ptrace */
+	case 26:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* recvmsg */
+	case 27:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sendmsg */
+	case 28:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* recvfrom */
+	case 29:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* accept */
+	case 30:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpeername */
+	case 31:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getsockname */
+	case 32:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* access */
+	case 33:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chflags */
+	case 34:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchflags */
+	case 35:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sync */
+	case 36:
+	/* kill */
+	case 37:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getppid */
+	case 39:
+	/* dup */
+	case 41:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pipe */
+	case 42:
+	/* getegid */
+	case 43:
+	/* profil */
+	case 44:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktrace */
+	case 45:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getgid */
+	case 47:
+	/* getlogin */
+	case 49:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setlogin */
+	case 50:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* acct */
+	case 51:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigaltstack */
+	case 53:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ioctl */
+	case 54:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* reboot */
+	case 55:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* revoke */
+	case 56:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* symlink */
+	case 57:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* readlink */
+	case 58:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* execve */
+	case 59:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* umask */
+	case 60:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* chroot */
+	case 61:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msync */
+	case 65:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* vfork */
+	case 66:
+	/* sbrk */
+	case 69:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sstk */
+	case 70:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ovadvise */
+	case 72:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munmap */
+	case 73:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mprotect */
+	case 74:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* madvise */
+	case 75:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mincore */
+	case 78:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getgroups */
+	case 79:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setgroups */
+	case 80:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpgrp */
+	case 81:
+	/* setpgid */
+	case 82:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setitimer */
+	case 83:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* swapon */
+	case 85:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getitimer */
+	case 86:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getdtablesize */
+	case 89:
+	/* dup2 */
+	case 90:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fcntl */
+	case 92:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* select */
+	case 93:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fsync */
+	case 95:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setpriority */
+	case 96:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* socket */
+	case 97:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* connect */
+	case 98:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpriority */
+	case 100:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* bind */
+	case 104:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setsockopt */
+	case 105:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* listen */
+	case 106:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* gettimeofday */
+	case 116:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getrusage */
+	case 117:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getsockopt */
+	case 118:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* readv */
+	case 120:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* writev */
+	case 121:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* settimeofday */
+	case 122:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchown */
+	case 123:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchmod */
+	case 124:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setreuid */
+	case 126:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setregid */
+	case 127:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rename */
+	case 128:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* flock */
+	case 131:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkfifo */
+	case 132:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sendto */
+	case 133:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shutdown */
+	case 134:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* socketpair */
+	case 135:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkdir */
+	case 136:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rmdir */
+	case 137:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* utimes */
+	case 138:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* adjtime */
+	case 140:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setsid */
+	case 147:
+	/* quotactl */
+	case 148:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nlm_syscall */
+	case 154:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nfssvc */
+	case 155:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lgetfh */
+	case 160:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getfh */
+	case 161:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sysarch */
+	case 165:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rtprio */
+	case 166:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* semsys */
+	case 169:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msgsys */
+	case 170:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmsys */
+	case 171:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd6_pread */
+	case 173:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* freebsd6_pwrite */
+	case 174:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* setfib */
+	case 175:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ntp_adjtime */
+	case 176:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setgid */
+	case 181:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setegid */
+	case 182:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* seteuid */
+	case 183:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* stat */
+	case 188:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fstat */
+	case 189:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lstat */
+	case 190:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pathconf */
+	case 191:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fpathconf */
+	case 192:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getrlimit */
+	case 194:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setrlimit */
+	case 195:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getdirentries */
+	case 196:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd6_mmap */
+	case 197:
+		if (ndx == 0 || ndx == 1)
+			p = "caddr_t";
+		break;
+	/* nosys */
+	case 198:
+	/* freebsd6_lseek */
+	case 199:
+		if (ndx == 0 || ndx == 1)
+			p = "off_t";
+		break;
+	/* freebsd6_truncate */
+	case 200:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd6_ftruncate */
+	case 201:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __sysctl */
+	case 202:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mlock */
+	case 203:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlock */
+	case 204:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* undelete */
+	case 205:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* futimes */
+	case 206:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getpgid */
+	case 207:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* poll */
+	case 209:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lkmnosys */
+	case 210:
+	/* lkmnosys */
+	case 211:
+	/* lkmnosys */
+	case 212:
+	/* lkmnosys */
+	case 213:
+	/* lkmnosys */
+	case 214:
+	/* lkmnosys */
+	case 215:
+	/* lkmnosys */
+	case 216:
+	/* lkmnosys */
+	case 217:
+	/* lkmnosys */
+	case 218:
+	/* lkmnosys */
+	case 219:
+	/* semget */
+	case 221:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* semop */
+	case 222:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msgget */
+	case 225:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msgsnd */
+	case 226:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msgrcv */
+	case 227:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmat */
+	case 228:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmdt */
+	case 230:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmget */
+	case 231:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* clock_gettime */
+	case 232:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* clock_settime */
+	case 233:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* clock_getres */
+	case 234:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktimer_create */
+	case 235:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktimer_delete */
+	case 236:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktimer_settime */
+	case 237:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktimer_gettime */
+	case 238:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ktimer_getoverrun */
+	case 239:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nanosleep */
+	case 240:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ffclock_getcounter */
+	case 241:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ffclock_setestimate */
+	case 242:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ffclock_getestimate */
+	case 243:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ntp_gettime */
+	case 248:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* minherit */
+	case 250:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rfork */
+	case 251:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* openbsd_poll */
+	case 252:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* issetugid */
+	case 253:
+	/* lchown */
+	case 254:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_read */
+	case 255:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_write */
+	case 256:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lio_listio */
+	case 257:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getdents */
+	case 272:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lchmod */
+	case 274:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lchown */
+	case 275:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lutimes */
+	case 276:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msync */
+	case 277:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nstat */
+	case 278:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nfstat */
+	case 279:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nlstat */
+	case 280:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* preadv */
+	case 289:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* pwritev */
+	case 290:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* fhopen */
+	case 298:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fhstat */
+	case 299:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modnext */
+	case 300:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modstat */
+	case 301:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modfnext */
+	case 302:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* modfind */
+	case 303:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldload */
+	case 304:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldunload */
+	case 305:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldfind */
+	case 306:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldnext */
+	case 307:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldstat */
+	case 308:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldfirstmod */
+	case 309:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getsid */
+	case 310:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setresuid */
+	case 311:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setresgid */
+	case 312:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_return */
+	case 314:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_suspend */
+	case 315:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_cancel */
+	case 316:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_error */
+	case 317:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* oaio_read */
+	case 318:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* oaio_write */
+	case 319:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* olio_listio */
+	case 320:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* yield */
+	case 321:
+	/* mlockall */
+	case 324:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* munlockall */
+	case 325:
+	/* __getcwd */
+	case 326:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_setparam */
+	case 327:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_getparam */
+	case 328:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_setscheduler */
+	case 329:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_getscheduler */
+	case 330:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_yield */
+	case 331:
+	/* sched_get_priority_max */
+	case 332:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_get_priority_min */
+	case 333:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sched_rr_get_interval */
+	case 334:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* utrace */
+	case 335:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldsym */
+	case 337:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail */
+	case 338:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nnpfs_syscall */
+	case 339:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigprocmask */
+	case 340:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigsuspend */
+	case 341:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigpending */
+	case 343:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigtimedwait */
+	case 345:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigwaitinfo */
+	case 346:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_get_file */
+	case 347:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_set_file */
+	case 348:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_get_fd */
+	case 349:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_set_fd */
+	case 350:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_delete_file */
+	case 351:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_delete_fd */
+	case 352:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_aclcheck_file */
+	case 353:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_aclcheck_fd */
+	case 354:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattrctl */
+	case 355:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_set_file */
+	case 356:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_get_file */
+	case 357:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_delete_file */
+	case 358:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_waitcomplete */
+	case 359:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresuid */
+	case 360:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getresgid */
+	case 361:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kqueue */
+	case 362:
+	/* kevent */
+	case 363:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_set_fd */
+	case 371:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_get_fd */
+	case 372:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_delete_fd */
+	case 373:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __setugid */
+	case 374:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* eaccess */
+	case 376:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* afs3_syscall */
+	case 377:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* nmount */
+	case 378:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_get_proc */
+	case 384:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_set_proc */
+	case 385:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_get_fd */
+	case 386:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_get_file */
+	case 387:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_set_fd */
+	case 388:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_set_file */
+	case 389:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kenv */
+	case 390:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lchflags */
+	case 391:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* uuidgen */
+	case 392:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sendfile */
+	case 393:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mac_syscall */
+	case 394:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getfsstat */
+	case 395:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* statfs */
+	case 396:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fstatfs */
+	case 397:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fhstatfs */
+	case 398:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_close */
+	case 400:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_post */
+	case 401:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_wait */
+	case 402:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_trywait */
+	case 403:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_init */
+	case 404:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_open */
+	case 405:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_unlink */
+	case 406:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_getvalue */
+	case 407:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ksem_destroy */
+	case 408:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_get_pid */
+	case 409:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_get_link */
+	case 410:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_set_link */
+	case 411:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_set_link */
+	case 412:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_get_link */
+	case 413:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_delete_link */
+	case 414:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __mac_execve */
+	case 415:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigaction */
+	case 416:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigreturn */
+	case 417:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getcontext */
+	case 421:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setcontext */
+	case 422:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* swapcontext */
+	case 423:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* swapoff */
+	case 424:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_get_link */
+	case 425:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_set_link */
+	case 426:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_delete_link */
+	case 427:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __acl_aclcheck_link */
+	case 428:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigwait */
+	case 429:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_create */
+	case 430:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_exit */
+	case 431:
+		if (ndx == 0 || ndx == 1)
+			p = "void";
+		break;
+	/* thr_self */
+	case 432:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_kill */
+	case 433:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* _umtx_lock */
+	case 434:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* _umtx_unlock */
+	case 435:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail_attach */
+	case 436:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* extattr_list_fd */
+	case 437:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_list_file */
+	case 438:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* extattr_list_link */
+	case 439:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* ksem_timedwait */
+	case 441:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_suspend */
+	case 442:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_wake */
+	case 443:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kldunloadf */
+	case 444:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* audit */
+	case 445:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* auditon */
+	case 446:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getauid */
+	case 447:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setauid */
+	case 448:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getaudit */
+	case 449:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setaudit */
+	case 450:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getaudit_addr */
+	case 451:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setaudit_addr */
+	case 452:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* auditctl */
+	case 453:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* _umtx_op */
+	case 454:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_new */
+	case 455:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sigqueue */
+	case 456:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_open */
+	case 457:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_setattr */
+	case 458:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_timedreceive */
+	case 459:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_timedsend */
+	case 460:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_notify */
+	case 461:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* kmq_unlink */
+	case 462:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* abort2 */
+	case 463:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_set_name */
+	case 464:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* aio_fsync */
+	case 465:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rtprio_thread */
+	case 466:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_peeloff */
+	case 471:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_generic_sendmsg */
+	case 472:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_generic_sendmsg_iov */
+	case 473:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* sctp_generic_recvmsg */
+	case 474:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pread */
+	case 475:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* pwrite */
+	case 476:
+		if (ndx == 0 || ndx == 1)
+			p = "ssize_t";
+		break;
+	/* mmap */
+	case 477:
+		if (ndx == 0 || ndx == 1)
+			p = "caddr_t";
+		break;
+	/* lseek */
+	case 478:
+		if (ndx == 0 || ndx == 1)
+			p = "off_t";
+		break;
+	/* truncate */
+	case 479:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* ftruncate */
+	case 480:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* thr_kill2 */
+	case 481:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shm_open */
+	case 482:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shm_unlink */
+	case 483:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cpuset */
+	case 484:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cpuset_setid */
+	case 485:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cpuset_getid */
+	case 486:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cpuset_getaffinity */
+	case 487:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cpuset_setaffinity */
+	case 488:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* faccessat */
+	case 489:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchmodat */
+	case 490:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fchownat */
+	case 491:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fexecve */
+	case 492:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* fstatat */
+	case 493:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* futimesat */
+	case 494:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* linkat */
+	case 495:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkdirat */
+	case 496:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mkfifoat */
+	case 497:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* mknodat */
+	case 498:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* openat */
+	case 499:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* readlinkat */
+	case 500:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* renameat */
+	case 501:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* symlinkat */
+	case 502:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* unlinkat */
+	case 503:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* posix_openpt */
+	case 504:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* gssd_syscall */
+	case 505:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail_get */
+	case 506:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail_set */
+	case 507:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* jail_remove */
+	case 508:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* closefrom */
+	case 509:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* __semctl */
+	case 510:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* msgctl */
+	case 511:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* shmctl */
+	case 512:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* lpathconf */
+	case 513:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cap_new */
+	case 514:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cap_getrights */
+	case 515:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* cap_enter */
+	case 516:
+	/* cap_getmode */
+	case 517:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pdfork */
+	case 518:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pdkill */
+	case 519:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pdgetpid */
+	case 520:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* pselect */
+	case 522:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* getloginclass */
+	case 523:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* setloginclass */
+	case 524:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_get_racct */
+	case 525:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_get_rules */
+	case 526:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_get_limits */
+	case 527:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_add_rule */
+	case 528:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* rctl_remove_rule */
+	case 529:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* posix_fallocate */
+	case 530:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* posix_fadvise */
+	case 531:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
 	default:
 		break;
 	};
diff --git a/sys/kern/uipc_mqueue.c b/sys/kern/uipc_mqueue.c
index 8a3ea79..696f974 100644
--- a/sys/kern/uipc_mqueue.c
+++ b/sys/kern/uipc_mqueue.c
@@ -186,7 +186,7 @@ struct mqueue_msg {
 	/* following real data... */
 };
 
-SYSCTL_NODE(_kern, OID_AUTO, mqueue, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_kern, OID_AUTO, mqueue, CTLFLAG_RW, 0,
 	"POSIX real time message queue");
 
 static int	default_maxmsg  = 10;
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index bbd4fad..2a1bf7f 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -2828,7 +2828,6 @@ bad:
 	return (error);
 }
 
-/* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */
 int
 soopt_getm(struct sockopt *sopt, struct mbuf **mp)
 {
@@ -2877,7 +2876,6 @@ soopt_getm(struct sockopt *sopt, struct mbuf **mp)
 	return (0);
 }
 
-/* XXX; copyin sopt data into mbuf chain for (__FreeBSD__ < 3) routines. */
 int
 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m)
 {
@@ -2906,7 +2904,6 @@ soopt_mcopyin(struct sockopt *sopt, struct mbuf *m)
 	return (0);
 }
 
-/* XXX; copyout mbuf chain data into soopt for (__FreeBSD__ < 3) routines. */
 int
 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m)
 {
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 801a244..3a439ec 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -159,10 +159,11 @@ static u_long	unpdg_recvspace = 4*1024;
 static u_long	unpsp_sendspace = PIPSIZ;	/* really max datagram size */
 static u_long	unpsp_recvspace = PIPSIZ;
 
-SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW, 0, "Local domain");
-SYSCTL_NODE(_net_local, SOCK_STREAM, stream, CTLFLAG_RW, 0, "SOCK_STREAM");
-SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, CTLFLAG_RW, 0, "SOCK_DGRAM");
-SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW, 0, "Local domain");
+static SYSCTL_NODE(_net_local, SOCK_STREAM, stream, CTLFLAG_RW, 0,
+    "SOCK_STREAM");
+static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, CTLFLAG_RW, 0, "SOCK_DGRAM");
+static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket, CTLFLAG_RW, 0,
     "SOCK_SEQPACKET");
 
 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW,
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 7b5b015..d3384ef 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -3760,10 +3760,9 @@ tryagain:
 		 * could interfere with paging I/O, no matter which
 		 * process we are.
 		 */
-		p = vm_page_alloc(NULL, pg >> PAGE_SHIFT, VM_ALLOC_NOOBJ |
-		    VM_ALLOC_SYSTEM | VM_ALLOC_WIRED |
-		    VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT));
-		if (!p) {
+		p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
+		    VM_ALLOC_WIRED | VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT));
+		if (p == NULL) {
 			VM_WAIT;
 			goto tryagain;
 		}
diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index ee4bf3c..177fd56 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -242,7 +242,8 @@ static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
 /*
  * Grab an atomic snapshot of the name cache hash chain lengths
  */
-SYSCTL_NODE(_debug, OID_AUTO, hashstat, CTLFLAG_RW, NULL, "hash table stats");
+static SYSCTL_NODE(_debug, OID_AUTO, hashstat, CTLFLAG_RW, NULL,
+    "hash table stats");
 
 static int
 sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
@@ -1067,16 +1068,8 @@ vn_vptocnp(struct vnode **vp, struct ucred *cred, char *buf, u_int *buflen)
 
 	CACHE_RLOCK();
 	error = vn_vptocnp_locked(vp, cred, buf, buflen);
-	if (error == 0) {
-		/*
-		 * vn_vptocnp_locked() dropped hold acquired by
-		 * VOP_VPTOCNP immediately after locking the
-		 * cache. Since we are going to drop the cache rlock,
-		 * re-hold the result.
-		 */
-		vhold(*vp);
+	if (error == 0)
 		CACHE_RUNLOCK();
-	}
 	return (error);
 }
 
@@ -1095,6 +1088,9 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 	if (ncp != NULL) {
 		if (*buflen < ncp->nc_nlen) {
 			CACHE_RUNLOCK();
+			vfslocked = VFS_LOCK_GIANT((*vp)->v_mount);
+			vrele(*vp);
+			VFS_UNLOCK_GIANT(vfslocked);
 			numfullpathfail4++;
 			error = ENOMEM;
 			SDT_PROBE(vfs, namecache, fullpath, return, error,
@@ -1105,18 +1101,23 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 		memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
 		SDT_PROBE(vfs, namecache, fullpath, hit, ncp->nc_dvp,
 		    ncp->nc_name, vp, 0, 0);
+		dvp = *vp;
 		*vp = ncp->nc_dvp;
+		vref(*vp);
+		CACHE_RUNLOCK();
+		vfslocked = VFS_LOCK_GIANT(dvp->v_mount);
+		vrele(dvp);
+		VFS_UNLOCK_GIANT(vfslocked);
+		CACHE_RLOCK();
 		return (0);
 	}
 	SDT_PROBE(vfs, namecache, fullpath, miss, vp, 0, 0, 0, 0);
 
-	vhold(*vp);
 	CACHE_RUNLOCK();
 	vfslocked = VFS_LOCK_GIANT((*vp)->v_mount);
 	vn_lock(*vp, LK_SHARED | LK_RETRY);
 	error = VOP_VPTOCNP(*vp, &dvp, cred, buf, buflen);
-	VOP_UNLOCK(*vp, 0);
-	vdrop(*vp);
+	vput(*vp);
 	VFS_UNLOCK_GIANT(vfslocked);
 	if (error) {
 		numfullpathfail2++;
@@ -1127,16 +1128,20 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 
 	*vp = dvp;
 	CACHE_RLOCK();
-	if ((*vp)->v_iflag & VI_DOOMED) {
+	if (dvp->v_iflag & VI_DOOMED) {
 		/* forced unmount */
 		CACHE_RUNLOCK();
-		vdrop(*vp);
+		vfslocked = VFS_LOCK_GIANT(dvp->v_mount);
+		vrele(dvp);
+		VFS_UNLOCK_GIANT(vfslocked);
 		error = ENOENT;
 		SDT_PROBE(vfs, namecache, fullpath, return, error, vp,
 		    NULL, 0, 0);
 		return (error);
 	}
-	vdrop(*vp);
+	/*
+	 * *vp has its use count incremented still.
+	 */
 
 	return (0);
 }
@@ -1148,10 +1153,11 @@ static int
 vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
     char *buf, char **retbuf, u_int buflen)
 {
-	int error, slash_prefixed;
+	int error, slash_prefixed, vfslocked;
 #ifdef KDTRACE_HOOKS
 	struct vnode *startvp = vp;
 #endif
+	struct vnode *vp1;
 
 	buflen--;
 	buf[buflen] = '\0';
@@ -1160,6 +1166,7 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 
 	SDT_PROBE(vfs, namecache, fullpath, entry, vp, 0, 0, 0, 0);
 	numfullpathcalls++;
+	vref(vp);
 	CACHE_RLOCK();
 	if (vp->v_type != VDIR) {
 		error = vn_vptocnp_locked(&vp, td->td_ucred, buf, &buflen);
@@ -1167,6 +1174,9 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 			return (error);
 		if (buflen == 0) {
 			CACHE_RUNLOCK();
+			vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+			vrele(vp);
+			VFS_UNLOCK_GIANT(vfslocked);
 			return (ENOMEM);
 		}
 		buf[--buflen] = '/';
@@ -1176,16 +1186,29 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 		if (vp->v_vflag & VV_ROOT) {
 			if (vp->v_iflag & VI_DOOMED) {	/* forced unmount */
 				CACHE_RUNLOCK();
+				vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+				vrele(vp);
+				VFS_UNLOCK_GIANT(vfslocked);
 				error = ENOENT;
 				SDT_PROBE(vfs, namecache, fullpath, return,
 				    error, vp, NULL, 0, 0);
 				break;
 			}
-			vp = vp->v_mount->mnt_vnodecovered;
+			vp1 = vp->v_mount->mnt_vnodecovered;
+			vref(vp1);
+			CACHE_RUNLOCK();
+			vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+			vrele(vp);
+			VFS_UNLOCK_GIANT(vfslocked);
+			vp = vp1;
+			CACHE_RLOCK();
 			continue;
 		}
 		if (vp->v_type != VDIR) {
 			CACHE_RUNLOCK();
+			vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+			vrele(vp);
+			VFS_UNLOCK_GIANT(vfslocked);
 			numfullpathfail1++;
 			error = ENOTDIR;
 			SDT_PROBE(vfs, namecache, fullpath, return,
@@ -1197,6 +1220,9 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 			break;
 		if (buflen == 0) {
 			CACHE_RUNLOCK();
+			vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+			vrele(vp);
+			VFS_UNLOCK_GIANT(vfslocked);
 			error = ENOMEM;
 			SDT_PROBE(vfs, namecache, fullpath, return, error,
 			    startvp, NULL, 0, 0);
@@ -1210,6 +1236,9 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 	if (!slash_prefixed) {
 		if (buflen == 0) {
 			CACHE_RUNLOCK();
+			vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+			vrele(vp);
+			VFS_UNLOCK_GIANT(vfslocked);
 			numfullpathfail4++;
 			SDT_PROBE(vfs, namecache, fullpath, return, ENOMEM,
 			    startvp, NULL, 0, 0);
@@ -1219,6 +1248,9 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 	}
 	numfullpathfound++;
 	CACHE_RUNLOCK();
+	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+	vrele(vp);
+	VFS_UNLOCK_GIANT(vfslocked);
 
 	SDT_PROBE(vfs, namecache, fullpath, return, 0, startvp, buf + buflen,
 	    0, 0);
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index b89d990..e47498e 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -96,6 +96,7 @@ struct vop_vector default_vnodeops = {
 
 	.vop_access =		vop_stdaccess,
 	.vop_accessx =		vop_stdaccessx,
+	.vop_advise =		vop_stdadvise,
 	.vop_advlock =		vop_stdadvlock,
 	.vop_advlockasync =	vop_stdadvlockasync,
 	.vop_advlockpurge =	vop_stdadvlockpurge,
@@ -843,7 +844,7 @@ out:
 	free(dirbuf, M_TEMP);
 	if (!error) {
 		*buflen = i;
-		vhold(*dvp);
+		vref(*dvp);
 	}
 	if (covered) {
 		vput(*dvp);
@@ -984,6 +985,58 @@ vop_stdallocate(struct vop_allocate_args *ap)
 	return (error);
 }
 
+int
+vop_stdadvise(struct vop_advise_args *ap)
+{
+	struct vnode *vp;
+	off_t start, end;
+	int error, vfslocked;
+
+	vp = ap->a_vp;
+	switch (ap->a_advice) {
+	case POSIX_FADV_WILLNEED:
+		/*
+		 * Do nothing for now.  Filesystems should provide a
+		 * custom method which starts an asynchronous read of
+		 * the requested region.
+		 */
+		error = 0;
+		break;
+	case POSIX_FADV_DONTNEED:
+		/*
+		 * Flush any open FS buffers and then remove pages
+		 * from the backing VM object.  Using vinvalbuf() here
+		 * is a bit heavy-handed as it flushes all buffers for
+		 * the given vnode, not just the buffers covering the
+		 * requested range.
+		 */
+		error = 0;
+		vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
+		if (vp->v_iflag & VI_DOOMED) {
+			VOP_UNLOCK(vp, 0);
+			VFS_UNLOCK_GIANT(vfslocked);
+			break;
+		}
+		vinvalbuf(vp, V_CLEANONLY, 0, 0);
+		if (vp->v_object != NULL) {
+			start = trunc_page(ap->a_start);
+			end = round_page(ap->a_end);
+			VM_OBJECT_LOCK(vp->v_object);
+			vm_object_page_cache(vp->v_object, OFF_TO_IDX(start),
+			    OFF_TO_IDX(end));
+			VM_OBJECT_UNLOCK(vp->v_object);
+		}
+		VOP_UNLOCK(vp, 0);
+		VFS_UNLOCK_GIANT(vfslocked);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	return (error);
+}
+
 /*
  * vfs default ops
  * used to fill the vfs function table to get reasonable default return values.
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index e8b89e5..873f425 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -37,6 +37,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_vfs_allow_nonmpsafe.h"
+
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/fcntl.h>
@@ -79,7 +81,7 @@ SYSCTL_INT(_vfs, OID_AUTO, usermount, CTLFLAG_RW, &usermount, 0,
     "Unprivileged users may mount and unmount file systems");
 
 MALLOC_DEFINE(M_MOUNT, "mount", "vfs mount structure");
-MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker");
+static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker");
 static uma_zone_t mount_zone;
 
 /* List of mounted filesystems. */
@@ -798,6 +800,14 @@ vfs_domount_first(
 	 * get.  No freeing of cn_pnbuf.
 	 */
 	error = VFS_MOUNT(mp);
+#ifndef VFS_ALLOW_NONMPSAFE
+	if (error == 0 && VFS_NEEDSGIANT(mp)) {
+		(void)VFS_UNMOUNT(mp, fsflags);
+		error = ENXIO;
+		printf("%s: Mounting non-MPSAFE fs (%s) is disabled\n",
+		    __func__, mp->mnt_vfc->vfc_name);
+	}
+#endif
 	if (error != 0) {
 		vfs_unbusy(mp);
 		vfs_mount_destroy(mp);
@@ -807,6 +817,11 @@ vfs_domount_first(
 		vrele(vp);
 		return (error);
 	}
+#ifdef VFS_ALLOW_NONMPSAFE
+	if (VFS_NEEDSGIANT(mp))
+		printf("%s: Mounting non-MPSAFE fs (%s) is deprecated\n",
+		    __func__, mp->mnt_vfc->vfc_name);
+#endif
 
 	if (mp->mnt_opt != NULL)
 		vfs_freeopts(mp->mnt_opt);
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 8101fce..2872f77 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -1191,7 +1191,7 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
 	do {
 		error = flushbuflist(&bo->bo_clean,
 		    flags, bo, slpflag, slptimeo);
-		if (error == 0)
+		if (error == 0 && !(flags & V_CLEANONLY))
 			error = flushbuflist(&bo->bo_dirty,
 			    flags, bo, slpflag, slptimeo);
 		if (error != 0 && error != EAGAIN) {
@@ -1220,7 +1220,8 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
 	/*
 	 * Destroy the copy in the VM cache, too.
 	 */
-	if (bo->bo_object != NULL && (flags & (V_ALT | V_NORMAL)) == 0) {
+	if (bo->bo_object != NULL &&
+	    (flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0) {
 		VM_OBJECT_LOCK(bo->bo_object);
 		vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
 		    OBJPR_CLEANONLY : 0);
@@ -1229,7 +1230,7 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
 
 #ifdef INVARIANTS
 	BO_LOCK(bo);
-	if ((flags & (V_ALT | V_NORMAL)) == 0 &&
+	if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0 &&
 	    (bo->bo_dirty.bv_cnt > 0 || bo->bo_clean.bv_cnt > 0))
 		panic("vinvalbuf: flush failed");
 	BO_UNLOCK(bo);
@@ -1255,7 +1256,7 @@ vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
  *
  */
 static int
-flushbuflist( struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
+flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
     int slptimeo)
 {
 	struct buf *bp, *nbp;
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index ec5ad06..fe012d5 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -86,6 +86,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_page.h>
 #include <vm/uma.h>
 
+static MALLOC_DEFINE(M_FADVISE, "fadvise", "posix_fadvise(2) information");
+
 SDT_PROVIDER_DEFINE(vfs);
 SDT_PROBE_DEFINE(vfs, , stat, mode, mode);
 SDT_PROBE_ARGTYPE(vfs, , stat, mode, 0, "char *");
@@ -2142,7 +2144,7 @@ vn_access(vp, user_flags, cred, td)
 #ifndef _SYS_SYSPROTO_H_
 struct access_args {
 	char	*path;
-	int	flags;
+	int	amode;
 };
 #endif
 int
@@ -2150,18 +2152,18 @@ sys_access(td, uap)
 	struct thread *td;
 	register struct access_args /* {
 		char *path;
-		int flags;
+		int amode;
 	} */ *uap;
 {
 
-	return (kern_access(td, uap->path, UIO_USERSPACE, uap->flags));
+	return (kern_access(td, uap->path, UIO_USERSPACE, uap->amode));
 }
 
 #ifndef _SYS_SYSPROTO_H_
 struct faccessat_args {
 	int	dirfd;
 	char	*path;
-	int	mode;
+	int	amode;
 	int	flag;
 }
 #endif
@@ -2172,19 +2174,19 @@ sys_faccessat(struct thread *td, struct faccessat_args *uap)
 	if (uap->flag & ~AT_EACCESS)
 		return (EINVAL);
 	return (kern_accessat(td, uap->fd, uap->path, UIO_USERSPACE, uap->flag,
-	    uap->mode));
+	    uap->amode));
 }
 
 int
-kern_access(struct thread *td, char *path, enum uio_seg pathseg, int mode)
+kern_access(struct thread *td, char *path, enum uio_seg pathseg, int amode)
 {
 
-	return (kern_accessat(td, AT_FDCWD, path, pathseg, 0, mode));
+	return (kern_accessat(td, AT_FDCWD, path, pathseg, 0, amode));
 }
 
 int
 kern_accessat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
-    int flags, int mode)
+    int flag, int amode)
 {
 	struct ucred *cred, *tmpcred;
 	struct vnode *vp;
@@ -2196,7 +2198,7 @@ kern_accessat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 	 * Create and modify a temporary credential instead of one that
 	 * is potentially shared.
 	 */
-	if (!(flags & AT_EACCESS)) {
+	if (!(flag & AT_EACCESS)) {
 		cred = td->td_ucred;
 		tmpcred = crdup(cred);
 		tmpcred->cr_uid = cred->cr_ruid;
@@ -2204,7 +2206,7 @@ kern_accessat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 		td->td_ucred = tmpcred;
 	} else
 		cred = tmpcred = td->td_ucred;
-	AUDIT_ARG_VALUE(mode);
+	AUDIT_ARG_VALUE(amode);
 	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | MPSAFE |
 	    AUDITVNODE1, pathseg, path, fd, CAP_FSTAT, td);
 	if ((error = namei(&nd)) != 0)
@@ -2212,12 +2214,12 @@ kern_accessat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 	vfslocked = NDHASGIANT(&nd);
 	vp = nd.ni_vp;
 
-	error = vn_access(vp, mode, tmpcred, td);
+	error = vn_access(vp, amode, tmpcred, td);
 	NDFREE(&nd, NDF_ONLY_PNBUF);
 	vput(vp);
 	VFS_UNLOCK_GIANT(vfslocked);
 out1:
-	if (!(flags & AT_EACCESS)) {
+	if (!(flag & AT_EACCESS)) {
 		td->td_ucred = cred;
 		crfree(tmpcred);
 	}
@@ -2230,7 +2232,7 @@ out1:
 #ifndef _SYS_SYSPROTO_H_
 struct eaccess_args {
 	char	*path;
-	int	flags;
+	int	amode;
 };
 #endif
 int
@@ -2238,18 +2240,18 @@ sys_eaccess(td, uap)
 	struct thread *td;
 	register struct eaccess_args /* {
 		char *path;
-		int flags;
+		int amode;
 	} */ *uap;
 {
 
-	return (kern_eaccess(td, uap->path, UIO_USERSPACE, uap->flags));
+	return (kern_eaccess(td, uap->path, UIO_USERSPACE, uap->amode));
 }
 
 int
-kern_eaccess(struct thread *td, char *path, enum uio_seg pathseg, int flags)
+kern_eaccess(struct thread *td, char *path, enum uio_seg pathseg, int amode)
 {
 
-	return (kern_accessat(td, AT_FDCWD, path, pathseg, AT_EACCESS, flags));
+	return (kern_accessat(td, AT_FDCWD, path, pathseg, AT_EACCESS, amode));
 }
 
 #if defined(COMPAT_43)
@@ -4751,7 +4753,7 @@ out:
 	return (error);
 }
 
-static int
+int
 kern_posix_fallocate(struct thread *td, int fd, off_t offset, off_t len)
 {
 	struct file *fp;
@@ -4845,3 +4847,140 @@ sys_posix_fallocate(struct thread *td, struct posix_fallocate_args *uap)
 
 	return (kern_posix_fallocate(td, uap->fd, uap->offset, uap->len));
 }
+
+/*
+ * Unlike madvise(2), we do not make a best effort to remember every
+ * possible caching hint.  Instead, we remember the last setting with
+ * the exception that we will allow POSIX_FADV_NORMAL to adjust the
+ * region of any current setting.
+ */
+int
+kern_posix_fadvise(struct thread *td, int fd, off_t offset, off_t len,
+    int advice)
+{
+	struct fadvise_info *fa, *new;
+	struct file *fp;
+	struct vnode *vp;
+	off_t end;
+	int error;
+
+	if (offset < 0 || len < 0 || offset > OFF_MAX - len)
+		return (EINVAL);
+	switch (advice) {
+	case POSIX_FADV_SEQUENTIAL:
+	case POSIX_FADV_RANDOM:
+	case POSIX_FADV_NOREUSE:
+		new = malloc(sizeof(*fa), M_FADVISE, M_WAITOK);
+		break;
+	case POSIX_FADV_NORMAL:
+	case POSIX_FADV_WILLNEED:
+	case POSIX_FADV_DONTNEED:
+		new = NULL;
+		break;
+	default:
+		return (EINVAL);
+	}
+	/* XXX: CAP_POSIX_FADVISE? */
+	error = fget(td, fd, 0, &fp);
+	if (error != 0)
+		goto out;
+	
+	switch (fp->f_type) {
+	case DTYPE_VNODE:
+		break;
+	case DTYPE_PIPE:
+	case DTYPE_FIFO:
+		error = ESPIPE;
+		goto out;
+	default:
+		error = ENODEV;
+		goto out;
+	}
+	vp = fp->f_vnode;
+	if (vp->v_type != VREG) {
+		error = ENODEV;
+		goto out;
+	}
+	if (len == 0)
+		end = OFF_MAX;
+	else
+		end = offset + len - 1;
+	switch (advice) {
+	case POSIX_FADV_SEQUENTIAL:
+	case POSIX_FADV_RANDOM:
+	case POSIX_FADV_NOREUSE:
+		/*
+		 * Try to merge any existing non-standard region with
+		 * this new region if possible, otherwise create a new
+		 * non-standard region for this request.
+		 */
+		mtx_pool_lock(mtxpool_sleep, fp);
+		fa = fp->f_advice;
+		if (fa != NULL && fa->fa_advice == advice &&
+		    ((fa->fa_start <= end && fa->fa_end >= offset) ||
+		    (end != OFF_MAX && fa->fa_start == end + 1) ||
+		    (fa->fa_end != OFF_MAX && fa->fa_end + 1 == offset))) {
+			if (offset < fa->fa_start)
+				fa->fa_start = offset;
+			if (end > fa->fa_end)
+				fa->fa_end = end;
+		} else {
+			new->fa_advice = advice;
+			new->fa_start = offset;
+			new->fa_end = end;
+			fp->f_advice = new;
+			new = fa;
+		}
+		mtx_pool_unlock(mtxpool_sleep, fp);
+		break;
+	case POSIX_FADV_NORMAL:
+		/*
+		 * If a the "normal" region overlaps with an existing
+		 * non-standard region, trim or remove the
+		 * non-standard region.
+		 */
+		mtx_pool_lock(mtxpool_sleep, fp);
+		fa = fp->f_advice;
+		if (fa != NULL) {
+			if (offset <= fa->fa_start && end >= fa->fa_end) {
+				new = fa;
+				fp->f_advice = NULL;
+			} else if (offset <= fa->fa_start &&
+ 			    end >= fa->fa_start)
+				fa->fa_start = end + 1;
+			else if (offset <= fa->fa_end && end >= fa->fa_end)
+				fa->fa_end = offset - 1;
+			else if (offset >= fa->fa_start && end <= fa->fa_end) {
+				/*
+				 * If the "normal" region is a middle
+				 * portion of the existing
+				 * non-standard region, just remove
+				 * the whole thing rather than picking
+				 * one side or the other to
+				 * preserve.
+				 */
+				new = fa;
+				fp->f_advice = NULL;
+			}
+		}
+		mtx_pool_unlock(mtxpool_sleep, fp);
+		break;
+	case POSIX_FADV_WILLNEED:
+	case POSIX_FADV_DONTNEED:
+		error = VOP_ADVISE(vp, offset, end, advice);
+		break;
+	}
+out:
+	if (fp != NULL)
+		fdrop(fp, td);
+	free(new, M_FADVISE);
+	return (error);
+}
+
+int
+sys_posix_fadvise(struct thread *td, struct posix_fadvise_args *uap)
+{
+
+	return (kern_posix_fadvise(td, uap->fd, uap->offset, uap->len,
+	    uap->advice));
+}
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 17dc5e7..e33592a 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -518,7 +518,7 @@ vn_read(fp, uio, active_cred, flags, td)
 	struct vnode *vp;
 	int error, ioflag;
 	struct mtx *mtxp;
-	int vfslocked;
+	int advice, vfslocked;
 
 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
 	    uio->uio_td, td));
@@ -529,27 +529,48 @@ vn_read(fp, uio, active_cred, flags, td)
 		ioflag |= IO_NDELAY;
 	if (fp->f_flag & O_DIRECT)
 		ioflag |= IO_DIRECT;
+	advice = POSIX_FADV_NORMAL;
 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
 	/*
 	 * According to McKusick the vn lock was protecting f_offset here.
 	 * It is now protected by the FOFFSET_LOCKED flag.
 	 */
-	if ((flags & FOF_OFFSET) == 0) {
+	if ((flags & FOF_OFFSET) == 0 || fp->f_advice != NULL) {
 		mtxp = mtx_pool_find(mtxpool_sleep, fp);
 		mtx_lock(mtxp);
-		while(fp->f_vnread_flags & FOFFSET_LOCKED) {
-			fp->f_vnread_flags |= FOFFSET_LOCK_WAITING;
-			msleep(&fp->f_vnread_flags, mtxp, PUSER -1,
-			    "vnread offlock", 0);
+		if ((flags & FOF_OFFSET) == 0) {
+			while (fp->f_vnread_flags & FOFFSET_LOCKED) {
+				fp->f_vnread_flags |= FOFFSET_LOCK_WAITING;
+				msleep(&fp->f_vnread_flags, mtxp, PUSER -1,
+				    "vnread offlock", 0);
+			}
+			fp->f_vnread_flags |= FOFFSET_LOCKED;
+			uio->uio_offset = fp->f_offset;
 		}
-		fp->f_vnread_flags |= FOFFSET_LOCKED;
+		if (fp->f_advice != NULL &&
+		    uio->uio_offset >= fp->f_advice->fa_start &&
+		    uio->uio_offset + uio->uio_resid <= fp->f_advice->fa_end)
+			advice = fp->f_advice->fa_advice;
 		mtx_unlock(mtxp);
-		vn_lock(vp, LK_SHARED | LK_RETRY);
-		uio->uio_offset = fp->f_offset;
-	} else
-		vn_lock(vp, LK_SHARED | LK_RETRY);
+	}
+	vn_lock(vp, LK_SHARED | LK_RETRY);
 
-	ioflag |= sequential_heuristic(uio, fp);
+	switch (advice) {
+	case POSIX_FADV_NORMAL:
+	case POSIX_FADV_SEQUENTIAL:
+		ioflag |= sequential_heuristic(uio, fp);
+		break;
+	case POSIX_FADV_RANDOM:
+		/* Disable read-ahead for random I/O. */
+		break;
+	case POSIX_FADV_NOREUSE:
+		/*
+		 * Request the underlying FS to discard the buffers
+		 * and pages after the I/O is complete.
+		 */
+		ioflag |= IO_DIRECT;
+		break;
+	}
 
 #ifdef MAC
 	error = mac_vnode_check_read(active_cred, fp->f_cred, vp);
@@ -584,7 +605,8 @@ vn_write(fp, uio, active_cred, flags, td)
 	struct vnode *vp;
 	struct mount *mp;
 	int error, ioflag, lock_flags;
-	int vfslocked;
+	struct mtx *mtxp;
+	int advice, vfslocked;
 
 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
 	    uio->uio_td, td));
@@ -618,7 +640,33 @@ vn_write(fp, uio, active_cred, flags, td)
 	vn_lock(vp, lock_flags | LK_RETRY);
 	if ((flags & FOF_OFFSET) == 0)
 		uio->uio_offset = fp->f_offset;
-	ioflag |= sequential_heuristic(uio, fp);
+	advice = POSIX_FADV_NORMAL;
+	if (fp->f_advice != NULL) {
+		mtxp = mtx_pool_find(mtxpool_sleep, fp);
+		mtx_lock(mtxp);
+		if (fp->f_advice != NULL &&
+		    uio->uio_offset >= fp->f_advice->fa_start &&
+		    uio->uio_offset + uio->uio_resid <= fp->f_advice->fa_end)
+			advice = fp->f_advice->fa_advice;
+		mtx_unlock(mtxp);
+	}
+	switch (advice) {
+	case POSIX_FADV_NORMAL:
+	case POSIX_FADV_SEQUENTIAL:
+		ioflag |= sequential_heuristic(uio, fp);
+		break;
+	case POSIX_FADV_RANDOM:
+		/* XXX: Is this correct? */
+		break;
+	case POSIX_FADV_NOREUSE:
+		/*
+		 * Request the underlying FS to discard the buffers
+		 * and pages after the I/O is complete.
+		 */
+		ioflag |= IO_DIRECT;
+		break;
+	}
+
 #ifdef MAC
 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
 	if (error == 0)
diff --git a/sys/kern/vnode_if.src b/sys/kern/vnode_if.src
index f75e7df..6f24d17 100644
--- a/sys/kern/vnode_if.src
+++ b/sys/kern/vnode_if.src
@@ -628,3 +628,12 @@ vop_allocate {
 	INOUT off_t *offset;
 	INOUT off_t *len;
 };
+
+%% advise	vp	U U U
+
+vop_advise {
+	IN struct vnode *vp;
+	IN off_t start;
+	IN off_t end;
+	IN int advice;
+};
diff --git a/sys/libkern/iconv.c b/sys/libkern/iconv.c
index 7700937..92b04c2 100644
--- a/sys/libkern/iconv.c
+++ b/sys/libkern/iconv.c
@@ -43,7 +43,7 @@ SYSCTL_DECL(_kern_iconv);
 SYSCTL_NODE(_kern, OID_AUTO, iconv, CTLFLAG_RW, NULL, "kernel iconv interface");
 
 MALLOC_DEFINE(M_ICONV, "iconv", "ICONV structures");
-MALLOC_DEFINE(M_ICONVDATA, "iconv_data", "ICONV data");
+static MALLOC_DEFINE(M_ICONVDATA, "iconv_data", "ICONV data");
 
 MODULE_VERSION(libiconv, 2);
 
@@ -377,6 +377,18 @@ iconv_sysctl_cslist(SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_kern_iconv, OID_AUTO, cslist, CTLFLAG_RD | CTLTYPE_OPAQUE,
 	    NULL, 0, iconv_sysctl_cslist, "S,xlat", "registered charset pairs");
 
+int
+iconv_add(const char *converter, const char *to, const char *from)
+{
+	struct iconv_converter_class *dcp;
+	struct iconv_cspair *csp;
+
+	if (iconv_lookupconv(converter, &dcp) != 0)
+		return EINVAL;
+
+	return iconv_register_cspair(to, from, dcp, NULL, &csp);
+}
+
 /*
  * Add new charset pair
  */
diff --git a/sys/libkern/iconv_ucs.c b/sys/libkern/iconv_ucs.c
new file mode 100644
index 0000000..c6f2823
--- /dev/null
+++ b/sys/libkern/iconv_ucs.c
@@ -0,0 +1,540 @@
+/*-
+ * Copyright (c) 2003, 2005 Ryuichiro Imura
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/iconv.h>
+
+#include "iconv_converter_if.h"
+
+/*
+ * "UCS" converter
+ */
+
+#define	KICONV_UCS_COMBINE	0x1
+#define	KICONV_UCS_FROM_UTF8	0x2
+#define	KICONV_UCS_TO_UTF8	0x4
+#define	KICONV_UCS_FROM_LE	0x8
+#define	KICONV_UCS_TO_LE	0x10
+#define	KICONV_UCS_FROM_UTF16	0x20
+#define	KICONV_UCS_TO_UTF16	0x40
+#define	KICONV_UCS_UCS4		0x80
+
+#define	ENCODING_UTF16	"UTF-16BE"
+#define	ENCODING_UTF8	"UTF-8"
+
+static struct {
+	const char *name;
+	int from_flag, to_flag;
+} unicode_family[] = {
+	{ "UTF-8",	KICONV_UCS_FROM_UTF8,	KICONV_UCS_TO_UTF8 },
+	{ "UCS-2LE",	KICONV_UCS_FROM_LE,	KICONV_UCS_TO_LE },
+	{ "UTF-16BE",	KICONV_UCS_FROM_UTF16,	KICONV_UCS_TO_UTF16 },
+	{ "UTF-16LE",	KICONV_UCS_FROM_UTF16|KICONV_UCS_FROM_LE,
+	    KICONV_UCS_TO_UTF16|KICONV_UCS_TO_LE },
+	{ NULL,		0,	0 }
+};
+
+static uint32_t utf8_to_ucs4(const char *src, size_t *utf8width, size_t srclen);
+static u_char *ucs4_to_utf8(uint32_t ucs4, char * dst, size_t *utf8width, size_t dstlen);
+static uint32_t encode_surrogate(uint32_t code);
+static uint32_t decode_surrogate(const u_char *ucs);
+
+#ifdef MODULE_DEPEND
+MODULE_DEPEND(iconv_ucs, libiconv, 2, 2, 2);
+#endif
+
+/*
+ * UCS converter instance
+ */
+struct iconv_ucs {
+	KOBJ_FIELDS;
+	int			convtype;
+	struct iconv_cspair *	d_csp;
+	struct iconv_cspair *	d_cspf;
+	void *			f_ctp;
+	void *			t_ctp;
+	void *			ctype;
+};
+
+static int
+iconv_ucs_open(struct iconv_converter_class *dcp,
+	struct iconv_cspair *csp, struct iconv_cspair *cspf, void **dpp)
+{
+	struct iconv_ucs *dp;
+	int i;
+	const char *from, *to;
+
+	dp = (struct iconv_ucs *)kobj_create((struct kobj_class*)dcp, M_ICONV, M_WAITOK);
+	to = csp->cp_to;
+	from = cspf ? cspf->cp_from : csp->cp_from;
+
+	dp->convtype = 0;
+
+	if (cspf)
+		dp->convtype |= KICONV_UCS_COMBINE;
+	for (i = 0; unicode_family[i].name; i++) {
+		if (strcmp(from, unicode_family[i].name) == 0)
+			dp->convtype |= unicode_family[i].from_flag;
+		if (strcmp(to, unicode_family[i].name) == 0)
+			dp->convtype |= unicode_family[i].to_flag;
+	}
+	if (strcmp(ENCODING_UNICODE, ENCODING_UTF16) == 0)
+		dp->convtype |= KICONV_UCS_UCS4;
+	else
+		dp->convtype &= ~KICONV_UCS_UCS4;
+
+	dp->f_ctp = dp->t_ctp = NULL;
+	if (dp->convtype & KICONV_UCS_COMBINE) {
+		if ((dp->convtype & KICONV_UCS_FROM_UTF8) == 0 &&
+		    (dp->convtype & KICONV_UCS_FROM_LE) == 0) {
+			iconv_open(ENCODING_UNICODE, from, &dp->f_ctp);
+		}
+		if ((dp->convtype & KICONV_UCS_TO_UTF8) == 0 &&
+		    (dp->convtype & KICONV_UCS_TO_LE) == 0) {
+			iconv_open(to, ENCODING_UNICODE, &dp->t_ctp);
+		}
+	}
+
+	dp->ctype = NULL;
+	if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_TO_UTF8))
+		iconv_open(KICONV_WCTYPE_NAME, ENCODING_UTF8, &dp->ctype);
+
+	dp->d_csp = csp;
+	if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_FROM_LE)) {
+		if (cspf) {
+			dp->d_cspf = cspf;
+			cspf->cp_refcount++;
+		} else
+			csp->cp_refcount++;
+	}
+	if (dp->convtype & (KICONV_UCS_TO_UTF8 | KICONV_UCS_TO_LE))
+		csp->cp_refcount++;
+	*dpp = (void*)dp;
+	return 0;
+}
+
+static int
+iconv_ucs_close(void *data)
+{
+	struct iconv_ucs *dp = data;
+
+	if (dp->f_ctp)
+		iconv_close(dp->f_ctp);
+	if (dp->t_ctp)
+		iconv_close(dp->t_ctp);
+	if (dp->ctype)
+		iconv_close(dp->ctype);
+	if (dp->d_cspf)
+		dp->d_cspf->cp_refcount--;
+	else if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_FROM_LE))
+		dp->d_csp->cp_refcount--;
+	if (dp->convtype & (KICONV_UCS_TO_UTF8 | KICONV_UCS_TO_LE))
+		dp->d_csp->cp_refcount--;
+	kobj_delete((struct kobj*)data, M_ICONV);
+	return 0;
+}
+
+static int
+iconv_ucs_conv(void *d2p, const char **inbuf,
+	size_t *inbytesleft, char **outbuf, size_t *outbytesleft,
+	int convchar, int casetype)
+{
+	struct iconv_ucs *dp = (struct iconv_ucs*)d2p;
+	int ret = 0, i;
+	size_t in, on, ir, or, inlen, outlen, ucslen;
+	const char *src, *p;
+	char *dst;
+	u_char ucs[4], *q;
+	uint32_t code;
+
+	if (inbuf == NULL || *inbuf == NULL || outbuf == NULL || *outbuf == NULL)
+		return 0;
+	ir = in = *inbytesleft;
+	or = on = *outbytesleft;
+	src = *inbuf;
+	dst = *outbuf;
+
+	while (ir > 0 && or > 0) {
+
+		/*
+		 * The first half of conversion.
+		 * (convert any code into ENCODING_UNICODE)
+		 */
+		code = 0;
+		p = src;
+		if (dp->convtype & KICONV_UCS_FROM_UTF8) {
+			/* convert UTF-8 to ENCODING_UNICODE */
+			inlen = 0;
+			code = utf8_to_ucs4(p, &inlen, ir);
+			if (code == 0) {
+				ret = -1;
+				break;
+			}
+
+			if (casetype == KICONV_FROM_LOWER && dp->ctype) {
+				code = towlower(code, dp->ctype);
+			} else if (casetype == KICONV_FROM_UPPER && dp->ctype) {
+				code = towupper(code, dp->ctype);
+			}
+
+			if ((code >= 0xd800 && code < 0xe000) || code >= 0x110000 ) {
+				/* reserved for utf-16 surrogate pair */
+				/* invalid unicode */
+				ret = -1;
+				break;
+			}
+
+			if (inlen == 4) {
+				if (dp->convtype & KICONV_UCS_UCS4) {
+					ucslen = 4;
+					code = encode_surrogate(code);
+				} else {
+					/* can't handle with ucs-2 */
+					ret = -1;
+					break;
+				}
+			} else {
+				ucslen = 2;
+			}
+
+			/* save UCS-4 into ucs[] */
+			for (q = ucs, i = ucslen - 1 ; i >= 0 ; i--)
+				*q++ = (code >> (i << 3)) & 0xff;
+
+		} else if (dp->convtype & KICONV_UCS_COMBINE && dp->f_ctp) {
+			/* convert local code to ENCODING_UNICODE */
+			ucslen = 4;
+			inlen = ir;
+			q = ucs;
+			ret = iconv_convchr_case(dp->f_ctp, &p, &inlen, (char **)&q,
+			    &ucslen, casetype & (KICONV_FROM_LOWER | KICONV_FROM_UPPER));
+			if (ret)
+				break;
+			inlen = ir - inlen;
+			ucslen = 4 - ucslen;
+
+		} else {
+			/* src code is a proper subset of ENCODING_UNICODE */
+			q = ucs;
+			if (dp->convtype & KICONV_UCS_FROM_LE) {
+				*q = *(p + 1);
+				*(q + 1) = *p;
+				p += 2;
+			} else {
+				*q = *p++;
+				*(q + 1) = *p++;
+			}
+			if ((*q & 0xfc) == 0xd8) {
+				if (dp->convtype & KICONV_UCS_UCS4 &&
+				    dp->convtype & KICONV_UCS_FROM_UTF16) {
+					inlen = ucslen = 4;
+				} else {
+					/* invalid unicode */
+					ret = -1;
+					break;
+				}
+			} else {
+				inlen = ucslen = 2;
+			}
+			if (ir < inlen) {
+				ret = -1;
+				break;
+			}
+			if (ucslen == 4) {
+				q += 2;
+				if (dp->convtype & KICONV_UCS_FROM_LE) {
+					*q = *(p + 1);
+					*(q + 1) = *p;
+				} else {
+					*q = *p++;
+					*(q + 1) = *p;
+				}
+				if ((*q & 0xfc) != 0xdc) {
+					/* invalid unicode */
+					ret = -1;
+					break;
+				}
+			}
+		}
+
+		/*
+		 * The second half of conversion.
+		 * (convert ENCODING_UNICODE into any code)
+		 */
+		p = ucs;
+		if (dp->convtype & KICONV_UCS_TO_UTF8) {
+			q = (u_char *)dst;
+			if (ucslen == 4 && dp->convtype & KICONV_UCS_UCS4) {
+				/* decode surrogate pair */
+				code = decode_surrogate(p);
+			} else {
+				code = (ucs[0] << 8) | ucs[1];
+			}
+
+			if (casetype == KICONV_LOWER && dp->ctype) {
+				code = towlower(code, dp->ctype);
+			} else if (casetype == KICONV_UPPER && dp->ctype) {
+				code = towupper(code, dp->ctype);
+			}
+
+			outlen = 0;
+			if (ucs4_to_utf8(code, q, &outlen, or) == NULL) {
+				ret = -1;
+				break;
+			}
+
+			src += inlen;
+			ir -= inlen;
+			dst += outlen;
+			or -= outlen;
+
+		} else if (dp->convtype & KICONV_UCS_COMBINE && dp->t_ctp) {
+			ret = iconv_convchr_case(dp->t_ctp, &p, &ucslen, &dst,
+			    &or, casetype & (KICONV_LOWER | KICONV_UPPER));
+			if (ret)
+				break;
+
+			src += inlen;
+			ir -= inlen;
+
+		} else {
+			/* dst code is a proper subset of ENCODING_UNICODE */
+			if (or < ucslen) {
+				ret = -1;
+				break;
+			}
+			src += inlen;
+			ir -= inlen;
+			or -= ucslen;
+			if (dp->convtype & KICONV_UCS_TO_LE) {
+				*dst++ = *(p + 1);
+				*dst++ = *p;
+				p += 2;
+			} else {
+				*dst++ = *p++;
+				*dst++ = *p++;
+			}
+			if (ucslen == 4) {
+				if ((dp->convtype & KICONV_UCS_UCS4) == 0 ||
+				    (dp->convtype & KICONV_UCS_TO_UTF16) == 0) {
+					ret = -1;
+					break;
+				}
+				if (dp->convtype & KICONV_UCS_TO_LE) {
+					*dst++ = *(p + 1);
+					*dst++ = *p;
+				} else {
+					*dst++ = *p++;
+					*dst++ = *p;
+				}
+			}
+		}
+
+		if (convchar == 1)
+			break;
+	}
+
+	*inbuf += in - ir;
+	*outbuf += on - or;
+	*inbytesleft -= in - ir;
+	*outbytesleft -= on - or;
+	return (ret);
+}
+
+static int
+iconv_ucs_init(struct iconv_converter_class *dcp)
+{
+	int error;
+
+	error = iconv_add(ENCODING_UNICODE, ENCODING_UNICODE, ENCODING_UTF8);
+	if (error)
+		return (error);
+	error = iconv_add(ENCODING_UNICODE, ENCODING_UTF8, ENCODING_UNICODE);
+	if (error)
+		return (error);
+	return (0);
+}
+
+static int
+iconv_ucs_done(struct iconv_converter_class *dcp)
+{
+	return (0);
+}
+
+static const char *
+iconv_ucs_name(struct iconv_converter_class *dcp)
+{
+	return (ENCODING_UNICODE);
+}
+
+static kobj_method_t iconv_ucs_methods[] = {
+	KOBJMETHOD(iconv_converter_open,	iconv_ucs_open),
+	KOBJMETHOD(iconv_converter_close,	iconv_ucs_close),
+	KOBJMETHOD(iconv_converter_conv,	iconv_ucs_conv),
+	KOBJMETHOD(iconv_converter_init,	iconv_ucs_init),
+	KOBJMETHOD(iconv_converter_done,	iconv_ucs_done),
+	KOBJMETHOD(iconv_converter_name,	iconv_ucs_name),
+	{0, 0}
+};
+
+KICONV_CONVERTER(ucs, sizeof(struct iconv_ucs));
+
+static uint32_t
+utf8_to_ucs4(const char *src, size_t *utf8width, size_t srclen)
+{
+	size_t i, w = 0;
+	uint32_t ucs4 = 0;
+
+	/*
+	 * get leading 1 byte from utf-8
+	 */
+	if ((*src & 0x80) == 0) {
+		/*
+		 * leading 1 bit is "0"
+		 *  utf-8: 0xxxxxxx
+		 *  ucs-4: 00000000 00000000 00000000 0xxxxxxx
+		 */
+		w = 1;
+		/* get trailing 7 bits */
+		ucs4 = *src & 0x7f;
+	} else if ((*src & 0xe0) == 0xc0) {
+		/*
+		 * leading 3 bits are "110"
+		 *  utf-8: 110xxxxx 10yyyyyy
+		 *  ucs-4: 00000000 00000000 00000xxx xxyyyyyy
+		 */
+		w = 2;
+		/* get trailing 5 bits */
+		ucs4 = *src & 0x1f;
+	} else if ((*src & 0xf0) == 0xe0) {
+		/*
+		 * leading 4 bits are "1110"
+		 *  utf-8: 1110xxxx 10yyyyyy 10zzzzzz
+		 *  ucs-4: 00000000 00000000 xxxxyyyy yyzzzzzz
+		 */
+		w = 3;
+		/* get trailing 4 bits */
+		ucs4 = *src & 0x0f;
+	} else if ((*src & 0xf8) == 0xf0) {
+		/*
+		 * leading 5 bits are "11110"
+		 *  utf-8: 11110www 10xxxxxx 10yyyyyy 10zzzzzz
+		 *  ucs-4: 00000000 000wwwxx xxxxyyyy yyzzzzzz
+		 */
+		w = 4;
+		/* get trailing 3 bits */
+		ucs4 = *src & 0x07;
+	} else {
+		/* out of utf-16 range or having illegal bits */
+		return (0);
+	}
+	if (w == 0)
+		return (0);
+
+	if (srclen < w)
+		return (0);
+
+	/*
+	 * get left parts from utf-8
+	 */
+	for (i = 1 ; i < w ; i++) {
+		if ((*(src + i) & 0xc0) != 0x80) {
+			/* invalid: leading 2 bits are not "10" */
+			return (0);
+		}
+		/* concatenate trailing 6 bits into ucs4 */
+		ucs4 <<= 6;
+		ucs4 |= *(src + i) & 0x3f;
+	}
+
+	*utf8width = w;
+	return (ucs4);
+}
+
+static u_char *
+ucs4_to_utf8(uint32_t ucs4, char *dst, size_t *utf8width, size_t dstlen)
+{
+	u_char lead, *p;
+	size_t i, w;
+
+	/*
+	 * determine utf-8 width and leading bits
+	 */
+	if (ucs4 < 0x80) {
+		w = 1;
+		lead = 0;	/* "0" */
+	} else if (ucs4 < 0x800) {
+		w = 2;
+		lead = 0xc0;	/* "11" */
+	} else if (ucs4 < 0x10000) {
+		w = 3;
+		lead = 0xe0;	/* "111" */
+	} else if (ucs4 < 0x200000) {
+		w = 4;
+		lead = 0xf0;	/* "1111" */
+	} else {
+		return (NULL);
+	}
+
+	if (dstlen < w)
+		return (NULL);
+
+	/*
+	 * construct utf-8
+	 */
+	p = dst;
+	for (i = w - 1 ; i >= 1 ; i--) {
+		/* get trailing 6 bits and put it with leading bit as "1" */
+		*(p + i) = (ucs4 & 0x3f) | 0x80;
+		ucs4 >>= 6;
+	}
+	*p = ucs4 | lead;
+
+	*utf8width = w;
+
+	return (p);
+}
+
+static uint32_t
+encode_surrogate(register uint32_t code)
+{
+	return ((((code - 0x10000) << 6) & 0x3ff0000) |
+	    ((code - 0x10000) & 0x3ff) | 0xd800dc00);
+}
+
+static uint32_t
+decode_surrogate(register const u_char *ucs)
+{
+	return ((((ucs[0] & 0x3) << 18) | (ucs[1] << 10) |
+	    ((ucs[2] & 0x3) << 8) | ucs[3]) + 0x10000);
+}
+
diff --git a/sys/mips/cavium/usb/octusb.c b/sys/mips/cavium/usb/octusb.c
index 7ecf232..d8ea9fb 100644
--- a/sys/mips/cavium/usb/octusb.c
+++ b/sys/mips/cavium/usb/octusb.c
@@ -81,7 +81,7 @@ __FBSDID("$FreeBSD$");
 #ifdef USB_DEBUG
 static int octusbdebug = 0;
 
-SYSCTL_NODE(_hw_usb, OID_AUTO, octusb, CTLFLAG_RW, 0, "OCTUSB");
+static SYSCTL_NODE(_hw_usb, OID_AUTO, octusb, CTLFLAG_RW, 0, "OCTUSB");
 SYSCTL_INT(_hw_usb_octusb, OID_AUTO, debug, CTLFLAG_RW,
     &octusbdebug, 0, "OCTUSB debug level");
 
diff --git a/sys/mips/conf/AR91XX_BASE b/sys/mips/conf/AR91XX_BASE
index 6f6f6cd..798ded8 100644
--- a/sys/mips/conf/AR91XX_BASE
+++ b/sys/mips/conf/AR91XX_BASE
@@ -35,16 +35,15 @@ options		_KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions
 
 # options		NFS_LEGACYRPC
 # Debugging for use in -current
-# options		INVARIANTS
-# options		INVARIANT_SUPPORT
-# options		WITNESS
-# options		WITNESS_SKIPSPIN
+options		INVARIANTS
+options		INVARIANT_SUPPORT
+options		WITNESS
+options		WITNESS_SKIPSPIN
 options         FFS                     #Berkeley Fast Filesystem
 options         SOFTUPDATES             #Enable FFS soft updates support
 options         UFS_ACL                 #Support for access control lists
 options         UFS_DIRHASH             #Improve performance on big directories
 
-
 # Wireless NIC cards
 options		IEEE80211_DEBUG
 options		IEEE80211_SUPPORT_MESH
diff --git a/sys/mips/conf/DEFAULTS b/sys/mips/conf/DEFAULTS
index f09ef54..f015e89 100644
--- a/sys/mips/conf/DEFAULTS
+++ b/sys/mips/conf/DEFAULTS
@@ -9,3 +9,5 @@ device		uart_ns8250
 
 options 	GEOM_PART_BSD
 options 	GEOM_PART_MBR
+
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/mips/conf/XLP b/sys/mips/conf/XLP
index 286f01f..b6cef20 100644
--- a/sys/mips/conf/XLP
+++ b/sys/mips/conf/XLP
@@ -23,78 +23,7 @@ ident           XLP
 options		ISA_MIPS32
 makeoptions	KERNLOADADDR=0x80100000
 
-include		"../nlm/std.xlp"
-makeoptions	MODULES_OVERRIDE=""
-makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
-#profile 	2
+include		"std.XLP"
+
 makeoptions	TRAMPLOADADDR=0xffffffff85000000
 makeoptions	TRAMP_ARCH_FLAGS="-mabi=64 -march=mips64"
-
-options 	SCHED_ULE		# ULE scheduler
-#options 	VERBOSE_SYSINIT
-#options 	SCHED_4BSD		# 4BSD scheduler
-options         SMP
-options 	PREEMPTION		# Enable kernel thread preemption
-#options 	FULL_PREEMPTION		# Enable kernel thread preemption
-#options 	MAXCPU=128		# XLP can probe 128 CPUs
-options 	INET			# InterNETworking
-options 	INET6			# IPv6 communications protocols
-options 	FFS			# Berkeley Fast Filesystem
-#options 	SOFTUPDATES		# Enable FFS soft updates support
-options 	UFS_ACL			# Support for access control lists
-options 	UFS_DIRHASH		# Improve performance on big directories
-options		NFSCL
-options		NFS_ROOT
-options		MSDOSFS			#MSDOS Filesystem
-#
-#options 	BOOTP
-#options 	BOOTP_NFSROOT
-#options 	BOOTP_NFSV3
-#options 	BOOTP_WIRED_TO=nlge0
-#options 	BOOTP_COMPAT
-#options 	ROOTDEVNAME=\"nfs:10.1.1.8:/usr/extra/nfsroot\"
-#
-options 	MD_ROOT			# MD is a potential root device
-options		MD_ROOT_SIZE=27000
-options		ROOTDEVNAME=\"ufs:md0\"
-options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
-options 	HZ=1000	
-options 	NO_SWAPPING
-
-#Debugging options
-options 	KTRACE			# ktrace(1) support
-options 	DDB
-options 	KDB
-options 	GDB
-options 	BREAK_TO_DEBUGGER
-options 	ALT_BREAK_TO_DEBUGGER
-#options 	DEADLKRES		#Enable the deadlock resolver
-#options 	INVARIANTS		#Enable calls of extra sanity checking
-#options 	INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
-#options 	WITNESS			#Enable checks to detect deadlocks and cycles
-#options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
-#options 	KTR			# ktr(4) and ktrdump(8) support
-#options 	KTR_COMPILE=(KTR_LOCK|KTR_PROC|KTR_INTR|KTR_CALLOUT|KTR_UMA|KTR_SYSC)
-#options 	KTR_ENTRIES=131072
-#options 	LOCK_DEBUG
-#options 	LOCK_PROFILING
-
-
-# Pseudo
-device		loop
-device		random
-device		md
-device		pty
-device		bpf
-
-# UART
-device		uart
-device		pci
-
-# Network
-device		ether
-#
-# FDT support 
-#options         FDT
-#options         FDT_DTB_STATIC
-#makeoptions     FDT_DTS_FILE=xlp-basic.dts
diff --git a/sys/mips/conf/XLP64 b/sys/mips/conf/XLP64
index 0b964a5..9080e24 100644
--- a/sys/mips/conf/XLP64
+++ b/sys/mips/conf/XLP64
@@ -24,79 +24,7 @@ options 	ISA_MIPS64
 makeoptions	ARCH_FLAGS="-march=mips64r2 -mabi=64"
 makeoptions	KERNLOADADDR=0xffffffff80100000
 
-include		"../nlm/std.xlp"
+include		"std.XLP"
 
-makeoptions	MODULES_OVERRIDE=""
-makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
-#profile 	2
 makeoptions	TRAMPLOADADDR=0xffffffff85000000
 makeoptions	TRAMP_ARCH_FLAGS="-mabi=64 -march=mips64"
-
-options 	SCHED_ULE		# ULE scheduler
-#options 	VERBOSE_SYSINIT
-#options 	SCHED_4BSD		# 4BSD scheduler
-options         SMP
-options 	PREEMPTION		# Enable kernel thread preemption
-#options 	FULL_PREEMPTION		# Enable kernel thread preemption
-#options 	MAXCPU=128		# XLP can probe 128 CPUs
-options 	INET			# InterNETworking
-options 	INET6			# IPv6 communications protocols
-options 	FFS			# Berkeley Fast Filesystem
-#options 	SOFTUPDATES		# Enable FFS soft updates support
-options 	UFS_ACL			# Support for access control lists
-options 	UFS_DIRHASH		# Improve performance on big directories
-options		NFSCL
-options		NFS_ROOT
-options		MSDOSFS			#MSDOS Filesystem
-#
-#options 	BOOTP
-#options 	BOOTP_NFSROOT
-#options 	BOOTP_NFSV3
-#options 	BOOTP_WIRED_TO=nlge0
-#options 	BOOTP_COMPAT
-#options 	ROOTDEVNAME=\"nfs:10.1.1.8:/usr/extra/nfsroot\"
-#
-options 	MD_ROOT			# MD is a potential root device
-options		MD_ROOT_SIZE=27000
-options		ROOTDEVNAME=\"ufs:md0\"
-options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
-options 	HZ=1000	
-options 	NO_SWAPPING
-
-#Debugging options
-options 	KTRACE			# ktrace(1) support
-options 	DDB
-options 	KDB
-options 	GDB
-options 	BREAK_TO_DEBUGGER
-options 	ALT_BREAK_TO_DEBUGGER
-#options 	DEADLKRES		#Enable the deadlock resolver
-#options 	INVARIANTS		#Enable calls of extra sanity checking
-#options 	INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
-#options 	WITNESS			#Enable checks to detect deadlocks and cycles
-#options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
-#options 	KTR			# ktr(4) and ktrdump(8) support
-#options 	KTR_COMPILE=(KTR_LOCK|KTR_PROC|KTR_INTR|KTR_CALLOUT|KTR_UMA|KTR_SYSC)
-#options 	KTR_ENTRIES=131072
-#options 	LOCK_DEBUG
-#options 	LOCK_PROFILING
-
-
-# Pseudo
-device		loop
-device		random
-device		md
-device		pty
-device		bpf
-
-# UART
-device		uart
-device		pci
-
-# Network
-device		ether
-
-# FDT support 
-#options         FDT
-#options         FDT_DTB_STATIC
-#makeoptions     FDT_DTS_FILE=xlp-basic.dts
diff --git a/sys/mips/conf/XLPN32 b/sys/mips/conf/XLPN32
index 02119a8..8adc95a 100644
--- a/sys/mips/conf/XLPN32
+++ b/sys/mips/conf/XLPN32
@@ -24,73 +24,7 @@ options 	ISA_MIPS64
 makeoptions	ARCH_FLAGS="-march=mips64 -mabi=n32"
 makeoptions	KERNLOADADDR=0x80100000
 
-include		"../nlm/std.xlp"
-makeoptions	MODULES_OVERRIDE=""
-makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
-#profile 	2
+include		"std.XLP"
+
 makeoptions	TRAMPLOADADDR=0xffffffff85000000
 makeoptions	TRAMP_ARCH_FLAGS="-mabi=64 -march=mips64"
-
-options 	SCHED_ULE		# ULE scheduler
-#options 	VERBOSE_SYSINIT
-#options 	SCHED_4BSD		# 4BSD scheduler
-options         SMP
-options 	PREEMPTION		# Enable kernel thread preemption
-#options 	FULL_PREEMPTION		# Enable kernel thread preemption
-#options 	MAXCPU=128		# XLP can probe 128 CPUs
-options 	INET			# InterNETworking
-options 	INET6			# IPv6 communications protocols
-options 	FFS			# Berkeley Fast Filesystem
-#options 	SOFTUPDATES		# Enable FFS soft updates support
-options 	UFS_ACL			# Support for access control lists
-options 	UFS_DIRHASH		# Improve performance on big directories
-options		NFSCL
-options		NFS_ROOT
-options		MSDOSFS			#MSDOS Filesystem
-#
-#options 	BOOTP
-#options 	BOOTP_NFSROOT
-#options 	BOOTP_NFSV3
-#options 	BOOTP_WIRED_TO=nlge0
-#options 	BOOTP_COMPAT
-#options 	ROOTDEVNAME=\"nfs:10.1.1.8:/usr/extra/nfsroot\"
-#
-options 	MD_ROOT			# MD is a potential root device
-options		MD_ROOT_SIZE=27000
-options		ROOTDEVNAME=\"ufs:md0\"
-options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
-options 	HZ=1000	
-options 	NO_SWAPPING
-
-#Debugging options
-options 	KTRACE			# ktrace(1) support
-#options 	DDB
-#options 	KDB
-#options 	GDB
-#options 	BREAK_TO_DEBUGGER
-#options 	ALT_BREAK_TO_DEBUGGER
-#options 	DEADLKRES		#Enable the deadlock resolver
-#options 	INVARIANTS		#Enable calls of extra sanity checking
-#options 	INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
-#options 	WITNESS			#Enable checks to detect deadlocks and cycles
-#options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
-#options 	KTR			# ktr(4) and ktrdump(8) support
-#options 	KTR_COMPILE=(KTR_LOCK|KTR_PROC|KTR_INTR|KTR_CALLOUT|KTR_UMA|KTR_SYSC)
-#options 	KTR_ENTRIES=131072
-#options 	LOCK_DEBUG
-#options 	LOCK_PROFILING
-
-
-# Pseudo
-device		loop
-device		random
-device		md
-device		pty
-device		bpf
-
-# UART
-device		uart
-device		pci
-
-# Network
-device		ether
diff --git a/sys/mips/conf/std.XLP b/sys/mips/conf/std.XLP
new file mode 100644
index 0000000..d1619e3
--- /dev/null
+++ b/sys/mips/conf/std.XLP
@@ -0,0 +1,89 @@
+# $FreeBSD$
+
+include		"../nlm/std.xlp"
+makeoptions	MODULES_OVERRIDE=""
+makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
+#profile 	2
+
+options 	SCHED_ULE		# ULE scheduler
+#options 	VERBOSE_SYSINIT
+#options 	SCHED_4BSD		# 4BSD scheduler
+options         SMP
+options 	PREEMPTION		# Enable kernel thread preemption
+#options 	FULL_PREEMPTION		# Enable kernel thread preemption
+options 	INET			# InterNETworking
+options 	INET6			# IPv6 communications protocols
+options 	FFS			# Berkeley Fast Filesystem
+#options 	SOFTUPDATES		# Enable FFS soft updates support
+options 	UFS_ACL			# Support for access control lists
+options 	UFS_DIRHASH		# Improve performance on big directories
+options		NFSCL
+options		NFS_ROOT
+options		MSDOSFS			#MSDOS Filesystem
+#
+#options 	BOOTP
+#options 	BOOTP_NFSROOT
+#options 	BOOTP_NFSV3
+#options 	BOOTP_WIRED_TO=nlge0
+#options 	BOOTP_COMPAT
+#options 	ROOTDEVNAME=\"nfs:10.1.1.8:/usr/extra/nfsroot\"
+
+options 	MD_ROOT			# MD is a potential root device
+options		MD_ROOT_SIZE=132000
+options		ROOTDEVNAME=\"ufs:md0\"
+options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
+options 	HZ=1000	
+options 	NO_SWAPPING
+
+# Debugging options
+options 	KTRACE			# ktrace(1) support
+options 	DDB
+options 	KDB
+options 	GDB
+options 	BREAK_TO_DEBUGGER
+options 	ALT_BREAK_TO_DEBUGGER
+#options 	DEADLKRES		# Enable the deadlock resolver
+#options 	INVARIANTS
+#options 	INVARIANT_SUPPORT
+#options 	WITNESS			# Detect deadlocks and cycles
+#options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+#options 	KTR			# ktr(4) and ktrdump(8) support
+#options 	KTR_COMPILE=(KTR_LOCK|KTR_PROC|KTR_INTR|KTR_CALLOUT|KTR_UMA|KTR_SYSC)
+#options 	KTR_ENTRIES=131072
+#options 	LOCK_DEBUG
+#options 	LOCK_PROFILING
+
+options 	GEOM_UZIP
+
+# Pseudo
+device		loop
+device		random
+device		md
+device		pty
+device		bpf
+
+# Network
+device		miibus
+device		ether
+#device		re
+device		msk
+device		em
+
+# Disks
+device		siis
+device		da
+device		scbus
+#device		ata
+#device		atadisk
+
+# USB
+device		usb			# USB Bus (required)
+device		ehci			# EHCI PCI->USB interface (USB 2.0)
+#options 	USB_DEBUG		# enable debug msgs
+#device		ugen			# Generic
+#device		uhid			# "Human Interface Devices"
+device		umass			# Requires scbus and da
+
+options 	FDT
+options 	FDT_DTB_STATIC
+makeoptions	FDT_DTS_FILE=xlp-basic.dts
diff --git a/sys/mips/include/asm.h b/sys/mips/include/asm.h
index 82d8837..12149c7 100644
--- a/sys/mips/include/asm.h
+++ b/sys/mips/include/asm.h
@@ -855,6 +855,15 @@ _C_LABEL(x):
  * For more info on CP0 hazards see Chapter 7 (p.99) of "MIPS32 Architecture 
  *    For Programmers Volume III: The MIPS32 Privileged Resource Architecture"
  */
+#if defined(CPU_NLM)
+#define	HAZARD_DELAY	sll $0,3
+#define	ITLBNOPFIX	sll $0,3
+#elif defined(CPU_RMI)
+#define	HAZARD_DELAY
+#define	ITLBNOPFIX
+#else
 #define	ITLBNOPFIX	nop;nop;nop;nop;nop;nop;nop;nop;nop;nop;
 #define	HAZARD_DELAY	nop;nop;nop;nop;nop;
+#endif
+
 #endif /* !_MACHINE_ASM_H_ */
diff --git a/sys/mips/include/cpufunc.h b/sys/mips/include/cpufunc.h
index f5c24b4..cd703ad 100644
--- a/sys/mips/include/cpufunc.h
+++ b/sys/mips/include/cpufunc.h
@@ -69,7 +69,7 @@
 static __inline void
 mips_barrier(void)
 {
-#ifdef CPU_CNMIPS
+#if defined(CPU_CNMIPS) || defined(CPU_RMI) || defined(CPU_NLM)
 	__asm __volatile("" : : : "memory");
 #else
 	__asm __volatile (".set noreorder\n\t"
@@ -272,6 +272,9 @@ MIPS_RW32_COP0(status, MIPS_COP_0_STATUS);
 MIPS_RW32_COP0(entryhi, MIPS_COP_0_TLB_HI);
 MIPS_RW32_COP0(pagemask, MIPS_COP_0_TLB_PG_MASK);
 #endif
+#ifdef CPU_NLM
+MIPS_RW32_COP0_SEL(pagegrain, MIPS_COP_0_TLB_PG_MASK, 1);
+#endif
 #if !defined(__mips_n64) && !defined(__mips_n32) /* !PHYSADDR_64_BIT */
 MIPS_RW32_COP0(entrylo0, MIPS_COP_0_TLB_LO0);
 MIPS_RW32_COP0(entrylo1, MIPS_COP_0_TLB_LO1);
diff --git a/sys/mips/include/cpuregs.h b/sys/mips/include/cpuregs.h
index 01d710d..b47b264 100644
--- a/sys/mips/include/cpuregs.h
+++ b/sys/mips/include/cpuregs.h
@@ -200,6 +200,8 @@
 /* CPU dependent mtc0 hazard hook */
 #if defined(CPU_CNMIPS) || defined(CPU_RMI)
 #define	COP0_SYNC
+#elif defined(CPU_NLM)
+#define	COP0_SYNC	.word 0xc0	/* ehb */
 #elif defined(CPU_SB1)
 #define COP0_SYNC  ssnop; ssnop; ssnop; ssnop; ssnop; ssnop; ssnop; ssnop; ssnop
 #else
diff --git a/sys/mips/mips/busdma_machdep.c b/sys/mips/mips/busdma_machdep.c
index 93ae8d2..cd95677 100644
--- a/sys/mips/mips/busdma_machdep.c
+++ b/sys/mips/mips/busdma_machdep.c
@@ -117,7 +117,7 @@ static int total_bpages;
 static int busdma_zonecount;
 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
 
-SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 	   "Total bounce pages");
 
diff --git a/sys/mips/mips/cache.c b/sys/mips/mips/cache.c
index 4e7f35f..6483319 100644
--- a/sys/mips/mips/cache.c
+++ b/sys/mips/mips/cache.c
@@ -101,13 +101,8 @@ mips_config_cache(struct mips_cpuinfo * cpuinfo)
 		break;
 	case 32:
 		mips_cache_ops.mco_icache_sync_all = mipsNN_icache_sync_all_32;
-#ifdef CPU_NLM
-		mips_cache_ops.mco_icache_sync_range =
-		    mipsNN_icache_sync_range_index_32;
-#else
 		mips_cache_ops.mco_icache_sync_range =
 		    mipsNN_icache_sync_range_32;
-#endif
 		mips_cache_ops.mco_icache_sync_range_index =
 		    mipsNN_icache_sync_range_index_32;
 		break;
@@ -155,18 +150,23 @@ mips_config_cache(struct mips_cpuinfo * cpuinfo)
 		mips_cache_ops.mco_pdcache_wbinv_all =
 		    mips_cache_ops.mco_intern_pdcache_wbinv_all =
 		    mipsNN_pdcache_wbinv_all_32;
-#ifdef CPU_NLM
-		mips_cache_ops.mco_pdcache_wbinv_range =
-		    mipsNN_pdcache_wbinv_range_index_32;
+#if defined(CPU_RMI) || defined(CPU_NLM)
+		mips_cache_ops.mco_pdcache_wbinv_range = cache_noop;
 #else
 		mips_cache_ops.mco_pdcache_wbinv_range =
 		    mipsNN_pdcache_wbinv_range_32;
 #endif
+#if defined(CPU_RMI) || defined(CPU_NLM)
+		mips_cache_ops.mco_pdcache_wbinv_range_index =
+		    mips_cache_ops.mco_intern_pdcache_wbinv_range_index = cache_noop;
+		mips_cache_ops.mco_pdcache_inv_range = cache_noop;
+#else
 		mips_cache_ops.mco_pdcache_wbinv_range_index =
 		    mips_cache_ops.mco_intern_pdcache_wbinv_range_index =
 		    mipsNN_pdcache_wbinv_range_index_32;
 		mips_cache_ops.mco_pdcache_inv_range =
 		    mipsNN_pdcache_inv_range_32;
+#endif
 #if defined(CPU_RMI) || defined(CPU_NLM)
 		mips_cache_ops.mco_pdcache_wb_range =
 		    mips_cache_ops.mco_intern_pdcache_wb_range = cache_noop;
diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c
index 7da76be..fc99035 100644
--- a/sys/mips/mips/pmap.c
+++ b/sys/mips/mips/pmap.c
@@ -181,7 +181,6 @@ static pt_entry_t init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot);
 
 #ifdef SMP
 static void pmap_invalidate_page_action(void *arg);
-static void pmap_invalidate_all_action(void *arg);
 static void pmap_update_page_action(void *arg);
 #endif
 
@@ -622,119 +621,94 @@ pmap_init(void)
  * Low level helper routines.....
  ***************************************************/
 
+#ifdef	SMP
 static __inline void
-pmap_invalidate_all_local(pmap_t pmap)
+pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg)
 {
-	u_int cpuid;
+	int	cpuid, cpu, self;
+	cpuset_t active_cpus;
 
+	sched_pin();
+	if (is_kernel_pmap(pmap)) {
+		smp_rendezvous(NULL, fn, NULL, arg);
+		goto out;
+	}
+	/* Force ASID update on inactive CPUs */
+	CPU_FOREACH(cpu) {
+		if (!CPU_ISSET(cpu, &pmap->pm_active))
+			pmap->pm_asid[cpu].gen = 0;
+	}
 	cpuid = PCPU_GET(cpuid);
+	/* 
+	 * XXX: barrier/locking for active? 
+	 *
+	 * Take a snapshot of active here, any further changes are ignored.
+	 * tlb update/invalidate should be harmless on inactive CPUs
+	 */
+	active_cpus = pmap->pm_active;
+	self = CPU_ISSET(cpuid, &active_cpus);
+	CPU_CLR(cpuid, &active_cpus);
+	/* Optimize for the case where this cpu is the only active one */
+	if (CPU_EMPTY(&active_cpus)) {
+		if (self)
+			fn(arg);
+	} else {
+		if (self)
+			CPU_SET(cpuid, &active_cpus);
+		smp_rendezvous_cpus(active_cpus, NULL, fn, NULL, arg);
+	}
+out:
+	sched_unpin();
+}
+#else /* !SMP */
+static __inline void
+pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg)
+{
+	int	cpuid;
 
-	if (pmap == kernel_pmap) {
-		tlb_invalidate_all();
+	if (is_kernel_pmap(pmap)) {
+		fn(arg);
 		return;
 	}
-	if (CPU_ISSET(cpuid, &pmap->pm_active))
-		tlb_invalidate_all_user(pmap);
-	else
+	cpuid = PCPU_GET(cpuid);
+	if (!CPU_ISSET(cpuid, &pmap->pm_active))
 		pmap->pm_asid[cpuid].gen = 0;
+	else
+		fn(arg);
 }
+#endif /* SMP */
 
-#ifdef SMP
 static void
 pmap_invalidate_all(pmap_t pmap)
 {
 
-	smp_rendezvous(0, pmap_invalidate_all_action, 0, pmap);
+	pmap_call_on_active_cpus(pmap,
+	    (void (*)(void *))tlb_invalidate_all_user, pmap);
 }
 
-static void
-pmap_invalidate_all_action(void *arg)
-{
-
-	pmap_invalidate_all_local((pmap_t)arg);
-}
-#else
-static void
-pmap_invalidate_all(pmap_t pmap)
-{
-
-	pmap_invalidate_all_local(pmap);
-}
-#endif
-
-static __inline void
-pmap_invalidate_page_local(pmap_t pmap, vm_offset_t va)
-{
-	u_int cpuid;
-
-	cpuid = PCPU_GET(cpuid);
-
-	if (is_kernel_pmap(pmap)) {
-		tlb_invalidate_address(pmap, va);
-		return;
-	}
-	if (pmap->pm_asid[cpuid].gen != PCPU_GET(asid_generation))
-		return;
-	else if (!CPU_ISSET(cpuid, &pmap->pm_active)) {
-		pmap->pm_asid[cpuid].gen = 0;
-		return;
-	}
-	tlb_invalidate_address(pmap, va);
-}
-
-#ifdef SMP
 struct pmap_invalidate_page_arg {
 	pmap_t pmap;
 	vm_offset_t va;
 };
 
 static void
-pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
-{
-	struct pmap_invalidate_page_arg arg;
-
-	arg.pmap = pmap;
-	arg.va = va;
-	smp_rendezvous(0, pmap_invalidate_page_action, 0, &arg);
-}
-
-static void
 pmap_invalidate_page_action(void *arg)
 {
 	struct pmap_invalidate_page_arg *p = arg;
 
-	pmap_invalidate_page_local(p->pmap, p->va);
+	tlb_invalidate_address(p->pmap, p->va);
 }
-#else
+
 static void
 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 {
+	struct pmap_invalidate_page_arg arg;
 
-	pmap_invalidate_page_local(pmap, va);
-}
-#endif
-
-static __inline void
-pmap_update_page_local(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
-{
-	u_int cpuid;
-
-	cpuid = PCPU_GET(cpuid);
-
-	if (is_kernel_pmap(pmap)) {
-		tlb_update(pmap, va, pte);
-		return;
-	}
-	if (pmap->pm_asid[cpuid].gen != PCPU_GET(asid_generation))
-		return;
-	else if (!CPU_ISSET(cpuid, &pmap->pm_active)) {
-		pmap->pm_asid[cpuid].gen = 0;
-		return;
-	}
-	tlb_update(pmap, va, pte);
+	arg.pmap = pmap;
+	arg.va = va;
+	pmap_call_on_active_cpus(pmap, pmap_invalidate_page_action, &arg);
 }
 
-#ifdef SMP
 struct pmap_update_page_arg {
 	pmap_t pmap;
 	vm_offset_t va;
@@ -742,31 +716,23 @@ struct pmap_update_page_arg {
 };
 
 static void
-pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
-{
-	struct pmap_update_page_arg arg;
-
-	arg.pmap = pmap;
-	arg.va = va;
-	arg.pte = pte;
-	smp_rendezvous(0, pmap_update_page_action, 0, &arg);
-}
-
-static void
 pmap_update_page_action(void *arg)
 {
 	struct pmap_update_page_arg *p = arg;
 
-	pmap_update_page_local(p->pmap, p->va, p->pte);
+	tlb_update(p->pmap, p->va, p->pte);
 }
-#else
+
 static void
 pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
 {
+	struct pmap_update_page_arg arg;
 
-	pmap_update_page_local(pmap, va, pte);
+	arg.pmap = pmap;
+	arg.va = va;
+	arg.pte = pte;
+	pmap_call_on_active_cpus(pmap, pmap_update_page_action, &arg);
 }
-#endif
 
 /*
  *	Routine:	pmap_extract
@@ -1077,7 +1043,8 @@ pmap_alloc_direct_page(unsigned int index, int req)
 {
 	vm_page_t m;
 
-	m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req);
+	m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED |
+	    VM_ALLOC_ZERO);
 	if (m == NULL)
 		return (NULL);
 
@@ -1085,8 +1052,6 @@ pmap_alloc_direct_page(unsigned int index, int req)
 		pmap_zero_page(m);
 
 	m->pindex = index;
-	atomic_add_int(&cnt.v_wire_count, 1);
-	m->wire_count = 1;
 	return (m);
 }
 
@@ -3214,7 +3179,7 @@ pmap_emulate_modified(pmap_t pmap, vm_offset_t va)
 #ifdef SMP
 	/* It is possible that some other CPU changed m-bit */
 	if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) {
-		pmap_update_page_local(pmap, va, *pte);
+		tlb_update(pmap, va, *pte);
 		PMAP_UNLOCK(pmap);
 		return (0);
 	}
@@ -3228,7 +3193,7 @@ pmap_emulate_modified(pmap_t pmap, vm_offset_t va)
 		return (1);
 	}
 	pte_set(pte, PTE_D);
-	pmap_update_page_local(pmap, va, *pte);
+	tlb_update(pmap, va, *pte);
 	pa = TLBLO_PTE_TO_PA(*pte);
 	if (!page_is_managed(pa))
 		panic("pmap_emulate_modified: unmanaged page");
diff --git a/sys/mips/nlm/cms.c b/sys/mips/nlm/cms.c
index a991add..4ba3209 100644
--- a/sys/mips/nlm/cms.c
+++ b/sys/mips/nlm/cms.c
@@ -75,8 +75,8 @@ __FBSDID("$FreeBSD$");
  * load
  */
 struct msgring_thread {
-	struct thread	*thread; /* msgring handler threads */
-	int	needed;		/* thread needs to wake up */
+	struct thread	*thread;	/* msgring handler threads */
+	int	needed;			/* thread needs to wake up */
 };
 static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS];
 static struct proc *msgring_proc;	/* all threads are under a proc */
@@ -91,142 +91,80 @@ struct tx_stn_handler {
 };
 static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
 static struct mtx	msgmap_lock;
-uint64_t xlp_cms_base;
 uint32_t xlp_msg_thread_mask;
-static int xlp_msg_threads_per_core = 3; /* Make tunable */
+static int xlp_msg_threads_per_core = 3;
 
 static void create_msgring_thread(int hwtid);
 static int msgring_process_fast_intr(void *arg);
-/*
- * Boot time init, called only once
- */
-void
-xlp_msgring_config(void)
-{
-	unsigned int thrmask, mask;
-	int i;
-
-	/* TODO: Add other nodes */
-	xlp_cms_base = nlm_get_cms_regbase(0);
-
-	mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
-	if (xlp_threads_per_core < xlp_msg_threads_per_core)
-		xlp_msg_threads_per_core = xlp_threads_per_core;
-	thrmask = ((1 << xlp_msg_threads_per_core) - 1);
-	/*thrmask <<= xlp_threads_per_core - xlp_msg_threads_per_core;*/
-	mask = 0;
-	for (i = 0; i < XLP_MAX_CORES; i++) {
-		mask <<= XLP_MAX_THREADS;
-		mask |= thrmask;
-	}
-	xlp_msg_thread_mask = xlp_hw_thread_mask & mask;
-	printf("Initializing CMS...@%jx, Message handler thread mask %#jx\n",
-	    (uintmax_t)xlp_cms_base, (uintmax_t)xlp_msg_thread_mask);
-}
 
-/*
- * Initialize the messaging subsystem.
- *
- * Message Stations are shared among all threads in a cpu core, this
- * has to be called once from every core which is online.
+/* Debug counters */
+static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS];
+static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS];
+static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS];
+static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4];
+static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS];
+
+/* Whether polled driver implementation */
+static int polled = 1;
+
+/* We do only i/o device credit setup here. CPU credit setup is now
+ * moved to xlp_msgring_cpu_init() so that the credits get setup 
+ * only if the CPU exists. xlp_msgring_cpu_init() gets called from
+ * platform_init_ap; and this makes it easy for us to setup CMS
+ * credits for various types of XLP chips, with varying number of
+ * cpu's and cores.
  */
-void
-xlp_msgring_iodi_config(void)
-{
-	void *cookie;
-
-	xlp_msgring_config();
-/*	nlm_cms_default_setup(0,0,0,0); */
-	nlm_cms_credit_setup(50);
-	create_msgring_thread(0);
-	cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL,
-	    NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie);
-}
-
-void
-nlm_cms_credit_setup(int credit)
+static void
+xlp_cms_credit_setup(int credit)
 {
+	uint64_t cmspcibase, cmsbase, pcibase;
+	uint32_t devoffset;
+	int dev, fn, maxqid;
 	int src, qid, i;
 
-#if 0
-	/* there are a total of 18 src stations on XLP. */
-	printf("Setting up CMS credits!\n");
-	for (src=0; src<18; src++) {
-		for(qid=0; qid<1024; qid++) {
-			nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-		}
-	}
+	for (i = 0; i < XLP_MAX_NODES; i++) {
+		cmspcibase = nlm_get_cms_pcibase(i);
+		if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i)))
+		    continue;
+		cmsbase = nlm_get_cms_regbase(i);
+		maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
+		for (dev = 0; dev < 8; dev++) {
+			for (fn = 0; fn < 8; fn++) {
+				devoffset = XLP_HDR_OFFSET(i, 0, dev, fn);
+				if (nlm_dev_exists(devoffset) == 0)
+					continue;
+				pcibase = nlm_pcicfg_base(devoffset);
+				src = nlm_qidstart(pcibase);
+				if (src == 0)
+				    continue;
+#if 0 /* Debug */
+				printf("Setup CMS credits for queues ");
+				printf("[%d to %d] from src %d\n", 0,
+				    maxqid, src);
 #endif
-	printf("Setting up CMS credits!\n");
-	/* CPU Credits */
-	for (i = 1; i < 8; i++) {
-		src = (i << 4);
-		for (qid = 0; qid < 1024; qid++)
-			nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-	}
-	/* PCIE Credits */
-	for(i = 0; i < 4; i++) {
-		src = (256 + (i * 2));
-		for(qid = 0; qid < 1024; qid++)
-			nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
+				for (qid = 0; qid < maxqid; qid++)
+					nlm_cms_setup_credits(cmsbase, qid,
+					    src, credit);
+			}
+		}
 	}
-	/* DTE Credits */
-	src = 264;
-	for (qid = 0; qid < 1024; qid++)
-		nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-	/* RSA Credits */
-	src = 272;
-	for (qid = 0; qid < 1024; qid++)
-		nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-
-	/* Crypto Credits */
-	src = 281;
-	for (qid = 0; qid < 1024; qid++)
-		nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-
-	/* CMP Credits */
-	src = 298;
-	for (qid = 0; qid < 1024; qid++)
-		nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-
-	/* POE Credits */
-	src = 384;
-	for(qid = 0; qid < 1024; qid++)
-		nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
-
-	/* NAE Credits */
-	src = 476;
-	for(qid = 0; qid < 1024; qid++)
-		nlm_cms_setup_credits(xlp_cms_base, qid, src, credit);
 }
 
 void
-xlp_msgring_cpu_init(uint32_t cpuid)
+xlp_msgring_cpu_init(int node, int cpu, int credit)
 {
-	int queue,i;
-
-	queue = CMS_CPU_PUSHQ(0, ((cpuid >> 2) & 0x7), (cpuid & 0x3), 0);
-	/* temp allocate 4 segments to each output queue */
-	nlm_cms_alloc_onchip_q(xlp_cms_base, queue, 4);
-	/* Enable high watermark and non empty interrupt */
-	nlm_cms_per_queue_level_intr(xlp_cms_base, queue,2,0);
-	for(i=0;i<8;i++) {
-		/* temp distribute the credits to all CPU stations */
-		nlm_cms_setup_credits(xlp_cms_base, queue, i * 16, 8);
-	}
-}
+	uint64_t cmspcibase = nlm_get_cms_pcibase(node);
+	uint64_t cmsbase = nlm_get_cms_regbase(node);
+	int qid, maxqid, src;
 
-void
-xlp_cpu_msgring_handler(int bucket, int size, int code, int stid,
-		    struct nlm_fmn_msg *msg, void *data)
-{
-	int i;
+	maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
 
-	printf("vc:%d srcid:%d size:%d\n",bucket,stid,size);
-	for(i=0;i<size;i++) {
-		printf("msg->msg[%d]:0x%jx ", i, (uintmax_t)msg->msg[i]);
+	/* cpu credit setup is done only from thread-0 of each core */
+	if((cpu % 4) == 0) {
+		src = cpu << 2; /* each thread has 4 vc's */
+		for (qid = 0; qid < maxqid; qid++)
+		    nlm_cms_setup_credits(cmsbase, qid, src, credit);
 	}
-	printf("\n");
 }
 
 /*
@@ -234,35 +172,100 @@ xlp_cpu_msgring_handler(int bucket, int size, int code, int stid,
  * Use max_msgs = 0 to drain out all messages.
  */
 int
-xlp_handle_msg_vc(int vc, int max_msgs)
+xlp_handle_msg_vc(u_int vcmask, int max_msgs)
 {
 	struct nlm_fmn_msg msg;
-	int i, srcid = 0, size = 0, code = 0;
+	int srcid = 0, size = 0, code = 0;
 	struct tx_stn_handler *he;
 	uint32_t mflags, status;
+	int n_msgs = 0, vc, m, hwtid;
+	u_int msgmask;
+
 
-	for (i = 0; i < max_msgs; i++) {
+	hwtid = nlm_cpuid();
+	for (;;) {
+		/* check if VC empty */
 		mflags = nlm_save_flags_cop2();
-		status = nlm_fmn_msgrcv(vc, &srcid, &size, &code, &msg);
+		status = nlm_read_c2_msgstatus1();
 		nlm_restore_flags(mflags);
-		if (status != 0) /* If there is no msg or error */
-			break;
-		if (srcid < 0 && srcid >= 1024) {
-			printf("[%s]: bad src id %d\n", __func__, srcid);
-			continue;
-		}
-		he = &msgmap[srcid];
-		if(he->action != NULL)
-			(he->action)(vc, size, code, srcid, &msg, he->arg);
-#if 0 /* debug */
-		else
-			printf("[%s]: No Handler for message from stn_id=%d,"
-			    " vc=%d, size=%d, msg0=%jx, dropping message\n",
-			    __func__, srcid, vc, size, (uintmax_t)msg.msg[0]);
+
+		msgmask = ((status >> 24) & 0xf) ^ 0xf;
+		msgmask &= vcmask;
+		if (msgmask == 0)
+			    break;
+		m = 0;
+		for (vc = 0; vc < 4; vc++) {
+			if ((msgmask & (1 << vc)) == 0)
+				continue;
+
+			mflags = nlm_save_flags_cop2();
+			status = nlm_fmn_msgrcv(vc, &srcid, &size, &code,
+		 	    &msg);
+			nlm_restore_flags(mflags);
+			if (status != 0) 	/*  no msg or error */
+				continue;
+			if (srcid < 0 && srcid >= 1024) {
+				printf("[%s]: bad src id %d\n", __func__,
+				    srcid);
+				continue;
+			}
+			he = &msgmap[srcid];
+			if(he->action != NULL)
+				(he->action)(vc, size, code, srcid, &msg, he->arg);
+#if 1 /* defined DEBUG */
+			else
+				printf("[%s]: No Handler for msg from stn %d,"
+				    " vc=%d, size=%d, msg0=%jx, droppinge\n",
+				    __func__, srcid, vc, size,
+				    (uintmax_t)msg.msg[0]);
 #endif
+			fmn_msgcount[hwtid][vc] += 1;
+			m++;	/* msgs handled in this iter */
+		}
+		if (m == 0)
+			break;	/* nothing done in this iter */
+		n_msgs += m;
+		if (max_msgs > 0 && n_msgs >= max_msgs)
+		    break;
+	}
+
+	return (n_msgs);
+}
+
+static void
+xlp_discard_msg_vc(u_int vcmask)
+{
+	struct nlm_fmn_msg msg;
+	int srcid = 0, size = 0, code = 0, vc;
+	uint32_t mflags, status;
+
+	for (vc = 0; vc < 4; vc++) {
+		for (;;) {
+			mflags = nlm_save_flags_cop2();
+			status = nlm_fmn_msgrcv(vc, &srcid,
+			    &size, &code, &msg);
+			nlm_restore_flags(mflags);
+
+			/* break if there is no msg or error */
+			if (status != 0)
+			    break;
+		}
 	}
+}
 
-	return (i);
+void
+xlp_cms_enable_intr(int node, int cpu, int type, int watermark)
+{
+	uint64_t cmsbase;
+	int i, qid;
+
+	cmsbase = nlm_get_cms_regbase(node);
+
+	for (i = 0; i < 4; i++) {
+		qid = (i + (cpu * 4)) & 0x7f;
+		nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark);
+		nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0);
+	}
 }
 
 static int
@@ -274,6 +277,7 @@ msgring_process_fast_intr(void *arg)
 
 	cpu = nlm_cpuid();
 	mthd = &msgring_threads[cpu];
+	msgring_nintr[cpu]++;
 	td = mthd->thread;
 
 	/* clear pending interrupts */
@@ -283,24 +287,24 @@ msgring_process_fast_intr(void *arg)
 	mthd->needed = 1;
 	thread_lock(td);
 	if (TD_AWAITING_INTR(td)) {
+		msgring_wakeup_sleep[cpu]++;
 		TD_CLR_IWAIT(td);
 		sched_add(td, SRQ_INTR);
-	}
+	} else
+		msgring_wakeup_nosleep[cpu]++;
 
 	thread_unlock(td);
+
 	return (FILTER_HANDLED);
 }
 
-u_int fmn_msgcount[32][4];
-u_int fmn_loops[32];
-
 static void
 msgring_process(void * arg)
 {
 	volatile struct msgring_thread *mthd;
 	struct thread *td;
-	uint32_t mflags;
-	int hwtid, vc, handled, nmsgs;
+	uint32_t mflags, msgstatus1;
+	int hwtid, nmsgs;
 
 	hwtid = (intptr_t)arg;
 	mthd = &msgring_threads[hwtid];
@@ -314,39 +318,47 @@ msgring_process(void * arg)
 	thread_unlock(td);
 
 	if (hwtid != nlm_cpuid())
-		printf("Misscheduled hwtid %d != cpuid %d\n", hwtid, nlm_cpuid());
-	mflags = nlm_save_flags_cop2();
-	nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0);
-	nlm_restore_flags(mflags);
+		printf("Misscheduled hwtid %d != cpuid %d\n", hwtid,
+		    nlm_cpuid());
+
+	xlp_discard_msg_vc(0xf);
+	xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT);
+	if (polled == 0) {
+		mflags = nlm_save_flags_cop2();
+		nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0);
+		nlm_restore_flags(mflags);
+		xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0);
+		/* clear pending interrupts.
+		 *  they will get re-raised if still valid */
+		nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
+	}
 
 	/* start processing messages */
-	for( ; ; ) {
-		/*atomic_store_rel_int(&mthd->needed, 0);*/
-
-	        /* enable cop2 access */
-		do {
-			handled = 0;
-			for (vc = 0; vc < 4; vc++) {
-				nmsgs = xlp_handle_msg_vc(vc, 1);
-				fmn_msgcount[hwtid][vc] += nmsgs;
-				handled += nmsgs;
-			}
-		} while (handled);
+	for (;;) {
+		atomic_store_rel_int(&mthd->needed, 0);
+		nmsgs = xlp_handle_msg_vc(0xf, 0);
 
 		/* sleep */
-#if 0
-		thread_lock(td);
-		if (mthd->needed) {
+		if (polled == 0) {
+			/* clear VC-pend bits */
+			mflags = nlm_save_flags_cop2();
+			msgstatus1 = nlm_read_c2_msgstatus1();
+			msgstatus1 |= (0xf << 16);
+			nlm_write_c2_msgstatus1(msgstatus1);
+			nlm_restore_flags(mflags);
+
+			thread_lock(td);
+			if (mthd->needed) {
+				thread_unlock(td);
+				continue;
+			}
+			sched_class(td, PRI_ITHD);
+			TD_SET_IWAIT(td);
+			mi_switch(SW_VOL, NULL);
 			thread_unlock(td);
-			continue;
-		}
-		sched_class(td, PRI_ITHD);
-		TD_SET_IWAIT(td);
-		mi_switch(SW_VOL, NULL);
-		thread_unlock(td);
-#else
-		pause("wmsg", 1);
-#endif
+		} else
+			pause("wmsg", 1);
+
 		fmn_loops[hwtid]++;
 	}
 }
@@ -370,7 +382,9 @@ create_msgring_thread(int hwtid)
 	sched_class(td, PRI_ITHD);
 	sched_add(td, SRQ_INTR);
 	thread_unlock(td);
-	CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name);
+	if (bootverbose)
+		printf("Msgring handler create on cpu %d (%s)\n",
+		    hwtid, td->td_name);
 }
 
 int
@@ -381,7 +395,7 @@ register_msgring_handler(int startb, int endb, msgring_handler action,
 
 	printf("Register handler %d-%d %p(%p)\n", startb, endb, action, arg);
 	KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
-	    ("Invalid value for for bucket range %d,%d", startb, endb));
+	    ("Invalid value for bucket range %d,%d", startb, endb));
 
 	mtx_lock_spin(&msgmap_lock);
 	for (i = startb; i <= endb; i++) {
@@ -395,6 +409,45 @@ register_msgring_handler(int startb, int endb, msgring_handler action,
 }
 
 /*
+ * Initialize the messaging subsystem.
+ *
+ * Message Stations are shared among all threads in a cpu core, this
+ * has to be called once from every core which is online.
+ */
+static void
+xlp_msgring_config(void *arg)
+{
+	void *cookie;
+	unsigned int thrmask, mask;
+	int i;
+
+	mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
+	if (xlp_threads_per_core < xlp_msg_threads_per_core)
+		xlp_msg_threads_per_core = xlp_threads_per_core;
+	thrmask = ((1 << xlp_msg_threads_per_core) - 1);
+	/*thrmask <<= xlp_threads_per_core - xlp_msg_threads_per_core;*/
+	mask = 0;
+	for (i = 0; i < XLP_MAX_CORES; i++) {
+		mask <<= XLP_MAX_THREADS;
+		mask |= thrmask;
+	}
+	xlp_msg_thread_mask = xlp_hw_thread_mask & mask;
+#if 0
+	printf("CMS Message handler thread mask %#jx\n",
+	    (uintmax_t)xlp_msg_thread_mask);
+#endif
+
+	if (nlm_is_xlp3xx())
+	    polled = 0;		/* switch to interrupt driven driver */
+
+/*	nlm_cms_default_setup(0,0,0,0); */
+	xlp_cms_credit_setup(CMS_DEFAULT_CREDIT);
+	create_msgring_thread(0);
+	cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL,
+	    NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie);
+}
+
+/*
  * Start message ring processing threads on other CPUs, after SMP start
  */
 static void
@@ -409,6 +462,8 @@ start_msgring_threads(void *arg)
 	}
 }
 
+SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST,
+    xlp_msgring_config, NULL);
 SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
     start_msgring_threads, NULL);
 
diff --git a/sys/mips/nlm/files.xlp b/sys/mips/nlm/files.xlp
index f9b2b71..682bbbf 100644
--- a/sys/mips/nlm/files.xlp
+++ b/sys/mips/nlm/files.xlp
@@ -1,4 +1,5 @@
 # $FreeBSD$
+mips/nlm/hal/nlm_hal.c				standard
 mips/nlm/hal/fmn.c				standard
 mips/nlm/xlp_machdep.c				standard
 mips/nlm/intr_machdep.c				standard
diff --git a/sys/mips/nlm/hal/bridge.h b/sys/mips/nlm/hal/bridge.h
index 543c365..0598809 100644
--- a/sys/mips/nlm/hal/bridge.h
+++ b/sys/mips/nlm/hal/bridge.h
@@ -30,7 +30,7 @@
  */
 
 #ifndef __NLM_HAL_BRIDGE_H__
-#define __NLM_HAL_BRIDGE_H__
+#define	__NLM_HAL_BRIDGE_H__
 
 /**
 * @file_name mio.h
@@ -44,141 +44,141 @@
  * These registers start after the PCIe header, which has 0x40
  * standard entries
  */
-#define BRIDGE_MODE			0x00
-#define BRIDGE_PCI_CFG_BASE		0x01
-#define BRIDGE_PCI_CFG_LIMIT		0x02
-#define BRIDGE_PCIE_CFG_BASE		0x03
-#define BRIDGE_PCIE_CFG_LIMIT		0x04
-#define BRIDGE_BUSNUM_BAR0		0x05
-#define BRIDGE_BUSNUM_BAR1		0x06
-#define BRIDGE_BUSNUM_BAR2		0x07
-#define BRIDGE_BUSNUM_BAR3		0x08
-#define BRIDGE_BUSNUM_BAR4		0x09
-#define BRIDGE_BUSNUM_BAR5		0x0a
-#define BRIDGE_BUSNUM_BAR6		0x0b
-#define BRIDGE_FLASH_BAR0		0x0c
-#define BRIDGE_FLASH_BAR1		0x0d
-#define BRIDGE_FLASH_BAR2		0x0e
-#define BRIDGE_FLASH_BAR3		0x0f
-#define BRIDGE_FLASH_LIMIT0		0x10
-#define BRIDGE_FLASH_LIMIT1		0x11
-#define BRIDGE_FLASH_LIMIT2		0x12
-#define BRIDGE_FLASH_LIMIT3		0x13
+#define	BRIDGE_MODE			0x00
+#define	BRIDGE_PCI_CFG_BASE		0x01
+#define	BRIDGE_PCI_CFG_LIMIT		0x02
+#define	BRIDGE_PCIE_CFG_BASE		0x03
+#define	BRIDGE_PCIE_CFG_LIMIT		0x04
+#define	BRIDGE_BUSNUM_BAR0		0x05
+#define	BRIDGE_BUSNUM_BAR1		0x06
+#define	BRIDGE_BUSNUM_BAR2		0x07
+#define	BRIDGE_BUSNUM_BAR3		0x08
+#define	BRIDGE_BUSNUM_BAR4		0x09
+#define	BRIDGE_BUSNUM_BAR5		0x0a
+#define	BRIDGE_BUSNUM_BAR6		0x0b
+#define	BRIDGE_FLASH_BAR0		0x0c
+#define	BRIDGE_FLASH_BAR1		0x0d
+#define	BRIDGE_FLASH_BAR2		0x0e
+#define	BRIDGE_FLASH_BAR3		0x0f
+#define	BRIDGE_FLASH_LIMIT0		0x10
+#define	BRIDGE_FLASH_LIMIT1		0x11
+#define	BRIDGE_FLASH_LIMIT2		0x12
+#define	BRIDGE_FLASH_LIMIT3		0x13
 
-#define BRIDGE_DRAM_BAR(i)		(0x14 + (i))
-#define BRIDGE_DRAM_BAR0		0x14
-#define BRIDGE_DRAM_BAR1		0x15
-#define BRIDGE_DRAM_BAR2		0x16
-#define BRIDGE_DRAM_BAR3		0x17
-#define BRIDGE_DRAM_BAR4		0x18
-#define BRIDGE_DRAM_BAR5		0x19
-#define BRIDGE_DRAM_BAR6		0x1a
-#define BRIDGE_DRAM_BAR7		0x1b
+#define	BRIDGE_DRAM_BAR(i)		(0x14 + (i))
+#define	BRIDGE_DRAM_BAR0		0x14
+#define	BRIDGE_DRAM_BAR1		0x15
+#define	BRIDGE_DRAM_BAR2		0x16
+#define	BRIDGE_DRAM_BAR3		0x17
+#define	BRIDGE_DRAM_BAR4		0x18
+#define	BRIDGE_DRAM_BAR5		0x19
+#define	BRIDGE_DRAM_BAR6		0x1a
+#define	BRIDGE_DRAM_BAR7		0x1b
 
-#define BRIDGE_DRAM_LIMIT(i)		(0x1c + (i))
-#define BRIDGE_DRAM_LIMIT0		0x1c
-#define BRIDGE_DRAM_LIMIT1		0x1d
-#define BRIDGE_DRAM_LIMIT2		0x1e
-#define BRIDGE_DRAM_LIMIT3		0x1f
-#define BRIDGE_DRAM_LIMIT4		0x20
-#define BRIDGE_DRAM_LIMIT5		0x21
-#define BRIDGE_DRAM_LIMIT6		0x22
-#define BRIDGE_DRAM_LIMIT7		0x23
+#define	BRIDGE_DRAM_LIMIT(i)		(0x1c + (i))
+#define	BRIDGE_DRAM_LIMIT0		0x1c
+#define	BRIDGE_DRAM_LIMIT1		0x1d
+#define	BRIDGE_DRAM_LIMIT2		0x1e
+#define	BRIDGE_DRAM_LIMIT3		0x1f
+#define	BRIDGE_DRAM_LIMIT4		0x20
+#define	BRIDGE_DRAM_LIMIT5		0x21
+#define	BRIDGE_DRAM_LIMIT6		0x22
+#define	BRIDGE_DRAM_LIMIT7		0x23
 
-#define BRIDGE_DRAM_NODE_TRANSLN0	0x24
-#define BRIDGE_DRAM_NODE_TRANSLN1	0x25
-#define BRIDGE_DRAM_NODE_TRANSLN2	0x26
-#define BRIDGE_DRAM_NODE_TRANSLN3	0x27
-#define BRIDGE_DRAM_NODE_TRANSLN4	0x28
-#define BRIDGE_DRAM_NODE_TRANSLN5	0x29
-#define BRIDGE_DRAM_NODE_TRANSLN6	0x2a
-#define BRIDGE_DRAM_NODE_TRANSLN7	0x2b
-#define BRIDGE_DRAM_CHNL_TRANSLN0	0x2c
-#define BRIDGE_DRAM_CHNL_TRANSLN1	0x2d
-#define BRIDGE_DRAM_CHNL_TRANSLN2	0x2e
-#define BRIDGE_DRAM_CHNL_TRANSLN3	0x2f
-#define BRIDGE_DRAM_CHNL_TRANSLN4	0x30
-#define BRIDGE_DRAM_CHNL_TRANSLN5	0x31
-#define BRIDGE_DRAM_CHNL_TRANSLN6	0x32
-#define BRIDGE_DRAM_CHNL_TRANSLN7	0x33
-#define BRIDGE_PCIEMEM_BASE0		0x34
-#define BRIDGE_PCIEMEM_BASE1		0x35
-#define BRIDGE_PCIEMEM_BASE2		0x36
-#define BRIDGE_PCIEMEM_BASE3		0x37
-#define BRIDGE_PCIEMEM_LIMIT0		0x38
-#define BRIDGE_PCIEMEM_LIMIT1		0x39
-#define BRIDGE_PCIEMEM_LIMIT2		0x3a
-#define BRIDGE_PCIEMEM_LIMIT3		0x3b
-#define BRIDGE_PCIEIO_BASE0		0x3c
-#define BRIDGE_PCIEIO_BASE1		0x3d
-#define BRIDGE_PCIEIO_BASE2		0x3e
-#define BRIDGE_PCIEIO_BASE3		0x3f
-#define BRIDGE_PCIEIO_LIMIT0		0x40
-#define BRIDGE_PCIEIO_LIMIT1		0x41
-#define BRIDGE_PCIEIO_LIMIT2		0x42
-#define BRIDGE_PCIEIO_LIMIT3		0x43
-#define BRIDGE_PCIEMEM_BASE4		0x44
-#define BRIDGE_PCIEMEM_BASE5		0x45
-#define BRIDGE_PCIEMEM_BASE6		0x46
-#define BRIDGE_PCIEMEM_LIMIT4		0x47
-#define BRIDGE_PCIEMEM_LIMIT5		0x48
-#define BRIDGE_PCIEMEM_LIMIT6		0x49
-#define BRIDGE_PCIEIO_BASE4		0x4a
-#define BRIDGE_PCIEIO_BASE5		0x4b
-#define BRIDGE_PCIEIO_BASE6		0x4c
-#define BRIDGE_PCIEIO_LIMIT4		0x4d
-#define BRIDGE_PCIEIO_LIMIT5		0x4e
-#define BRIDGE_PCIEIO_LIMIT6		0x4f
-#define BRIDGE_NBU_EVENT_CNT_CTL	0x50
-#define BRIDGE_EVNTCTR1_LOW		0x51
-#define BRIDGE_EVNTCTR1_HI		0x52
-#define BRIDGE_EVNT_CNT_CTL2		0x53
-#define BRIDGE_EVNTCTR2_LOW		0x54
-#define BRIDGE_EVNTCTR2_HI		0x55
-#define BRIDGE_TRACEBUF_MATCH0		0x56
-#define BRIDGE_TRACEBUF_MATCH1		0x57
-#define BRIDGE_TRACEBUF_MATCH_LOW	0x58
-#define BRIDGE_TRACEBUF_MATCH_HI	0x59
-#define BRIDGE_TRACEBUF_CTRL		0x5a
-#define BRIDGE_TRACEBUF_INIT		0x5b
-#define BRIDGE_TRACEBUF_ACCESS		0x5c
-#define BRIDGE_TRACEBUF_READ_DATA0	0x5d
-#define BRIDGE_TRACEBUF_READ_DATA1	0x5d
-#define BRIDGE_TRACEBUF_READ_DATA2	0x5f
-#define BRIDGE_TRACEBUF_READ_DATA3	0x60
-#define BRIDGE_TRACEBUF_STATUS		0x61
-#define BRIDGE_ADDRESS_ERROR0		0x62
-#define BRIDGE_ADDRESS_ERROR1		0x63
-#define BRIDGE_ADDRESS_ERROR2		0x64
-#define BRIDGE_TAG_ECC_ADDR_ERROR0	0x65
-#define BRIDGE_TAG_ECC_ADDR_ERROR1	0x66
-#define BRIDGE_TAG_ECC_ADDR_ERROR2	0x67
-#define BRIDGE_LINE_FLUSH0		0x68
-#define BRIDGE_LINE_FLUSH1		0x69
-#define BRIDGE_NODE_ID			0x6a
-#define BRIDGE_ERROR_INTERRUPT_EN	0x6b
-#define BRIDGE_PCIE0_WEIGHT		0x2c0
-#define BRIDGE_PCIE1_WEIGHT		0x2c1
-#define BRIDGE_PCIE2_WEIGHT		0x2c2
-#define BRIDGE_PCIE3_WEIGHT		0x2c3
-#define BRIDGE_USB_WEIGHT		0x2c4
-#define BRIDGE_NET_WEIGHT		0x2c5
-#define BRIDGE_POE_WEIGHT		0x2c6
-#define BRIDGE_CMS_WEIGHT		0x2c7
-#define BRIDGE_DMAENG_WEIGHT		0x2c8
-#define BRIDGE_SEC_WEIGHT		0x2c9
-#define BRIDGE_COMP_WEIGHT		0x2ca
-#define BRIDGE_GIO_WEIGHT		0x2cb
-#define BRIDGE_FLASH_WEIGHT		0x2cc
+#define	BRIDGE_DRAM_NODE_TRANSLN0	0x24
+#define	BRIDGE_DRAM_NODE_TRANSLN1	0x25
+#define	BRIDGE_DRAM_NODE_TRANSLN2	0x26
+#define	BRIDGE_DRAM_NODE_TRANSLN3	0x27
+#define	BRIDGE_DRAM_NODE_TRANSLN4	0x28
+#define	BRIDGE_DRAM_NODE_TRANSLN5	0x29
+#define	BRIDGE_DRAM_NODE_TRANSLN6	0x2a
+#define	BRIDGE_DRAM_NODE_TRANSLN7	0x2b
+#define	BRIDGE_DRAM_CHNL_TRANSLN0	0x2c
+#define	BRIDGE_DRAM_CHNL_TRANSLN1	0x2d
+#define	BRIDGE_DRAM_CHNL_TRANSLN2	0x2e
+#define	BRIDGE_DRAM_CHNL_TRANSLN3	0x2f
+#define	BRIDGE_DRAM_CHNL_TRANSLN4	0x30
+#define	BRIDGE_DRAM_CHNL_TRANSLN5	0x31
+#define	BRIDGE_DRAM_CHNL_TRANSLN6	0x32
+#define	BRIDGE_DRAM_CHNL_TRANSLN7	0x33
+#define	BRIDGE_PCIEMEM_BASE0		0x34
+#define	BRIDGE_PCIEMEM_BASE1		0x35
+#define	BRIDGE_PCIEMEM_BASE2		0x36
+#define	BRIDGE_PCIEMEM_BASE3		0x37
+#define	BRIDGE_PCIEMEM_LIMIT0		0x38
+#define	BRIDGE_PCIEMEM_LIMIT1		0x39
+#define	BRIDGE_PCIEMEM_LIMIT2		0x3a
+#define	BRIDGE_PCIEMEM_LIMIT3		0x3b
+#define	BRIDGE_PCIEIO_BASE0		0x3c
+#define	BRIDGE_PCIEIO_BASE1		0x3d
+#define	BRIDGE_PCIEIO_BASE2		0x3e
+#define	BRIDGE_PCIEIO_BASE3		0x3f
+#define	BRIDGE_PCIEIO_LIMIT0		0x40
+#define	BRIDGE_PCIEIO_LIMIT1		0x41
+#define	BRIDGE_PCIEIO_LIMIT2		0x42
+#define	BRIDGE_PCIEIO_LIMIT3		0x43
+#define	BRIDGE_PCIEMEM_BASE4		0x44
+#define	BRIDGE_PCIEMEM_BASE5		0x45
+#define	BRIDGE_PCIEMEM_BASE6		0x46
+#define	BRIDGE_PCIEMEM_LIMIT4		0x47
+#define	BRIDGE_PCIEMEM_LIMIT5		0x48
+#define	BRIDGE_PCIEMEM_LIMIT6		0x49
+#define	BRIDGE_PCIEIO_BASE4		0x4a
+#define	BRIDGE_PCIEIO_BASE5		0x4b
+#define	BRIDGE_PCIEIO_BASE6		0x4c
+#define	BRIDGE_PCIEIO_LIMIT4		0x4d
+#define	BRIDGE_PCIEIO_LIMIT5		0x4e
+#define	BRIDGE_PCIEIO_LIMIT6		0x4f
+#define	BRIDGE_NBU_EVENT_CNT_CTL	0x50
+#define	BRIDGE_EVNTCTR1_LOW		0x51
+#define	BRIDGE_EVNTCTR1_HI		0x52
+#define	BRIDGE_EVNT_CNT_CTL2		0x53
+#define	BRIDGE_EVNTCTR2_LOW		0x54
+#define	BRIDGE_EVNTCTR2_HI		0x55
+#define	BRIDGE_TRACEBUF_MATCH0		0x56
+#define	BRIDGE_TRACEBUF_MATCH1		0x57
+#define	BRIDGE_TRACEBUF_MATCH_LOW	0x58
+#define	BRIDGE_TRACEBUF_MATCH_HI	0x59
+#define	BRIDGE_TRACEBUF_CTRL		0x5a
+#define	BRIDGE_TRACEBUF_INIT		0x5b
+#define	BRIDGE_TRACEBUF_ACCESS		0x5c
+#define	BRIDGE_TRACEBUF_READ_DATA0	0x5d
+#define	BRIDGE_TRACEBUF_READ_DATA1	0x5d
+#define	BRIDGE_TRACEBUF_READ_DATA2	0x5f
+#define	BRIDGE_TRACEBUF_READ_DATA3	0x60
+#define	BRIDGE_TRACEBUF_STATUS		0x61
+#define	BRIDGE_ADDRESS_ERROR0		0x62
+#define	BRIDGE_ADDRESS_ERROR1		0x63
+#define	BRIDGE_ADDRESS_ERROR2		0x64
+#define	BRIDGE_TAG_ECC_ADDR_ERROR0	0x65
+#define	BRIDGE_TAG_ECC_ADDR_ERROR1	0x66
+#define	BRIDGE_TAG_ECC_ADDR_ERROR2	0x67
+#define	BRIDGE_LINE_FLUSH0		0x68
+#define	BRIDGE_LINE_FLUSH1		0x69
+#define	BRIDGE_NODE_ID			0x6a
+#define	BRIDGE_ERROR_INTERRUPT_EN	0x6b
+#define	BRIDGE_PCIE0_WEIGHT		0x2c0
+#define	BRIDGE_PCIE1_WEIGHT		0x2c1
+#define	BRIDGE_PCIE2_WEIGHT		0x2c2
+#define	BRIDGE_PCIE3_WEIGHT		0x2c3
+#define	BRIDGE_USB_WEIGHT		0x2c4
+#define	BRIDGE_NET_WEIGHT		0x2c5
+#define	BRIDGE_POE_WEIGHT		0x2c6
+#define	BRIDGE_CMS_WEIGHT		0x2c7
+#define	BRIDGE_DMAENG_WEIGHT		0x2c8
+#define	BRIDGE_SEC_WEIGHT		0x2c9
+#define	BRIDGE_COMP_WEIGHT		0x2ca
+#define	BRIDGE_GIO_WEIGHT		0x2cb
+#define	BRIDGE_FLASH_WEIGHT		0x2cc
 
 #if !defined(LOCORE) && !defined(__ASSEMBLY__)
 
-#define nlm_read_bridge_reg(b, r)	nlm_read_reg(b, r)
-#define nlm_write_bridge_reg(b, r, v)	nlm_write_reg(b, r, v)
+#define	nlm_read_bridge_reg(b, r)	nlm_read_reg(b, r)
+#define	nlm_write_bridge_reg(b, r, v)	nlm_write_reg(b, r, v)
 #define	nlm_get_bridge_pcibase(node)	\
-				nlm_pcicfg_base(XLP_IO_BRIDGE_OFFSET(node))
+			nlm_pcicfg_base(XLP_IO_BRIDGE_OFFSET(node))
 #define	nlm_get_bridge_regbase(node)	\
-				(nlm_get_bridge_pcibase(node) + XLP_IO_PCI_HDRSZ)
+			(nlm_get_bridge_pcibase(node) + XLP_IO_PCI_HDRSZ)
 
 #endif
 #endif
diff --git a/sys/mips/nlm/hal/cop2.h b/sys/mips/nlm/hal/cop2.h
index 6b79d3f..9cbd266 100644
--- a/sys/mips/nlm/hal/cop2.h
+++ b/sys/mips/nlm/hal/cop2.h
@@ -30,29 +30,29 @@
  */
 
 #ifndef __NLM_HAL_COP2_H__
-#define __NLM_HAL_COP2_H__
+#define	__NLM_HAL_COP2_H__
 
-#define COP2_TX_BUF		0
-#define COP2_RX_BUF		1
-#define COP2_TXMSGSTATUS	2
-#define COP2_RXMSGSTATUS	3
-#define COP2_MSGSTATUS1		4
-#define COP2_MSGCONFIG		5
-#define COP2_MSGCONFIG1		6
+#define	COP2_TX_BUF		0
+#define	COP2_RX_BUF		1
+#define	COP2_TXMSGSTATUS	2
+#define	COP2_RXMSGSTATUS	3
+#define	COP2_MSGSTATUS1		4
+#define	COP2_MSGCONFIG		5
+#define	COP2_MSGERROR		6
 
-#define CROSSTHR_POPQ_EN	0x01
-#define VC0_POPQ_EN		0x02
-#define VC1_POPQ_EN		0x04
-#define VC2_POPQ_EN		0x08
-#define VC3_POPQ_EN		0x10
-#define ALL_VC_POPQ_EN		0x1E
-#define ALL_VC_CT_POPQ_EN	0x1F
+#define	CROSSTHR_POPQ_EN	0x01
+#define	VC0_POPQ_EN		0x02
+#define	VC1_POPQ_EN		0x04
+#define	VC2_POPQ_EN		0x08
+#define	VC3_POPQ_EN		0x10
+#define	ALL_VC_POPQ_EN		0x1E
+#define	ALL_VC_CT_POPQ_EN	0x1F
 
 struct nlm_fmn_msg {
 	uint64_t msg[4];
 };
 
-#define NLM_DEFINE_COP2_ACCESSORS32(name, reg, sel)		\
+#define	NLM_DEFINE_COP2_ACCESSORS32(name, reg, sel)		\
 static inline uint32_t nlm_read_c2_##name(void)			\
 {								\
 	uint32_t __rv;						\
@@ -79,7 +79,7 @@ static inline void nlm_write_c2_##name(uint32_t val)		\
 } struct __hack
 
 #if (__mips == 64)
-#define NLM_DEFINE_COP2_ACCESSORS64(name, reg, sel)		\
+#define	NLM_DEFINE_COP2_ACCESSORS64(name, reg, sel)		\
 static inline uint64_t nlm_read_c2_##name(void)			\
 {								\
 	uint64_t __rv;						\
@@ -107,7 +107,7 @@ static inline void nlm_write_c2_##name(uint64_t val)		\
 
 #else
 
-#define NLM_DEFINE_COP2_ACCESSORS64(name, reg, sel)		\
+#define	NLM_DEFINE_COP2_ACCESSORS64(name, reg, sel)		\
 static inline uint64_t nlm_read_c2_##name(void)			\
 {								\
 	uint32_t __high, __low;					\
@@ -160,7 +160,10 @@ NLM_DEFINE_COP2_ACCESSORS32(txmsgstatus, COP2_TXMSGSTATUS, 0);
 NLM_DEFINE_COP2_ACCESSORS32(rxmsgstatus, COP2_RXMSGSTATUS, 0);
 NLM_DEFINE_COP2_ACCESSORS32(msgstatus1, COP2_MSGSTATUS1, 0);
 NLM_DEFINE_COP2_ACCESSORS32(msgconfig, COP2_MSGCONFIG, 0);
-NLM_DEFINE_COP2_ACCESSORS32(msgconfig1, COP2_MSGCONFIG1, 0);
+NLM_DEFINE_COP2_ACCESSORS32(msgerror0, COP2_MSGERROR, 0);
+NLM_DEFINE_COP2_ACCESSORS32(msgerror1, COP2_MSGERROR, 1);
+NLM_DEFINE_COP2_ACCESSORS32(msgerror2, COP2_MSGERROR, 2);
+NLM_DEFINE_COP2_ACCESSORS32(msgerror3, COP2_MSGERROR, 3);
 
 /* successful completion returns 1, else 0 */
 static inline int
@@ -279,7 +282,7 @@ nlm_fmn_msgrcv(int vc, int *srcid, int *size, int *code, struct nlm_fmn_msg *m)
 }
 
 static inline void
-nlm_fmn_cpu_init(int int_vec, int ctpe, int v0pe, int v1pe, int v2pe, int v3pe)
+nlm_fmn_cpu_init(int int_vec, int ecc_en, int v0pe, int v1pe, int v2pe, int v3pe)
 {
 	uint32_t val = nlm_read_c2_msgconfig();
 
@@ -287,12 +290,12 @@ nlm_fmn_cpu_init(int int_vec, int ctpe, int v0pe, int v1pe, int v2pe, int v3pe)
 	 * in msgconfig register of cop2.
 	 * As per chip/cpu RTL, [16:20] bits consist of int_vec.
 	 */
-	val |= ((int_vec & 0x1f) << 16) |
+	val |= (((int_vec & 0x1f) << 16) |
+		((ecc_en & 0x1) << 8) |
 		((v3pe & 0x1) << 4) |
 		((v2pe & 0x1) << 3) |
 		((v1pe & 0x1) << 2) |
-		((v0pe & 0x1) << 1) |
-		(ctpe & 0x1);
+		((v0pe & 0x1) << 1));
 
 	nlm_write_c2_msgconfig(val);
 }
diff --git a/sys/mips/nlm/hal/cpucontrol.h b/sys/mips/nlm/hal/cpucontrol.h
index 715cd53..041f9d2 100644
--- a/sys/mips/nlm/hal/cpucontrol.h
+++ b/sys/mips/nlm/hal/cpucontrol.h
@@ -30,52 +30,52 @@
  */
 
 #ifndef __NLM_HAL_CPUCONTROL_H__
-#define __NLM_HAL_CPUCONTROL_H__
-
-#define CPU_BLOCKID_IFU		0
-#define CPU_BLOCKID_ICU		1
-#define CPU_BLOCKID_IEU		2
-#define CPU_BLOCKID_LSU		3
-#define CPU_BLOCKID_MMU		4
-#define CPU_BLOCKID_PRF		5
-#define CPU_BLOCKID_SCH		7
-#define CPU_BLOCKID_SCU		8
-#define CPU_BLOCKID_FPU		9
-#define CPU_BLOCKID_MAP		10
-
-#define LSU_DEFEATURE		0x304
-#define LSU_CERRLOG_REGID	0x09
-#define SCHED_DEFEATURE		0x700
+#define	__NLM_HAL_CPUCONTROL_H__
+
+#define	CPU_BLOCKID_IFU		0
+#define	CPU_BLOCKID_ICU		1
+#define	CPU_BLOCKID_IEU		2
+#define	CPU_BLOCKID_LSU		3
+#define	CPU_BLOCKID_MMU		4
+#define	CPU_BLOCKID_PRF		5
+#define	CPU_BLOCKID_SCH		7
+#define	CPU_BLOCKID_SCU		8
+#define	CPU_BLOCKID_FPU		9
+#define	CPU_BLOCKID_MAP		10
+
+#define	LSU_DEFEATURE		0x304
+#define	LSU_CERRLOG_REGID	0x09
+#define	SCHED_DEFEATURE		0x700
 
 /* Offsets of interest from the 'MAP' Block */
-#define MAP_THREADMODE			0x00
-#define MAP_EXT_EBASE_ENABLE		0x04
-#define MAP_CCDI_CONFIG			0x08
-#define MAP_THRD0_CCDI_STATUS		0x0c
-#define MAP_THRD1_CCDI_STATUS		0x10
-#define MAP_THRD2_CCDI_STATUS		0x14
-#define MAP_THRD3_CCDI_STATUS		0x18
-#define MAP_THRD0_DEBUG_MODE		0x1c
-#define MAP_THRD1_DEBUG_MODE		0x20
-#define MAP_THRD2_DEBUG_MODE		0x24
-#define MAP_THRD3_DEBUG_MODE		0x28
-#define MAP_MISC_STATE			0x60
-#define MAP_DEBUG_READ_CTL		0x64
-#define MAP_DEBUG_READ_REG0		0x68
-#define MAP_DEBUG_READ_REG1		0x6c
-
-#define MMU_SETUP		0x400
-#define MMU_LFSRSEED		0x401
-#define MMU_HPW_NUM_PAGE_LVL	0x410
-#define MMU_PGWKR_PGDBASE	0x411
-#define MMU_PGWKR_PGDSHFT	0x412
-#define MMU_PGWKR_PGDMASK	0x413
-#define MMU_PGWKR_PUDSHFT	0x414
-#define MMU_PGWKR_PUDMASK	0x415
-#define MMU_PGWKR_PMDSHFT	0x416
-#define MMU_PGWKR_PMDMASK	0x417
-#define MMU_PGWKR_PTESHFT	0x418
-#define MMU_PGWKR_PTEMASK	0x419
+#define	MAP_THREADMODE			0x00
+#define	MAP_EXT_EBASE_ENABLE		0x04
+#define	MAP_CCDI_CONFIG			0x08
+#define	MAP_THRD0_CCDI_STATUS		0x0c
+#define	MAP_THRD1_CCDI_STATUS		0x10
+#define	MAP_THRD2_CCDI_STATUS		0x14
+#define	MAP_THRD3_CCDI_STATUS		0x18
+#define	MAP_THRD0_DEBUG_MODE		0x1c
+#define	MAP_THRD1_DEBUG_MODE		0x20
+#define	MAP_THRD2_DEBUG_MODE		0x24
+#define	MAP_THRD3_DEBUG_MODE		0x28
+#define	MAP_MISC_STATE			0x60
+#define	MAP_DEBUG_READ_CTL		0x64
+#define	MAP_DEBUG_READ_REG0		0x68
+#define	MAP_DEBUG_READ_REG1		0x6c
+
+#define	MMU_SETUP		0x400
+#define	MMU_LFSRSEED		0x401
+#define	MMU_HPW_NUM_PAGE_LVL	0x410
+#define	MMU_PGWKR_PGDBASE	0x411
+#define	MMU_PGWKR_PGDSHFT	0x412
+#define	MMU_PGWKR_PGDMASK	0x413
+#define	MMU_PGWKR_PUDSHFT	0x414
+#define	MMU_PGWKR_PUDMASK	0x415
+#define	MMU_PGWKR_PMDSHFT	0x416
+#define	MMU_PGWKR_PMDMASK	0x417
+#define	MMU_PGWKR_PTESHFT	0x418
+#define	MMU_PGWKR_PTEMASK	0x419
 
 
 #if !defined(LOCORE) && !defined(__ASSEMBLY__)
diff --git a/sys/mips/nlm/hal/fmn.c b/sys/mips/nlm/hal/fmn.c
index e58a2a5..6d40134 100644
--- a/sys/mips/nlm/hal/fmn.c
+++ b/sys/mips/nlm/hal/fmn.c
@@ -67,7 +67,6 @@ uint64_t nlm_cms_spill_total_messages = 1 * 1024;
 * For all 4 nodes, there are 18*4 = 72 FMN stations
 */
 uint32_t nlm_cms_total_stations = 18 * 4 /*xlp_num_nodes*/;
-uint32_t cms_onchip_seg_availability[CMS_ON_CHIP_PER_QUEUE_SPACE];
 
 /**
  * Takes inputs as node, queue_size and maximum number of queues.
@@ -146,114 +145,6 @@ void nlm_cms_setup_credits(uint64_t base, int destid, int srcid, int credit)
 
 }
 
-int nlm_cms_config_onchip_queue (uint64_t base, uint64_t spill_base,
-					int qid, int spill_en)
-{
-
-	/* Configure 32 as onchip queue depth */
-	nlm_cms_alloc_onchip_q(base, qid, 1);
-
-	/* Spill configuration */
-	if (spill_en) {
-		/* Configure 4*4KB = 16K as spill size */
-		nlm_cms_alloc_spill_q(base, qid, spill_base, 4);
-	}
-
-#if 0
-	/* configure credits for src cpu0, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU0_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu1, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU1_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu2, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU2_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu3, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU3_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu4, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU4_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu5, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU5_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu6, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU6_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cpu7, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CPU7_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src pcie0, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_PCIE0_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src pcie1, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_PCIE1_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src pcie2, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_PCIE2_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src pcie3, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_PCIE3_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src dte, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_DTE_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src rsa_ecc, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_RSA_ECC_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src crypto, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CRYPTO_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src cmp, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_CMP_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src poe, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_POE_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-
-	/* configure credits for src nae, on this queue */
-	nlm_cms_setup_credits(base, qid, CMS_NAE_SRC_STID,
-		CMS_DEFAULT_CREDIT(nlm_cms_total_stations,
-			nlm_cms_spill_total_messages));
-#endif
-
-	return 0;
-}
-
 /*
  * base		- CMS module base address for this node.
  * qid		- is the output queue id otherwise called as vc id
@@ -268,7 +159,7 @@ int nlm_cms_alloc_spill_q(uint64_t base, int qid, uint64_t spill_base,
 	uint64_t queue_config;
 	uint32_t spill_start;
 
-	if(nsegs > CMS_MAX_SPILL_SEGMENTS_PER_QUEUE) {
+	if (nsegs > CMS_MAX_SPILL_SEGMENTS_PER_QUEUE) {
 		return 1;
 	}
 
@@ -286,152 +177,6 @@ int nlm_cms_alloc_spill_q(uint64_t base, int qid, uint64_t spill_base,
 	return 0;
 }
 
-/*
- * base		- CMS module base address for this node.
- * qid		- is the output queue id otherwise called as vc id
- * nsegs	- No of segments where a "1" indicates 32 credits. On chip
- *                credits must be a multiple of 32.
- */
-int nlm_cms_alloc_onchip_q(uint64_t base, int qid, int nsegs)
-{
-	static uint32_t curr_end = 0;
-	uint64_t queue_config;
-	int onchipbase, start, last;
-	uint8_t i;
-
-        if( ((curr_end + nsegs) > CMS_MAX_ONCHIP_SEGMENTS) ||
-		(nsegs > CMS_ON_CHIP_PER_QUEUE_SPACE) ) {
-		/* Invalid configuration */
-                return 1;
-        }
-        if(((curr_end % 32) + nsegs - 1) <= 31) {
-                onchipbase = (curr_end / 32);
-                start  = (curr_end % 32);
-                curr_end += nsegs;
-        } else {
-                onchipbase = (curr_end / 32) + 1;
-                start  = 0;
-                curr_end = ((onchipbase * 32) + nsegs);
-        }
-        last   = start + nsegs - 1;
-
-	for(i = start;i <= last;i++) {
-		if(cms_onchip_seg_availability[onchipbase] & (1 << i)) {
-			/* Conflict!!! segment is already allocated */
-			return 1;
-		}
-	}
-	/* Update the availability bitmap as consumed */
-	for(i = start; i <= last; i++) {
-		cms_onchip_seg_availability[onchipbase] |= (1 << i);
-	}
-
-	queue_config = nlm_read_cms_reg(base,(CMS_OUTPUTQ_CONFIG(qid)));
-
-	/* On chip configuration */
-	queue_config = (((uint64_t)CMS_QUEUE_ENA << 63) |
-			((onchipbase & 0x1f) << 10) |
-			((last & 0x1f) << 5) |
-			(start & 0x1f));
-
-	nlm_write_cms_reg(base,(CMS_OUTPUTQ_CONFIG(qid)),queue_config);
-
-	return 0;
-}
-
-void nlm_cms_default_setup(int node, uint64_t spill_base, int spill_en,
-				int popq_en)
-{
-	int j, k, vc;
-	int queue;
-	uint64_t base;
-
-	base = nlm_get_cms_regbase(node);
-	for(j=0; j<1024; j++) {
-		printf("Qid:0x%04d Val:0x%016jx\n",j, 
-		    (uintmax_t)nlm_cms_get_onchip_queue (base, j));
-	}
-	/* Enable all cpu push queues */
-	for (j=0; j<XLP_MAX_CORES; j++)
-		for (k=0; k<XLP_MAX_THREADS; k++)
-			for (vc=0; vc<CMS_MAX_VCPU_VC; vc++) {
-		/* TODO : remove this once SMP works */
-		if( (j == 0) && (k == 0) )
-			continue;
-		queue = CMS_CPU_PUSHQ(node, j, k, vc);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable pcie 0 push queue */
-	for (j=CMS_PCIE0_QID(0); j<CMS_PCIE0_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable pcie 1 push queue */
-	for (j=CMS_PCIE1_QID(0); j<CMS_PCIE1_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable pcie 2 push queue */
-	for (j=CMS_PCIE2_QID(0); j<CMS_PCIE2_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable pcie 3 push queue */
-	for (j=CMS_PCIE3_QID(0); j<CMS_PCIE3_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable DTE push queue */
-	for (j=CMS_DTE_QID(0); j<CMS_DTE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable RSA/ECC push queue */
-	for (j=CMS_RSA_ECC_QID(0); j<CMS_RSA_ECC_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable crypto push queue */
-	for (j=CMS_CRYPTO_QID(0); j<CMS_CRYPTO_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable CMP push queue */
-	for (j=CMS_CMP_QID(0); j<CMS_CMP_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable POE push queue */
-	for (j=CMS_POE_QID(0); j<CMS_POE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable NAE push queue */
-	for (j=CMS_NAE_QID(0); j<CMS_NAE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_config_onchip_queue(base, spill_base, queue, spill_en);
-	}
-
-	/* Enable all pop queues */
-	if (popq_en) {
-		for (j=CMS_POPQ_QID(0); j<CMS_POPQ_MAXQID; j++) {
-			queue = CMS_POPQ(node, j);
-			nlm_cms_config_onchip_queue(base, spill_base, queue,
-							spill_en);
-		}
-	}
-}
-
 uint64_t nlm_cms_get_onchip_queue (uint64_t base, int qid)
 {
 	return nlm_read_cms_reg(base, CMS_OUTPUTQ_CONFIG(qid));
@@ -453,94 +198,14 @@ void nlm_cms_per_queue_level_intr(uint64_t base, int qid, int sub_type,
 
 	val = nlm_read_cms_reg(base, CMS_OUTPUTQ_CONFIG(qid));
 
+	val &= ~((0x7ULL << 56) | (0x3ULL << 54));
+
 	val |= (((uint64_t)sub_type<<54) |
 		((uint64_t)intr_val<<56));
 
 	nlm_write_cms_reg(base, CMS_OUTPUTQ_CONFIG(qid), val);
 }
 
-void nlm_cms_level_intr(int node, int sub_type, int intr_val)
-{
-	int j, k, vc;
-	int queue;
-	uint64_t base;
-
-	base = nlm_get_cms_regbase(node);
-	/* setup level intr config on all cpu push queues */
-	for (j=0; j<XLP_MAX_CORES; j++)
-		for (k=0; k<XLP_MAX_THREADS; k++)
-			for (vc=0; vc<CMS_MAX_VCPU_VC; vc++) {
-		queue = CMS_CPU_PUSHQ(node, j, k, vc);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all pcie 0 push queue */
-	for (j=CMS_PCIE0_QID(0); j<CMS_PCIE0_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all pcie 1 push queue */
-	for (j=CMS_PCIE1_QID(0); j<CMS_PCIE1_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all pcie 2 push queue */
-	for (j=CMS_PCIE2_QID(0); j<CMS_PCIE2_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all pcie 3 push queue */
-	for (j=CMS_PCIE3_QID(0); j<CMS_PCIE3_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all DTE push queue */
-	for (j=CMS_DTE_QID(0); j<CMS_DTE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all RSA/ECC push queue */
-	for (j=CMS_RSA_ECC_QID(0); j<CMS_RSA_ECC_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all crypto push queue */
-	for (j=CMS_CRYPTO_QID(0); j<CMS_CRYPTO_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all CMP push queue */
-	for (j=CMS_CMP_QID(0); j<CMS_CMP_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all POE push queue */
-	for (j=CMS_POE_QID(0); j<CMS_POE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all NAE push queue */
-	for (j=CMS_NAE_QID(0); j<CMS_NAE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup level intr config on all pop queues */
-	for (j=CMS_POPQ_QID(0); j<CMS_POPQ_MAXQID; j++) {
-		queue = CMS_POPQ(node, j);
-		nlm_cms_per_queue_level_intr(base, queue, sub_type, intr_val);
-	}
-}
-
 void nlm_cms_per_queue_timer_intr(uint64_t base, int qid, int sub_type,
 					int intr_val)
 {
@@ -548,94 +213,14 @@ void nlm_cms_per_queue_timer_intr(uint64_t base, int qid, int sub_type,
 
 	val = nlm_read_cms_reg(base, CMS_OUTPUTQ_CONFIG(qid));
 
+	val &= ~((0x7ULL << 51) | (0x3ULL << 49));
+
 	val |= (((uint64_t)sub_type<<49) |
 		((uint64_t)intr_val<<51));
 
 	nlm_write_cms_reg(base, CMS_OUTPUTQ_CONFIG(qid), val);
 }
 
-void nlm_cms_timer_intr(int node, int en, int sub_type, int intr_val)
-{
-	int j, k, vc;
-	int queue;
-	uint64_t base;
-
-	base = nlm_get_cms_regbase(node);
-	/* setup timer intr config on all cpu push queues */
-	for (j=0; j<XLP_MAX_CORES; j++)
-		for (k=0; k<XLP_MAX_THREADS; k++)
-			for (vc=0; vc<CMS_MAX_VCPU_VC; vc++) {
-		queue = CMS_CPU_PUSHQ(node, j, k, vc);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all pcie 0 push queue */
-	for (j=CMS_PCIE0_QID(0); j<CMS_PCIE0_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all pcie 1 push queue */
-	for (j=CMS_PCIE1_QID(0); j<CMS_PCIE1_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all pcie 2 push queue */
-	for (j=CMS_PCIE2_QID(0); j<CMS_PCIE2_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all pcie 3 push queue */
-	for (j=CMS_PCIE3_QID(0); j<CMS_PCIE3_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all DTE push queue */
-	for (j=CMS_DTE_QID(0); j<CMS_DTE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all RSA/ECC push queue */
-	for (j=CMS_RSA_ECC_QID(0); j<CMS_RSA_ECC_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all crypto push queue */
-	for (j=CMS_CRYPTO_QID(0); j<CMS_CRYPTO_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all CMP push queue */
-	for (j=CMS_CMP_QID(0); j<CMS_CMP_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all POE push queue */
-	for (j=CMS_POE_QID(0); j<CMS_POE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all NAE push queue */
-	for (j=CMS_NAE_QID(0); j<CMS_NAE_MAXQID; j++) {
-		queue = CMS_IO_PUSHQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-
-	/* setup timer intr config on all pop queues */
-	for (j=CMS_POPQ_QID(0); j<CMS_POPQ_MAXQID; j++) {
-		queue = CMS_POPQ(node, j);
-		nlm_cms_per_queue_timer_intr(base, queue, sub_type, intr_val);
-	}
-}
-
 /* returns 1 if interrupt has been generated for this output queue */
 int nlm_cms_outputq_intr_check(uint64_t base, int qid)
 {
diff --git a/sys/mips/nlm/hal/fmn.h b/sys/mips/nlm/hal/fmn.h
index 88ba113..92f3bd7 100644
--- a/sys/mips/nlm/hal/fmn.h
+++ b/sys/mips/nlm/hal/fmn.h
@@ -30,7 +30,7 @@
  */
 
 #ifndef __NLM_FMNV2_H__
-#define __NLM_FMNV2_H__
+#define	__NLM_FMNV2_H__
 
 /**
 * @file_name fmn.h
@@ -39,156 +39,126 @@
 */
 
 /* FMN configuration registers */
-#define CMS_OUTPUTQ_CONFIG(i)		((i)*2)
-#define CMS_MAX_OUTPUTQ			1024
-#define CMS_OUTPUTQ_CREDIT_CFG		(0x2000/4)
-#define CMS_MSG_CONFIG			(0x2008/4)
-#define CMS_MSG_ERR			(0x2010/4)
-#define CMS_TRACE_CONFIG		(0x2018/4)
-#define CMS_TRACE_BASE_ADDR		(0x2020/4)
-#define CMS_TRACE_LIMIT_ADDR		(0x2028/4)
-#define CMS_TRACE_CURRENT_ADDR		(0x2030/4)
-#define CMS_MSG_ENDIAN_SWAP		(0x2038/4)
-
-#define CMS_CPU_PUSHQ(node, core, thread, vc)	\
+#define	CMS_OUTPUTQ_CONFIG(i)		((i)*2)
+#define	CMS_MAX_OUTPUTQ			1024
+#define	CMS_OUTPUTQ_CREDIT_CFG		(0x2000/4)
+#define	CMS_MSG_CONFIG			(0x2008/4)
+#define	CMS_MSG_ERR			(0x2010/4)
+#define	CMS_TRACE_CONFIG		(0x2018/4)
+#define	CMS_TRACE_BASE_ADDR		(0x2020/4)
+#define	CMS_TRACE_LIMIT_ADDR		(0x2028/4)
+#define	CMS_TRACE_CURRENT_ADDR		(0x2030/4)
+#define	CMS_MSG_ENDIAN_SWAP		(0x2038/4)
+
+#define	CMS_CPU_PUSHQ(node, core, thread, vc)	\
 		(((node)<<10) | ((core)<<4) | ((thread)<<2) | ((vc)<<0))
-#define CMS_POPQ(node, queue)	(((node)<<10) | (queue))
-#define CMS_IO_PUSHQ(node, queue)	(((node)<<10) | (queue))
-
-#define CMS_POPQ_QID(i)		(128+(i))
-#define CMS_POPQ_MAXQID		255
-#define CMS_PCIE0_QID(i)	(256+(i))
-#define CMS_PCIE0_MAXQID	257
-#define CMS_PCIE1_QID(i)	(258+(i))
-#define CMS_PCIE1_MAXQID	259
-#define CMS_PCIE2_QID(i)	(260+(i))
-#define CMS_PCIE2_MAXQID	261
-#define CMS_PCIE3_QID(i)	(262+(i))
-#define CMS_PCIE3_MAXQID	263
-#define CMS_DTE_QID(i)		(264+(i))
-#define CMS_DTE_MAXQID		267
-#define CMS_RSA_ECC_QID(i)	(272+(i))
-#define CMS_RSA_ECC_MAXQID	280
-#define CMS_CRYPTO_QID(i)	(281+(i))
-#define CMS_CRYPTO_MAXQID	296
-/* TODO PCI header register 0x3C says CMP starts at 297(0x129) VERIFY */
-#define CMS_CMP_QID(i)		(298+(i))
-#define CMS_CMP_MAXQID		305
-#define CMS_POE_QID(i)		(384+(i))
-#define CMS_POE_MAXQID		391
-#define CMS_NAE_QID(i)		(476+(i))
-#define CMS_NAE_MAXQID		1023
+#define	CMS_POPQ(node, queue)	(((node)<<10) | (queue))
+#define	CMS_IO_PUSHQ(node, queue)	(((node)<<10) | (queue))
 
-#define CMS_NAE_TX_VC_BASE	476
-#define CMS_NAE_TX_VC_LIMIT	999
-#define CMS_NAE_RX_VC_BASE	1000
-#define CMS_NAE_RX_VC_LIMIT	1019
-
-#define MAX_CMS_QUEUES	1024
+#define	CMS_POPQ_QID(i)		(128+(i))
 
 /* FMN Level Interrupt Type */
-#define CMS_LVL_INTR_DISABLE	0
-#define CMS_LVL_LOW_WATERMARK	1
-#define CMS_LVL_HI_WATERMARK	2
+#define	CMS_LVL_INTR_DISABLE	0
+#define	CMS_LVL_LOW_WATERMARK	1
+#define	CMS_LVL_HI_WATERMARK	2
 
 /* FMN Level interrupt trigger values */
-#define CMS_QUEUE_NON_EMPTY		0
-#define CMS_QUEUE_QUARTER_FULL		1
-#define CMS_QUEUE_HALF_FULL		2
-#define CMS_QUEUE_THREE_QUARTER_FULL	3
-#define CMS_QUEUE_FULL			4
+#define	CMS_QUEUE_NON_EMPTY		0
+#define	CMS_QUEUE_QUARTER_FULL		1
+#define	CMS_QUEUE_HALF_FULL		2
+#define	CMS_QUEUE_THREE_QUARTER_FULL	3
+#define	CMS_QUEUE_FULL			4
 
 /* FMN Timer Interrupt Type */
-#define CMS_TIMER_INTR_DISABLE	0
-#define CMS_TIMER_CONSUMER		1
-#define CMS_TIMER_PRODUCER		1
+#define	CMS_TIMER_INTR_DISABLE	0
+#define	CMS_TIMER_CONSUMER		1
+#define	CMS_TIMER_PRODUCER		1
 
 /* FMN timer interrupt trigger values */
-#define CMS_TWO_POW_EIGHT_CYCLES	0
-#define CMS_TWO_POW_TEN_CYCLES		1
-#define CMS_TWO_POW_TWELVE_CYCLES	2
-#define CMS_TWO_POW_FOURTEEN_CYCLES	3
-#define CMS_TWO_POW_SIXTEEN_CYCLES	4
-#define CMS_TWO_POW_EIGHTTEEN_CYCLES	5
-#define CMS_TWO_POW_TWENTY_CYCLES	6
-#define CMS_TWO_POW_TWENTYTWO_CYCLES	7
-
-#define CMS_QUEUE_ENA		1ULL
-#define CMS_QUEUE_DIS		0
-#define CMS_SPILL_ENA		1ULL
-#define CMS_SPILL_DIS		0
-
-#define CMS_MAX_VCPU_VC		4
+#define	CMS_TWO_POW_EIGHT_CYCLES	0
+#define	CMS_TWO_POW_TEN_CYCLES		1
+#define	CMS_TWO_POW_TWELVE_CYCLES	2
+#define	CMS_TWO_POW_FOURTEEN_CYCLES	3
+#define	CMS_TWO_POW_SIXTEEN_CYCLES	4
+#define	CMS_TWO_POW_EIGHTTEEN_CYCLES	5
+#define	CMS_TWO_POW_TWENTY_CYCLES	6
+#define	CMS_TWO_POW_TWENTYTWO_CYCLES	7
+
+#define	CMS_QUEUE_ENA		1ULL
+#define	CMS_QUEUE_DIS		0
+#define	CMS_SPILL_ENA		1ULL
+#define	CMS_SPILL_DIS		0
+
+#define	CMS_MAX_VCPU_VC		4
 
 /* Each XLP chip can hold upto 32K messages on the chip itself */
-#define CMS_ON_CHIP_MESG_SPACE	(32*1024)
-#define CMS_ON_CHIP_PER_QUEUE_SPACE	\
-		((CMS_ON_CHIP_MESG_SPACE)/(MAX_CMS_QUEUES))
-#define CMS_MAX_ONCHIP_SEGMENTS	1024
-#define CMS_MAX_SPILL_SEGMENTS_PER_QUEUE 	64
+#define	CMS_ON_CHIP_MESG_SPACE	(32*1024)
+#define	CMS_MAX_ONCHIP_SEGMENTS	1024
+#define	CMS_MAX_SPILL_SEGMENTS_PER_QUEUE 	64
 
 /* FMN Network error */
-#define CMS_ILLEGAL_DST_ERROR		0x100
-#define CMS_BIU_TIMEOUT_ERROR		0x080
-#define CMS_BIU_ERROR			0x040
-#define CMS_SPILL_FILL_UNCORRECT_ECC_ERROR	0x020
-#define CMS_SPILL_FILL_CORRECT_ECC_ERROR	0x010
-#define CMS_SPILL_UNCORRECT_ECC_ERROR	0x008
-#define CMS_SPILL_CORRECT_ECC_ERROR		0x004
-#define CMS_OUTPUTQ_UNCORRECT_ECC_ERROR	0x002
-#define CMS_OUTPUTQ_CORRECT_ECC_ERROR	0x001
+#define	CMS_ILLEGAL_DST_ERROR		0x100
+#define	CMS_BIU_TIMEOUT_ERROR		0x080
+#define	CMS_BIU_ERROR			0x040
+#define	CMS_SPILL_FILL_UNCORRECT_ECC_ERROR	0x020
+#define	CMS_SPILL_FILL_CORRECT_ECC_ERROR	0x010
+#define	CMS_SPILL_UNCORRECT_ECC_ERROR	0x008
+#define	CMS_SPILL_CORRECT_ECC_ERROR		0x004
+#define	CMS_OUTPUTQ_UNCORRECT_ECC_ERROR	0x002
+#define	CMS_OUTPUTQ_CORRECT_ECC_ERROR	0x001
 
 /* worst case, a single entry message consists of a 4 byte header
  * and an 8-byte entry = 12 bytes in total
  */
-#define CMS_SINGLE_ENTRY_MSG_SIZE	12
+#define	CMS_SINGLE_ENTRY_MSG_SIZE	12
 /* total spill memory needed for one FMN queue */
-#define CMS_PER_QUEUE_SPILL_MEM(spilltotmsgs)		\
+#define	CMS_PER_QUEUE_SPILL_MEM(spilltotmsgs)		\
 		((spilltotmsgs) * (CMS_SINGLE_ENTRY_MSG_SIZE))
-/* total spill memory needed */
-#define CMS_TOTAL_SPILL_MEM(spilltotmsgs)			\
-		((CMS_PER_QUEUE_SPILL_MEM(spilltotmsgs)) *	\
-		(MAX_CMS_QUEUES))
-/* total number of FMN messages possible in a queue */
-#define CMS_TOTAL_QUEUE_SIZE(spilltotmsgs)			\
-		((spilltotmsgs) + (CMS_ON_CHIP_PER_QUEUE_SPACE))
 
 /* FMN Src station id's */
-#define CMS_CPU0_SRC_STID		(0 << 4)
-#define CMS_CPU1_SRC_STID		(1 << 4)
-#define CMS_CPU2_SRC_STID		(2 << 4)
-#define CMS_CPU3_SRC_STID		(3 << 4)
-#define CMS_CPU4_SRC_STID		(4 << 4)
-#define CMS_CPU5_SRC_STID		(5 << 4)
-#define CMS_CPU6_SRC_STID		(6 << 4)
-#define CMS_CPU7_SRC_STID		(7 << 4)
-#define CMS_PCIE0_SRC_STID		256
-#define CMS_PCIE1_SRC_STID		258
-#define CMS_PCIE2_SRC_STID		260
-#define CMS_PCIE3_SRC_STID		262
-#define CMS_DTE_SRC_STID		264
-#define CMS_RSA_ECC_SRC_STID		272
-#define CMS_CRYPTO_SRC_STID		281
-#define CMS_CMP_SRC_STID		298
-#define CMS_POE_SRC_STID		384
-#define CMS_NAE_SRC_STID		476
-#if 0
-#define CMS_DEFAULT_CREDIT(cmstotstns,spilltotmsgs)	\
-		((CMS_TOTAL_QUEUE_SIZE(spilltotmsgs)) /	\
-		(cmstotstns))
-#endif
-#define CMS_DEFAULT_CREDIT(cmstotstns,spilltotmsgs) 8
+#define	CMS_CPU0_SRC_STID		(0 << 4)
+#define	CMS_CPU1_SRC_STID		(1 << 4)
+#define	CMS_CPU2_SRC_STID		(2 << 4)
+#define	CMS_CPU3_SRC_STID		(3 << 4)
+#define	CMS_CPU4_SRC_STID		(4 << 4)
+#define	CMS_CPU5_SRC_STID		(5 << 4)
+#define	CMS_CPU6_SRC_STID		(6 << 4)
+#define	CMS_CPU7_SRC_STID		(7 << 4)
+#define	CMS_PCIE0_SRC_STID		256
+#define	CMS_PCIE1_SRC_STID		258
+#define	CMS_PCIE2_SRC_STID		260
+#define	CMS_PCIE3_SRC_STID		262
+#define	CMS_DTE_SRC_STID		264
+#define	CMS_RSA_ECC_SRC_STID		272
+#define	CMS_CRYPTO_SRC_STID		281
+#define	CMS_CMP_SRC_STID		298
+#define	CMS_POE_SRC_STID		384
+#define	CMS_NAE_SRC_STID		476
 
 /* POPQ related defines */
-#define CMS_POPQID_START	128
-#define CMS_POPQID_END		255
+#define	CMS_POPQID_START	128
+#define	CMS_POPQID_END		255
 
-#define CMS_INT_RCVD		0x800000000000000ULL
+#define	CMS_INT_RCVD		0x800000000000000ULL
 
 #define	nlm_read_cms_reg(b, r)	nlm_read_reg64_xkphys(b,r)
 #define	nlm_write_cms_reg(b, r, v)	nlm_write_reg64_xkphys(b,r,v)
-#define nlm_get_cms_pcibase(node)	nlm_pcicfg_base(XLP_IO_CMS_OFFSET(node))
-#define nlm_get_cms_regbase(node)	nlm_xkphys_map_pcibar0(nlm_get_cms_pcibase(node))
+#define	nlm_get_cms_pcibase(node)	\
+	nlm_pcicfg_base(XLP_IO_CMS_OFFSET(node))
+#define	nlm_get_cms_regbase(node)	\
+	nlm_xkphys_map_pcibar0(nlm_get_cms_pcibase(node))
+
+#define	XLP_CMS_ON_CHIP_PER_QUEUE_SPACE(node)			\
+		((XLP_CMS_ON_CHIP_MESG_SPACE)/			\
+		(nlm_read_reg(nlm_pcibase_cms(node),		\
+		XLP_PCI_DEVINFO_REG0))
+/* total spill memory needed */
+#define	XLP_CMS_TOTAL_SPILL_MEM(node, spilltotmsgs)		\
+               ((XLP_CMS_PER_QUEUE_SPILL_MEM(spilltotmsgs)) *	\
+		(nlm_read_reg(nlm_pcibase_cms(node),		\
+		XLP_PCI_DEVINFO_REG0))
+#define	CMS_TOTAL_QUEUE_SIZE(node, spilltotmsgs)		\
+		((spilltotmsgs) + (CMS_ON_CHIP_PER_QUEUE_SPACE(node)))
 
 enum fmn_swcode {
 	FMN_SWCODE_CPU0=1,
@@ -238,7 +208,6 @@ enum fmn_swcode {
 
 extern uint64_t nlm_cms_spill_total_messages;
 extern uint32_t nlm_cms_total_stations;
-extern uint32_t cms_onchip_seg_availability[CMS_ON_CHIP_PER_QUEUE_SPACE];
 
 extern uint64_t cms_base_addr(int node);
 extern int nlm_cms_verify_credit_config (int spill_en, int tot_credit);
diff --git a/sys/mips/nlm/hal/haldefs.h b/sys/mips/nlm/hal/haldefs.h
index a8e1342..40a6e84 100644
--- a/sys/mips/nlm/hal/haldefs.h
+++ b/sys/mips/nlm/hal/haldefs.h
@@ -30,7 +30,7 @@
  */
 
 #ifndef __NLM_HAL_MMIO_H__
-#define __NLM_HAL_MMIO_H__
+#define	__NLM_HAL_MMIO_H__
 
 /*
  * This file contains platform specific memory mapped IO implementation
diff --git a/sys/mips/nlm/hal/iomap.h b/sys/mips/nlm/hal/iomap.h
index 9f4a52c..56b585b 100644
--- a/sys/mips/nlm/hal/iomap.h
+++ b/sys/mips/nlm/hal/iomap.h
@@ -30,93 +30,97 @@
  */
 
 #ifndef __NLM_HAL_IOMAP_H__
-#define __NLM_HAL_IOMAP_H__
+#define	__NLM_HAL_IOMAP_H__
 
-#define XLP_DEFAULT_IO_BASE             0x18000000
-#define NMI_BASE			0xbfc00000
+#define	XLP_DEFAULT_IO_BASE             0x18000000
+#define	NMI_BASE			0xbfc00000
 #define	XLP_IO_CLK			133333333
 
-#define XLP_PCIE_CFG_SIZE		0x1000		/* 4K */
-#define XLP_PCIE_DEV_BLK_SIZE		(8 * XLP_PCIE_CFG_SIZE)
-#define XLP_PCIE_BUS_BLK_SIZE		(256 * XLP_PCIE_DEV_BLK_SIZE)
-#define XLP_IO_SIZE			(64 << 20)	/* ECFG space size */
-#define XLP_IO_PCI_HDRSZ		0x100
-#define XLP_IO_DEV(node, dev)		((dev) + (node) * 8)
-#define XLP_HDR_OFFSET(node, bus, dev, fn)	(((bus) << 20) | \
+#define	XLP_PCIE_CFG_SIZE		0x1000		/* 4K */
+#define	XLP_PCIE_DEV_BLK_SIZE		(8 * XLP_PCIE_CFG_SIZE)
+#define	XLP_PCIE_BUS_BLK_SIZE		(256 * XLP_PCIE_DEV_BLK_SIZE)
+#define	XLP_IO_SIZE			(64 << 20)	/* ECFG space size */
+#define	XLP_IO_PCI_HDRSZ		0x100
+#define	XLP_IO_DEV(node, dev)		((dev) + (node) * 8)
+#define	XLP_HDR_OFFSET(node, bus, dev, fn)	(((bus) << 20) | \
 				((XLP_IO_DEV(node, dev)) << 15) | ((fn) << 12))
 
-#define XLP_IO_BRIDGE_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 0)
+#define	XLP_IO_BRIDGE_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 0)
 /* coherent inter chip */
-#define XLP_IO_CIC0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 1)
-#define XLP_IO_CIC1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 2)
-#define XLP_IO_CIC2_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 3)
-#define XLP_IO_PIC_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 0, 4)
-
-#define XLP_IO_PCIE_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 1, i)
-#define XLP_IO_PCIE0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 0)
-#define XLP_IO_PCIE1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 1)
-#define XLP_IO_PCIE2_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 2)
-#define XLP_IO_PCIE3_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 3)
-
-#define XLP_IO_USB_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 2, i)
-#define XLP_IO_USB_EHCI0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 0)
-#define XLP_IO_USB_OHCI0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 1)
-#define XLP_IO_USB_OHCI1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 2)
-#define XLP_IO_USB_EHCI1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 3)
-#define XLP_IO_USB_OHCI2_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 4)
-#define XLP_IO_USB_OHCI3_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 5)
-
-#define XLP_IO_NAE_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 3, 0)
-#define XLP_IO_POE_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 3, 1)
-
-#define XLP_IO_CMS_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 4, 0)
-
-#define XLP_IO_DMA_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 1)
-#define XLP_IO_SEC_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 2)
-#define XLP_IO_CMP_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 3)
-
-#define XLP_IO_UART_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 6, i)
-#define XLP_IO_UART0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 0)
-#define XLP_IO_UART1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 1)
-#define XLP_IO_I2C_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 6, 2 + i)
-#define XLP_IO_I2C0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 2)
-#define XLP_IO_I2C1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 3)
-#define XLP_IO_GPIO_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 4)
+#define	XLP_IO_CIC0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 1)
+#define	XLP_IO_CIC1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 2)
+#define	XLP_IO_CIC2_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 0, 3)
+#define	XLP_IO_PIC_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 0, 4)
+
+#define	XLP_IO_PCIE_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 1, i)
+#define	XLP_IO_PCIE0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 0)
+#define	XLP_IO_PCIE1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 1)
+#define	XLP_IO_PCIE2_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 2)
+#define	XLP_IO_PCIE3_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 1, 3)
+
+#define	XLP_IO_USB_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 2, i)
+#define	XLP_IO_USB_EHCI0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 0)
+#define	XLP_IO_USB_OHCI0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 1)
+#define	XLP_IO_USB_OHCI1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 2)
+#define	XLP_IO_USB_EHCI1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 3)
+#define	XLP_IO_USB_OHCI2_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 4)
+#define	XLP_IO_USB_OHCI3_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 2, 5)
+
+#define	XLP_IO_NAE_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 3, 0)
+#define	XLP_IO_POE_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 3, 1)
+#define	XLP_IO_SATA_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 3, 2)
+
+#define	XLP_IO_CMS_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 4, 0)
+
+#define	XLP_IO_DMA_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 0)
+#define	XLP_IO_SEC_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 1)
+#define	XLP_IO_RSA_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 2)
+#define	XLP_IO_CMP_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 5, 3)
+#define	XLP_IO_SRIO_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 5, 4)
+#define	XLP_IO_REGEX_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 5, 5)
+
+#define	XLP_IO_UART_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 6, i)
+#define	XLP_IO_UART0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 0)
+#define	XLP_IO_UART1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 1)
+#define	XLP_IO_I2C_OFFSET(node, i)	XLP_HDR_OFFSET(node, 0, 6, 2 + i)
+#define	XLP_IO_I2C0_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 2)
+#define	XLP_IO_I2C1_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 3)
+#define	XLP_IO_GPIO_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 4)
 /* system management */
-#define XLP_IO_SYS_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 6, 5)
-#define XLP_IO_JTAG_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 6)
+#define	XLP_IO_SYS_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 6, 5)
+#define	XLP_IO_JTAG_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 6, 6)
 
-#define XLP_IO_NOR_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 7, 0)
-#define XLP_IO_NAND_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 7, 1)
-#define XLP_IO_SPI_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 7, 2)
+#define	XLP_IO_NOR_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 7, 0)
+#define	XLP_IO_NAND_OFFSET(node)	XLP_HDR_OFFSET(node, 0, 7, 1)
+#define	XLP_IO_SPI_OFFSET(node)		XLP_HDR_OFFSET(node, 0, 7, 2)
 /* SD flash */
-#define XLP_IO_SD_OFFSET(node)          XLP_HDR_OFFSET(node, 0, 7, 3)
-#define XLP_IO_MMC_OFFSET(node, slot)   \
+#define	XLP_IO_SD_OFFSET(node)          XLP_HDR_OFFSET(node, 0, 7, 3)
+#define	XLP_IO_MMC_OFFSET(node, slot)   \
 		((XLP_IO_SD_OFFSET(node))+(slot*0x100)+XLP_IO_PCI_HDRSZ)
 
 /* PCI config header register id's */
-#define XLP_PCI_CFGREG0			0x00
-#define XLP_PCI_CFGREG1			0x01
-#define XLP_PCI_CFGREG2			0x02
-#define XLP_PCI_CFGREG3			0x03
-#define XLP_PCI_CFGREG4			0x04
-#define XLP_PCI_CFGREG5			0x05
-#define XLP_PCI_DEVINFO_REG0		0x30
-#define XLP_PCI_DEVINFO_REG1		0x31
-#define XLP_PCI_DEVINFO_REG2		0x32
-#define XLP_PCI_DEVINFO_REG3		0x33
-#define XLP_PCI_DEVINFO_REG4		0x34
-#define XLP_PCI_DEVINFO_REG5		0x35
-#define XLP_PCI_DEVINFO_REG6		0x36
-#define XLP_PCI_DEVINFO_REG7		0x37
-#define XLP_PCI_DEVSCRATCH_REG0		0x38
-#define XLP_PCI_DEVSCRATCH_REG1		0x39
-#define XLP_PCI_DEVSCRATCH_REG2		0x3a
-#define XLP_PCI_DEVSCRATCH_REG3		0x3b
-#define XLP_PCI_MSGSTN_REG		0x3c
-#define XLP_PCI_IRTINFO_REG		0x3d
-#define XLP_PCI_UCODEINFO_REG		0x3e
-#define XLP_PCI_SBB_WT_REG		0x3f
+#define	XLP_PCI_CFGREG0			0x00
+#define	XLP_PCI_CFGREG1			0x01
+#define	XLP_PCI_CFGREG2			0x02
+#define	XLP_PCI_CFGREG3			0x03
+#define	XLP_PCI_CFGREG4			0x04
+#define	XLP_PCI_CFGREG5			0x05
+#define	XLP_PCI_DEVINFO_REG0		0x30
+#define	XLP_PCI_DEVINFO_REG1		0x31
+#define	XLP_PCI_DEVINFO_REG2		0x32
+#define	XLP_PCI_DEVINFO_REG3		0x33
+#define	XLP_PCI_DEVINFO_REG4		0x34
+#define	XLP_PCI_DEVINFO_REG5		0x35
+#define	XLP_PCI_DEVINFO_REG6		0x36
+#define	XLP_PCI_DEVINFO_REG7		0x37
+#define	XLP_PCI_DEVSCRATCH_REG0		0x38
+#define	XLP_PCI_DEVSCRATCH_REG1		0x39
+#define	XLP_PCI_DEVSCRATCH_REG2		0x3a
+#define	XLP_PCI_DEVSCRATCH_REG3		0x3b
+#define	XLP_PCI_MSGSTN_REG		0x3c
+#define	XLP_PCI_IRTINFO_REG		0x3d
+#define	XLP_PCI_UCODEINFO_REG		0x3e
+#define	XLP_PCI_SBB_WT_REG		0x3f
 
 /* PCI IDs for SoC device */
 #define	PCI_VENDOR_NETLOGIC		0x184e
@@ -142,11 +146,50 @@
 
 #if !defined(LOCORE) && !defined(__ASSEMBLY__)
 
-#define nlm_read_pci_reg(b, r)		nlm_read_reg(b, r)
-#define nlm_write_pci_reg(b, r, v)	nlm_write_reg(b, r, v)
+#define	nlm_read_pci_reg(b, r)		nlm_read_reg(b, r)
+#define	nlm_write_pci_reg(b, r, v)	nlm_write_reg(b, r, v)
 
 extern uint64_t xlp_sys_base;
 extern uint64_t xlp_pic_base;
+
+static __inline__ int
+nlm_dev_exists(uint32_t devoffset)
+{
+	uint64_t pcibase = nlm_pcicfg_base(devoffset);
+
+	return (nlm_read_reg(pcibase, XLP_PCI_CFGREG0) != 0xffffffff);
+}
+
+static __inline__ int
+nlm_qidstart(uint64_t pcibase)
+{
+	return (nlm_read_reg(pcibase, XLP_PCI_MSGSTN_REG) & 0xffff);
+}
+
+static __inline__ int
+nlm_qnum(uint64_t pcibase)
+{
+	return (nlm_read_reg(pcibase, XLP_PCI_MSGSTN_REG) >> 16);
+}
+
+static __inline__ int
+nlm_irtstart(uint64_t pcibase)
+{
+	return (nlm_read_reg(pcibase, XLP_PCI_IRTINFO_REG) & 0xffff);
+}
+
+static __inline__ int
+nlm_irtnum(uint64_t pcibase)
+{
+	return (nlm_read_reg(pcibase, XLP_PCI_IRTINFO_REG) >> 16);
+}
+
+static __inline__ int
+nlm_uenginenum(uint64_t pcibase)
+{
+	return nlm_read_reg(pcibase, XLP_PCI_UCODEINFO_REG);
+}
+
 #endif /* !LOCORE or !__ASSEMBLY */
 
 #endif /* __NLM_HAL_IOMAP_H__ */
diff --git a/sys/mips/nlm/hal/mips-extns.h b/sys/mips/nlm/hal/mips-extns.h
index 839aa73..76dcaa6 100644
--- a/sys/mips/nlm/hal/mips-extns.h
+++ b/sys/mips/nlm/hal/mips-extns.h
@@ -30,7 +30,7 @@
  */
 
 #ifndef __NLM_MIPS_EXTNS_H__
-#define __NLM_MIPS_EXTNS_H__
+#define	__NLM_MIPS_EXTNS_H__
 
 #if !defined(LOCORE) && !defined(__ASSEMBLY__)
 static __inline__ int32_t nlm_swapw(int32_t *loc, int32_t val)
@@ -115,7 +115,7 @@ nlm_ldaddwu(unsigned int value, unsigned int *addr)
 /*
  * 32 bit read write for c0
  */
-#define read_c0_register32(reg, sel)				\
+#define	read_c0_register32(reg, sel)				\
 ({								\
 	 uint32_t __rv;						\
 	__asm__ __volatile__(					\
@@ -127,7 +127,7 @@ nlm_ldaddwu(unsigned int value, unsigned int *addr)
 	__rv;							\
  })
 
-#define write_c0_register32(reg,  sel, value)			\
+#define	write_c0_register32(reg,  sel, value)			\
 	__asm__ __volatile__(					\
 	    ".set	push\n\t"				\
 	    ".set	mips32\n\t"				\
@@ -139,7 +139,7 @@ nlm_ldaddwu(unsigned int value, unsigned int *addr)
 /*
  * On 64 bit compilation, the operations are simple
  */
-#define read_c0_register64(reg, sel)				\
+#define	read_c0_register64(reg, sel)				\
 ({								\
 	uint64_t __rv;						\
 	__asm__ __volatile__(					\
@@ -151,7 +151,7 @@ nlm_ldaddwu(unsigned int value, unsigned int *addr)
 	__rv;							\
  })
 
-#define write_c0_register64(reg,  sel, value)			\
+#define	write_c0_register64(reg,  sel, value)			\
 	__asm__ __volatile__(					\
 	    ".set	push\n\t"				\
 	    ".set	mips64\n\t"				\
@@ -163,7 +163,7 @@ nlm_ldaddwu(unsigned int value, unsigned int *addr)
 /*
  * 32 bit compilation, 64 bit values has to split 
  */
-#define read_c0_register64(reg, sel)				\
+#define	read_c0_register64(reg, sel)				\
 ({								\
 	uint32_t __high, __low;					\
 	__asm__ __volatile__(					\
@@ -179,7 +179,7 @@ nlm_ldaddwu(unsigned int value, unsigned int *addr)
 	((uint64_t)__high << 32) | __low;			\
 })
 
-#define write_c0_register64(reg, sel, value)			\
+#define	write_c0_register64(reg, sel, value)			\
 do {								\
        uint32_t __high = value >> 32;				\
        uint32_t __low = value & 0xffffffff;			\
@@ -267,8 +267,8 @@ nlm_coreid(void)
 }
 #endif
 
-#define XLP_MAX_NODES	4
-#define XLP_MAX_CORES	8
-#define XLP_MAX_THREADS	4
+#define	XLP_MAX_NODES	4
+#define	XLP_MAX_CORES	8
+#define	XLP_MAX_THREADS	4
 
 #endif
diff --git a/sys/mips/nlm/hal/mmu.h b/sys/mips/nlm/hal/mmu.h
index ebb4b5d..c74e743 100644
--- a/sys/mips/nlm/hal/mmu.h
+++ b/sys/mips/nlm/hal/mmu.h
@@ -30,7 +30,7 @@
  */
 
 #ifndef __XLP_MMU_H__
-#define __XLP_MMU_H__
+#define	__XLP_MMU_H__
 
 #include <mips/nlm/hal/mips-extns.h>
 
diff --git a/sys/mips/nlm/hal/nlm_hal.c b/sys/mips/nlm/hal/nlm_hal.c
new file mode 100644
index 0000000..049593c
--- /dev/null
+++ b/sys/mips/nlm/hal/nlm_hal.c
@@ -0,0 +1,170 @@
+/*-
+ * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE 
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * NETLOGIC_BSD */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+
+#include <mips/nlm/hal/mips-extns.h>
+#include <mips/nlm/hal/haldefs.h>
+#include <mips/nlm/hal/iomap.h>
+#include <mips/nlm/hal/sys.h>
+#include <mips/nlm/hal/pic.h>
+#include <mips/nlm/xlp.h>
+
+#include <mips/nlm/hal/uart.h>
+#include <mips/nlm/hal/mmu.h>
+#include <mips/nlm/hal/pcibus.h>
+#include <mips/nlm/hal/usb.h>
+
+int pic_irt_ehci0;
+int pic_irt_ehci1;
+int pic_irt_uart0;
+int pic_irt_uart1;
+int pic_irt_pcie_lnk0;
+int pic_irt_pcie_lnk1;
+int pic_irt_pcie_lnk2;
+int pic_irt_pcie_lnk3;
+
+uint32_t
+xlp_get_cpu_frequency(int core)
+{
+	uint64_t sysbase = nlm_get_sys_regbase(nlm_nodeid());
+	uint64_t num;
+	unsigned int pll_divf, pll_divr, dfs_div, ext_div;
+	unsigned int rstval, dfsval, denom;
+
+	rstval = nlm_read_sys_reg(sysbase, SYS_POWER_ON_RESET_CFG);
+	dfsval = nlm_read_sys_reg(sysbase, SYS_CORE_DFS_DIV_VALUE);
+	pll_divf = ((rstval >> 10) & 0x7f) + 1;
+	pll_divr = ((rstval >> 8)  & 0x3) + 1;
+	ext_div  = ((rstval >> 30) & 0x3) + 1;
+	dfs_div  = ((dfsval >> (core * 4)) & 0xf) + 1;
+
+	num = 800000000ULL * pll_divf;
+	denom = 3 * pll_divr * ext_div * dfs_div;
+	num = num/denom;
+	return (num);
+}
+
+void
+nlm_pic_irt_init(void)
+{
+	pic_irt_ehci0 = nlm_irtstart(nlm_get_usb_pcibase(nlm_nodeid(), 0));
+	pic_irt_ehci1 = nlm_irtstart(nlm_get_usb_pcibase(nlm_nodeid(), 3));
+	pic_irt_uart0 = nlm_irtstart(nlm_get_uart_pcibase(nlm_nodeid(), 0));
+	pic_irt_uart1 = nlm_irtstart(nlm_get_uart_pcibase(nlm_nodeid(), 1));
+
+	/* Hardcoding the PCIE IRT information as PIC doesn't 
+	   understand any value other than 78,79,80,81 for PCIE0/1/2/3 */
+	pic_irt_pcie_lnk0 = 78;
+	pic_irt_pcie_lnk1 = 79;
+	pic_irt_pcie_lnk2 = 80;
+	pic_irt_pcie_lnk3 = 81;
+}
+/*
+ * Find the IRQ for the link, each link has a different interrupt 
+ * at the XLP pic
+ */
+int xlp_pcie_link_irt(int link)
+{
+
+	if( (link < 0) || (link > 3))
+		return (-1);
+
+	return (pic_irt_pcie_lnk0 + link);
+}
+
+int
+xlp_irt_to_irq(int irt)
+{
+	if (irt == pic_irt_ehci0)
+	    return PIC_EHCI_0_IRQ;
+	else if (irt == pic_irt_ehci1)
+	    return PIC_EHCI_1_IRQ;
+	else if (irt == pic_irt_uart0)
+	    return PIC_UART_0_IRQ;
+	else if (irt == pic_irt_uart1)
+	    return PIC_UART_1_IRQ;
+	else if (irt == pic_irt_pcie_lnk0)
+	    return PIC_PCIE_0_IRQ;
+	else if (irt == pic_irt_pcie_lnk1)
+	    return PIC_PCIE_1_IRQ;
+	else if (irt == pic_irt_pcie_lnk2)
+	    return PIC_PCIE_2_IRQ;
+	else if (irt == pic_irt_pcie_lnk3)
+	    return PIC_PCIE_3_IRQ;
+	else {
+	    printf("Cannot find irq for IRT %d\n", irt);
+	    return 0;
+	 }
+}
+
+int
+xlp_irq_to_irt(int irq)
+{
+	switch (irq) {
+ 		case PIC_EHCI_0_IRQ :
+ 			return pic_irt_ehci0;
+ 		case PIC_EHCI_1_IRQ :
+ 			return pic_irt_ehci1;
+		case PIC_UART_0_IRQ :
+			return pic_irt_uart0;
+		case PIC_UART_1_IRQ :
+			return pic_irt_uart1;
+		case PIC_PCIE_0_IRQ :
+			return pic_irt_pcie_lnk0;
+		case PIC_PCIE_1_IRQ :
+			return pic_irt_pcie_lnk1;
+		case PIC_PCIE_2_IRQ :
+			return pic_irt_pcie_lnk2;
+		case PIC_PCIE_3_IRQ :
+			return pic_irt_pcie_lnk3;
+		default: panic("Bad IRQ %d\n", irq);
+	}
+}
+
+int
+xlp_irq_is_picintr(int irq)
+{
+	switch (irq) {
+		case PIC_MMC_IRQ : return 1;
+ 		case PIC_EHCI_0_IRQ : return 1;
+ 		case PIC_EHCI_1_IRQ : return 1;
+		case PIC_UART_0_IRQ : return 1;
+		case PIC_UART_1_IRQ : return 1;
+		case PIC_PCIE_0_IRQ : return 1;
+		case PIC_PCIE_1_IRQ : return 1;
+		case PIC_PCIE_2_IRQ : return 1;
+		case PIC_PCIE_3_IRQ : return 1;
+		default: return 0;
+	}
+}
diff --git a/sys/mips/nlm/hal/pcibus.h b/sys/mips/nlm/hal/pcibus.h
index 6275ff1..c7e5810 100644
--- a/sys/mips/nlm/hal/pcibus.h
+++ b/sys/mips/nlm/hal/pcibus.h
@@ -30,60 +30,74 @@
  */
 
 #ifndef __XLP_PCIBUS_H__
-#define __XLP_PCIBUS_H__
+#define	__XLP_PCIBUS_H__
 
-#define MSI_MIPS_ADDR_BASE              0xfee00000
+#define	MSI_MIPS_ADDR_BASE              0xfee00000
 /* MSI support */
-#define MSI_MIPS_ADDR_DEST		0x000ff000
-#define MSI_MIPS_ADDR_RH		0x00000008
-#define MSI_MIPS_ADDR_RH_OFF		0x00000000
-#define MSI_MIPS_ADDR_RH_ON		0x00000008
-#define MSI_MIPS_ADDR_DM		0x00000004
-#define MSI_MIPS_ADDR_DM_PHYSICAL	0x00000000
-#define MSI_MIPS_ADDR_DM_LOGICAL	0x00000004
+#define	MSI_MIPS_ADDR_DEST		0x000ff000
+#define	MSI_MIPS_ADDR_RH		0x00000008
+#define	MSI_MIPS_ADDR_RH_OFF		0x00000000
+#define	MSI_MIPS_ADDR_RH_ON		0x00000008
+#define	MSI_MIPS_ADDR_DM		0x00000004
+#define	MSI_MIPS_ADDR_DM_PHYSICAL	0x00000000
+#define	MSI_MIPS_ADDR_DM_LOGICAL	0x00000004
 
 /* Fields in data for Intel MSI messages. */
-#define MSI_MIPS_DATA_TRGRMOD		0x00008000	/* Trigger mode */
-#define MSI_MIPS_DATA_TRGREDG		0x00000000	/* edge */
-#define MSI_MIPS_DATA_TRGRLVL		0x00008000	/* level */
+#define	MSI_MIPS_DATA_TRGRMOD		0x00008000	/* Trigger mode */
+#define	MSI_MIPS_DATA_TRGREDG		0x00000000	/* edge */
+#define	MSI_MIPS_DATA_TRGRLVL		0x00008000	/* level */
 
-#define MSI_MIPS_DATA_LEVEL		0x00004000	/* Polarity. */
-#define MSI_MIPS_DATA_DEASSERT		0x00000000
-#define MSI_MIPS_DATA_ASSERT		0x00004000
+#define	MSI_MIPS_DATA_LEVEL		0x00004000	/* Polarity. */
+#define	MSI_MIPS_DATA_DEASSERT		0x00000000
+#define	MSI_MIPS_DATA_ASSERT		0x00004000
 
-#define MSI_MIPS_DATA_DELMOD		0x00000700	/* Delivery Mode */
-#define MSI_MIPS_DATA_DELFIXED		0x00000000	/* fixed */
-#define MSI_MIPS_DATA_DELLOPRI		0x00000100	/* lowest priority */
+#define	MSI_MIPS_DATA_DELMOD		0x00000700	/* Delivery Mode */
+#define	MSI_MIPS_DATA_DELFIXED		0x00000000	/* fixed */
+#define	MSI_MIPS_DATA_DELLOPRI		0x00000100	/* lowest priority */
 
-#define MSI_MIPS_DATA_INTVEC		0x000000ff
+#define	MSI_MIPS_DATA_INTVEC		0x000000ff
+
+#define	PCIE_BRIDGE_CMD		0x1
+#define	PCIE_BRIDGE_MSI_CAP	0x14
+#define	PCIE_BRIDGE_MSI_ADDRL	0x15
+#define	PCIE_BRIDGE_MSI_ADDRH	0x16
+#define	PCIE_BRIDGE_MSI_DATA	0x17
+
+/* XLP Global PCIE configuration space registers */
+#define	PCIE_MSI_STATUS		0x25A
+#define	PCIE_MSI_EN		0x25B
+#define	PCIE_INT_EN0		0x261
+
+/* PCIE_MSI_EN */
+#define	PCIE_MSI_VECTOR_INT_EN		0xFFFFFFFF
+
+/* PCIE_INT_EN0 */
+#define	PCIE_MSI_INT_EN			(1 << 9)
+
+#if !defined(LOCORE) && !defined(__ASSEMBLY__)
+
+#define	nlm_read_pcie_reg(b, r)		nlm_read_reg(b, r)
+#define	nlm_write_pcie_reg(b, r, v)	nlm_write_reg(b, r, v)
+#define	nlm_get_pcie_base(node, inst)	\
+				nlm_pcicfg_base(XLP_IO_PCIE_OFFSET(node, inst))
+#define	nlm_get_pcie_regbase(node, inst)	\
+				(nlm_get_pcie_base(node, inst) + XLP_IO_PCI_HDRSZ)
 
 /*
  * Build Intel MSI message and data values from a source.  AMD64 systems
  * seem to be compatible, so we use the same function for both.
  */
-#define MIPS_MSI_ADDR(cpu)					       \
+#define	MIPS_MSI_ADDR(cpu)					       \
         (MSI_MIPS_ADDR_BASE | (cpu) << 12 |			       \
 	 MSI_MIPS_ADDR_RH_OFF | MSI_MIPS_ADDR_DM_PHYSICAL)
 
-#define MIPS_MSI_DATA(irq)					       \
+#define	MIPS_MSI_DATA(irq)					       \
         (MSI_MIPS_DATA_TRGRLVL | MSI_MIPS_DATA_DELFIXED |	       \
 	 MSI_MIPS_DATA_ASSERT | (irq))
 
-#define PCIE_BRIDGE_CMD		0x1
-#define PCIE_BRIDGE_MSI_CAP	0x14
-#define PCIE_BRIDGE_MSI_ADDRL	0x15
-#define PCIE_BRIDGE_MSI_ADDRH	0x16
-#define PCIE_BRIDGE_MSI_DATA	0x17
-
-/* XLP Global PCIE configuration space registers */
-#define PCIE_MSI_STATUS		0x25A
-#define PCIE_MSI_EN		0x25B
-#define PCIE_INT_EN0		0x261
-
-/* PCIE_MSI_EN */
-#define PCIE_MSI_VECTOR_INT_EN		0xFFFFFFFF
-
-/* PCIE_INT_EN0 */
-#define PCIE_MSI_INT_EN			(1 << 9)
+#endif
 
+#ifndef LOCORE
+int xlp_pcie_link_irt(int link);
+#endif
 #endif /* __XLP_PCIBUS_H__ */
diff --git a/sys/mips/nlm/hal/pic.h b/sys/mips/nlm/hal/pic.h
index efc676e..7313e5c 100644
--- a/sys/mips/nlm/hal/pic.h
+++ b/sys/mips/nlm/hal/pic.h
@@ -30,234 +30,163 @@
  */
 
 #ifndef _NLM_HAL_PIC_H
-#define _NLM_HAL_PIC_H
+#define	_NLM_HAL_PIC_H
 
 /* PIC Specific registers */
-#define PIC_CTRL                0x00
+#define	PIC_CTRL                0x00
 
 /* PIC control register defines */
-#define PIC_CTRL_ITV		32 /* interrupt timeout value */
-#define PIC_CTRL_ICI		19 /* ICI interrupt timeout enable */
-#define PIC_CTRL_ITE		18 /* interrupt timeout enable */
-#define PIC_CTRL_STE		10 /* system timer interrupt enable */
-#define PIC_CTRL_WWR1		8  /* watchdog 1 wraparound count for reset */
-#define PIC_CTRL_WWR0		6  /* watchdog 0 wraparound count for reset */
-#define PIC_CTRL_WWN1		4  /* watchdog 1 wraparound count for NMI */
-#define PIC_CTRL_WWN0		2  /* watchdog 0 wraparound count for NMI */
-#define PIC_CTRL_WTE		0  /* watchdog timer enable */
+#define	PIC_CTRL_ITV		32 /* interrupt timeout value */
+#define	PIC_CTRL_ICI		19 /* ICI interrupt timeout enable */
+#define	PIC_CTRL_ITE		18 /* interrupt timeout enable */
+#define	PIC_CTRL_STE		10 /* system timer interrupt enable */
+#define	PIC_CTRL_WWR1		8  /* watchdog 1 wraparound count for reset */
+#define	PIC_CTRL_WWR0		6  /* watchdog 0 wraparound count for reset */
+#define	PIC_CTRL_WWN1		4  /* watchdog 1 wraparound count for NMI */
+#define	PIC_CTRL_WWN0		2  /* watchdog 0 wraparound count for NMI */
+#define	PIC_CTRL_WTE		0  /* watchdog timer enable */
 
 /* PIC Status register defines */
-#define PIC_ICI_STATUS		33 /* ICI interrupt timeout status */
-#define PIC_ITE_STATUS		32 /* interrupt timeout status */
-#define PIC_STS_STATUS		4  /* System timer interrupt status */
-#define PIC_WNS_STATUS		2  /* NMI status for watchdog timers */
-#define PIC_WIS_STATUS		0  /* Interrupt status for watchdog timers */
+#define	PIC_ICI_STATUS		33 /* ICI interrupt timeout status */
+#define	PIC_ITE_STATUS		32 /* interrupt timeout status */
+#define	PIC_STS_STATUS		4  /* System timer interrupt status */
+#define	PIC_WNS_STATUS		2  /* NMI status for watchdog timers */
+#define	PIC_WIS_STATUS		0  /* Interrupt status for watchdog timers */
 
 /* PIC IPI control register offsets */
-#define PIC_IPICTRL_NMI		32
-#define PIC_IPICTRL_RIV		20 /* received interrupt vector */
-#define PIC_IPICTRL_IDB		16 /* interrupt destination base */
-#define PIC_IPICTRL_DTE		 0 /* interrupt destination thread enables */
+#define	PIC_IPICTRL_NMI		32
+#define	PIC_IPICTRL_RIV		20 /* received interrupt vector */
+#define	PIC_IPICTRL_IDB		16 /* interrupt destination base */
+#define	PIC_IPICTRL_DTE		 0 /* interrupt destination thread enables */
 
 /* PIC IRT register offsets */
-#define PIC_IRT_ENABLE		31
-#define PIC_IRT_NMI		29
-#define PIC_IRT_SCH		28 /* Scheduling scheme */
-#define PIC_IRT_RVEC		20 /* Interrupt receive vectors */
-#define PIC_IRT_DT		19 /* Destination type */
-#define PIC_IRT_DB		16 /* Destination base */
-#define PIC_IRT_DTE		0  /* Destination thread enables */
-
-#define PIC_BYTESWAP		0x02
-#define PIC_STATUS		0x04
-#define PIC_INTR_TIMEOUT	0x06
-#define PIC_ICI0_INTR_TIMEOUT	0x08
-#define PIC_ICI1_INTR_TIMEOUT	0x0a
-#define PIC_ICI2_INTR_TIMEOUT	0x0c
-#define PIC_IPI_CTL		0x0e
-#define PIC_INT_ACK		0x10
-#define PIC_INT_PENDING0	0x12
-#define PIC_INT_PENDING1	0x14
-#define PIC_INT_PENDING2	0x16
-
-#define PIC_WDOG0_MAXVAL	0x18
-#define PIC_WDOG0_COUNT		0x1a
-#define PIC_WDOG0_ENABLE0	0x1c
-#define PIC_WDOG0_ENABLE1	0x1e
-#define PIC_WDOG0_BEATCMD	0x20
-#define PIC_WDOG0_BEAT0		0x22
-#define PIC_WDOG0_BEAT1		0x24
-
-#define PIC_WDOG1_MAXVAL	0x26
-#define PIC_WDOG1_COUNT		0x28
-#define PIC_WDOG1_ENABLE0	0x2a
-#define PIC_WDOG1_ENABLE1	0x2c
-#define PIC_WDOG1_BEATCMD	0x2e
-#define PIC_WDOG1_BEAT0		0x30
-#define PIC_WDOG1_BEAT1		0x32
-
-#define PIC_WDOG_MAXVAL(i)	(PIC_WDOG0_MAXVAL + ((i) ? 7 : 0))
-#define PIC_WDOG_COUNT(i)	(PIC_WDOG0_COUNT + ((i) ? 7 : 0))
-#define PIC_WDOG_ENABLE0(i)	(PIC_WDOG0_ENABLE0 + ((i) ? 7 : 0))
-#define PIC_WDOG_ENABLE1(i)	(PIC_WDOG0_ENABLE1 + ((i) ? 7 : 0))
-#define PIC_WDOG_BEATCMD(i)	(PIC_WDOG0_BEATCMD + ((i) ? 7 : 0))
-#define PIC_WDOG_BEAT0(i)	(PIC_WDOG0_BEAT0 + ((i) ? 7 : 0))
-#define PIC_WDOG_BEAT1(i)	(PIC_WDOG0_BEAT1 + ((i) ? 7 : 0))
-
-#define PIC_TIMER0_MAXVAL	0x34
-#define PIC_TIMER1_MAXVAL	0x36
-#define PIC_TIMER2_MAXVAL	0x38
-#define PIC_TIMER3_MAXVAL	0x3a
-#define PIC_TIMER4_MAXVAL	0x3c
-#define PIC_TIMER5_MAXVAL	0x3e
-#define PIC_TIMER6_MAXVAL	0x40
-#define PIC_TIMER7_MAXVAL	0x42
-#define PIC_TIMER_MAXVAL(i)	(PIC_TIMER0_MAXVAL + ((i) * 2))
-
-#define PIC_TIMER0_COUNT	0x44
-#define PIC_TIMER1_COUNT	0x46
-#define PIC_TIMER2_COUNT	0x48
-#define PIC_TIMER3_COUNT	0x4a
-#define PIC_TIMER4_COUNT	0x4c
-#define PIC_TIMER5_COUNT	0x4e
-#define PIC_TIMER6_COUNT	0x50
-#define PIC_TIMER7_COUNT	0x52
-#define PIC_TIMER_COUNT(i)	(PIC_TIMER0_COUNT + ((i) * 2))
-
-#define PIC_ITE0_N0_N1		0x54
-#define PIC_ITE1_N0_N1		0x58
-#define PIC_ITE2_N0_N1		0x5c
-#define PIC_ITE3_N0_N1		0x60
-#define PIC_ITE4_N0_N1		0x64
-#define PIC_ITE5_N0_N1		0x68
-#define PIC_ITE6_N0_N1		0x6c
-#define PIC_ITE7_N0_N1		0x70
-#define PIC_ITE_N0_N1(i)	(PIC_ITE0_N0_N1 + ((i) * 4))
-
-#define PIC_ITE0_N2_N3		0x56
-#define PIC_ITE1_N2_N3		0x5a
-#define PIC_ITE2_N2_N3		0x5e
-#define PIC_ITE3_N2_N3		0x62
-#define PIC_ITE4_N2_N3		0x66
-#define PIC_ITE5_N2_N3		0x6a
-#define PIC_ITE6_N2_N3		0x6e
-#define PIC_ITE7_N2_N3		0x72
-#define PIC_ITE_N2_N3(i)		(PIC_ITE0_N2_N3 + ((i) * 4))
-
-#define PIC_IRT0		0x74
-#define PIC_IRT(i)		(PIC_IRT0 + ((i) * 2))
-
-#define TIMER_CYCLES_MAXVAL	0xffffffffffffffffULL
+#define	PIC_IRT_ENABLE		31
+#define	PIC_IRT_NMI		29
+#define	PIC_IRT_SCH		28 /* Scheduling scheme */
+#define	PIC_IRT_RVEC		20 /* Interrupt receive vectors */
+#define	PIC_IRT_DT		19 /* Destination type */
+#define	PIC_IRT_DB		16 /* Destination base */
+#define	PIC_IRT_DTE		0  /* Destination thread enables */
+
+#define	PIC_BYTESWAP		0x02
+#define	PIC_STATUS		0x04
+#define	PIC_INTR_TIMEOUT	0x06
+#define	PIC_ICI0_INTR_TIMEOUT	0x08
+#define	PIC_ICI1_INTR_TIMEOUT	0x0a
+#define	PIC_ICI2_INTR_TIMEOUT	0x0c
+#define	PIC_IPI_CTL		0x0e
+#define	PIC_INT_ACK		0x10
+#define	PIC_INT_PENDING0	0x12
+#define	PIC_INT_PENDING1	0x14
+#define	PIC_INT_PENDING2	0x16
+
+#define	PIC_WDOG0_MAXVAL	0x18
+#define	PIC_WDOG0_COUNT		0x1a
+#define	PIC_WDOG0_ENABLE0	0x1c
+#define	PIC_WDOG0_ENABLE1	0x1e
+#define	PIC_WDOG0_BEATCMD	0x20
+#define	PIC_WDOG0_BEAT0		0x22
+#define	PIC_WDOG0_BEAT1		0x24
+
+#define	PIC_WDOG1_MAXVAL	0x26
+#define	PIC_WDOG1_COUNT		0x28
+#define	PIC_WDOG1_ENABLE0	0x2a
+#define	PIC_WDOG1_ENABLE1	0x2c
+#define	PIC_WDOG1_BEATCMD	0x2e
+#define	PIC_WDOG1_BEAT0		0x30
+#define	PIC_WDOG1_BEAT1		0x32
+
+#define	PIC_WDOG_MAXVAL(i)	(PIC_WDOG0_MAXVAL + ((i) ? 7 : 0))
+#define	PIC_WDOG_COUNT(i)	(PIC_WDOG0_COUNT + ((i) ? 7 : 0))
+#define	PIC_WDOG_ENABLE0(i)	(PIC_WDOG0_ENABLE0 + ((i) ? 7 : 0))
+#define	PIC_WDOG_ENABLE1(i)	(PIC_WDOG0_ENABLE1 + ((i) ? 7 : 0))
+#define	PIC_WDOG_BEATCMD(i)	(PIC_WDOG0_BEATCMD + ((i) ? 7 : 0))
+#define	PIC_WDOG_BEAT0(i)	(PIC_WDOG0_BEAT0 + ((i) ? 7 : 0))
+#define	PIC_WDOG_BEAT1(i)	(PIC_WDOG0_BEAT1 + ((i) ? 7 : 0))
+
+#define	PIC_TIMER0_MAXVAL	0x34
+#define	PIC_TIMER1_MAXVAL	0x36
+#define	PIC_TIMER2_MAXVAL	0x38
+#define	PIC_TIMER3_MAXVAL	0x3a
+#define	PIC_TIMER4_MAXVAL	0x3c
+#define	PIC_TIMER5_MAXVAL	0x3e
+#define	PIC_TIMER6_MAXVAL	0x40
+#define	PIC_TIMER7_MAXVAL	0x42
+#define	PIC_TIMER_MAXVAL(i)	(PIC_TIMER0_MAXVAL + ((i) * 2))
+
+#define	PIC_TIMER0_COUNT	0x44
+#define	PIC_TIMER1_COUNT	0x46
+#define	PIC_TIMER2_COUNT	0x48
+#define	PIC_TIMER3_COUNT	0x4a
+#define	PIC_TIMER4_COUNT	0x4c
+#define	PIC_TIMER5_COUNT	0x4e
+#define	PIC_TIMER6_COUNT	0x50
+#define	PIC_TIMER7_COUNT	0x52
+#define	PIC_TIMER_COUNT(i)	(PIC_TIMER0_COUNT + ((i) * 2))
+
+#define	PIC_ITE0_N0_N1		0x54
+#define	PIC_ITE1_N0_N1		0x58
+#define	PIC_ITE2_N0_N1		0x5c
+#define	PIC_ITE3_N0_N1		0x60
+#define	PIC_ITE4_N0_N1		0x64
+#define	PIC_ITE5_N0_N1		0x68
+#define	PIC_ITE6_N0_N1		0x6c
+#define	PIC_ITE7_N0_N1		0x70
+#define	PIC_ITE_N0_N1(i)	(PIC_ITE0_N0_N1 + ((i) * 4))
+
+#define	PIC_ITE0_N2_N3		0x56
+#define	PIC_ITE1_N2_N3		0x5a
+#define	PIC_ITE2_N2_N3		0x5e
+#define	PIC_ITE3_N2_N3		0x62
+#define	PIC_ITE4_N2_N3		0x66
+#define	PIC_ITE5_N2_N3		0x6a
+#define	PIC_ITE6_N2_N3		0x6e
+#define	PIC_ITE7_N2_N3		0x72
+#define	PIC_ITE_N2_N3(i)		(PIC_ITE0_N2_N3 + ((i) * 4))
+
+#define	PIC_IRT0		0x74
+#define	PIC_IRT(i)		(PIC_IRT0 + ((i) * 2))
+
+#define	TIMER_CYCLES_MAXVAL	0xffffffffffffffffULL
 
 /*
  *    IRT Map
  */
-#define PIC_NUM_IRTS		160
-
-#define PIC_IRT_WD_0_INDEX	0
-#define PIC_IRT_WD_1_INDEX	1
-#define PIC_IRT_WD_NMI_0_INDEX	2
-#define PIC_IRT_WD_NMI_1_INDEX	3
-#define PIC_IRT_TIMER_0_INDEX	4
-#define PIC_IRT_TIMER_1_INDEX	5
-#define PIC_IRT_TIMER_2_INDEX	6
-#define PIC_IRT_TIMER_3_INDEX	7
-#define PIC_IRT_TIMER_4_INDEX	8
-#define PIC_IRT_TIMER_5_INDEX	9
-#define PIC_IRT_TIMER_6_INDEX	10
-#define PIC_IRT_TIMER_7_INDEX	11
-#define PIC_IRT_CLOCK_INDEX	PIC_IRT_TIMER_7_INDEX
-#define PIC_IRT_TIMER_INDEX(num)	((num) + PIC_IRT_TIMER_0_INDEX)
-
-
-/* 11 and 12 */
-#define PIC_NUM_MSG_Q_IRTS	32
-#define PIC_IRT_MSG_Q0_INDEX	12
-#define PIC_IRT_MSG_Q_INDEX(qid)	((qid) + PIC_IRT_MSG_Q0_INDEX)
-/* 12 to 43 */
-#define PIC_IRT_MSG_0_INDEX	44
-#define PIC_IRT_MSG_1_INDEX	45
-/* 44 and 45 */
-#define PIC_NUM_PCIE_MSIX_IRTS	32
-#define PIC_IRT_PCIE_MSIX_0_INDEX	46
-#define PIC_IRT_PCIE_MSIX_INDEX(num)	((num) + PIC_IRT_PCIE_MSIX_0_INDEX)
-/* 46 to 77 */
-#define PIC_NUM_PCIE_LINK_IRTS		4
-#define PIC_IRT_PCIE_LINK_0_INDEX	78
-#define PIC_IRT_PCIE_LINK_1_INDEX	79
-#define PIC_IRT_PCIE_LINK_2_INDEX	80
-#define PIC_IRT_PCIE_LINK_3_INDEX	81
-#define PIC_IRT_PCIE_LINK_INDEX(num)	((num) + PIC_IRT_PCIE_LINK_0_INDEX)
-/* 78 to 81 */
-#define PIC_NUM_NA_IRTS			32
-/* 82 to 113 */
-#define PIC_IRT_NA_0_INDEX		82
-#define PIC_IRT_NA_INDEX(num)		((num) + PIC_IRT_NA_0_INDEX)
-#define PIC_IRT_POE_INDEX		114
-
-#define PIC_NUM_USB_IRTS		6
-#define PIC_IRT_USB_0_INDEX		115
-#define PIC_IRT_EHCI_0_INDEX		115
-#define PIC_IRT_EHCI_1_INDEX		118
-#define PIC_IRT_USB_INDEX(num)		((num) + PIC_IRT_USB_0_INDEX)
-/* 115 to 120 */
-#define PIC_IRT_GDX_INDEX		121
-#define PIC_IRT_SEC_INDEX		122
-#define PIC_IRT_RSA_INDEX		123
-
-#define PIC_NUM_COMP_IRTS		4
-#define PIC_IRT_COMP_0_INDEX		124
-#define PIC_IRT_COMP_INDEX(num)		((num) + PIC_IRT_COMP_0_INDEX)
-/* 124 to 127 */
-#define PIC_IRT_GBU_INDEX		128
-#define PIC_IRT_ICC_0_INDEX		129 /* ICC - Inter Chip Coherency */
-#define PIC_IRT_ICC_1_INDEX		130
-#define PIC_IRT_ICC_2_INDEX		131
-#define PIC_IRT_CAM_INDEX		132
-#define PIC_IRT_UART_0_INDEX		133
-#define PIC_IRT_UART_1_INDEX		134
-#define PIC_IRT_I2C_0_INDEX		135
-#define PIC_IRT_I2C_1_INDEX		136
-#define PIC_IRT_SYS_0_INDEX		137
-#define PIC_IRT_SYS_1_INDEX		138
-#define PIC_IRT_JTAG_INDEX		139
-#define PIC_IRT_PIC_INDEX		140
-#define PIC_IRT_NBU_INDEX		141
-#define PIC_IRT_TCU_INDEX		142
-#define PIC_IRT_GCU_INDEX		143 /* GBC - Global Coherency */
-#define PIC_IRT_DMC_0_INDEX		144
-#define PIC_IRT_DMC_1_INDEX		145
-
-#define PIC_NUM_GPIO_IRTS		4
-#define PIC_IRT_GPIO_0_INDEX		146
-#define PIC_IRT_GPIO_INDEX(num)		((num) + PIC_IRT_GPIO_0_INDEX)
-
-/* 146 to 149 */
-#define PIC_IRT_NOR_INDEX		150
-#define PIC_IRT_NAND_INDEX		151
-#define PIC_IRT_SPI_INDEX		152
-#define PIC_IRT_MMC_INDEX		153
-
-#define PIC_CLOCK_TIMER			7
-#define PIC_IRQ_BASE			8
+#define	PIC_IRT_WD_0_INDEX	0
+#define	PIC_IRT_WD_1_INDEX	1
+#define	PIC_IRT_WD_NMI_0_INDEX	2
+#define	PIC_IRT_WD_NMI_1_INDEX	3
+#define	PIC_IRT_TIMER_0_INDEX	4
+#define	PIC_IRT_TIMER_1_INDEX	5
+#define	PIC_IRT_TIMER_2_INDEX	6
+#define	PIC_IRT_TIMER_3_INDEX	7
+#define	PIC_IRT_TIMER_4_INDEX	8
+#define	PIC_IRT_TIMER_5_INDEX	9
+#define	PIC_IRT_TIMER_6_INDEX	10
+#define	PIC_IRT_TIMER_7_INDEX	11
+#define	PIC_IRT_CLOCK_INDEX	PIC_IRT_TIMER_7_INDEX
+#define	PIC_IRT_TIMER_INDEX(num)	((num) + PIC_IRT_TIMER_0_INDEX)
+
+#define	PIC_CLOCK_TIMER			7
+#define	PIC_IRQ_BASE			8
 
 #if !defined(LOCORE) && !defined(__ASSEMBLY__)
 
-#define PIC_IRT_FIRST_IRQ		(PIC_IRQ_BASE)
-#define PIC_IRT_LAST_IRQ		63
-#define PIC_IRQ_IS_IRT(irq)		((irq) >= PIC_IRT_FIRST_IRQ)
+#define	PIC_IRT_FIRST_IRQ		(PIC_IRQ_BASE)
+#define	PIC_IRT_LAST_IRQ		63
+#define	PIC_IRQ_IS_IRT(irq)		((irq) >= PIC_IRT_FIRST_IRQ)
 
 /*
  *   Misc
  */
-#define PIC_IRT_VALID			1
-#define PIC_LOCAL_SCHEDULING		1
-#define PIC_GLOBAL_SCHEDULING		0
+#define	PIC_IRT_VALID			1
+#define	PIC_LOCAL_SCHEDULING		1
+#define	PIC_GLOBAL_SCHEDULING		0
 
-#define nlm_read_pic_reg(b, r)	nlm_read_reg64(b, r)
-#define nlm_write_pic_reg(b, r, v) nlm_write_reg64(b, r, v)
-#define nlm_get_pic_pcibase(node) nlm_pcicfg_base(XLP_IO_PIC_OFFSET(node))
-#define nlm_get_pic_regbase(node) (nlm_get_pic_pcibase(node) + XLP_IO_PCI_HDRSZ)
+#define	nlm_read_pic_reg(b, r)	nlm_read_reg64(b, r)
+#define	nlm_write_pic_reg(b, r, v) nlm_write_reg64(b, r, v)
+#define	nlm_get_pic_pcibase(node) nlm_pcicfg_base(XLP_IO_PIC_OFFSET(node))
+#define	nlm_get_pic_regbase(node) (nlm_get_pic_pcibase(node) + XLP_IO_PCI_HDRSZ)
 
 /* IRT and h/w interrupt routines */
 static inline int
@@ -363,15 +292,22 @@ nlm_pic_write_timer(uint64_t base, int timer, uint64_t value)
 static inline void
 nlm_pic_set_timer(uint64_t base, int timer, uint64_t value, int irq, int cpu)
 {
-	uint64_t pic_ctrl = nlm_read_pic_reg(base, PIC_CTRL);
-	int en;
-
-	en = (irq > 0);
+	uint64_t pic_ctrl;
+	int en, nmi;
+
+	en = nmi = 0;
+	if (irq > 0)
+		en = 1;
+	else if (irq < 0) {
+		en = nmi = 1;
+		irq = -irq;
+	}
 	nlm_write_pic_reg(base, PIC_TIMER_MAXVAL(timer), value);
 	nlm_pic_write_irt_direct(base, PIC_IRT_TIMER_INDEX(timer),
-		en, 0, 0, irq, cpu);
+		en, nmi, 0, irq, cpu);
 
 	/* enable the timer */
+	pic_ctrl = nlm_read_pic_reg(base, PIC_CTRL);
 	pic_ctrl |= (1 << (PIC_CTRL_STE + timer));
 	nlm_write_pic_reg(base, PIC_CTRL, pic_ctrl);
 }
diff --git a/sys/mips/nlm/hal/uart.h b/sys/mips/nlm/hal/uart.h
index ce11c84..e7e68d0 100644
--- a/sys/mips/nlm/hal/uart.h
+++ b/sys/mips/nlm/hal/uart.h
@@ -30,69 +30,69 @@
  */
 
 #ifndef __XLP_HAL_UART_H__
-#define __XLP_HAL_UART_H__
+#define	__XLP_HAL_UART_H__
 
 /* UART Specific registers */
-#define UART_RX_DATA		0x00
-#define UART_TX_DATA		0x00
+#define	UART_RX_DATA		0x00
+#define	UART_TX_DATA		0x00
 
-#define UART_INT_EN		0x01
-#define UART_INT_ID		0x02
-#define UART_FIFO_CTL		0x02
-#define UART_LINE_CTL		0x03
-#define UART_MODEM_CTL		0x04
-#define UART_LINE_STS		0x05
-#define UART_MODEM_STS		0x06
+#define	UART_INT_EN		0x01
+#define	UART_INT_ID		0x02
+#define	UART_FIFO_CTL		0x02
+#define	UART_LINE_CTL		0x03
+#define	UART_MODEM_CTL		0x04
+#define	UART_LINE_STS		0x05
+#define	UART_MODEM_STS		0x06
 
-#define UART_DIVISOR0		0x00
-#define UART_DIVISOR1		0x01
+#define	UART_DIVISOR0		0x00
+#define	UART_DIVISOR1		0x01
 
-#define BASE_BAUD		(XLP_IO_CLK/16)
-#define BAUD_DIVISOR(baud)	(BASE_BAUD / baud)
+#define	BASE_BAUD		(XLP_IO_CLK/16)
+#define	BAUD_DIVISOR(baud)	(BASE_BAUD / baud)
 
 /* LCR mask values */
-#define LCR_5BITS		0x00
-#define LCR_6BITS		0x01
-#define LCR_7BITS		0x02
-#define LCR_8BITS		0x03
-#define LCR_STOPB		0x04
-#define LCR_PENAB		0x08
-#define LCR_PODD		0x00
-#define LCR_PEVEN		0x10
-#define LCR_PONE		0x20
-#define LCR_PZERO		0x30
-#define LCR_SBREAK		0x40
-#define LCR_EFR_ENABLE		0xbf
-#define LCR_DLAB		0x80
+#define	LCR_5BITS		0x00
+#define	LCR_6BITS		0x01
+#define	LCR_7BITS		0x02
+#define	LCR_8BITS		0x03
+#define	LCR_STOPB		0x04
+#define	LCR_PENAB		0x08
+#define	LCR_PODD		0x00
+#define	LCR_PEVEN		0x10
+#define	LCR_PONE		0x20
+#define	LCR_PZERO		0x30
+#define	LCR_SBREAK		0x40
+#define	LCR_EFR_ENABLE		0xbf
+#define	LCR_DLAB		0x80
 
 /* MCR mask values */
-#define MCR_DTR			0x01
-#define MCR_RTS			0x02
-#define MCR_DRS			0x04
-#define MCR_IE			0x08
-#define MCR_LOOPBACK		0x10
+#define	MCR_DTR			0x01
+#define	MCR_RTS			0x02
+#define	MCR_DRS			0x04
+#define	MCR_IE			0x08
+#define	MCR_LOOPBACK		0x10
 
 /* FCR mask values */
-#define FCR_RCV_RST		0x02
-#define FCR_XMT_RST		0x04
-#define FCR_RX_LOW		0x00
-#define FCR_RX_MEDL		0x40
-#define FCR_RX_MEDH		0x80
-#define FCR_RX_HIGH		0xc0
+#define	FCR_RCV_RST		0x02
+#define	FCR_XMT_RST		0x04
+#define	FCR_RX_LOW		0x00
+#define	FCR_RX_MEDL		0x40
+#define	FCR_RX_MEDH		0x80
+#define	FCR_RX_HIGH		0xc0
 
 /* IER mask values */
-#define IER_ERXRDY		0x1
-#define IER_ETXRDY		0x2
-#define IER_ERLS		0x4
-#define IER_EMSC		0x8
+#define	IER_ERXRDY		0x1
+#define	IER_ETXRDY		0x2
+#define	IER_ERLS		0x4
+#define	IER_EMSC		0x8
 
 #if !defined(LOCORE) && !defined(__ASSEMBLY__)
 
 #define	nlm_read_uart_reg(b, r)		nlm_read_reg(b, r)
 #define	nlm_write_uart_reg(b, r, v)	nlm_write_reg(b, r, v)
-#define nlm_get_uart_pcibase(node, inst)	\
+#define	nlm_get_uart_pcibase(node, inst)	\
 		nlm_pcicfg_base(XLP_IO_UART_OFFSET(node, inst))
-#define nlm_get_uart_regbase(node, inst)	\
+#define	nlm_get_uart_regbase(node, inst)	\
 			(nlm_get_uart_pcibase(node, inst) + XLP_IO_PCI_HDRSZ)
 
 static inline void
diff --git a/sys/mips/nlm/mpreset.S b/sys/mips/nlm/mpreset.S
index 95b2440..b35df63 100644
--- a/sys/mips/nlm/mpreset.S
+++ b/sys/mips/nlm/mpreset.S
@@ -45,6 +45,42 @@
 	.set	noreorder
 	.set	mips64
 
+#define	MFCR(rt,rs)	.word	((0x1c<<26)|((rs)<<21)|((rt)<<16)|(0x18))
+#define	MTCR(rt,rs)	.word	((0x1c<<26)|((rs)<<21)|((rt)<<16)|(0x19))
+/*
+ * We need to do this to really flush the dcache before splitting it
+ */
+.macro flush_l1_dcache
+	.set	push
+	.set	noreorder
+	li	$8, LSU_DEBUG_DATA0 /* use register number to handle */
+	li	$9, LSU_DEBUG_ADDR  /* different ABIs */
+	li	t2, 0
+	li	t3, 0x200
+1:
+	sll	v0, t2, 5
+	MTCR(0, 8)
+	ori	v1, v0, 0x3
+	MTCR(3, 9)
+2:
+	MFCR(3, 9)
+	andi	v1, 0x1
+	bnez	v1, 2b
+	nop
+	MTCR(0, 8)
+	ori	v1, v0, 0x7
+	MTCR(3, 9)
+3:
+	MFCR(3, 9)
+	andi	v1, 0x1
+	bnez	v1, 3b
+	nop
+	addi	t2, 1
+	bne	t3, t2, 1b
+	nop
+	.set pop
+.endm
+
 VECTOR(XLPResetEntry, unknown)
 	mfc0	t0, MIPS_COP_0_STATUS
 	li	t1, 0x80000
@@ -111,11 +147,14 @@ LEAF(xlp_enable_threads)
 	sd	t1, 72(sp)
 	sd	gp, 80(sp)
 	sd	ra, 88(sp)
+
+	flush_l1_dcache
+
 	/* Use register number to work in o32 and n32 */
 	li	$9, ((CPU_BLOCKID_MAP << 8) | MAP_THREADMODE)
 	move	$8, a0	
 	sync
-	.word	0x71280019	/* mtcr    t0, t1*/
+	MTCR(8, 9)
 	mfc0	t0, MIPS_COP_0_PRID, 1
 	andi	t0, 0x3
 	beqz	t0, 2f
@@ -134,8 +173,8 @@ LEAF(xlp_enable_threads)
 	 */
 	li	$9, 0x400
 	li	$8, 0
-	.word	0x71280019	/* mtcr    $8, $9*/
-	.word	0x000000c0	/* ehb */
+	MTCR(8, 9)
+	sll	zero,3		/* ehb */
 #endif
 	dmfc0	t0, $4, 2	/* SP saved in UserLocal */
 	ori	sp, t0, 0x7
diff --git a/sys/mips/nlm/msgring.h b/sys/mips/nlm/msgring.h
index e57a0fe..cf5f66e 100644
--- a/sys/mips/nlm/msgring.h
+++ b/sys/mips/nlm/msgring.h
@@ -29,15 +29,12 @@
  * $FreeBSD$
  */
 
+#define	CMS_DEFAULT_CREDIT	50
+
 extern uint32_t xlp_msg_thread_mask;
 typedef void (*msgring_handler)(int, int, int, int, struct nlm_fmn_msg *, void *);
 int register_msgring_handler(int startb, int endb, msgring_handler action,
 		                    void *arg);
-int xlp_handle_msg_vc(int vc, int max_msgs);
-void xlp_msgring_cpu_init(uint32_t);
-void xlp_msgring_config(void);
-void xlp_cpu_msgring_handler(int bucket, int size, int code, int stid,
-		    struct nlm_fmn_msg *msg, void *data);
-
-void nlm_cms_credit_setup(int credit);
-void xlp_msgring_iodi_config(void);
+int xlp_handle_msg_vc(u_int vcmask, int max_msgs);
+void xlp_msgring_cpu_init(int, int, int);
+void xlp_cms_enable_intr(int , int , int , int);
diff --git a/sys/mips/nlm/std.xlp b/sys/mips/nlm/std.xlp
index 01c9245..cef006b 100644
--- a/sys/mips/nlm/std.xlp
+++ b/sys/mips/nlm/std.xlp
@@ -2,3 +2,6 @@
 files	"../nlm/files.xlp"
 cpu	CPU_NLM
 
+# Devices needed always
+device		uart
+device		pci
diff --git a/sys/mips/nlm/uart_bus_xlp_iodi.c b/sys/mips/nlm/uart_bus_xlp_iodi.c
deleted file mode 100644
index ca68d68..0000000
--- a/sys/mips/nlm/uart_bus_xlp_iodi.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/*-
- * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights
- * reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- * 
- * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE 
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- *
- * NETLOGIC_BSD */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/rman.h>
-
-#include <machine/bus.h>
-#include <machine/resource.h>
-
-#include <mips/nlm/hal/haldefs.h>
-#include <mips/nlm/hal/iomap.h>
-#include <mips/nlm/hal/uart.h>
-
-#include <dev/uart/uart.h>
-#include <dev/uart/uart_bus.h>
-#include <dev/uart/uart_cpu.h>
-
-static int uart_iodi_probe(device_t dev);
-
-static device_method_t uart_iodi_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe, uart_iodi_probe),
-	DEVMETHOD(device_attach, uart_bus_attach),
-	DEVMETHOD(device_detach, uart_bus_detach),
-	{0, 0}
-};
-
-static driver_t uart_iodi_driver = {
-	uart_driver_name,
-	uart_iodi_methods,
-	sizeof(struct uart_softc),
-};
-
-extern SLIST_HEAD(uart_devinfo_list, uart_devinfo) uart_sysdevs;
-
-static int
-uart_iodi_probe(device_t dev)
-{
-	struct uart_softc *sc;
-
-	sc = device_get_softc(dev);
-	sc->sc_sysdev = SLIST_FIRST(&uart_sysdevs);
-	sc->sc_class = &uart_ns8250_class;
-	bcopy(&sc->sc_sysdev->bas, &sc->sc_bas, sizeof(sc->sc_bas));
-	sc->sc_sysdev->bas.bst = rmi_bus_space;
-	sc->sc_sysdev->bas.bsh = nlm_get_uart_regbase(0, 0);
-	sc->sc_bas.bst = rmi_bus_space;
-	sc->sc_bas.bsh = nlm_get_uart_regbase(0, 0);
-	/* regshft = 2, rclk = 66000000, rid = 0, chan = 0 */
-	return (uart_bus_probe(dev, 2, 133000000, 0, 0));
-}
-
-DRIVER_MODULE(uart, iodi, uart_iodi_driver, uart_devclass, 0, 0);
diff --git a/sys/mips/nlm/uart_pci_xlp.c b/sys/mips/nlm/uart_pci_xlp.c
index 16c795c..4f75081 100644
--- a/sys/mips/nlm/uart_pci_xlp.c
+++ b/sys/mips/nlm/uart_pci_xlp.c
@@ -69,13 +69,11 @@ static int
 uart_soc_probe(device_t dev)
 {
 	struct uart_softc *sc;
-	uint64_t ubase;
 
 	if (pci_get_vendor(dev) != PCI_VENDOR_NETLOGIC ||
 	    pci_get_device(dev) != PCI_DEVICE_ID_NLM_UART)
 		return (ENXIO);
 
-	ubase = nlm_get_uart_regbase(0, 0);
 	sc = device_get_softc(dev);
 	sc->sc_class = &uart_ns8250_class;
 	device_set_desc(dev, "Netlogic SoC UART");
diff --git a/sys/mips/nlm/usb_init.c b/sys/mips/nlm/usb_init.c
index 32a6165..cb4d762 100644
--- a/sys/mips/nlm/usb_init.c
+++ b/sys/mips/nlm/usb_init.c
@@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$");
 
 #include <mips/nlm/hal/haldefs.h>
 #include <mips/nlm/hal/iomap.h>
-#include <mips/nlm/hal/cpuinfo.h>
 #include <mips/nlm/hal/usb.h>
 
 #include <mips/nlm/xlp.h>
diff --git a/sys/mips/nlm/xlp.h b/sys/mips/nlm/xlp.h
index b400891..c80c80a 100644
--- a/sys/mips/nlm/xlp.h
+++ b/sys/mips/nlm/xlp.h
@@ -31,7 +31,8 @@
 
 #ifndef __NLM_XLP_H__
 #define __NLM_XLP_H__
-#include <mips/nlm/hal/pic.h>
+#include <mips/nlm/hal/mips-extns.h>
+#include <mips/nlm/hal/iomap.h>
 
 #define PIC_UART_0_IRQ	9
 #define PIC_UART_1_IRQ	10
@@ -43,7 +44,17 @@
 
 #define PIC_EHCI_0_IRQ	39 
 #define PIC_EHCI_1_IRQ	42 
-#define PIC_MMC_IRQ		43
+#define PIC_MMC_IRQ	43
+
+/* XLP 8xx/4xx A0, A1, A2 CPU COP0 PRIDs */
+#define	CHIP_PROCESSOR_ID_XLP_8XX		0x10
+#define	CHIP_PROCESSOR_ID_XLP_3XX		0x11
+
+/* Revision id's */
+#define	XLP_REVISION_A0				0x00
+#define	XLP_REVISION_A1				0x01
+#define	XLP_REVISION_A2				0x02
+#define	XLP_REVISION_B0				0x03
 
 #ifndef LOCORE
 /*
@@ -58,74 +69,25 @@ extern int xlp_hwtid_to_cpuid[];
 #ifdef SMP
 extern void xlp_enable_threads(int code);
 #endif
+uint32_t xlp_get_cpu_frequency(int core);
+void nlm_pic_irt_init(void);
+int xlp_irt_to_irq(int irt);
+int xlp_irq_to_irt(int irq);
+int xlp_irq_is_picintr(int irq);
 
-static __inline__ int
-xlp_irt_to_irq(int irt)
+static __inline int nlm_is_xlp3xx(void)
 {
-	switch (irt) {
-		case PIC_IRT_MMC_INDEX :
-			return PIC_MMC_IRQ;
-		case PIC_IRT_EHCI_0_INDEX :
- 			return PIC_EHCI_0_IRQ;
- 		case PIC_IRT_EHCI_1_INDEX :
- 			return PIC_EHCI_1_IRQ;
-		case PIC_IRT_UART_0_INDEX :
-			return PIC_UART_0_IRQ;
-		case PIC_IRT_UART_1_INDEX :
-			return PIC_UART_1_IRQ;
-		case PIC_IRT_PCIE_LINK_0_INDEX :
-			return PIC_PCIE_0_IRQ;
-		case PIC_IRT_PCIE_LINK_1_INDEX :
-			return PIC_PCIE_1_IRQ;
-		case PIC_IRT_PCIE_LINK_2_INDEX :
-			return PIC_PCIE_2_IRQ;
-		case PIC_IRT_PCIE_LINK_3_INDEX :
-			return PIC_PCIE_3_IRQ;
-		default: panic("Bad IRT %d\n", irt);
-	}
-}
+	int prid = (mips_rd_prid() >> 8) & 0xff;
 
-static __inline__ int
-xlp_irq_to_irt(int irq)
-{
-	switch (irq) {
-		case PIC_MMC_IRQ :
-			return PIC_IRT_MMC_INDEX;
- 		case PIC_EHCI_0_IRQ :
- 			return PIC_IRT_EHCI_0_INDEX;
- 		case PIC_EHCI_1_IRQ :
- 			return PIC_IRT_EHCI_1_INDEX;
-		case PIC_UART_0_IRQ :
-			return PIC_IRT_UART_0_INDEX;
-		case PIC_UART_1_IRQ :
-			return PIC_IRT_UART_1_INDEX;
-		case PIC_PCIE_0_IRQ :
-			return PIC_IRT_PCIE_LINK_0_INDEX;
-		case PIC_PCIE_1_IRQ :
-			return PIC_IRT_PCIE_LINK_1_INDEX;
-		case PIC_PCIE_2_IRQ :
-			return PIC_IRT_PCIE_LINK_2_INDEX;
-		case PIC_PCIE_3_IRQ :
-			return PIC_IRT_PCIE_LINK_3_INDEX;
-		default: panic("Bad IRQ %d\n", irq);
-	}
+	return (prid == CHIP_PROCESSOR_ID_XLP_3XX);
 }
 
-static __inline__ int
-xlp_irq_is_picintr(int irq)
+static __inline int nlm_is_xlp8xx(void)
 {
-	switch (irq) {
-		case PIC_MMC_IRQ : return 1;
- 		case PIC_EHCI_0_IRQ : return 1;
- 		case PIC_EHCI_1_IRQ : return 1;
-		case PIC_UART_0_IRQ : return 1;
-		case PIC_UART_1_IRQ : return 1;
-		case PIC_PCIE_0_IRQ : return 1;
-		case PIC_PCIE_1_IRQ : return 1;
-		case PIC_PCIE_2_IRQ : return 1;
-		case PIC_PCIE_3_IRQ : return 1;
-		default: return 0;
-	}
+	int prid = (mips_rd_prid() >> 8) & 0xff;
+
+	return (prid == CHIP_PROCESSOR_ID_XLP_8XX);
 }
+
 #endif /* LOCORE */
 #endif /* __NLM_XLP_H__ */
diff --git a/sys/mips/nlm/xlp_machdep.c b/sys/mips/nlm/xlp_machdep.c
index 5724df0..0744f5b 100644
--- a/sys/mips/nlm/xlp_machdep.c
+++ b/sys/mips/nlm/xlp_machdep.c
@@ -80,11 +80,13 @@ __FBSDID("$FreeBSD$");
 #include <mips/nlm/hal/mmu.h>
 #include <mips/nlm/hal/bridge.h>
 #include <mips/nlm/hal/cpucontrol.h>
+#include <mips/nlm/hal/cop2.h>
 
 #include <mips/nlm/clock.h>
 #include <mips/nlm/interrupt.h>
 #include <mips/nlm/board.h>
 #include <mips/nlm/xlp.h>
+#include <mips/nlm/msgring.h>
 
 #ifdef FDT
 #include <dev/fdt/fdt_common.h>
@@ -137,19 +139,30 @@ xlp_setup_core(void)
 static void 
 xlp_setup_mmu(void)
 {
+	uint32_t pagegrain;
 
-	nlm_setup_extended_pagemask(0); /* pagemask = 0 for 4K pages */
-	nlm_large_variable_tlb_en(0);
-	nlm_extended_tlb_en(1);
-	nlm_mmu_setup(0, 0, 0);
+	if (nlm_threadid() == 0) {
+		nlm_setup_extended_pagemask(0);
+		nlm_large_variable_tlb_en(1);
+		nlm_extended_tlb_en(1);
+		nlm_mmu_setup(0, 0, 0);
+	}
+
+	/* Enable no-read, no-exec, large-physical-address */
+	pagegrain = mips_rd_pagegrain();
+	pagegrain |= (1 << 31)	|	/* RIE */
+	    (1 << 30)		|	/* XIE */
+	    (1 << 29);			/* ELPA */
+	mips_wr_pagegrain(pagegrain);
 }
 
 static void
 xlp_parse_mmu_options(void)
 {
-	int i, j, k;
+	uint64_t sysbase;
 	uint32_t cpu_map = xlp_hw_thread_mask;
-	uint32_t core0_thr_mask, core_thr_mask;
+	uint32_t core0_thr_mask, core_thr_mask, cpu_rst_mask;
+	int i, j, k;
 
 #ifdef SMP
 	if (cpu_map == 0)
@@ -183,41 +196,39 @@ xlp_parse_mmu_options(void)
 		goto unsupp;
 	}
 
-	/* Verify other cores CPU masks */
+	/* Take out cores which do not exist on chip */
+	sysbase = nlm_get_sys_regbase(0);
+	cpu_rst_mask = nlm_read_sys_reg(sysbase, SYS_CPU_RESET) & 0xff;
 	for (i = 1; i < XLP_MAX_CORES; i++) {
-		core_thr_mask = (cpu_map >> (i*4)) & 0xf;
-		if (core_thr_mask) {
-			if (core_thr_mask != core0_thr_mask)
-				goto unsupp; 
-			xlp_ncores++;
-		}
+		if ((cpu_rst_mask & (1 << i)) == 0)
+			cpu_map &= ~(0xfu << (4 * i));
+	}
+
+ 	/* Verify other cores' CPU masks */
+	for (i = 1; i < XLP_MAX_CORES; i++) {
+		core_thr_mask = (cpu_map >> (4 * i)) & 0xf;
+		if (core_thr_mask == 0)
+	       		continue;
+		if (core_thr_mask != core0_thr_mask)
+			goto unsupp; 
+		xlp_ncores++;
 	}
 
 	xlp_hw_thread_mask = cpu_map;
 	/* setup hardware processor id to cpu id mapping */
 	for (i = 0; i< MAXCPU; i++)
 		xlp_cpuid_to_hwtid[i] = 
-		    xlp_hwtid_to_cpuid [i] = -1;
+		    xlp_hwtid_to_cpuid[i] = -1;
 	for (i = 0, k = 0; i < XLP_MAX_CORES; i++) {
-		if (((cpu_map >> (i*4)) & 0xf) == 0)
+		if (((cpu_map >> (i * 4)) & 0xf) == 0)
 			continue;
 		for (j = 0; j < xlp_threads_per_core; j++) {
-			xlp_cpuid_to_hwtid[k] = i*4 + j;
-			xlp_hwtid_to_cpuid[i*4 + j] = k;
+			xlp_cpuid_to_hwtid[k] = i * 4 + j;
+			xlp_hwtid_to_cpuid[i * 4 + j] = k;
 			k++;
 		}
 	}
 
-#ifdef SMP
-	/* 
-	 * We will enable the other threads in core 0 here
-	 * so that the TLB and cache info is correct when
-	 * mips_init runs
-	 */
-	xlp_enable_threads(xlp_mmuval);
-#endif
-	/* setup for the startup core */
-	xlp_setup_mmu();
 	return;
 
 unsupp:
@@ -285,10 +296,11 @@ xlp_bootargs_init(__register_t arg)
 	OF_interpret("perform-fixup", 0);
 
 	chosen = OF_finddevice("/chosen");
-	if (OF_getprop(chosen, "cpumask", &mask, sizeof(mask)) == 0)
+	if (OF_getprop(chosen, "cpumask", &mask, sizeof(mask)) != -1) {
 		xlp_hw_thread_mask = mask;
+	}
 
-	if (OF_getprop(chosen, "bootargs", buf, sizeof(buf)) == 0)
+	if (OF_getprop(chosen, "bootargs", buf, sizeof(buf)) != -1)
 		xlp_parse_bootargs(buf);
 }
 #else
@@ -303,6 +315,14 @@ xlp_bootargs_init(__register_t arg)
 	char	*p, *v, *n;
 	uint32_t mask;
 
+	/*
+	 * provide backward compat for passing cpu mask as arg
+	 */
+	if (arg & 1) {
+		xlp_hw_thread_mask = arg;
+		return;
+	}
+
 	p = (void *)(intptr_t)arg;
 	while (*p != '\0') {
 		strlcpy(buf, p, sizeof(buf));
@@ -368,14 +388,17 @@ xlp_pic_init(void)
 		2000,                   /* quality (adjusted in code) */
 	};
         int i;
+	int maxirt;
 
 	xlp_pic_base = nlm_get_pic_regbase(0);  /* TOOD: Add other nodes */
-        printf("Initializing PIC...@%jx\n", (uintmax_t)xlp_pic_base);
+	maxirt = nlm_read_reg(nlm_get_pic_pcibase(nlm_nodeid()),
+	    XLP_PCI_DEVINFO_REG0);
+        printf("Initializing PIC...@%jx %d IRTs\n", (uintmax_t)xlp_pic_base,
+	    maxirt);
 	/* Bind all PIC irqs to cpu 0 */
-        for(i = 0; i < PIC_NUM_IRTS; i++) {
-                nlm_pic_write_irt(xlp_pic_base, i, 0, 0, 1, 0,
-		    1, 0, 0x1);
-        }
+        for (i = 0; i < maxirt; i++)
+	    nlm_pic_write_irt(xlp_pic_base, i, 0, 0, 1, 0,
+	    1, 0, 0x1);
 
 	nlm_pic_set_timer(xlp_pic_base, PIC_CLOCK_TIMER, ~0ULL, 0, 0);
 	platform_timecounter = &pic_timecounter;
@@ -469,30 +492,12 @@ xlp_mem_init(void)
 	phys_avail[j] = phys_avail[j + 1] = 0;
 
 	/* copy phys_avail to dump_avail */
-	for(i = 0; i <= j + 1; i++) 
+	for (i = 0; i <= j + 1; i++) 
 		dump_avail[i] = phys_avail[i];
 
 	realmem = physmem = btoc(physsz);
 }
 
-static uint32_t
-xlp_get_cpu_frequency(void)
-{
-	uint64_t  sysbase = nlm_get_sys_regbase(0);
-	unsigned int pll_divf, pll_divr, dfs_div, num, denom;
-	uint32_t val;
-	       
-	val = nlm_read_sys_reg(sysbase, SYS_POWER_ON_RESET_CFG);
-	pll_divf = (val >> 10) & 0x7f;
-	pll_divr = (val >> 8)  & 0x3;
-	dfs_div  = (val >> 17) & 0x3;
-
-	num = pll_divf + 1;
-	denom = 3 * (pll_divr + 1) * (1<< (dfs_div + 1));
-	val = 800000000ULL * num / denom;
-	return (val);
-}
-
 void
 platform_start(__register_t a0 __unused,
     __register_t a1 __unused,
@@ -506,11 +511,12 @@ platform_start(__register_t a0 __unused,
 	/* initialize console so that we have printf */
 	boothowto |= (RB_SERIAL | RB_MULTIPLE);	/* Use multiple consoles */
 
+	nlm_pic_irt_init(); /* complete before interrupts or console init */
 	init_static_kenv(boot1_env, sizeof(boot1_env));
 	xlp_bootargs_init(a0);
 
 	/* clockrate used by delay, so initialize it here */
-	xlp_cpu_frequency = xlp_get_cpu_frequency();
+	xlp_cpu_frequency = xlp_get_cpu_frequency(0);
 	cpu_clock = xlp_cpu_frequency / 1000000;
 	mips_timer_early_init(xlp_cpu_frequency);
 
@@ -525,10 +531,18 @@ platform_start(__register_t a0 __unused,
 
 	bcopy(XLPResetEntry, (void *)MIPS_RESET_EXC_VEC,
               XLPResetEntryEnd - XLPResetEntry);
-
-	/*
-	 * MIPS generic init 
+#ifdef SMP
+	/* 
+	 * We will enable the other threads in core 0 here
+	 * so that the TLB and cache info is correct when
+	 * mips_init runs
 	 */
+	xlp_enable_threads(xlp_mmuval);
+#endif
+	/* setup for the startup core */
+	xlp_setup_mmu();
+
+	/* MIPS generic init */
 	mips_init();
 
 	/*
@@ -568,7 +582,7 @@ platform_reset(void)
 	uint64_t sysbase = nlm_get_sys_regbase(0);
 
 	nlm_write_sys_reg(sysbase, SYS_CHIP_RESET, 1);
-	for(;;)
+	for( ; ; )
 		__asm __volatile("wait");
 }
 
@@ -637,7 +651,6 @@ platform_init_ap(int cpuid)
 	if (thr == 0) {
 		xlp_setup_core();
 		xlp_enable_threads(xlp_mmuval);
-		xlp_setup_mmu();
 	} else {
 		/*
 		 * FIXME busy wait here eats too many cycles, especially 
@@ -648,6 +661,7 @@ platform_init_ap(int cpuid)
 		thr_unblock[thr] = 0;
 	}
 
+	xlp_setup_mmu();
 	stat = mips_rd_status();
 	KASSERT((stat & MIPS_SR_INT_IE) == 0,
 	    ("Interrupts enabled in %s!", __func__));
diff --git a/sys/mips/nlm/xlp_pci.c b/sys/mips/nlm/xlp_pci.c
index 09e214e..bb98c80 100644
--- a/sys/mips/nlm/xlp_pci.c
+++ b/sys/mips/nlm/xlp_pci.c
@@ -109,7 +109,6 @@ xlp_pci_init_resources(void)
 	if (rman_init(&emul_rman)
 	    || rman_manage_region(&emul_rman, 0x18000000ULL, 0x18ffffffULL))
 		panic("pci_init_resources emul_rman");
-
 }
 
 static int
@@ -117,7 +116,6 @@ xlp_pcib_probe(device_t dev)
 {
 
 	device_set_desc(dev, "XLP PCI bus");
-
 	xlp_pci_init_resources();
 	return (0);
 }
@@ -184,7 +182,6 @@ xlp_pcib_read_config(device_t dev, u_int b, u_int s, u_int f,
 		    (dev == 6 || dev == 2 || dev == 7))
 			data |= 0x1 << 8;	/* Fake int pin */
 	}
-
 	if (width == 1)
 		return ((data >> ((reg & 3) << 3)) & 0xff);
 	else if (width == 2)
@@ -220,7 +217,6 @@ xlp_pcib_write_config(device_t dev, u_int b, u_int s, u_int f,
 	}
 
 	nlm_write_pci_reg(cfgaddr, regindex, data);
-
 	return;
 }
 
@@ -253,8 +249,10 @@ xlp_pcie_link(device_t pcib, device_t dev)
 	device_t parent, tmp;
 
 	/* find the lane on which the slot is connected to */
+#if 0 /* Debug */
 	printf("xlp_pcie_link : bus %s dev %s\n", device_get_nameunit(pcib),
 		device_get_nameunit(dev));
+#endif
 	tmp = dev;
 	while (1) {
 		parent = device_get_parent(tmp);
@@ -269,20 +267,6 @@ xlp_pcie_link(device_t pcib, device_t dev)
 	return (pci_get_function(tmp));
 }
 
-/*
- * Find the IRQ for the link, each link has a different interrupt 
- * at the XLP pic
- */
-static int
-xlp_pcie_link_irt(int link)
-{
-
-	if( (link < 0) || (link > 3))
-		return (-1);
-
-	return PIC_IRT_PCIE_LINK_INDEX(link);
-}
-
 static int
 xlp_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs)
 {
@@ -346,7 +330,7 @@ bridge_pcie_ack(int irq)
 	node = nlm_nodeid();
 	reg = PCIE_MSI_STATUS;
 
-	switch(irq) {
+	switch (irq) {
 		case PIC_PCIE_0_IRQ:
 			base = nlm_pcicfg_base(XLP_IO_PCIE0_OFFSET(node));
 			break;
@@ -364,7 +348,6 @@ bridge_pcie_ack(int irq)
 	}
 
 	nlm_write_pci_reg(base, reg, 0xFFFFFFFF);
-
 	return;
 }
 
@@ -375,7 +358,6 @@ mips_platform_pci_setup_intr(device_t dev, device_t child,
 {
 	int error = 0;
 	int xlpirq;
-	int node,base,val,link;
 	void *extra_ack;
 
 	error = rman_activate_resource(irq);
@@ -399,6 +381,9 @@ mips_platform_pci_setup_intr(device_t dev, device_t child,
 	 * link, and assign the link interrupt to the device interrupt
 	 */
 	if (xlpirq >= 64) {
+		int node, val, link;
+		uint64_t base;
+
 		xlpirq -= 64;
 		if (xlpirq % 32 != 0)
 			return (0);
@@ -438,6 +423,7 @@ mips_platform_pci_setup_intr(device_t dev, device_t child,
 		xlpirq = xlp_irt_to_irq(xlpirq);
 	}
 	/* Set all irqs to CPU 0 for now */
+	printf("set up intr %d->%d(%d)\n", xlp_irq_to_irt(xlpirq), xlpirq, (int)rman_get_start(irq));
 	nlm_pic_write_irt_direct(xlp_pic_base, xlp_irq_to_irt(xlpirq), 1, 0,
 				 PIC_LOCAL_SCHEDULING, xlpirq, 0);
 	extra_ack = NULL;
@@ -468,41 +454,28 @@ assign_soc_resource(device_t child, int type, u_long *startp, u_long *endp,
 	int devid = pci_get_device(child);
 	int inst = pci_get_function(child);
 	int node = pci_get_slot(child) / 8;
+	int dev = pci_get_slot(child) % 8;
 
 	*rm = NULL;
 	*va = 0;
 	*bst = 0;
-	switch (devid) {
-	case PCI_DEVICE_ID_NLM_UART:
-		switch (type) {
-		case SYS_RES_IRQ:
-			*startp = *endp = PIC_UART_0_IRQ + inst;
-			*countp = 1;
-			break;
-		case SYS_RES_MEMORY: 
+	if (type == SYS_RES_IRQ) {
+		printf("%s: %d %d %d : start %d, end %d\n", __func__,
+				node, dev, inst, (int)*startp, (int)*endp);
+	} else if (type == SYS_RES_MEMORY) {
+		switch (devid) {
+		case PCI_DEVICE_ID_NLM_UART:
 			*va = nlm_get_uart_regbase(node, inst);
 			*startp = MIPS_KSEG1_TO_PHYS(va);
 			*countp = 0x100;
 			*rm = &emul_rman;
 			*bst = uart_bus_space_mem;
 			break;
-		};
-		break;
-
-	case PCI_DEVICE_ID_NLM_EHCI:
-		if (type == SYS_RES_IRQ) {
-			if (inst == 0)
-				*startp = *endp = PIC_EHCI_0_IRQ;
-			else if (inst == 3)
-				*startp = *endp = PIC_EHCI_1_IRQ;
-			else
-				device_printf(child, "bad instance %d\n", inst);
-
-			*countp = 1; 
 		}
-		break;
-	}
 
+	} else
+		printf("Unknown type %d in req for [%x%x]\n",
+			type, devid, inst);
 	/* default to rmi_bus_space for SoC resources */
 	if (type == SYS_RES_MEMORY && *bst == 0)
 		*bst = rmi_bus_space;
@@ -614,12 +587,40 @@ mips_pci_route_interrupt(device_t bus, device_t dev, int pin)
 	/*
 	 * Validate requested pin number.
 	 */
-	device_printf(bus, "route  %s %d", device_get_nameunit(dev), pin);
 	if ((pin < 1) || (pin > 4))
 		return (255);
 
-	link = xlp_pcie_link(bus, dev);
-	irt = xlp_pcie_link_irt(link);
+	device_printf(bus, "route  %s %d", device_get_nameunit(dev), pin);
+	if (pci_get_bus(dev) == 0 &&
+	    pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC) {
+		/* SoC devices */
+		uint64_t pcibase;
+		int f, n, d, num;
+
+		f = pci_get_function(dev);
+		n = pci_get_slot(dev) / 8;
+		d = pci_get_slot(dev) % 8;
+
+		/*
+		 * For PCIe links, return link IRT, for other SoC devices
+		 * get the IRT from its PCIe header
+		 */
+		if (d == 1) {
+			irt = xlp_pcie_link_irt(f);
+		} else {
+			pcibase = nlm_pcicfg_base(XLP_HDR_OFFSET(n, 0, d, f));
+			irt = nlm_irtstart(pcibase);
+			num = nlm_irtnum(pcibase);
+			if (num != 1)
+				device_printf(bus, "[%d:%d:%d] Error %d IRQs\n",
+				    n, d, f, num);
+		}
+	} else {
+		/* Regular PCI devices */
+		link = xlp_pcie_link(bus, dev);
+		irt = xlp_pcie_link_irt(link);
+	}
+
 	if (irt != -1)
 		return (xlp_irt_to_irq(irt));
 
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index f51b560..e47daba 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -90,6 +90,7 @@ SUBDIR=	${_3dfx} \
 	en \
 	${_ep} \
 	${_epic} \
+	esp \
 	${_et} \
 	${_ex} \
 	${_exca} \
@@ -137,7 +138,7 @@ SUBDIR=	${_3dfx} \
 	${_io} \
 	ipdivert \
 	${_ipfilter} \
-	ipfw \
+	${_ipfw} \
 	ipfw_nat \
 	${_ipmi} \
 	ip_mroute_mod \
@@ -255,6 +256,7 @@ SUBDIR=	${_3dfx} \
 	${_pst} \
 	pty  \
 	puc \
+	${_qlxgb} \
 	ral \
 	ralfw \
 	${_random} \
@@ -274,6 +276,7 @@ SUBDIR=	${_3dfx} \
 	sem \
 	send \
 	sf \
+	${_sfxge} \
 	sge \
 	siba_bwn \
 	siftr \
@@ -315,6 +318,7 @@ SUBDIR=	${_3dfx} \
 	usb \
 	utopia \
 	${_vesa} \
+	${_virtio} \
 	vge \
 	vkbd \
 	${_vmm} \
@@ -379,6 +383,10 @@ _if_carp=	if_carp
 _ipfilter=	ipfilter
 .endif
 
+.if ${MK_INET_SUPPORT} != "no" || defined(ALL_MODULES)
+_ipfw=		ipfw
+.endif
+
 .if ${MK_NETGRAPH} != "no" || defined(ALL_MODULES)
 _netgraph=	netgraph
 .endif
@@ -532,6 +540,7 @@ _padlock=	padlock
 _s3=		s3
 _twa=		twa
 _vesa=		vesa
+_virtio=	virtio
 _x86bios=	x86bios
 .elif ${MACHINE} == "pc98"
 _canbepm=	canbepm
@@ -619,10 +628,12 @@ _opensolaris=	opensolaris
 _padlock=	padlock
 .endif
 _pccard=	pccard
+_qlxgb=		qlxgb
 _rdma=		rdma	
 _s3=		s3
 _safe=		safe
 _scsi_low=	scsi_low
+_sfxge=		sfxge
 _smbfs=		smbfs
 _sound=		sound
 _speaker=	speaker
@@ -631,6 +642,7 @@ _sppp=		sppp
 _tpm=		tpm
 _twa=		twa
 _vesa=		vesa
+_virtio=	virtio
 _vmm=		vmm
 _vxge=  	vxge
 _x86bios=	x86bios
diff --git a/sys/modules/ath/Makefile b/sys/modules/ath/Makefile
index 49f7fa6..50571aa 100644
--- a/sys/modules/ath/Makefile
+++ b/sys/modules/ath/Makefile
@@ -75,7 +75,7 @@ SRCS+=	ar2425.c
 SRCS+=	ar5413.c
 
 #
-# AR5416, AR9130, AR9160, AR9220, AR9280, AR9285 support.
+# AR5416, AR9130, AR9160, AR9220, AR9280, AR9285, AR9287 support.
 
 # Note enabling this support requires defining AH_SUPPORT_AR5416
 # in opt_ah.h so the 11n tx/rx descriptor format is handled.
diff --git a/sys/modules/ath_ahb/Makefile b/sys/modules/ath_ahb/Makefile
index 698514a..9d31841 100644
--- a/sys/modules/ath_ahb/Makefile
+++ b/sys/modules/ath_ahb/Makefile
@@ -34,7 +34,7 @@
 
 KMOD=	if_ath_ahb
 SRCS=	if_ath_ahb.c
-SRCS+=	device_if.h bus_if.h
+SRCS+=	device_if.h bus_if.h opt_wlan.h opt_ath.h opt_ah.h
 
 CFLAGS+=  -I. -I${.CURDIR}/../../dev/ath -I${.CURDIR}/../../dev/ath/ath_hal
 
diff --git a/sys/modules/ath_pci/Makefile b/sys/modules/ath_pci/Makefile
index 9c06b49..5f9e0a1 100644
--- a/sys/modules/ath_pci/Makefile
+++ b/sys/modules/ath_pci/Makefile
@@ -34,7 +34,7 @@
 
 KMOD=	if_ath_pci
 SRCS=	if_ath_pci.c
-SRCS+=	device_if.h bus_if.h pci_if.h opt_ah.h
+SRCS+=	device_if.h bus_if.h pci_if.h opt_wlan.h opt_ath.h opt_ah.h
 
 CFLAGS+=  -I. -I${.CURDIR}/../../dev/ath -I${.CURDIR}/../../dev/ath/ath_hal
 
diff --git a/sys/modules/esp/Makefile b/sys/modules/esp/Makefile
index 2171d85..ccefca5 100644
--- a/sys/modules/esp/Makefile
+++ b/sys/modules/esp/Makefile
@@ -3,7 +3,8 @@
 .PATH: ${.CURDIR}/../../dev/esp
 
 KMOD=	esp
-SRCS=	device_if.h ${esp_sbus} bus_if.h ncr53c9x.c ${ofw_bus_if} opt_cam.h
+SRCS=	device_if.h esp_pci.c ${esp_sbus} bus_if.h ncr53c9x.c ${ofw_bus_if}
+SRCS+=	opt_cam.h pci_if.h
 
 .if ${MACHINE} == "sparc64"
 ofw_bus_if=	ofw_bus_if.h
diff --git a/sys/modules/ipfw/Makefile b/sys/modules/ipfw/Makefile
index 60ab848..575a6ce 100644
--- a/sys/modules/ipfw/Makefile
+++ b/sys/modules/ipfw/Makefile
@@ -8,7 +8,7 @@ KMOD=	ipfw
 SRCS=	ip_fw2.c ip_fw_pfil.c
 SRCS+=	ip_fw_dynamic.c ip_fw_log.c
 SRCS+=	ip_fw_sockopt.c ip_fw_table.c
-SRCS+=	opt_inet.h opt_inet6.h opt_ipfw.h opt_ipsec.h
+SRCS+=	opt_inet.h opt_inet6.h opt_ipdivert.h opt_ipfw.h opt_ipsec.h
 
 CFLAGS+= -DIPFIREWALL
 CFLAGS+= -I${.CURDIR}/../../contrib/pf
diff --git a/sys/modules/libiconv/Makefile b/sys/modules/libiconv/Makefile
index 6646064..1b51970 100644
--- a/sys/modules/libiconv/Makefile
+++ b/sys/modules/libiconv/Makefile
@@ -3,12 +3,13 @@
 .PATH: ${.CURDIR}/../../libkern ${.CURDIR}/../../sys
 
 KMOD=	libiconv
-SRCS=	iconv.c iconv_xlat.c iconv_xlat16.c
+SRCS=	iconv.c iconv_ucs.c iconv_xlat.c iconv_xlat16.c
 SRCS+=	iconv.h 
 SRCS+=	iconv_converter_if.c iconv_converter_if.h
 MFILES=	libkern/iconv_converter_if.m
 
-EXPORT_SYMS=	iconv_open	\
+EXPORT_SYMS=	iconv_add	\
+		iconv_open	\
 		iconv_close	\
 		iconv_conv	\
 		iconv_conv_case	\
diff --git a/sys/modules/libmchain/Makefile b/sys/modules/libmchain/Makefile
index fd151ec..b8ab48e 100644
--- a/sys/modules/libmchain/Makefile
+++ b/sys/modules/libmchain/Makefile
@@ -11,6 +11,7 @@ EXPORT_SYMS=	mb_init		\
 		mb_detach	\
 		mb_fixhdr	\
 		mb_reserve	\
+		mb_put_padbyte	\
 		mb_put_uint8	\
 		mb_put_uint16be	\
 		mb_put_uint16le	\
diff --git a/sys/modules/mii/Makefile b/sys/modules/mii/Makefile
index b0cfc12..68b01e2 100644
--- a/sys/modules/mii/Makefile
+++ b/sys/modules/mii/Makefile
@@ -5,14 +5,17 @@
 KMOD=	miibus
 SRCS=	acphy.c amphy.c atphy.c axphy.c bmtphy.c brgphy.c bus_if.h
 SRCS+=	ciphy.c device_if.h
-SRCS+=	e1000phy.c gentbi.c icsphy.c ip1000phy.c jmphy.c
-SRCS+=	lxtphy.c miibus_if.c miibus_if.h mii.c miidevs.h mii_physubr.c
+SRCS+=	e1000phy.c gentbi.c icsphy.c ip1000phy.c jmphy.c lxtphy.c
+SRCS+=	miibus_if.c miibus_if.h mii.c miidevs.h mii_bitbang.c mii_physubr.c
 SRCS+=	mlphy.c nsgphy.c nsphy.c nsphyter.c pci_if.h pnaphy.c qsphy.c
 SRCS+=	rdcphy.c rgephy.c rlphy.c tdkphy.c tlphy.c truephy.c
 SRCS+=	ukphy.c ukphy_subr.c
 SRCS+=	xmphy.c
 
 EXPORT_SYMS=	mii_attach	\
+		mii_bitbang_readreg \
+		mii_bitbang_sync \
+		mii_bitbang_writereg \
 		mii_mediachg	\
 		mii_pollstat	\
 		mii_tick
diff --git a/sys/modules/qlxgb/Makefile b/sys/modules/qlxgb/Makefile
new file mode 100644
index 0000000..330cef9
--- /dev/null
+++ b/sys/modules/qlxgb/Makefile
@@ -0,0 +1,43 @@
+#-
+# Copyright (c) 2010-2011 Qlogic Corporation
+# All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGE.
+#
+# File : Makefile
+# Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656.
+#
+# $FreeBSD$
+#
+
+.PATH: ${.CURDIR}/../../dev/qlxgb
+
+KMOD=	if_qlxgb
+SRCS=	qla_os.c qla_dbg.c qla_hw.c qla_misc.c qla_isr.c qla_ioctl.c
+SRCS+=	device_if.h bus_if.h pci_if.h
+
+clean:
+	rm -f opt_bdg.h device_if.h bus_if.h pci_if.h export_syms
+	rm -f *.o *.kld *.ko
+	rm -f @ machine
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/sfxge/Makefile b/sys/modules/sfxge/Makefile
new file mode 100644
index 0000000..f18e8ad
--- /dev/null
+++ b/sys/modules/sfxge/Makefile
@@ -0,0 +1,25 @@
+# $FreeBSD$
+
+KMOD=	sfxge
+
+SFXGE= ${.CURDIR}/../../dev/sfxge
+
+SRCS=	device_if.h bus_if.h pci_if.h
+SRCS+=	opt_inet.h opt_zero.h opt_sched.h
+
+.PATH: ${.CURDIR}/../../dev/sfxge
+SRCS+=	sfxge.c sfxge_dma.c sfxge_ev.c
+SRCS+=	sfxge_intr.c sfxge_mcdi.c
+SRCS+=	sfxge_port.c sfxge_rx.c sfxge_tx.c
+
+.PATH: ${.CURDIR}/../../dev/sfxge/common
+SRCS+=	efx_ev.c efx_intr.c efx_mac.c efx_mcdi.c efx_nic.c 
+SRCS+=	efx_nvram.c efx_phy.c efx_port.c efx_rx.c efx_sram.c efx_tx.c
+SRCS+=	efx_vpd.c efx_wol.c
+
+SRCS+=	siena_mac.c siena_nic.c siena_nvram.c siena_phy.c
+SRCS+=	siena_sram.c siena_vpd.c 
+
+DEBUG_FLAGS= -g -DDEBUG=1
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/virtio/Makefile b/sys/modules/virtio/Makefile
new file mode 100644
index 0000000..9c9457a
--- /dev/null
+++ b/sys/modules/virtio/Makefile
@@ -0,0 +1,28 @@
+#
+# $FreeBSD$
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+SUBDIR=	virtio pci network block balloon
+
+.include <bsd.subdir.mk>
diff --git a/sys/modules/virtio/balloon/Makefile b/sys/modules/virtio/balloon/Makefile
new file mode 100644
index 0000000..dc14cbc
--- /dev/null
+++ b/sys/modules/virtio/balloon/Makefile
@@ -0,0 +1,36 @@
+#
+# $FreeBSD$
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+.PATH: ${.CURDIR}/../../../dev/virtio/balloon
+
+KMOD=	virtio_balloon
+SRCS=	virtio_balloon.c
+SRCS+=	virtio_bus_if.h virtio_if.h
+SRCS+=	bus_if.h device_if.h 
+
+MFILES=	kern/bus_if.m kern/device_if.m \
+	dev/virtio/virtio_bus_if.m dev/virtio/virtio_if.m
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/virtio/block/Makefile b/sys/modules/virtio/block/Makefile
new file mode 100644
index 0000000..5df9eab
--- /dev/null
+++ b/sys/modules/virtio/block/Makefile
@@ -0,0 +1,36 @@
+#
+# $FreeBSD$
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+.PATH: ${.CURDIR}/../../../dev/virtio/block
+
+KMOD=	virtio_blk
+SRCS=	virtio_blk.c
+SRCS+=	virtio_bus_if.h virtio_if.h
+SRCS+=	bus_if.h device_if.h 
+
+MFILES=	kern/bus_if.m kern/device_if.m \
+	dev/virtio/virtio_bus_if.m dev/virtio/virtio_if.m
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/virtio/network/Makefile b/sys/modules/virtio/network/Makefile
new file mode 100644
index 0000000..8463309c
--- /dev/null
+++ b/sys/modules/virtio/network/Makefile
@@ -0,0 +1,36 @@
+#
+# $FreeBSD$
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+.PATH: ${.CURDIR}/../../../dev/virtio/network
+
+KMOD=	if_vtnet
+SRCS=	if_vtnet.c
+SRCS+=	virtio_bus_if.h virtio_if.h
+SRCS+=	bus_if.h device_if.h 
+
+MFILES=	kern/bus_if.m kern/device_if.m \
+	dev/virtio/virtio_bus_if.m dev/virtio/virtio_if.m
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/virtio/pci/Makefile b/sys/modules/virtio/pci/Makefile
new file mode 100644
index 0000000..a58d64c
--- /dev/null
+++ b/sys/modules/virtio/pci/Makefile
@@ -0,0 +1,36 @@
+#
+# $FreeBSD$
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+.PATH: ${.CURDIR}/../../../dev/virtio/pci
+
+KMOD=	virtio_pci
+SRCS=	virtio_pci.c
+SRCS+=	virtio_bus_if.h virtio_if.h 
+SRCS+=	bus_if.h device_if.h pci_if.h
+
+MFILES=	kern/bus_if.m kern/device_if.m dev/pci/pci_if.m \
+	dev/virtio/virtio_bus_if.m dev/virtio/virtio_if.m
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/virtio/virtio/Makefile b/sys/modules/virtio/virtio/Makefile
new file mode 100644
index 0000000..e8973c0
--- /dev/null
+++ b/sys/modules/virtio/virtio/Makefile
@@ -0,0 +1,38 @@
+#
+# $FreeBSD$
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+.PATH: ${.CURDIR}/../../../dev/virtio
+
+KMOD=	virtio
+
+SRCS=	virtio.c virtqueue.c
+SRCS+=	virtio_bus_if.c virtio_bus_if.h
+SRCS+=	virtio_if.c virtio_if.h
+SRCS+=	bus_if.h device_if.h 
+
+MFILES=	kern/bus_if.m kern/device_if.m \
+	dev/virtio/virtio_bus_if.m dev/virtio/virtio_if.m
+
+.include <bsd.kmod.mk>
diff --git a/sys/net/bpf.c b/sys/net/bpf.c
index 79c77a9..f7b73cc 100644
--- a/sys/net/bpf.c
+++ b/sys/net/bpf.c
@@ -1,12 +1,17 @@
 /*-
  * Copyright (c) 1990, 1991, 1993
- *	The Regents of the University of California.  All rights reserved.
+ *	The Regents of the University of California.
+ * Copyright (c) 2011 The FreeBSD Foundation.
+ * All rights reserved.
  *
  * This code is derived from the Stanford/CMU enet packet filter,
  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
  * Berkeley Laboratory.
  *
+ * Portions of this software were developed by Julien Ridoux at the University
+ * of Melbourne under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,6 +44,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_bpf.h"
 #include "opt_compat.h"
+#include "opt_ffclock.h"
 #include "opt_netgraph.h"
 
 #include <sys/types.h>
@@ -55,6 +61,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/signalvar.h>
 #include <sys/filio.h>
 #include <sys/sockio.h>
+#ifdef FFCLOCK
+#include <sys/timeffc.h>
+#endif
 #include <sys/ttycom.h>
 #include <sys/uio.h>
 
@@ -90,8 +99,13 @@ MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
 
 #define PRINET  26			/* interruptible */
 
+#ifdef FFCLOCK
+#define SIZEOF_BPF_HDR(type)	\
+    (offsetof(type, ffcount_stamp) + sizeof(((type *)0)->ffcount_stamp))
+#else
 #define	SIZEOF_BPF_HDR(type)	\
     (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
+#endif
 
 #ifdef COMPAT_FREEBSD32
 #include <sys/mount.h>
@@ -111,6 +125,9 @@ struct bpf_hdr32 {
 	uint32_t	bh_datalen;	/* original length of packet */
 	uint16_t	bh_hdrlen;	/* length of bpf header (this struct
 					   plus alignment padding) */
+#ifdef FFCLOCK
+	ffcounter	ffcount_stamp;	/* ffcounter timestamp of packet */
+#endif
 };
 #endif
 
@@ -151,9 +168,16 @@ static int	bpf_setif(struct bpf_d *, struct ifreq *);
 static void	bpf_timed_out(void *);
 static __inline void
 		bpf_wakeup(struct bpf_d *);
+#ifdef FFCLOCK
+static void	catchpacket(struct bpf_d *, u_char *, unsigned int,
+		    unsigned int, void (*)(struct bpf_d *, caddr_t,
+		    unsigned int, void *, unsigned int), struct bintime *,
+		    ffcounter *);
+#else
 static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
 		    void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
 		    struct bintime *);
+#endif
 static void	reset_d(struct bpf_d *);
 static int	 bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
@@ -170,8 +194,14 @@ SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
 static int bpf_zerocopy_enable = 0;
 SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
     &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
-SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
+static SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
     bpf_stats_sysctl, "bpf statistics portal");
+#ifdef FFCLOCK
+static int bpf_ffclock_tstamp = 0;
+SYSCTL_INT(_net_bpf, OID_AUTO, ffclock_tstamp, CTLFLAG_RW,
+    &bpf_ffclock_tstamp, 0,
+    "Set BPF to timestamp using Feed-Forward clock by default");
+#endif
 
 static	d_open_t	bpfopen;
 static	d_read_t	bpfread;
@@ -698,6 +728,15 @@ bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	callout_init_mtx(&d->bd_callout, &d->bd_mtx, 0);
 	knlist_init_mtx(&d->bd_sel.si_note, &d->bd_mtx);
 
+#ifdef FFCLOCK
+	/*
+	 * Set the timestamping mode for this device, i.e. which clock is used.
+	 * The default option is to use the feedback/ntpd system clock.
+	 */
+	if (bpf_ffclock_tstamp)
+		d->bd_tstamp = d->bd_tstamp | BPF_T_FFCLOCK;
+#endif
+
 	return (0);
 }
 
@@ -1776,8 +1815,13 @@ bpf_ts_quality(int tstype)
 	return (BPF_TSTAMP_NORMAL);
 }
 
+#ifdef FFCLOCK
+static int
+bpf_gettime(struct bintime *bt, ffcounter *ffcount, int tstype, struct mbuf *m)
+#else
 static int
 bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
+#endif
 {
 	struct m_tag *tag;
 	int quality;
@@ -1793,11 +1837,31 @@ bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
 			return (BPF_TSTAMP_EXTERN);
 		}
 	}
-	if (quality == BPF_TSTAMP_NORMAL)
-		binuptime(bt);
-	else
-		getbinuptime(bt);
-
+	if (quality == BPF_TSTAMP_NORMAL) {
+#ifdef FFCLOCK
+		if ((tstype & BPF_T_FFCLOCK) == 0) {
+			ffclock_read_counter(ffcount);
+#endif
+			binuptime(bt);
+#ifdef FFCLOCK
+		} else {
+			if ((tstype & BPF_T_MONOTONIC) == 0)
+				ffclock_abstime(ffcount, bt, NULL,
+				    (FFCLOCK_LERP | FFCLOCK_LEAPSEC));
+			else
+				ffclock_abstime(ffcount, bt, NULL,
+				    (FFCLOCK_LERP | FFCLOCK_UPTIME));
+		}
+#endif
+	} else {
+#ifdef FFCLOCK
+		if ((tstype & BPF_T_FFCLOCK) == BPF_T_FFCLOCK)
+			ffclock_abstime(ffcount, bt, NULL,
+			    (FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST));
+		else
+#endif
+			getbinuptime(bt);
+	}
 	return (quality);
 }
 
@@ -1817,6 +1881,9 @@ bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
 #endif
 	u_int slen;
 	int gottime;
+#ifdef FFCLOCK
+	ffcounter ffcount;
+#endif
 
 	gottime = BPF_TSTAMP_NONE;
 	BPFIF_LOCK(bp);
@@ -1838,13 +1905,24 @@ bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
 		if (slen != 0) {
 			d->bd_fcount++;
-			if (gottime < bpf_ts_quality(d->bd_tstamp))
+			if (gottime < bpf_ts_quality(d->bd_tstamp)) {
+#ifdef FFCLOCK
+				gottime = bpf_gettime(&bt, &ffcount,
+				    d->bd_tstamp, NULL);
+#else
 				gottime = bpf_gettime(&bt, d->bd_tstamp, NULL);
+#endif
+			}
 #ifdef MAC
 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
 #endif
+#ifdef FFCLOCK
+				catchpacket(d, pkt, pktlen, slen,
+				    bpf_append_bytes, &bt, &ffcount);
+#else
 				catchpacket(d, pkt, pktlen, slen,
 				    bpf_append_bytes, &bt);
+#endif
 		}
 		BPFD_UNLOCK(d);
 	}
@@ -1868,6 +1946,9 @@ bpf_mtap(struct bpf_if *bp, struct mbuf *m)
 #endif
 	u_int pktlen, slen;
 	int gottime;
+#ifdef FFCLOCK
+	ffcounter ffcount;
+#endif
 
 	/* Skip outgoing duplicate packets. */
 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
@@ -1895,12 +1976,24 @@ bpf_mtap(struct bpf_if *bp, struct mbuf *m)
 		if (slen != 0) {
 			d->bd_fcount++;
 			if (gottime < bpf_ts_quality(d->bd_tstamp))
+			{
+#ifdef FFCLOCK
+				gottime = bpf_gettime(&bt, &ffcount,
+				    d->bd_tstamp, m);
+#else
 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
+#endif
+			}
 #ifdef MAC
 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
 #endif
+#ifdef FFCLOCK
+				catchpacket(d, (u_char *)m, pktlen, slen,
+				    bpf_append_mbuf, &bt, &ffcount);
+#else
 				catchpacket(d, (u_char *)m, pktlen, slen,
 				    bpf_append_mbuf, &bt);
+#endif
 		}
 		BPFD_UNLOCK(d);
 	}
@@ -1919,6 +2012,9 @@ bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
 	struct bpf_d *d;
 	u_int pktlen, slen;
 	int gottime;
+#ifdef FFCLOCK
+	ffcounter ffcount;
+#endif
 
 	/* Skip outgoing duplicate packets. */
 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
@@ -1948,12 +2044,24 @@ bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
 		if (slen != 0) {
 			d->bd_fcount++;
 			if (gottime < bpf_ts_quality(d->bd_tstamp))
+			{
+#ifdef FFCLOCK
+				gottime = bpf_gettime(&bt, &ffcount,
+				    d->bd_tstamp, m);
+#else
 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
+#endif
+			}
 #ifdef MAC
 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
 #endif
+#ifdef FFCLOCK
+				catchpacket(d, (u_char *)m, pktlen, slen,
+				    bpf_append_mbuf, &bt, &ffcount);
+#else
 				catchpacket(d, (u_char *)&mb, pktlen, slen,
 				    bpf_append_mbuf, &bt);
+#endif
 		}
 		BPFD_UNLOCK(d);
 	}
@@ -2002,11 +2110,17 @@ bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
 	struct timeval tsm;
 	struct timespec tsn;
 
-	if ((tstype & BPF_T_MONOTONIC) == 0) {
-		bt2 = *bt;
-		bintime_add(&bt2, &boottimebin);
-		bt = &bt2;
+#ifdef FFCLOCK
+	if ((tstype & BPF_T_FFCLOCK) == 0) {
+#endif
+		if ((tstype & BPF_T_MONOTONIC) == 0) {
+			bt2 = *bt;
+			bintime_add(&bt2, &boottimebin);
+			bt = &bt2;
+		}
+#ifdef FFCLOCK
 	}
+#endif
 	switch (BPF_T_FORMAT(tstype)) {
 	case BPF_T_MICROTIME:
 		bintime2timeval(bt, &tsm);
@@ -2032,10 +2146,17 @@ bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
  * bpf_append_mbuf is passed in to copy mbuf chains.  In the latter case,
  * pkt is really an mbuf.
  */
+#ifdef FFCLOCK
+static void
+catchpacket(struct bpf_d *d, u_char *pkt, unsigned int pktlen,
+    unsigned int snaplen, void (*cpfn)(struct bpf_d *, caddr_t, unsigned int,
+    void *, unsigned int), struct bintime *bt, ffcounter *ffcount)
+#else
 static void
 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
     void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
     struct bintime *bt)
+#endif
 {
 	struct bpf_xhdr hdr;
 #ifndef BURN_BRIDGES
@@ -2123,6 +2244,9 @@ catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
 		if (d->bd_compat32) {
 			bzero(&hdr32_old, sizeof(hdr32_old));
 			if (do_timestamp) {
+#ifdef FFCLOCK
+				hdr32_old.ffcount_stamp = *ffcount;
+#endif
 				hdr32_old.bh_tstamp.tv_sec = ts.bt_sec;
 				hdr32_old.bh_tstamp.tv_usec = ts.bt_frac;
 			}
@@ -2136,6 +2260,9 @@ catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
 #endif
 		bzero(&hdr_old, sizeof(hdr_old));
 		if (do_timestamp) {
+#ifdef FFCLOCK
+			hdr_old.ffcount_stamp = *ffcount;
+#endif
 			hdr_old.bh_tstamp.tv_sec = ts.bt_sec;
 			hdr_old.bh_tstamp.tv_usec = ts.bt_frac;
 		}
@@ -2154,7 +2281,12 @@ catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
 	 */
 	bzero(&hdr, sizeof(hdr));
 	if (do_timestamp)
+	{
+#ifdef FFCLOCK
+		hdr.ffcount_stamp = *ffcount;
+#endif
 		bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype);
+	}
 	hdr.bh_datalen = pktlen;
 	hdr.bh_hdrlen = hdrlen;
 	hdr.bh_caplen = caplen;
diff --git a/sys/net/bpf.h b/sys/net/bpf.h
index d4c369d..8a80514 100644
--- a/sys/net/bpf.h
+++ b/sys/net/bpf.h
@@ -1,12 +1,17 @@
 /*-
  * Copyright (c) 1990, 1991, 1993
- *	The Regents of the University of California.  All rights reserved.
+ *	The Regents of the University of California.
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
  *
  * This code is derived from the Stanford/CMU enet packet filter,
  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
  * Berkeley Laboratory.
  *
+ * Portions of this software were developed by Julien Ridoux at the University
+ * of Melbourne under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -170,11 +175,21 @@ enum bpf_direction {
 #define	BPF_T_MONOTONIC		0x0200
 #define	BPF_T_MONOTONIC_FAST	(BPF_T_FAST | BPF_T_MONOTONIC)
 #define	BPF_T_FLAG_MASK		0x0300
+#ifdef FFCLOCK
+#define	BPF_T_FFCLOCK		0x8000
+#define	BPF_T_CLOCK_MASK	0x8000
+#endif
 #define	BPF_T_FORMAT(t)		((t) & BPF_T_FORMAT_MASK)
 #define	BPF_T_FLAG(t)		((t) & BPF_T_FLAG_MASK)
+#ifdef FFCLOCK
+#define	BPF_T_VALID(t)						\
+    ((t) == BPF_T_NONE || (BPF_T_FORMAT(t) != BPF_T_NONE &&	\
+    ((t) & ~(BPF_T_FORMAT_MASK | BPF_T_FLAG_MASK | BPF_T_CLOCK_MASK)) == 0))
+#else
 #define	BPF_T_VALID(t)						\
     ((t) == BPF_T_NONE || (BPF_T_FORMAT(t) != BPF_T_NONE &&	\
     ((t) & ~(BPF_T_FORMAT_MASK | BPF_T_FLAG_MASK)) == 0))
+#endif
 
 #define	BPF_T_MICROTIME_FAST		(BPF_T_MICROTIME | BPF_T_FAST)
 #define	BPF_T_NANOTIME_FAST		(BPF_T_NANOTIME | BPF_T_FAST)
@@ -199,6 +214,9 @@ struct bpf_xhdr {
 	bpf_u_int32	bh_datalen;	/* original length of packet */
 	u_short		bh_hdrlen;	/* length of bpf header (this struct
 					   plus alignment padding) */
+#ifdef FFCLOCK
+	ffcounter	ffcount_stamp;	/* feed-forward counter timestamp */
+#endif
 };
 /* Obsolete */
 struct bpf_hdr {
@@ -207,6 +225,9 @@ struct bpf_hdr {
 	bpf_u_int32	bh_datalen;	/* original length of packet */
 	u_short		bh_hdrlen;	/* length of bpf header (this struct
 					   plus alignment padding) */
+#ifdef FFCLOCK
+	ffcounter	ffcount_stamp;	/* feed-forward counter timestamp */
+#endif
 };
 #ifdef _KERNEL
 #define	MTAG_BPF		0x627066
diff --git a/sys/net/flowtable.c b/sys/net/flowtable.c
index 198c4c3..7814e3a 100644
--- a/sys/net/flowtable.c
+++ b/sys/net/flowtable.c
@@ -248,7 +248,8 @@ static VNET_DEFINE(int, flowtable_ready) = 0;
 #define	V_flowtable_nmbflows		VNET(flowtable_nmbflows)
 #define	V_flowtable_ready		VNET(flowtable_ready)
 
-SYSCTL_NODE(_net_inet, OID_AUTO, flowtable, CTLFLAG_RD, NULL, "flowtable");
+static SYSCTL_NODE(_net_inet, OID_AUTO, flowtable, CTLFLAG_RD, NULL,
+    "flowtable");
 SYSCTL_VNET_INT(_net_inet_flowtable, OID_AUTO, debug, CTLFLAG_RW,
     &VNET_NAME(flowtable_debug), 0, "print debug info.");
 SYSCTL_VNET_INT(_net_inet_flowtable, OID_AUTO, enable, CTLFLAG_RW,
diff --git a/sys/net/if.c b/sys/net/if.c
index 2b7a24a..322759a 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -118,7 +118,7 @@ SYSCTL_UINT(_net, OID_AUTO, ifdescr_maxlen, CTLFLAG_RW,
 	&ifdescr_maxlen, 0,
 	"administrative maximum length for interface description");
 
-MALLOC_DEFINE(M_IFDESCR, "ifdescr", "ifnet descriptions");
+static MALLOC_DEFINE(M_IFDESCR, "ifdescr", "ifnet descriptions");
 
 /* global sx for non-critical path ifdescr */
 static struct sx ifdescr_sx;
@@ -215,7 +215,7 @@ struct sx ifnet_sxlock;
 static	if_com_alloc_t *if_com_alloc[256];
 static	if_com_free_t *if_com_free[256];
 
-MALLOC_DEFINE(M_IFNET, "ifnet", "interface internals");
+static MALLOC_DEFINE(M_IFNET, "ifnet", "interface internals");
 MALLOC_DEFINE(M_IFADDR, "ifaddr", "interface address");
 MALLOC_DEFINE(M_IFMADDR, "ether_multi", "link-level multicast address");
 
diff --git a/sys/net/if_arcsubr.c b/sys/net/if_arcsubr.c
index 2a58d5a..03f3a1c 100644
--- a/sys/net/if_arcsubr.c
+++ b/sys/net/if_arcsubr.c
@@ -837,7 +837,7 @@ arc_resolvemulti(struct ifnet *ifp, struct sockaddr **llsa,
 	}
 }
 
-MALLOC_DEFINE(M_ARCCOM, "arccom", "ARCNET interface internals");
+static MALLOC_DEFINE(M_ARCCOM, "arccom", "ARCNET interface internals");
 
 static void*
 arc_alloc(u_char type, struct ifnet *ifp)
diff --git a/sys/net/if_atmsubr.c b/sys/net/if_atmsubr.c
index fb63fd5..bb691e1 100644
--- a/sys/net/if_atmsubr.c
+++ b/sys/net/if_atmsubr.c
@@ -98,7 +98,7 @@ void	(*atm_harp_event_p)(struct ifnet *, uint32_t, void *);
 
 SYSCTL_NODE(_hw, OID_AUTO, atm, CTLFLAG_RW, 0, "ATM hardware");
 
-MALLOC_DEFINE(M_IFATM, "ifatm", "atm interface internals");
+static MALLOC_DEFINE(M_IFATM, "ifatm", "atm interface internals");
 
 #ifndef ETHERTYPE_IPV6
 #define	ETHERTYPE_IPV6	0x86dd
diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index c251653..8793cf7 100644
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -344,7 +344,7 @@ static struct bstp_cb_ops bridge_ops = {
 };
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_BRIDGE, bridge, CTLFLAG_RW, 0, "Bridge");
+static SYSCTL_NODE(_net_link, IFT_BRIDGE, bridge, CTLFLAG_RW, 0, "Bridge");
 
 static int pfil_onlyip = 1; /* only pass IP[46] packets when pfil is enabled */
 static int pfil_bridge = 1; /* run pfil hooks on the bridge interface */
@@ -676,7 +676,7 @@ bridge_clone_destroy(struct ifnet *ifp)
 
 	bstp_detach(&sc->sc_stp);
 	ether_ifdetach(ifp);
-	if_free_type(ifp, IFT_ETHER);
+	if_free(ifp);
 
 	/* Tear down the routing table. */
 	bridge_rtable_fini(sc);
diff --git a/sys/net/if_enc.c b/sys/net/if_enc.c
index eee3733..9506172 100644
--- a/sys/net/if_enc.c
+++ b/sys/net/if_enc.c
@@ -102,9 +102,9 @@ IFC_SIMPLE_DECLARE(enc, 1);
  * Before and after are relative to when we are stripping the
  * outer IP header.
  */
-SYSCTL_NODE(_net, OID_AUTO, enc, CTLFLAG_RW, 0, "enc sysctl");
+static SYSCTL_NODE(_net, OID_AUTO, enc, CTLFLAG_RW, 0, "enc sysctl");
 
-SYSCTL_NODE(_net_enc, OID_AUTO, in, CTLFLAG_RW, 0, "enc input sysctl");
+static SYSCTL_NODE(_net_enc, OID_AUTO, in, CTLFLAG_RW, 0, "enc input sysctl");
 static int ipsec_filter_mask_in = ENC_BEFORE;
 SYSCTL_INT(_net_enc_in, OID_AUTO, ipsec_filter_mask, CTLFLAG_RW,
 	&ipsec_filter_mask_in, 0, "IPsec input firewall filter mask");
@@ -112,7 +112,7 @@ static int ipsec_bpf_mask_in = ENC_BEFORE;
 SYSCTL_INT(_net_enc_in, OID_AUTO, ipsec_bpf_mask, CTLFLAG_RW,
 	&ipsec_bpf_mask_in, 0, "IPsec input bpf mask");
 
-SYSCTL_NODE(_net_enc, OID_AUTO, out, CTLFLAG_RW, 0, "enc output sysctl");
+static SYSCTL_NODE(_net_enc, OID_AUTO, out, CTLFLAG_RW, 0, "enc output sysctl");
 static int ipsec_filter_mask_out = ENC_BEFORE;
 SYSCTL_INT(_net_enc_out, OID_AUTO, ipsec_filter_mask, CTLFLAG_RW,
 	&ipsec_filter_mask_out, 0, "IPsec output firewall filter mask");
diff --git a/sys/net/if_epair.c b/sys/net/if_epair.c
index dff9efc..fe22ed7 100644
--- a/sys/net/if_epair.c
+++ b/sys/net/if_epair.c
@@ -75,7 +75,7 @@ __FBSDID("$FreeBSD$");
 #define	EPAIRNAME	"epair"
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW, 0, "epair sysctl");
+static SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW, 0, "epair sysctl");
 
 #ifdef EPAIR_DEBUG
 static int epair_debug = 0;
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 1d16ae6..9eae683 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -134,7 +134,7 @@ static	void ether_reassign(struct ifnet *, struct vnet *, char *);
 #endif
 
 /* XXX: should be in an arp support file, not here */
-MALLOC_DEFINE(M_ARPCOM, "arpcom", "802.* interface internals");
+static MALLOC_DEFINE(M_ARPCOM, "arpcom", "802.* interface internals");
 
 #define	ETHER_IS_BROADCAST(addr) \
 	(bcmp(etherbroadcastaddr, (addr), ETHER_ADDR_LEN) == 0)
diff --git a/sys/net/if_fwsubr.c b/sys/net/if_fwsubr.c
index 7c57feb..5414d79 100644
--- a/sys/net/if_fwsubr.c
+++ b/sys/net/if_fwsubr.c
@@ -63,7 +63,7 @@
 
 #include <security/mac/mac_framework.h>
 
-MALLOC_DEFINE(M_FWCOM, "fw_com", "firewire interface internals");
+static MALLOC_DEFINE(M_FWCOM, "fw_com", "firewire interface internals");
 
 struct fw_hwaddr firewire_broadcastaddr = {
 	0xffffffff,
diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c
index 2b6b2b4..350611d 100644
--- a/sys/net/if_gif.c
+++ b/sys/net/if_gif.c
@@ -112,7 +112,7 @@ IFC_SIMPLE_DECLARE(gif, 0);
 static int gifmodevent(module_t, int, void *);
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_GIF, gif, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_link, IFT_GIF, gif, CTLFLAG_RW, 0,
     "Generic Tunnel Interface");
 #ifndef MAX_GIF_NEST
 /*
diff --git a/sys/net/if_gre.c b/sys/net/if_gre.c
index 5f8156b..8b250d7 100644
--- a/sys/net/if_gre.c
+++ b/sys/net/if_gre.c
@@ -148,7 +148,7 @@ static const struct protosw in_mobile_protosw = {
 #endif
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_TUNNEL, gre, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_link, IFT_TUNNEL, gre, CTLFLAG_RW, 0,
     "Generic Routing Encapsulation");
 #ifndef MAX_GRE_NEST
 /*
diff --git a/sys/net/if_iso88025subr.c b/sys/net/if_iso88025subr.c
index 7961df0..91d67fd 100644
--- a/sys/net/if_iso88025subr.c
+++ b/sys/net/if_iso88025subr.c
@@ -781,7 +781,7 @@ iso88025_resolvemulti (ifp, llsa, sa)
 	return (0);
 }
 
-MALLOC_DEFINE(M_ISO88025, "arpcom", "802.5 interface internals");
+static MALLOC_DEFINE(M_ISO88025, "arpcom", "802.5 interface internals");
 
 static void*
 iso88025_alloc(u_char type, struct ifnet *ifp)
diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c
index 6a3eb93..730047f 100644
--- a/sys/net/if_lagg.c
+++ b/sys/net/if_lagg.c
@@ -165,7 +165,8 @@ static const struct {
 };
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, OID_AUTO, lagg, CTLFLAG_RW, 0, "Link Aggregation");
+static SYSCTL_NODE(_net_link, OID_AUTO, lagg, CTLFLAG_RW, 0,
+    "Link Aggregation");
 
 static int lagg_failover_rx_all = 0; /* Allow input on any failover links */
 SYSCTL_INT(_net_link_lagg, OID_AUTO, failover_rx_all, CTLFLAG_RW,
@@ -274,7 +275,7 @@ lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 		if (lagg_protos[i].ti_proto == LAGG_PROTO_DEFAULT) {
 			sc->sc_proto = lagg_protos[i].ti_proto;
 			if ((error = lagg_protos[i].ti_attach(sc)) != 0) {
-				if_free_type(ifp, IFT_ETHER);
+				if_free(ifp);
 				free(sc, M_DEVBUF);
 				return (error);
 			}
@@ -292,7 +293,6 @@ lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
 
 	if_initname(ifp, ifc->ifc_name, unit);
-	ifp->if_type = IFT_ETHER;
 	ifp->if_softc = sc;
 	ifp->if_start = lagg_start;
 	ifp->if_init = lagg_init;
@@ -304,7 +304,7 @@ lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	IFQ_SET_READY(&ifp->if_snd);
 
 	/*
-	 * Attach as an ordinary ethernet device, childs will be attached
+	 * Attach as an ordinary ethernet device, children will be attached
 	 * as special device IFT_IEEE8023ADLAG.
 	 */
 	ether_ifattach(ifp, eaddr);
@@ -351,7 +351,7 @@ lagg_clone_destroy(struct ifnet *ifp)
 
 	ifmedia_removeall(&sc->sc_media);
 	ether_ifdetach(ifp);
-	if_free_type(ifp, IFT_ETHER);
+	if_free(ifp);
 
 	mtx_lock(&lagg_list_mtx);
 	SLIST_REMOVE(&lagg_list, sc, lagg_softc, sc_entries);
diff --git a/sys/net/if_mib.c b/sys/net/if_mib.c
index c334ac7..97a2d40 100644
--- a/sys/net/if_mib.c
+++ b/sys/net/if_mib.c
@@ -63,7 +63,7 @@
  */
 
 SYSCTL_DECL(_net_link_generic);
-SYSCTL_NODE(_net_link_generic, IFMIB_SYSTEM, system, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_link_generic, IFMIB_SYSTEM, system, CTLFLAG_RW, 0,
 	    "Variables global to all interfaces");
 
 SYSCTL_VNET_INT(_net_link_generic_system, IFMIB_IFCOUNT, ifcount, CTLFLAG_RD,
@@ -164,6 +164,6 @@ out:
 	return error;
 }
 
-SYSCTL_NODE(_net_link_generic, IFMIB_IFDATA, ifdata, CTLFLAG_RW,
+static SYSCTL_NODE(_net_link_generic, IFMIB_IFDATA, ifdata, CTLFLAG_RW,
 	    sysctl_ifdata, "Interface table");
 
diff --git a/sys/net/if_spppfr.c b/sys/net/if_spppfr.c
index ca84656..ee5339e 100644
--- a/sys/net/if_spppfr.c
+++ b/sys/net/if_spppfr.c
@@ -25,7 +25,7 @@
 
 #include <sys/param.h>
 
-#if defined(__FreeBSD__) && __FreeBSD__ >= 3
+#if defined(__FreeBSD__)
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipx.h"
@@ -45,7 +45,7 @@
 #include <sys/sockio.h>
 #include <sys/socket.h>
 #include <sys/syslog.h>
-#if defined(__FreeBSD__) && __FreeBSD__ >= 3
+#if defined(__FreeBSD__)
 #include <sys/random.h>
 #endif
 #include <sys/malloc.h>
@@ -149,7 +149,7 @@ struct arp_req {
 	unsigned short  ptarget2;
 } __packed;
 
-#if defined(__FreeBSD__) && __FreeBSD__ >= 3 && __FreeBSD_version < 501113
+#if defined(__FreeBSD__) && __FreeBSD_version < 501113
 #define	SPP_FMT		"%s%d: "
 #define	SPP_ARGS(ifp)	(ifp)->if_name, (ifp)->if_unit
 #else
diff --git a/sys/net/if_spppsubr.c b/sys/net/if_spppsubr.c
index 363d9aa..7c9c230 100644
--- a/sys/net/if_spppsubr.c
+++ b/sys/net/if_spppsubr.c
@@ -174,7 +174,7 @@
 #define STATE_ACK_SENT	8
 #define STATE_OPENED	9
 
-MALLOC_DEFINE(M_SPPP, "sppp", "synchronous PPP interface internals");
+static MALLOC_DEFINE(M_SPPP, "sppp", "synchronous PPP interface internals");
 
 struct ppp_header {
 	u_char address;
diff --git a/sys/net/if_stf.c b/sys/net/if_stf.c
index e32956e..fa8ee64 100644
--- a/sys/net/if_stf.c
+++ b/sys/net/if_stf.c
@@ -121,7 +121,7 @@
 #include <security/mac/mac_framework.h>
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_STF, stf, CTLFLAG_RW, 0, "6to4 Interface");
+static SYSCTL_NODE(_net_link, IFT_STF, stf, CTLFLAG_RW, 0, "6to4 Interface");
 
 static int stf_route_cache = 1;
 SYSCTL_INT(_net_link_stf, OID_AUTO, route_cache, CTLFLAG_RW,
diff --git a/sys/net/if_tap.c b/sys/net/if_tap.c
index 4ae5a3d..4ff5a49 100644
--- a/sys/net/if_tap.c
+++ b/sys/net/if_tap.c
@@ -161,7 +161,7 @@ MALLOC_DEFINE(M_TAP, CDEV_NAME, "Ethernet tunnel interface");
 SYSCTL_INT(_debug, OID_AUTO, if_tap_debug, CTLFLAG_RW, &tapdebug, 0, "");
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, OID_AUTO, tap, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_link, OID_AUTO, tap, CTLFLAG_RW, 0,
     "Ethernet tunnel software network interface");
 SYSCTL_INT(_net_link_tap, OID_AUTO, user_open, CTLFLAG_RW, &tapuopen, 0,
 	"Allow user to open /dev/tap (based on node permissions)");
@@ -218,7 +218,7 @@ tap_destroy(struct tap_softc *tp)
 	knlist_destroy(&tp->tap_rsel.si_note);
 	destroy_dev(tp->tap_dev);
 	ether_ifdetach(ifp);
-	if_free_type(ifp, IFT_ETHER);
+	if_free(ifp);
 
 	mtx_destroy(&tp->tap_mtx);
 	free(tp, M_TAP);
diff --git a/sys/net/if_tun.c b/sys/net/if_tun.c
index 57a11f9..f8eda08 100644
--- a/sys/net/if_tun.c
+++ b/sys/net/if_tun.c
@@ -115,7 +115,7 @@ static TAILQ_HEAD(,tun_softc)	tunhead = TAILQ_HEAD_INITIALIZER(tunhead);
 SYSCTL_INT(_debug, OID_AUTO, if_tun_debug, CTLFLAG_RW, &tundebug, 0, "");
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, OID_AUTO, tun, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_link, OID_AUTO, tun, CTLFLAG_RW, 0,
     "IP tunnel software network interface.");
 SYSCTL_INT(_net_link_tun, OID_AUTO, devfs_cloning, CTLFLAG_RW, &tundclone, 0,
     "Enable legacy devfs interface creation.");
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 2dcb6f9..e06a086 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -321,6 +321,18 @@ void	if_maddr_runlock(struct ifnet *ifp);	/* if_multiaddrs */
 	IF_UNLOCK(ifq); 					\
 } while (0)
 
+#define	_IF_DEQUEUE_ALL(ifq, m) do {				\
+	(m) = (ifq)->ifq_head;					\
+	(ifq)->ifq_head = (ifq)->ifq_tail = NULL;		\
+	(ifq)->ifq_len = 0;					\
+} while (0)
+
+#define	IF_DEQUEUE_ALL(ifq, m) do {				\
+	IF_LOCK(ifq); 						\
+	_IF_DEQUEUE_ALL(ifq, m);				\
+	IF_UNLOCK(ifq); 					\
+} while (0)
+
 #define	_IF_POLL(ifq, m)	((m) = (ifq)->ifq_head)
 #define	IF_POLL(ifq, m)		_IF_POLL(ifq, m)
 
diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c
index 9dd54e2..083086e 100644
--- a/sys/net/if_vlan.c
+++ b/sys/net/if_vlan.c
@@ -131,8 +131,10 @@ static struct {
 };
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_L2VLAN, vlan, CTLFLAG_RW, 0, "IEEE 802.1Q VLAN");
-SYSCTL_NODE(_net_link_vlan, PF_LINK, link, CTLFLAG_RW, 0, "for consistency");
+static SYSCTL_NODE(_net_link, IFT_L2VLAN, vlan, CTLFLAG_RW, 0,
+    "IEEE 802.1Q VLAN");
+static SYSCTL_NODE(_net_link_vlan, PF_LINK, link, CTLFLAG_RW, 0,
+    "for consistency");
 
 static int soft_pad = 0;
 SYSCTL_INT(_net_link_vlan, OID_AUTO, soft_pad, CTLFLAG_RW, &soft_pad, 0,
@@ -965,7 +967,7 @@ vlan_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params)
 			 */
 			ether_ifdetach(ifp);
 			vlan_unconfig(ifp);
-			if_free_type(ifp, IFT_ETHER);
+			if_free(ifp);
 			ifc_free_unit(ifc, unit);
 			free(ifv, M_VLAN);
 
@@ -987,7 +989,7 @@ vlan_clone_destroy(struct if_clone *ifc, struct ifnet *ifp)
 
 	ether_ifdetach(ifp);	/* first, remove it from system-wide lists */
 	vlan_unconfig(ifp);	/* now it can be unconfigured and freed */
-	if_free_type(ifp, IFT_ETHER);
+	if_free(ifp);
 	free(ifv, M_VLAN);
 	ifc_free_unit(ifc, unit);
 
diff --git a/sys/net/netisr.c b/sys/net/netisr.c
index 127cf67..534d80c 100644
--- a/sys/net/netisr.c
+++ b/sys/net/netisr.c
@@ -124,7 +124,7 @@ static struct rmlock	netisr_rmlock;
 #define	NETISR_WUNLOCK()	rm_wunlock(&netisr_rmlock)
 /* #define	NETISR_LOCKING */
 
-SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr");
+static SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr");
 
 /*-
  * Three global direct dispatch policies are supported:
diff --git a/sys/net/netmap.h b/sys/net/netmap.h
new file mode 100644
index 0000000..be9c686
--- /dev/null
+++ b/sys/net/netmap.h
@@ -0,0 +1,281 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ * 
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the
+ *      distribution.
+ * 
+ *   3. Neither the name of the authors nor the names of their contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY MATTEO LANDI AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL MATTEO LANDI OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: netmap.h 9662 2011-11-16 13:18:06Z luigi $
+ *
+ * This header contains the definitions of the constants and the
+ * structures needed by the ``netmap'' module, both kernel and
+ * userspace.
+ */
+
+#ifndef _NET_NETMAP_H_
+#define _NET_NETMAP_H_
+
+/*
+ * --- Netmap data structures ---
+ *
+ * The data structures used by netmap are shown below. Those in
+ * capital letters are in an mmapp()ed area shared with userspace,
+ * while others are private to the kernel.
+ * Shared structures do not contain pointers but only relative
+ * offsets, so that addressing is portable between kernel and userspace.
+ *
+ * The 'softc' of each interface is extended with a struct netmap_adapter
+ * containing information to support netmap operation. In addition to
+ * the fixed fields, it has two pointers to reach the arrays of
+ * 'struct netmap_kring' which in turn reaches the various
+ * struct netmap_ring, shared with userspace.
+
+
+ softc
++----------------+
+| standard fields|
+| if_pspare[0] ----------+
++----------------+       |
+                         |
++----------------+<------+
+|(netmap_adapter)|
+|                |                             netmap_kring
+| tx_rings *--------------------------------->+-------------+
+|                |       netmap_kring         | ring *---------> ...
+| rx_rings *---------->+--------------+       | nr_hwcur    |
++----------------+     | ring    *-------+    | nr_hwavail  |
+                       | nr_hwcur     |  |    | selinfo     |
+                       | nr_hwavail   |  |    +-------------+
+                       | selinfo      |  |    |     ...     |
+                       +--------------+  |  (na_num_rings+1 entries)
+                       |    ....      |  |    |             |
+                    (na_num_rings+1 entries)  +-------------+
+                       |              |  |
+                       +--------------+  |
+                                         |      NETMAP_RING
+                                         +---->+-------------+
+                                             / | cur         |
+   NETMAP_IF  (nifp, one per file desc.)    /  | avail       |
+    +---------------+                      /   | buf_ofs     |
+    | ni_num_queues |                     /    +=============+
+    |               |                    /     | buf_idx     | slot[0]
+    |               |                   /      | len, flags  |
+    |               |                  /       +-------------+
+    +===============+                 /        | buf_idx     | slot[1]
+    | txring_ofs[0] | (rel.to nifp)--'         | len, flags  |
+    | txring_ofs[1] |                          +-------------+
+  (num_rings+1 entries)                     (nr_num_slots entries)
+    | txring_ofs[n] |                          | buf_idx     | slot[n-1]
+    +---------------+                          | len, flags  |
+    | rxring_ofs[0] |                          +-------------+
+    | rxring_ofs[1] |
+  (num_rings+1 entries)
+    | txring_ofs[n] |
+    +---------------+
+
+ * The NETMAP_RING is the shadow ring that mirrors the NIC rings.
+ * Each slot has the index of a buffer, its length and some flags.
+ * In user space, the buffer address is computed as
+ *	(char *)ring + buf_ofs + index*MAX_BUF_SIZE
+ * In the kernel, buffers do not necessarily need to be contiguous,
+ * and the virtual and physical addresses are derived through
+ * a lookup table. When userspace wants to use a different buffer
+ * in a location, it must set the NS_BUF_CHANGED flag to make
+ * sure that the kernel recomputes updates the hardware ring and
+ * other fields (bus_dmamap, etc.) as needed.
+ *
+ * Normally the driver is not requested to report the result of
+ * transmissions (this can dramatically speed up operation).
+ * However the user may request to report completion by setting
+ * NS_REPORT.
+ */
+struct netmap_slot {
+	uint32_t buf_idx; /* buffer index */
+	uint16_t len;	/* packet length, to be copied to/from the hw ring */
+	uint16_t flags;	/* buf changed, etc. */
+#define	NS_BUF_CHANGED	0x0001	/* must resync the map, buffer changed */
+#define	NS_REPORT	0x0002	/* ask the hardware to report results
+				 * e.g. by generating an interrupt
+				 */
+};
+
+/*
+ * Netmap representation of a TX or RX ring (also known as "queue").
+ * This is a queue implemented as a fixed-size circular array.
+ * At the software level, two fields are important: avail and cur.
+ *
+ * In TX rings:
+ *	avail	indicates the number of slots available for transmission.
+ *		It is decremented by the application when it appends a
+ *		packet, and set to nr_hwavail (see below) on a
+ *		NIOCTXSYNC to reflect the actual state of the queue
+ *		(keeping track of completed transmissions).
+ *	cur	indicates the empty slot to use for the next packet
+ *		to send (i.e. the "tail" of the queue).
+ *		It is incremented by the application.
+ *
+ *   The kernel side of netmap uses two additional fields in its own
+ *   private ring structure, netmap_kring:
+ *	nr_hwcur is a copy of nr_cur on an NIOCTXSYNC.
+ *	nr_hwavail is the number of slots known as available by the
+ *		hardware. It is updated on an INTR (inc by the
+ *		number of packets sent) and on a NIOCTXSYNC
+ *		(decrease by nr_cur - nr_hwcur)
+ *		A special case, nr_hwavail is -1 if the transmit
+ *		side is idle (no pending transmits).
+ *
+ * In RX rings:
+ *	avail	is the number of packets available (possibly 0).
+ *		It is decremented by the software when it consumes
+ *		a packet, and set to nr_hwavail on a NIOCRXSYNC
+ *	cur	indicates the first slot that contains a packet
+ *		(the "head" of the queue).
+ *		It is incremented by the software when it consumes
+ *		a packet.
+ *
+ *   The kernel side of netmap uses two additional fields in the kring:
+ *	nr_hwcur is a copy of nr_cur on an NIOCRXSYNC
+ *	nr_hwavail is the number of packets available. It is updated
+ *		on INTR (inc by the number of new packets arrived)
+ *		and on NIOCRXSYNC (decreased by nr_cur - nr_hwcur).
+ *
+ * DATA OWNERSHIP/LOCKING:
+ *	The netmap_ring is owned by the user program and it is only
+ *	accessed or modified in the upper half of the kernel during
+ *	a system call.
+ *
+ *	The netmap_kring is only modified by the upper half of the kernel.
+ */
+struct netmap_ring {
+	/*
+	 * nr_buf_base_ofs is meant to be used through macros.
+	 * It contains the offset of the buffer region from this
+	 * descriptor.
+	 */
+	const ssize_t	buf_ofs;
+	const uint32_t	num_slots;	/* number of slots in the ring. */
+	uint32_t	avail;		/* number of usable slots */
+	uint32_t	cur;		/* 'current' r/w position */
+
+	const uint16_t	nr_buf_size;
+	uint16_t	flags;
+	/*
+	 * When a ring is reinitialized, the kernel sets kflags.
+	 * On exit from a syscall, if the flag is found set, we
+	 * also reinitialize the nr_* variables. The kflag is then
+	 * unconditionally copied to nr_flags and cleared.
+	 */
+#define	NR_REINIT	0x0001		/* ring reinitialized! */
+#define	NR_TIMESTAMP	0x0002		/* set timestamp on *sync() */
+
+	struct timeval	ts;		/* time of last *sync() */
+
+	/* the slots follow. This struct has variable size */
+	struct netmap_slot slot[0]; /* array of slots. */
+};
+
+
+/*
+ * Netmap representation of an interface and its queue(s).
+ * There is one netmap_if for each file descriptor on which we want
+ * to select/poll.  We assume that on each interface has the same number
+ * of receive and transmit queues.
+ * select/poll operates on one or all pairs depending on the value of
+ * nmr_queueid passed on the ioctl.
+ */
+struct netmap_if {
+	char ni_name[IFNAMSIZ]; /* name of the interface. */
+	const u_int ni_version;	/* API version, currently unused */
+	const u_int ni_num_queues; /* number of queue pairs (TX/RX). */
+	const u_int ni_rx_queues;  /* if zero, use ni_num_queues */
+	/*
+	 * the following array contains the offset of the
+	 * each netmap ring from this structure. The first num_queues+1
+	 * refer to the tx rings, the next n+1 refer to the rx rings.
+	 * The area is filled up by the kernel on NIOCREG,
+	 * and then only read by userspace code.
+	 * entries 0..ni_num_queues-1 indicate the hardware queues,
+	 * entry ni_num_queues is the queue from/to the stack.
+	 */
+	const ssize_t	ring_ofs[0];
+};
+
+#ifndef IFCAP_NETMAP	/* this should go in net/if.h */
+#define	IFCAP_NETMAP	0x100000
+#endif
+
+#ifndef NIOCREGIF	
+/*
+ * ioctl names and related fields
+ *
+ * NIOCGINFO takes a struct ifreq, the interface name is the input,
+ *	the outputs are number of queues and number of descriptor
+ *	for each queue (useful to set number of threads etc.).
+ *
+ * NIOCREGIF takes an interface name within a struct ifreq,
+ *	and activates netmap mode on the interface (if possible).
+ *
+ * NIOCUNREGIF unregisters the interface associated to the fd.
+ *
+ * NIOCTXSYNC, NIOCRXSYNC synchronize tx or rx queues,
+ *	whose identity is set in NIOCREGIF through nr_ringid
+ */
+
+/*
+ * struct nmreq overlays a struct ifreq
+ */
+struct nmreq {
+	char		nr_name[IFNAMSIZ];
+	uint32_t	nr_version;	/* API version (unused) */
+	uint32_t	nr_offset;	/* nifp offset in the shared region */
+	uint32_t	nr_memsize;	/* size of the shared region */
+	uint32_t	nr_numslots;	/* descriptors per queue */
+	uint16_t	nr_numrings;
+	uint16_t	nr_ringid;	/* ring(s) we care about */
+#define NETMAP_HW_RING	0x4000		/* low bits indicate one hw ring */
+#define NETMAP_SW_RING	0x2000		/* we process the sw ring */
+#define NETMAP_NO_TX_POLL	0x1000	/* no gratuitous txsync on poll */
+#define NETMAP_RING_MASK 0xfff		/* the ring number */
+};
+
+/*
+ * default buf size is 2048, but it may make sense to have
+ * it shorter for better cache usage.
+ */
+
+#define	NETMAP_BUF_SIZE	(2048)
+#define NIOCGINFO	_IOWR('i', 145, struct nmreq) /* return IF info */
+#define NIOCREGIF	_IOWR('i', 146, struct nmreq) /* interface register */
+#define NIOCUNREGIF	_IO('i', 147) /* interface unregister */
+#define NIOCTXSYNC	_IO('i', 148) /* sync tx queues */
+#define NIOCRXSYNC	_IO('i', 149) /* sync rx queues */
+#endif /* !NIOCREGIF */
+
+#endif /* _NET_NETMAP_H_ */
diff --git a/sys/net/netmap_user.h b/sys/net/netmap_user.h
new file mode 100644
index 0000000..c9443b8
--- /dev/null
+++ b/sys/net/netmap_user.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ * 
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the
+ *      distribution.
+ * 
+ *   3. Neither the name of the authors nor the names of their contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY MATTEO LANDI AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL MATTEO LANDI OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: netmap_user.h 9495 2011-10-18 15:28:23Z luigi $
+ *
+ * This header contains the macros used to manipulate netmap structures
+ * and packets in userspace. See netmap(4) for more information.
+ *
+ * The address of the struct netmap_if, say nifp, is determined
+ * by the value returned from ioctl(.., NIOCREG, ...) and the mmap
+ * region:
+ *	ioctl(fd, NIOCREG, &req);
+ *	mem = mmap(0, ... );
+ *	nifp = NETMAP_IF(mem, req.nr_nifp);
+ *		(so simple, we could just do it manually)
+ *
+ * From there:
+ *	struct netmap_ring *NETMAP_TXRING(nifp, index)
+ *	struct netmap_ring *NETMAP_RXRING(nifp, index)
+ *		we can access ring->nr_cur, ring->nr_avail, ring->nr_flags
+ *
+ *	ring->slot[i] gives us the i-th slot (we can access
+ *		directly plen, flags, bufindex)
+ *
+ *	char *buf = NETMAP_BUF(ring, index) returns a pointer to
+ *		the i-th buffer
+ *
+ * Since rings are circular, we have macros to compute the next index
+ *	i = NETMAP_RING_NEXT(ring, i);
+ */
+
+#ifndef _NET_NETMAP_USER_H_
+#define _NET_NETMAP_USER_H_
+
+#define NETMAP_IF(b, o)	(struct netmap_if *)((char *)(b) + (o))
+
+#define NETMAP_TXRING(nifp, index)			\
+	((struct netmap_ring *)((char *)(nifp) +	\
+		(nifp)->ring_ofs[index] ) )
+
+#define NETMAP_RXRING(nifp, index)			\
+	((struct netmap_ring *)((char *)(nifp) +	\
+	    (nifp)->ring_ofs[index + (nifp)->ni_num_queues+1] ) )
+
+#if NETMAP_BUF_SIZE != 2048
+#error cannot handle odd size
+#define NETMAP_BUF(ring, index)				\
+	((char *)(ring) + (ring)->buf_ofs + ((index)*NETMAP_BUF_SIZE))
+#else
+#define NETMAP_BUF(ring, index)				\
+	((char *)(ring) + (ring)->buf_ofs + ((index)<<11))
+#endif
+
+#define	NETMAP_RING_NEXT(r, i)				\
+	((i)+1 == (r)->num_slots ? 0 : (i) + 1 )
+
+/*
+ * Return 1 if the given tx ring is empty.
+ *
+ * @r netmap_ring descriptor pointer.
+ * Special case, a negative value in hwavail indicates that the
+ * transmit queue is idle.
+ * XXX revise
+ */
+#define NETMAP_TX_RING_EMPTY(r)	((r)->avail >= (r)->num_slots - 1)
+
+#endif /* _NET_NETMAP_USER_H_ */
diff --git a/sys/net/raw_cb.c b/sys/net/raw_cb.c
index c430d97..c90f537 100644
--- a/sys/net/raw_cb.c
+++ b/sys/net/raw_cb.c
@@ -59,7 +59,8 @@
 struct mtx rawcb_mtx;
 VNET_DEFINE(struct rawcb_list_head, rawcb_list);
 
-SYSCTL_NODE(_net, OID_AUTO, raw, CTLFLAG_RW, 0, "Raw socket infrastructure");
+static SYSCTL_NODE(_net, OID_AUTO, raw, CTLFLAG_RW, 0,
+    "Raw socket infrastructure");
 
 static u_long	raw_sendspace = RAWSNDQ;
 SYSCTL_ULONG(_net_raw, OID_AUTO, sendspace, CTLFLAG_RW, &raw_sendspace, 0,
diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c
index fdd017c..b2313c0 100644
--- a/sys/net/rtsock.c
+++ b/sys/net/rtsock.c
@@ -143,7 +143,7 @@ MTX_SYSINIT(rtsock, &rtsock_mtx, "rtsock route_cb lock", MTX_DEF);
 #define	RTSOCK_UNLOCK()	mtx_unlock(&rtsock_mtx)
 #define	RTSOCK_LOCK_ASSERT()	mtx_assert(&rtsock_mtx, MA_OWNED)
 
-SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RD, 0, "");
+static SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RD, 0, "");
 
 struct walkarg {
 	int	w_tmemsize;
@@ -166,7 +166,7 @@ static void	rt_setmetrics(u_long which, const struct rt_metrics *in,
 			struct rt_metrics_lite *out);
 static void	rt_getmetrics(const struct rt_metrics_lite *in,
 			struct rt_metrics *out);
-static void	rt_dispatch(struct mbuf *, const struct sockaddr *);
+static void	rt_dispatch(struct mbuf *, sa_family_t);
 
 static struct netisr_handler rtsock_nh = {
 	.nh_name = "rtsock",
@@ -545,6 +545,7 @@ route_output(struct mbuf *m, struct socket *so)
 	int len, error = 0;
 	struct ifnet *ifp = NULL;
 	union sockaddr_union saun;
+	sa_family_t saf = AF_UNSPEC;
 
 #define senderr(e) { error = e; goto flush;}
 	if (m == NULL || ((m->m_len < sizeof(long)) &&
@@ -581,6 +582,7 @@ route_output(struct mbuf *m, struct socket *so)
 	    (info.rti_info[RTAX_GATEWAY] != NULL &&
 	     info.rti_info[RTAX_GATEWAY]->sa_family >= AF_MAX))
 		senderr(EINVAL);
+	saf = info.rti_info[RTAX_DST]->sa_family;
 	/*
 	 * Verify that the caller has the appropriate privilege; RTM_GET
 	 * is the only operation the non-superuser is allowed.
@@ -926,10 +928,10 @@ flush:
 			 */
 			unsigned short family = rp->rcb_proto.sp_family;
 			rp->rcb_proto.sp_family = 0;
-			rt_dispatch(m, info.rti_info[RTAX_DST]);
+			rt_dispatch(m, saf);
 			rp->rcb_proto.sp_family = family;
 		} else
-			rt_dispatch(m, info.rti_info[RTAX_DST]);
+			rt_dispatch(m, saf);
 	}
 	/* info.rti_info[RTAX_DST] (used above) can point inside of rtm */
 	if (rtm)
@@ -1185,7 +1187,7 @@ rt_missmsg_fib(int type, struct rt_addrinfo *rtinfo, int flags, int error,
 	rtm->rtm_flags = RTF_DONE | flags;
 	rtm->rtm_errno = error;
 	rtm->rtm_addrs = rtinfo->rti_addrs;
-	rt_dispatch(m, sa);
+	rt_dispatch(m, sa ? sa->sa_family : AF_UNSPEC);
 }
 
 void
@@ -1217,7 +1219,7 @@ rt_ifmsg(struct ifnet *ifp)
 	ifm->ifm_flags = ifp->if_flags | ifp->if_drv_flags;
 	ifm->ifm_data = ifp->if_data;
 	ifm->ifm_addrs = 0;
-	rt_dispatch(m, NULL);
+	rt_dispatch(m, AF_UNSPEC);
 }
 
 /*
@@ -1295,7 +1297,7 @@ rt_newaddrmsg_fib(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt,
 			M_SETFIB(m, fibnum);
 			m->m_flags |= RTS_FILTER_FIB;
 		}
-		rt_dispatch(m, sa);
+		rt_dispatch(m, sa ? sa->sa_family : AF_UNSPEC);
 	}
 }
 
@@ -1338,7 +1340,7 @@ rt_newmaddrmsg(int cmd, struct ifmultiaddr *ifma)
 	    __func__));
 	ifmam->ifmam_index = ifp->if_index;
 	ifmam->ifmam_addrs = info.rti_addrs;
-	rt_dispatch(m, ifma->ifma_addr);
+	rt_dispatch(m, ifma->ifma_addr ? ifma->ifma_addr->sa_family : AF_UNSPEC);
 }
 
 static struct mbuf *
@@ -1398,7 +1400,7 @@ rt_ieee80211msg(struct ifnet *ifp, int what, void *data, size_t data_len)
 		if (m->m_flags & M_PKTHDR)
 			m->m_pkthdr.len += data_len;
 		mtod(m, struct if_announcemsghdr *)->ifan_msglen += data_len;
-		rt_dispatch(m, NULL);
+		rt_dispatch(m, AF_UNSPEC);
 	}
 }
 
@@ -1414,11 +1416,11 @@ rt_ifannouncemsg(struct ifnet *ifp, int what)
 
 	m = rt_makeifannouncemsg(ifp, RTM_IFANNOUNCE, what, &info);
 	if (m != NULL)
-		rt_dispatch(m, NULL);
+		rt_dispatch(m, AF_UNSPEC);
 }
 
 static void
-rt_dispatch(struct mbuf *m, const struct sockaddr *sa)
+rt_dispatch(struct mbuf *m, sa_family_t saf)
 {
 	struct m_tag *tag;
 
@@ -1427,14 +1429,14 @@ rt_dispatch(struct mbuf *m, const struct sockaddr *sa)
 	 * use when injecting the mbuf into the routing socket buffer from
 	 * the netisr.
 	 */
-	if (sa != NULL) {
+	if (saf != AF_UNSPEC) {
 		tag = m_tag_get(PACKET_TAG_RTSOCKFAM, sizeof(unsigned short),
 		    M_NOWAIT);
 		if (tag == NULL) {
 			m_freem(m);
 			return;
 		}
-		*(unsigned short *)(tag + 1) = sa->sa_family;
+		*(unsigned short *)(tag + 1) = saf;
 		m_tag_prepend(m, tag);
 	}
 #ifdef VIMAGE
@@ -1735,7 +1737,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 	return (error);
 }
 
-SYSCTL_NODE(_net, PF_ROUTE, routetable, CTLFLAG_RD, sysctl_rtsock, "");
+static SYSCTL_NODE(_net, PF_ROUTE, routetable, CTLFLAG_RD, sysctl_rtsock, "");
 
 /*
  * Definitions of protocols supported in the ROUTE domain.
diff --git a/sys/net/vnet.c b/sys/net/vnet.c
index b9eebd9..e8864aa 100644
--- a/sys/net/vnet.c
+++ b/sys/net/vnet.c
@@ -81,7 +81,7 @@ __FBSDID("$FreeBSD$");
 
 FEATURE(vimage, "VIMAGE kernel virtualization");
 
-MALLOC_DEFINE(M_VNET, "vnet", "network stack control block");
+static MALLOC_DEFINE(M_VNET, "vnet", "network stack control block");
 
 /*
  * The virtual network stack list has two read-write locks, one sleepable and
@@ -161,7 +161,7 @@ struct vnet *vnet0;
  */
 #define	VNET_BYTES	(VNET_STOP - VNET_START)
 
-MALLOC_DEFINE(M_VNET_DATA, "vnet_data", "VNET data");
+static MALLOC_DEFINE(M_VNET_DATA, "vnet_data", "VNET data");
 
 /*
  * VNET_MODMIN is the minimum number of bytes we will reserve for the sum of
@@ -203,7 +203,8 @@ struct vnet_data_free {
 	TAILQ_ENTRY(vnet_data_free) vnd_link;
 };
 
-MALLOC_DEFINE(M_VNET_DATA_FREE, "vnet_data_free", "VNET resource accounting");
+static MALLOC_DEFINE(M_VNET_DATA_FREE, "vnet_data_free",
+    "VNET resource accounting");
 static TAILQ_HEAD(, vnet_data_free) vnet_data_free_head =
 	    TAILQ_HEAD_INITIALIZER(vnet_data_free_head);
 static struct sx vnet_data_free_lock;
diff --git a/sys/net80211/ieee80211.h b/sys/net80211/ieee80211.h
index 5d8d8fa..028afec 100644
--- a/sys/net80211/ieee80211.h
+++ b/sys/net80211/ieee80211.h
@@ -759,6 +759,18 @@ struct ieee80211_country_ie {
 /*
  * 802.11h Channel Switch Announcement (CSA).
  */
+struct ieee80211_quiet_ie {
+	uint8_t		quiet_ie;		/* IEEE80211_ELEMID_QUIET */
+	uint8_t		len;
+	uint8_t		tbttcount;		/* quiet start */
+	uint8_t		period;			/* beacon intervals between quiets */
+	uint16_t	duration;		/* TUs of each quiet*/
+	uint16_t	offset;			/* TUs of from TBTT of quiet start */
+} __packed;
+
+/*
+ * 802.11h Channel Switch Announcement (CSA).
+ */
 struct ieee80211_csa_ie {
 	uint8_t		csa_ie;		/* IEEE80211_ELEMID_CHANSWITCHANN */
 	uint8_t		csa_len;
diff --git a/sys/net80211/ieee80211_acl.c b/sys/net80211/ieee80211_acl.c
index da505e3..d39c428 100644
--- a/sys/net80211/ieee80211_acl.c
+++ b/sys/net80211/ieee80211_acl.c
@@ -87,7 +87,7 @@ struct aclstate {
 #define	ACL_HASH(addr)	\
 	(((const uint8_t *)(addr))[IEEE80211_ADDR_LEN - 1] % ACL_HASHSIZE)
 
-MALLOC_DEFINE(M_80211_ACL, "acl", "802.11 station acl");
+static MALLOC_DEFINE(M_80211_ACL, "acl", "802.11 station acl");
 
 static	int acl_free_all(struct ieee80211vap *);
 
diff --git a/sys/net80211/ieee80211_dfs.c b/sys/net80211/ieee80211_dfs.c
index 19e552d..ffb05e9 100644
--- a/sys/net80211/ieee80211_dfs.c
+++ b/sys/net80211/ieee80211_dfs.c
@@ -52,7 +52,7 @@ __FBSDID("$FreeBSD$");
 
 #include <net80211/ieee80211_var.h>
 
-MALLOC_DEFINE(M_80211_DFS, "80211dfs", "802.11 DFS state");
+static MALLOC_DEFINE(M_80211_DFS, "80211dfs", "802.11 DFS state");
 
 static	int ieee80211_nol_timeout = 30*60;		/* 30 minutes */
 SYSCTL_INT(_net_wlan, OID_AUTO, nol_timeout, CTLFLAG_RW,
@@ -64,6 +64,12 @@ SYSCTL_INT(_net_wlan, OID_AUTO, cac_timeout, CTLFLAG_RW,
 	&ieee80211_cac_timeout, 0, "CAC timeout (secs)");
 #define	CAC_TIMEOUT	msecs_to_ticks(ieee80211_cac_timeout*1000)
 
+static int
+null_set_quiet(struct ieee80211_node *ni, u_int8_t *quiet_elm)
+{
+	return ENOSYS;
+}
+
 void
 ieee80211_dfs_attach(struct ieee80211com *ic)
 {
@@ -71,6 +77,8 @@ ieee80211_dfs_attach(struct ieee80211com *ic)
 
 	callout_init_mtx(&dfs->nol_timer, IEEE80211_LOCK_OBJ(ic), 0);
 	callout_init_mtx(&dfs->cac_timer, IEEE80211_LOCK_OBJ(ic), 0);
+
+	ic->ic_set_quiet = null_set_quiet;
 }
 
 void
diff --git a/sys/net80211/ieee80211_freebsd.c b/sys/net80211/ieee80211_freebsd.c
index 523ae1a..10ef1f6 100644
--- a/sys/net80211/ieee80211_freebsd.c
+++ b/sys/net80211/ieee80211_freebsd.c
@@ -63,7 +63,7 @@ SYSCTL_INT(_net_wlan, OID_AUTO, debug, CTLFLAG_RW, &ieee80211_debug,
 	    0, "debugging printfs");
 #endif
 
-MALLOC_DEFINE(M_80211_COM, "80211com", "802.11 com state");
+static MALLOC_DEFINE(M_80211_COM, "80211com", "802.11 com state");
 
 /*
  * Allocate/free com structure in conjunction with ifnet;
@@ -571,8 +571,8 @@ ieee80211_notify_replay_failure(struct ieee80211vap *vap,
 	struct ifnet *ifp = vap->iv_ifp;
 
 	IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_CRYPTO, wh->i_addr2,
-	    "%s replay detected <rsc %ju, csc %ju, keyix %u rxkeyix %u>",
-	    k->wk_cipher->ic_name, (intmax_t) rsc,
+	    "%s replay detected tid %d <rsc %ju, csc %ju, keyix %u rxkeyix %u>",
+	    tid, k->wk_cipher->ic_name, (intmax_t) rsc,
 	    (intmax_t) k->wk_keyrsc[tid],
 	    k->wk_keyix, k->wk_rxkeyix);
 
diff --git a/sys/net80211/ieee80211_hwmp.c b/sys/net80211/ieee80211_hwmp.c
index 78729fc..93a9e55 100644
--- a/sys/net80211/ieee80211_hwmp.c
+++ b/sys/net80211/ieee80211_hwmp.c
@@ -161,7 +161,7 @@ struct ieee80211_hwmp_state {
 	uint8_t			hs_maxhops;	/* max hop count */
 };
 
-SYSCTL_NODE(_net_wlan, OID_AUTO, hwmp, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_net_wlan, OID_AUTO, hwmp, CTLFLAG_RD, 0,
     "IEEE 802.11s HWMP parameters");
 static int	ieee80211_hwmp_targetonly = 0;
 SYSCTL_INT(_net_wlan_hwmp, OID_AUTO, targetonly, CTLTYPE_INT | CTLFLAG_RW,
diff --git a/sys/net80211/ieee80211_input.c b/sys/net80211/ieee80211_input.c
index a18da4a..6fdc499 100644
--- a/sys/net80211/ieee80211_input.c
+++ b/sys/net80211/ieee80211_input.c
@@ -522,6 +522,9 @@ ieee80211_parse_beacon(struct ieee80211_node *ni, struct mbuf *m,
 		case IEEE80211_ELEMID_CSA:
 			scan->csa = frm;
 			break;
+		case IEEE80211_ELEMID_QUIET:
+			scan->quiet = frm;
+			break;
 		case IEEE80211_ELEMID_FHPARMS:
 			if (ic->ic_phytype == IEEE80211_T_FH) {
 				scan->fhdwell = LE_READ_2(&frm[2]);
diff --git a/sys/net80211/ieee80211_ioctl.c b/sys/net80211/ieee80211_ioctl.c
index f8e1785..2e388ae 100644
--- a/sys/net80211/ieee80211_ioctl.c
+++ b/sys/net80211/ieee80211_ioctl.c
@@ -972,6 +972,21 @@ ieee80211_ioctl_get80211(struct ieee80211vap *vap, u_long cmd,
 	case IEEE80211_IOC_PUREG:
 		ireq->i_val = (vap->iv_flags & IEEE80211_F_PUREG) != 0;
 		break;
+	case IEEE80211_IOC_QUIET:
+		ireq->i_val = vap->iv_quiet;
+		break;
+	case IEEE80211_IOC_QUIET_COUNT:
+		ireq->i_val = vap->iv_quiet_count;
+		break;
+	case IEEE80211_IOC_QUIET_PERIOD:
+		ireq->i_val = vap->iv_quiet_period;
+		break;
+	case IEEE80211_IOC_QUIET_DUR:
+		ireq->i_val = vap->iv_quiet_duration;
+		break;
+	case IEEE80211_IOC_QUIET_OFFSET:
+		ireq->i_val = vap->iv_quiet_offset;
+		break;
 	case IEEE80211_IOC_BGSCAN:
 		ireq->i_val = (vap->iv_flags & IEEE80211_F_BGSCAN) != 0;
 		break;
@@ -2939,6 +2954,24 @@ ieee80211_ioctl_set80211(struct ieee80211vap *vap, u_long cmd, struct ieee80211r
 		if (isvap11g(vap))
 			error = ENETRESET;
 		break;
+	case IEEE80211_IOC_QUIET:
+		vap->iv_quiet= ireq->i_val;
+		break;
+	case IEEE80211_IOC_QUIET_COUNT:
+		vap->iv_quiet_count=ireq->i_val;
+		break;
+	case IEEE80211_IOC_QUIET_PERIOD:
+		vap->iv_quiet_period=ireq->i_val;
+		break;
+	case IEEE80211_IOC_QUIET_OFFSET:
+		vap->iv_quiet_offset=ireq->i_val;
+		break;
+	case IEEE80211_IOC_QUIET_DUR:
+		if(ireq->i_val < vap->iv_bss->ni_intval)
+			vap->iv_quiet_duration = ireq->i_val;
+		else
+			error = EINVAL;
+		break;
 	case IEEE80211_IOC_BGSCAN:
 		if (ireq->i_val) {
 			if ((vap->iv_caps & IEEE80211_C_BGSCAN) == 0)
diff --git a/sys/net80211/ieee80211_ioctl.h b/sys/net80211/ieee80211_ioctl.h
index cad5576..0f11bc6 100644
--- a/sys/net80211/ieee80211_ioctl.h
+++ b/sys/net80211/ieee80211_ioctl.h
@@ -715,6 +715,11 @@ struct ieee80211req {
 #define	IEEE80211_IOC_TDMA_SLOTLEN	203	/* TDMA: slot length (usecs) */
 #define	IEEE80211_IOC_TDMA_BINTERVAL	204	/* TDMA: beacon intvl (slots) */
 
+#define	IEEE80211_IOC_QUIET		205	/* Quiet Enable/Disable */
+#define	IEEE80211_IOC_QUIET_PERIOD	206	/* Quiet Period */
+#define	IEEE80211_IOC_QUIET_OFFSET	207	/* Quiet Offset */
+#define	IEEE80211_IOC_QUIET_DUR		208	/* Quiet Duration */
+#define	IEEE80211_IOC_QUIET_COUNT	209	/* Quiet Count */
 /*
  * Parameters for controlling a scan requested with
  * IEEE80211_IOC_SCAN_REQ.
diff --git a/sys/net80211/ieee80211_mesh.c b/sys/net80211/ieee80211_mesh.c
index 571a733..cfffd87 100644
--- a/sys/net80211/ieee80211_mesh.c
+++ b/sys/net80211/ieee80211_mesh.c
@@ -97,7 +97,7 @@ uint32_t	mesh_airtime_calc(struct ieee80211_node *);
 /*
  * Timeout values come from the specification and are in milliseconds.
  */
-SYSCTL_NODE(_net_wlan, OID_AUTO, mesh, CTLFLAG_RD, 0,
+static SYSCTL_NODE(_net_wlan, OID_AUTO, mesh, CTLFLAG_RD, 0,
     "IEEE 802.11s parameters");
 static int ieee80211_mesh_retrytimeout = -1;
 SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, retrytimeout, CTLTYPE_INT | CTLFLAG_RW,
diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c
index bcd3c2b..0e45e8c 100644
--- a/sys/net80211/ieee80211_output.c
+++ b/sys/net80211/ieee80211_output.c
@@ -1661,6 +1661,33 @@ ieee80211_add_supportedchannels(uint8_t *frm, struct ieee80211com *ic)
 }
 
 /*
+ * Add an 11h Quiet time element to a frame.
+ */
+static uint8_t *
+ieee80211_add_quiet(uint8_t *frm, struct ieee80211vap *vap)
+{
+	struct ieee80211_quiet_ie *quiet = (struct ieee80211_quiet_ie *) frm;
+
+	quiet->quiet_ie = IEEE80211_ELEMID_QUIET;
+	quiet->len = 6;
+	if (vap->iv_quiet_count_value == 1)
+		vap->iv_quiet_count_value = vap->iv_quiet_count;
+	else if (vap->iv_quiet_count_value > 1)
+		vap->iv_quiet_count_value--;
+
+	if (vap->iv_quiet_count_value == 0) {
+		/* value 0 is reserved as per 802.11h standerd */
+		vap->iv_quiet_count_value = 1;
+	}
+
+	quiet->tbttcount = vap->iv_quiet_count_value;
+	quiet->period = vap->iv_quiet_period;
+	quiet->duration = htole16(vap->iv_quiet_duration);
+	quiet->offset = htole16(vap->iv_quiet_offset);
+	return frm + sizeof(*quiet);
+}
+
+/*
  * Add an 11h Channel Switch Announcement element to a frame.
  * Note that we use the per-vap CSA count to adjust the global
  * counter so we can use this routine to form probe response
@@ -2253,6 +2280,7 @@ ieee80211_alloc_proberesp(struct ieee80211_node *bss, int legacy)
 	       + IEEE80211_COUNTRY_MAX_SIZE
 	       + 3
 	       + sizeof(struct ieee80211_csa_ie)
+	       + sizeof(struct ieee80211_quiet_ie)
 	       + 3
 	       + 2 + (IEEE80211_RATE_MAXSIZE - IEEE80211_RATE_SIZE)
 	       + sizeof(struct ieee80211_ie_wpa)
@@ -2319,6 +2347,13 @@ ieee80211_alloc_proberesp(struct ieee80211_node *bss, int legacy)
 		if (ic->ic_flags & IEEE80211_F_CSAPENDING)
 			frm = ieee80211_add_csa(frm, vap);
 	}
+	if (vap->iv_flags & IEEE80211_F_DOTH) {
+		if (IEEE80211_IS_CHAN_DFS(ic->ic_bsschan) &&
+		    (vap->iv_flags_ext & IEEE80211_FEXT_DFS)) {
+			if (vap->iv_quiet)
+				frm = ieee80211_add_quiet(frm, vap);
+		}
+	}
 	if (IEEE80211_IS_CHAN_ANYG(bss->ni_chan))
 		frm = ieee80211_add_erp(frm, ic);
 	frm = ieee80211_add_xrates(frm, rs);
@@ -2617,9 +2652,20 @@ ieee80211_beacon_construct(struct mbuf *m, uint8_t *frm,
 			frm = ieee80211_add_powerconstraint(frm, vap);
 		bo->bo_csa = frm;
 		if (ic->ic_flags & IEEE80211_F_CSAPENDING)
-			frm = ieee80211_add_csa(frm, vap);
+			frm = ieee80211_add_csa(frm, vap);	
 	} else
 		bo->bo_csa = frm;
+
+	if (vap->iv_flags & IEEE80211_F_DOTH) {
+		bo->bo_quiet = frm;
+		if (IEEE80211_IS_CHAN_DFS(ic->ic_bsschan) &&
+		    (vap->iv_flags_ext & IEEE80211_FEXT_DFS)) {
+			if (vap->iv_quiet)
+				frm = ieee80211_add_quiet(frm,vap);
+		}
+	} else
+		bo->bo_quiet = frm;
+
 	if (IEEE80211_IS_CHAN_ANYG(ni->ni_chan)) {
 		bo->bo_erp = frm;
 		frm = ieee80211_add_erp(frm, ic);
@@ -2733,7 +2779,8 @@ ieee80211_beacon_alloc(struct ieee80211_node *ni,
 		 + 2 + 4 + vap->iv_tim_len		/* DTIM/IBSSPARMS */
 		 + IEEE80211_COUNTRY_MAX_SIZE		/* country */
 		 + 2 + 1				/* power control */
-	         + sizeof(struct ieee80211_csa_ie)	/* CSA */
+		 + sizeof(struct ieee80211_csa_ie)	/* CSA */
+		 + sizeof(struct ieee80211_quiet_ie)	/* Quiet */
 		 + 2 + 1				/* ERP */
 	         + 2 + (IEEE80211_RATE_MAXSIZE - IEEE80211_RATE_SIZE)
 		 + (vap->iv_caps & IEEE80211_C_WPA ?	/* WPA 1+2 */
@@ -2953,6 +3000,7 @@ ieee80211_beacon_update(struct ieee80211_node *ni,
 				bo->bo_appie += adjust;
 				bo->bo_wme += adjust;
 				bo->bo_csa += adjust;
+				bo->bo_quiet += adjust;
 				bo->bo_tim_len = timlen;
 
 				/* update information element */
@@ -3006,6 +3054,7 @@ ieee80211_beacon_update(struct ieee80211_node *ni,
 #endif
 				bo->bo_appie += sizeof(*csa);
 				bo->bo_csa_trailer_len += sizeof(*csa);
+				bo->bo_quiet += sizeof(*csa);
 				bo->bo_tim_trailer_len += sizeof(*csa);
 				m->m_len += sizeof(*csa);
 				m->m_pkthdr.len += sizeof(*csa);
@@ -3016,6 +3065,11 @@ ieee80211_beacon_update(struct ieee80211_node *ni,
 			vap->iv_csa_count++;
 			/* NB: don't clear IEEE80211_BEACON_CSA */
 		}
+		if (IEEE80211_IS_CHAN_DFS(ic->ic_bsschan) &&
+		    (vap->iv_flags_ext & IEEE80211_FEXT_DFS) ){
+			if (vap->iv_quiet)
+				ieee80211_add_quiet(bo->bo_quiet, vap);
+		}
 		if (isset(bo->bo_flags, IEEE80211_BEACON_ERP)) {
 			/*
 			 * ERP element needs updating.
diff --git a/sys/net80211/ieee80211_power.c b/sys/net80211/ieee80211_power.c
index 201eea0..11eb141 100644
--- a/sys/net80211/ieee80211_power.c
+++ b/sys/net80211/ieee80211_power.c
@@ -48,7 +48,7 @@ __FBSDID("$FreeBSD$");
 static void ieee80211_update_ps(struct ieee80211vap *, int);
 static int ieee80211_set_tim(struct ieee80211_node *, int);
 
-MALLOC_DEFINE(M_80211_POWER, "80211power", "802.11 power save state");
+static MALLOC_DEFINE(M_80211_POWER, "80211power", "802.11 power save state");
 
 void
 ieee80211_power_attach(struct ieee80211com *ic)
diff --git a/sys/net80211/ieee80211_proto.h b/sys/net80211/ieee80211_proto.h
index 7e88216..54af007 100644
--- a/sys/net80211/ieee80211_proto.h
+++ b/sys/net80211/ieee80211_proto.h
@@ -344,6 +344,7 @@ struct ieee80211_beacon_offsets {
 	uint16_t	bo_appie_len;	/* AppIE length in bytes */
 	uint16_t	bo_csa_trailer_len;
 	uint8_t		*bo_csa;	/* start of CSA element */
+	uint8_t		*bo_quiet;	/* start of Quiet element */
 	uint8_t		*bo_meshconf;	/* start of MESHCONF element */
 	uint8_t		*bo_spare[3];
 };
diff --git a/sys/net80211/ieee80211_scan.h b/sys/net80211/ieee80211_scan.h
index 4c5e869..3f13e17 100644
--- a/sys/net80211/ieee80211_scan.h
+++ b/sys/net80211/ieee80211_scan.h
@@ -213,6 +213,7 @@ struct ieee80211_scanparams {
 	uint8_t		*ath;
 	uint8_t		*tdma;
 	uint8_t		*csa;
+	uint8_t		*quiet;
 	uint8_t		*meshid;
 	uint8_t		*meshconf;
 	uint8_t		*spare[3];
diff --git a/sys/net80211/ieee80211_sta.c b/sys/net80211/ieee80211_sta.c
index 97a9dbc..89b1934 100644
--- a/sys/net80211/ieee80211_sta.c
+++ b/sys/net80211/ieee80211_sta.c
@@ -50,6 +50,8 @@ __FBSDID("$FreeBSD$");
 #include <net/if.h>
 #include <net/if_media.h>
 #include <net/if_llc.h>
+#include <net/if_dl.h>
+#include <net/if_var.h>
 #include <net/ethernet.h>
 
 #include <net/bpf.h>
@@ -584,6 +586,30 @@ sta_input(struct ieee80211_node *ni, struct mbuf *m, int rssi, int nf)
 			vap->iv_stats.is_rx_wrongbss++;
 			goto out;
 		}
+
+		/*
+		 * Some devices may be in a promiscuous mode
+		 * where they receive frames for multiple station
+		 * addresses.
+		 *
+		 * If we receive a data frame that isn't
+		 * destined to our VAP MAC, drop it.
+		 *
+		 * XXX TODO: This is only enforced when not scanning;
+		 * XXX it assumes a software-driven scan will put the NIC
+		 * XXX into a "no data frames" mode before setting this
+		 * XXX flag. Otherwise it may be possible that we'll still
+		 * XXX process data frames whilst scanning.
+		 */
+		if ((! IEEE80211_IS_MULTICAST(wh->i_addr1))
+		    && (! IEEE80211_ADDR_EQ(wh->i_addr1, IF_LLADDR(ifp)))) {
+			IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT,
+			    bssid, NULL, "not to cur sta: lladdr=%6D, addr1=%6D",
+			    IF_LLADDR(ifp), ":", wh->i_addr1, ":");
+			vap->iv_stats.is_rx_wrongbss++;
+			goto out;
+		}
+
 		IEEE80211_RSSI_LPF(ni->ni_avgrssi, rssi);
 		ni->ni_noise = nf;
 		if (HAS_SEQ(type) && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
@@ -1347,6 +1373,8 @@ sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 				    scan.htcap, scan.htinfo);
 				/* XXX state changes? */
 			}
+			if (scan.quiet)
+				ic->ic_set_quiet(ni, scan.quiet);
 			if (scan.tim != NULL) {
 				struct ieee80211_tim_ie *tim =
 				    (struct ieee80211_tim_ie *) scan.tim;
diff --git a/sys/net80211/ieee80211_var.h b/sys/net80211/ieee80211_var.h
index a3dcd9f..60fbacb 100644
--- a/sys/net80211/ieee80211_var.h
+++ b/sys/net80211/ieee80211_var.h
@@ -242,6 +242,10 @@ struct ieee80211com {
 	int			(*ic_setregdomain)(struct ieee80211com *,
 				    struct ieee80211_regdomain *,
 				    int, struct ieee80211_channel []);
+
+	int			(*ic_set_quiet)(struct ieee80211_node *,
+				    u_int8_t *quiet_elm);
+
 	/* send/recv 802.11 management frame */
 	int			(*ic_send_mgmt)(struct ieee80211_node *,
 				     int, int);
@@ -403,6 +407,12 @@ struct ieee80211vap {
 	uint8_t			iv_dtim_period;	/* DTIM period */
 	uint8_t			iv_dtim_count;	/* DTIM count from last bcn */
 						/* set/unset aid pwrsav state */
+	uint8_t			iv_quiet;	/* Quiet Element */
+	uint8_t			iv_quiet_count;	/* constant count for Quiet Element */
+	uint8_t			iv_quiet_count_value;	/* variable count for Quiet Element */
+	uint8_t			iv_quiet_period;	/* period for Quiet Element */
+	uint16_t		iv_quiet_duration;	/* duration for Quiet Element */
+	uint16_t		iv_quiet_offset;	/* offset for Quiet Element */
 	int			iv_csa_count;	/* count for doing CSA */
 
 	struct ieee80211_node	*iv_bss;	/* information for this node */
diff --git a/sys/netgraph/atm/ng_atm.c b/sys/netgraph/atm/ng_atm.c
index 4b7d7c0..32bde57 100644
--- a/sys/netgraph/atm/ng_atm.c
+++ b/sys/netgraph/atm/ng_atm.c
@@ -75,7 +75,8 @@ extern void	(*ng_atm_event_p)(struct ifnet *, uint32_t, void *);
 /*
  * Sysctl stuff.
  */
-SYSCTL_NODE(_net_graph, OID_AUTO, atm, CTLFLAG_RW, 0, "atm related stuff");
+static SYSCTL_NODE(_net_graph, OID_AUTO, atm, CTLFLAG_RW, 0,
+    "atm related stuff");
 
 #ifdef NGATM_DEBUG
 static int allow_shutdown;
diff --git a/sys/netgraph/atm/ngatmbase.c b/sys/netgraph/atm/ngatmbase.c
index ecb23e6..c5b0ced 100644
--- a/sys/netgraph/atm/ngatmbase.c
+++ b/sys/netgraph/atm/ngatmbase.c
@@ -61,8 +61,8 @@ DECLARE_MODULE(ngatmbase, ngatm_data, SI_SUB_EXEC, SI_ORDER_ANY);
 /*
  * UNI Stack message handling functions
  */
-MALLOC_DEFINE(M_UNIMSG, "unimsg", "uni message buffers");
-MALLOC_DEFINE(M_UNIMSGHDR, "unimsghdr", "uni message headers");
+static MALLOC_DEFINE(M_UNIMSG, "unimsg", "uni message buffers");
+static MALLOC_DEFINE(M_UNIMSGHDR, "unimsghdr", "uni message headers");
 
 #define EXTRA	128
 
diff --git a/sys/netgraph/atm/uni/ng_uni.c b/sys/netgraph/atm/uni/ng_uni.c
index 2fc9599..980b81a 100644
--- a/sys/netgraph/atm/uni/ng_uni.c
+++ b/sys/netgraph/atm/uni/ng_uni.c
@@ -60,8 +60,8 @@ __FBSDID("$FreeBSD$");
 #include <netgraph/atm/ng_sscfu.h>
 #include <netgraph/atm/ng_uni.h>
 
-MALLOC_DEFINE(M_NG_UNI, "netgraph_uni_node", "netgraph uni node");
-MALLOC_DEFINE(M_UNI, "netgraph_uni_data", "uni protocol data");
+static MALLOC_DEFINE(M_NG_UNI, "netgraph_uni_node", "netgraph uni node");
+static MALLOC_DEFINE(M_UNI, "netgraph_uni_data", "uni protocol data");
 
 MODULE_DEPEND(ng_uni, ngatmbase, 1, 1, 1);
 
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c b/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c
index d8c32b1..df18913 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c
@@ -61,7 +61,7 @@
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_HCI_RAW, "netgraph_btsocks_hci_raw",
+static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_HCI_RAW, "netgraph_btsocks_hci_raw",
 	"Netgraph Bluetooth raw HCI sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_HCI_RAW M_NETGRAPH
@@ -123,7 +123,7 @@ static int					ng_btsocket_hci_raw_curpps;
  
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_hci_sockets);
-SYSCTL_NODE(_net_bluetooth_hci_sockets, OID_AUTO, raw, CTLFLAG_RW,
+static SYSCTL_NODE(_net_bluetooth_hci_sockets, OID_AUTO, raw, CTLFLAG_RW,
         0, "Bluetooth raw HCI sockets family");
 SYSCTL_UINT(_net_bluetooth_hci_sockets_raw, OID_AUTO, debug_level, CTLFLAG_RW,
         &ng_btsocket_hci_raw_debug_level, NG_BTSOCKET_WARN_LEVEL,
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
index efca5389..0066a85 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
@@ -63,7 +63,7 @@
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_L2CAP, "netgraph_btsocks_l2cap",
+static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_L2CAP, "netgraph_btsocks_l2cap",
 		"Netgraph Bluetooth L2CAP sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_L2CAP M_NETGRAPH
@@ -111,7 +111,7 @@ static int					ng_btsocket_l2cap_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_l2cap_sockets);
-SYSCTL_NODE(_net_bluetooth_l2cap_sockets, OID_AUTO, seq, CTLFLAG_RW,
+static SYSCTL_NODE(_net_bluetooth_l2cap_sockets, OID_AUTO, seq, CTLFLAG_RW,
 	0, "Bluetooth SEQPACKET L2CAP sockets family");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_seq, OID_AUTO, debug_level,
 	CTLFLAG_RW,
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c
index 72339f6..29cca80 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c
@@ -60,8 +60,8 @@
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_L2CAP_RAW, "netgraph_btsocks_l2cap_raw",
-		"Netgraph Bluetooth raw L2CAP sockets");
+static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_L2CAP_RAW,
+    "netgraph_btsocks_l2cap_raw", "Netgraph Bluetooth raw L2CAP sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_L2CAP_RAW M_NETGRAPH
 #endif /* NG_SEPARATE_MALLOC */
@@ -123,7 +123,7 @@ static int					ng_btsocket_l2cap_raw_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_l2cap_sockets);
-SYSCTL_NODE(_net_bluetooth_l2cap_sockets, OID_AUTO, raw, CTLFLAG_RW,
+static SYSCTL_NODE(_net_bluetooth_l2cap_sockets, OID_AUTO, raw, CTLFLAG_RW,
 	0, "Bluetooth raw L2CAP sockets family");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_raw, OID_AUTO, debug_level,
 	CTLFLAG_RW,
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
index e90cab2..a0e266e 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
@@ -66,7 +66,7 @@
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_RFCOMM, "netgraph_btsocks_rfcomm",
+static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_RFCOMM, "netgraph_btsocks_rfcomm",
 		"Netgraph Bluetooth RFCOMM sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_RFCOMM M_NETGRAPH
@@ -203,7 +203,7 @@ static int					ng_btsocket_rfcomm_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_rfcomm_sockets);
-SYSCTL_NODE(_net_bluetooth_rfcomm_sockets, OID_AUTO, stream, CTLFLAG_RW,
+static SYSCTL_NODE(_net_bluetooth_rfcomm_sockets, OID_AUTO, stream, CTLFLAG_RW,
 	0, "Bluetooth STREAM RFCOMM sockets family");
 SYSCTL_UINT(_net_bluetooth_rfcomm_sockets_stream, OID_AUTO, debug_level,
 	CTLFLAG_RW,
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c b/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
index 85b68dc..12786fa 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
@@ -63,7 +63,7 @@
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_SCO, "netgraph_btsocks_sco",
+static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_SCO, "netgraph_btsocks_sco",
 		"Netgraph Bluetooth SCO sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_SCO M_NETGRAPH
@@ -110,7 +110,7 @@ static int					ng_btsocket_sco_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_sco_sockets);
-SYSCTL_NODE(_net_bluetooth_sco_sockets, OID_AUTO, seq, CTLFLAG_RW,
+static SYSCTL_NODE(_net_bluetooth_sco_sockets, OID_AUTO, seq, CTLFLAG_RW,
 	0, "Bluetooth SEQPACKET SCO sockets family");
 SYSCTL_UINT(_net_bluetooth_sco_sockets_seq, OID_AUTO, debug_level,
 	CTLFLAG_RW,
diff --git a/sys/netgraph/netgraph.h b/sys/netgraph/netgraph.h
index 7b67c94e..d649c6c 100644
--- a/sys/netgraph/netgraph.h
+++ b/sys/netgraph/netgraph.h
@@ -57,6 +57,7 @@
 
 #ifdef HAVE_KERNEL_OPTION_HEADERS
 #include "opt_netgraph.h"
+#include "opt_kdb.h"
 #endif
 
 /* debugging options */
@@ -190,7 +191,7 @@ static __inline void
 _chkhook(hook_p hook, char *file, int line)
 {
 	if (hook->hk_magic != HK_MAGIC) {
-		printf("Accessing freed hook ");
+		printf("Accessing freed ");
 		dumphook(hook, file, line);
 	}
 	hook->lastline = line;
@@ -458,7 +459,7 @@ static __inline void
 _chknode(node_p node, char *file, int line)
 {
 	if (node->nd_magic != ND_MAGIC) {
-		printf("Accessing freed node ");
+		printf("Accessing freed ");
 		dumpnode(node, file, line);
 	}
 	node->lastline = line;
@@ -1134,7 +1135,7 @@ SYSCTL_DECL(_net_graph);
  */
 int	ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr);
 int	ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr);
-int	ng_address_path(node_p here, item_p item, char *address, ng_ID_t raddr);
+int	ng_address_path(node_p here, item_p item, const char *address, ng_ID_t raddr);
 int	ng_bypass(hook_p hook1, hook_p hook2);
 hook_p	ng_findhook(node_p node, const char *name);
 struct	ng_type *ng_findtype(const char *type);
diff --git a/sys/netgraph/ng_async.c b/sys/netgraph/ng_async.c
index 283ba15..217590b 100644
--- a/sys/netgraph/ng_async.c
+++ b/sys/netgraph/ng_async.c
@@ -61,7 +61,7 @@
 #include <net/ppp_defs.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_ASYNC, "netgraph_async", "netgraph async node ");
+static MALLOC_DEFINE(M_NETGRAPH_ASYNC, "netgraph_async", "netgraph async node");
 #else
 #define M_NETGRAPH_ASYNC M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_atmllc.c b/sys/netgraph/ng_atmllc.c
index 336d4b1..3ba9dff 100644
--- a/sys/netgraph/ng_atmllc.c
+++ b/sys/netgraph/ng_atmllc.c
@@ -153,7 +153,7 @@ ng_atmllc_rcvdata(hook_p hook, item_p item)
 	int	error;
 
 	priv = NG_NODE_PRIVATE(NG_HOOK_NODE(hook));
-	m = NGI_M(item);
+	NGI_GET_M(item, m);
 	outhook = NULL;
 	padding = 0;
 
@@ -170,6 +170,7 @@ ng_atmllc_rcvdata(hook_p hook, item_p item)
 		if (m->m_len < sizeof(struct atmllc) + ETHER_HDR_LEN) {
 			m = m_pullup(m, sizeof(struct atmllc) + ETHER_HDR_LEN);
 			if (m == NULL) {
+				NG_FREE_ITEM(item);
 				return (ENOMEM);
 			}
 		}
@@ -236,6 +237,7 @@ ng_atmllc_rcvdata(hook_p hook, item_p item)
 	}
 
 	if (outhook == NULL) {
+		NG_FREE_M(m);
 		NG_FREE_ITEM(item);
 		return (0);
 	}
diff --git a/sys/netgraph/ng_base.c b/sys/netgraph/ng_base.c
index 0918193..2f8e99f 100644
--- a/sys/netgraph/ng_base.c
+++ b/sys/netgraph/ng_base.c
@@ -232,10 +232,13 @@ void	ng_unname(node_p node);
 
 /* Our own netgraph malloc type */
 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages");
-MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", "netgraph hook structures");
-MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", "netgraph node structures");
-MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", "netgraph item structures");
 MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage");
+static MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook",
+    "netgraph hook structures");
+static MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node",
+    "netgraph node structures");
+static MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item",
+    "netgraph item structures");
 
 /* Should not be visible outside this file */
 
@@ -3167,6 +3170,9 @@ dumphook (hook_p hook, char *file, int line)
 		hook->lastfile, hook->lastline);
 	if (line) {
 		printf(" problem discovered at file %s, line %d\n", file, line);
+#ifdef KDB
+		kdb_backtrace();
+#endif
 	}
 }
 
@@ -3181,6 +3187,9 @@ dumpnode(node_p node, char *file, int line)
 		node->lastfile, node->lastline);
 	if (line) {
 		printf(" problem discovered at file %s, line %d\n", file, line);
+#ifdef KDB
+		kdb_backtrace();
+#endif
 	}
 }
 
@@ -3509,7 +3518,7 @@ ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr)
 }
 
 int
-ng_address_path(node_p here, item_p item, char *address, ng_ID_t retaddr)
+ng_address_path(node_p here, item_p item, const char *address, ng_ID_t retaddr)
 {
 	node_p	dest = NULL;
 	hook_p	hook = NULL;
diff --git a/sys/netgraph/ng_bpf.c b/sys/netgraph/ng_bpf.c
index 50a6bf7..49db81e 100644
--- a/sys/netgraph/ng_bpf.c
+++ b/sys/netgraph/ng_bpf.c
@@ -74,7 +74,7 @@
 #include <netgraph/ng_bpf.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BPF, "netgraph_bpf", "netgraph bpf node ");
+static MALLOC_DEFINE(M_NETGRAPH_BPF, "netgraph_bpf", "netgraph bpf node");
 #else
 #define M_NETGRAPH_BPF M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_bridge.c b/sys/netgraph/ng_bridge.c
index e9631b8..738441d 100644
--- a/sys/netgraph/ng_bridge.c
+++ b/sys/netgraph/ng_bridge.c
@@ -84,7 +84,8 @@
 #include <netgraph/ng_bridge.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_BRIDGE, "netgraph_bridge", "netgraph bridge node");
+static MALLOC_DEFINE(M_NETGRAPH_BRIDGE, "netgraph_bridge",
+    "netgraph bridge node");
 #else
 #define M_NETGRAPH_BRIDGE M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_cisco.c b/sys/netgraph/ng_cisco.c
index 76dc1ec..aa12b75 100644
--- a/sys/netgraph/ng_cisco.c
+++ b/sys/netgraph/ng_cisco.c
@@ -359,12 +359,13 @@ cisco_rcvdata(hook_p hook, item_p item)
 
 	/* OK so it came from a protocol, heading out. Prepend general data
 	   packet header. For now, IP,IPX only  */
-	m = NGI_M(item); /* still associated with item */
+	NGI_GET_M(item, m);
 	M_PREPEND(m, CISCO_HEADER_LEN, M_DONTWAIT);
 	if (!m) {
 		error = ENOBUFS;
 		goto out;
 	}
+	NGI_M(item) = m;
 	h = mtod(m, struct cisco_header *);
 	h->address = CISCO_UNICAST;
 	h->control = 0;
diff --git a/sys/netgraph/ng_deflate.c b/sys/netgraph/ng_deflate.c
index f3ce1c0..5fa6363 100644
--- a/sys/netgraph/ng_deflate.c
+++ b/sys/netgraph/ng_deflate.c
@@ -49,7 +49,8 @@
 
 #include "opt_netgraph.h"
 
-MALLOC_DEFINE(M_NETGRAPH_DEFLATE, "netgraph_deflate", "netgraph deflate node ");
+static MALLOC_DEFINE(M_NETGRAPH_DEFLATE, "netgraph_deflate",
+    "netgraph deflate node");
 
 /* DEFLATE header length */
 #define DEFLATE_HDRLEN		2
diff --git a/sys/netgraph/ng_fec.c b/sys/netgraph/ng_fec.c
index eff70ea..53f59e7 100644
--- a/sys/netgraph/ng_fec.c
+++ b/sys/netgraph/ng_fec.c
@@ -1332,7 +1332,7 @@ ng_fec_shutdown(node_p node)
 	}
 
 	ether_ifdetach(priv->ifp);
-	if_free_type(priv->ifp, IFT_ETHER);
+	if_free(priv->ifp);
 	ifmedia_removeall(&priv->ifmedia);
 	ng_fec_free_unit(priv->unit);
 	free(priv, M_NETGRAPH);
diff --git a/sys/netgraph/ng_gif_demux.c b/sys/netgraph/ng_gif_demux.c
index 857f10f..646e37d 100644
--- a/sys/netgraph/ng_gif_demux.c
+++ b/sys/netgraph/ng_gif_demux.c
@@ -89,7 +89,7 @@
 #include <netgraph/ng_gif_demux.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_GIF_DEMUX, "netgraph_gif_demux",
+static MALLOC_DEFINE(M_NETGRAPH_GIF_DEMUX, "netgraph_gif_demux",
     "netgraph gif demux node");
 #else
 #define M_NETGRAPH_GIF_DEMUX M_NETGRAPH
diff --git a/sys/netgraph/ng_hub.c b/sys/netgraph/ng_hub.c
index f013aac..2a8c2d2 100644
--- a/sys/netgraph/ng_hub.c
+++ b/sys/netgraph/ng_hub.c
@@ -37,7 +37,7 @@
 #include <netgraph/netgraph.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_HUB, "netgraph_hub", "netgraph hub node");
+static MALLOC_DEFINE(M_NETGRAPH_HUB, "netgraph_hub", "netgraph hub node");
 #else
 #define M_NETGRAPH_HUB M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_iface.c b/sys/netgraph/ng_iface.c
index e774098..3ce0261 100644
--- a/sys/netgraph/ng_iface.c
+++ b/sys/netgraph/ng_iface.c
@@ -87,7 +87,7 @@
 #include <netgraph/ng_cisco.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_IFACE, "netgraph_iface", "netgraph iface node ");
+static MALLOC_DEFINE(M_NETGRAPH_IFACE, "netgraph_iface", "netgraph iface node");
 #else
 #define M_NETGRAPH_IFACE M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_ksocket.c b/sys/netgraph/ng_ksocket.c
index 6caade0..20121ba 100644
--- a/sys/netgraph/ng_ksocket.c
+++ b/sys/netgraph/ng_ksocket.c
@@ -69,7 +69,8 @@
 #include <netatalk/at.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_KSOCKET, "netgraph_ksock", "netgraph ksock node ");
+static MALLOC_DEFINE(M_NETGRAPH_KSOCKET, "netgraph_ksock",
+    "netgraph ksock node");
 #else
 #define M_NETGRAPH_KSOCKET M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_l2tp.c b/sys/netgraph/ng_l2tp.c
index 201fc45..3b91b84 100644
--- a/sys/netgraph/ng_l2tp.c
+++ b/sys/netgraph/ng_l2tp.c
@@ -62,7 +62,7 @@
 #include <netgraph/ng_l2tp.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_L2TP, "netgraph_l2tp", "netgraph l2tp node");
+static MALLOC_DEFINE(M_NETGRAPH_L2TP, "netgraph_l2tp", "netgraph l2tp node");
 #else
 #define M_NETGRAPH_L2TP M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_mppc.c b/sys/netgraph/ng_mppc.c
index cb65316..6754be7 100644
--- a/sys/netgraph/ng_mppc.c
+++ b/sys/netgraph/ng_mppc.c
@@ -74,7 +74,7 @@
 #endif
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_MPPC, "netgraph_mppc", "netgraph mppc node ");
+static MALLOC_DEFINE(M_NETGRAPH_MPPC, "netgraph_mppc", "netgraph mppc node");
 #else
 #define M_NETGRAPH_MPPC M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_parse.c b/sys/netgraph/ng_parse.c
index fbbefc6..e8cbc9a 100644
--- a/sys/netgraph/ng_parse.c
+++ b/sys/netgraph/ng_parse.c
@@ -62,7 +62,7 @@
 #include <netgraph/ng_parse.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_PARSE, "netgraph_parse", "netgraph parse info");
+static MALLOC_DEFINE(M_NETGRAPH_PARSE, "netgraph_parse", "netgraph parse info");
 #else
 #define M_NETGRAPH_PARSE M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_ppp.c b/sys/netgraph/ng_ppp.c
index a0a3c38..b26163a 100644
--- a/sys/netgraph/ng_ppp.c
+++ b/sys/netgraph/ng_ppp.c
@@ -108,7 +108,7 @@
 #include <netgraph/ng_vjc.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_PPP, "netgraph_ppp", "netgraph ppp node");
+static MALLOC_DEFINE(M_NETGRAPH_PPP, "netgraph_ppp", "netgraph ppp node");
 #else
 #define M_NETGRAPH_PPP M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_pppoe.c b/sys/netgraph/ng_pppoe.c
index 4d5e24e..3b953b3 100644
--- a/sys/netgraph/ng_pppoe.c
+++ b/sys/netgraph/ng_pppoe.c
@@ -58,7 +58,7 @@
 #include <netgraph/ng_ether.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_PPPOE, "netgraph_pppoe", "netgraph pppoe node");
+static MALLOC_DEFINE(M_NETGRAPH_PPPOE, "netgraph_pppoe", "netgraph pppoe node");
 #else
 #define M_NETGRAPH_PPPOE M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_pred1.c b/sys/netgraph/ng_pred1.c
index 5f01e88..52efe63 100644
--- a/sys/netgraph/ng_pred1.c
+++ b/sys/netgraph/ng_pred1.c
@@ -46,7 +46,7 @@
 
 #include "opt_netgraph.h"
 
-MALLOC_DEFINE(M_NETGRAPH_PRED1, "netgraph_pred1", "netgraph pred1 node ");
+static MALLOC_DEFINE(M_NETGRAPH_PRED1, "netgraph_pred1", "netgraph pred1 node");
 
 /* PRED1 header length */
 #define PRED1_HDRLEN		2
diff --git a/sys/netgraph/ng_sample.c b/sys/netgraph/ng_sample.c
index 3d644db..f0e4d50 100644
--- a/sys/netgraph/ng_sample.c
+++ b/sys/netgraph/ng_sample.c
@@ -58,7 +58,7 @@
 /* If you do complicated mallocs you may want to do this */
 /* and use it for your mallocs */
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_XXX, "netgraph_xxx", "netgraph xxx node ");
+static MALLOC_DEFINE(M_NETGRAPH_XXX, "netgraph_xxx", "netgraph xxx node");
 #else
 #define M_NETGRAPH_XXX M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_socket.c b/sys/netgraph/ng_socket.c
index 8c819c0..d2053e3 100644
--- a/sys/netgraph/ng_socket.c
+++ b/sys/netgraph/ng_socket.c
@@ -76,8 +76,8 @@
 #include <netgraph/ng_socket.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_PATH, "netgraph_path", "netgraph path info ");
-MALLOC_DEFINE(M_NETGRAPH_SOCK, "netgraph_sock", "netgraph socket info ");
+static MALLOC_DEFINE(M_NETGRAPH_PATH, "netgraph_path", "netgraph path info");
+static MALLOC_DEFINE(M_NETGRAPH_SOCK, "netgraph_sock", "netgraph socket info");
 #else
 #define M_NETGRAPH_PATH M_NETGRAPH
 #define M_NETGRAPH_SOCK M_NETGRAPH
@@ -1172,8 +1172,8 @@ ngs_mod_event(module_t mod, int event, void *data)
 VNET_DOMAIN_SET(ng);
 
 SYSCTL_INT(_net_graph, OID_AUTO, family, CTLFLAG_RD, 0, AF_NETGRAPH, "");
-SYSCTL_NODE(_net_graph, OID_AUTO, data, CTLFLAG_RW, 0, "DATA");
+static SYSCTL_NODE(_net_graph, OID_AUTO, data, CTLFLAG_RW, 0, "DATA");
 SYSCTL_INT(_net_graph_data, OID_AUTO, proto, CTLFLAG_RD, 0, NG_DATA, "");
-SYSCTL_NODE(_net_graph, OID_AUTO, control, CTLFLAG_RW, 0, "CONTROL");
+static SYSCTL_NODE(_net_graph, OID_AUTO, control, CTLFLAG_RW, 0, "CONTROL");
 SYSCTL_INT(_net_graph_control, OID_AUTO, proto, CTLFLAG_RD, 0, NG_CONTROL, "");
 
diff --git a/sys/netgraph/ng_sppp.c b/sys/netgraph/ng_sppp.c
index ad96146..4ad89ec 100644
--- a/sys/netgraph/ng_sppp.c
+++ b/sys/netgraph/ng_sppp.c
@@ -44,7 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <netgraph/ng_sppp.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_SPPP, "netgraph_sppp", "netgraph sppp node ");
+static MALLOC_DEFINE(M_NETGRAPH_SPPP, "netgraph_sppp", "netgraph sppp node");
 #else
 #define M_NETGRAPH_SPPP M_NETGRAPH
 #endif
diff --git a/sys/netgraph/ng_tag.c b/sys/netgraph/ng_tag.c
index b183285..150a0f2 100644
--- a/sys/netgraph/ng_tag.c
+++ b/sys/netgraph/ng_tag.c
@@ -69,7 +69,7 @@
 #include <netgraph/ng_tag.h>
 
 #ifdef NG_SEPARATE_MALLOC
-MALLOC_DEFINE(M_NETGRAPH_TAG, "netgraph_tag", "netgraph tag node ");
+static MALLOC_DEFINE(M_NETGRAPH_TAG, "netgraph_tag", "netgraph tag node");
 #else
 #define M_NETGRAPH_TAG M_NETGRAPH
 #endif
diff --git a/sys/netinet/accf_http.c b/sys/netinet/accf_http.c
index b3761ef..41e442c 100644
--- a/sys/netinet/accf_http.c
+++ b/sys/netinet/accf_http.c
@@ -69,7 +69,7 @@ DECLARE_MODULE(accf_http, accf_http_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
 
 static int parse_http_version = 1;
 
-SYSCTL_NODE(_net_inet_accf, OID_AUTO, http, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_inet_accf, OID_AUTO, http, CTLFLAG_RW, 0,
 "HTTP accept filter");
 SYSCTL_INT(_net_inet_accf_http, OID_AUTO, parsehttpversion, CTLFLAG_RW,
 &parse_http_version, 1,
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index f766fc4..60fa944 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -77,8 +77,8 @@ __FBSDID("$FreeBSD$");
 #define SDL(s) ((struct sockaddr_dl *)s)
 
 SYSCTL_DECL(_net_link_ether);
-SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
-SYSCTL_NODE(_net_link_ether, PF_ARP, arp, CTLFLAG_RW, 0, "");
+static SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
+static SYSCTL_NODE(_net_link_ether, PF_ARP, arp, CTLFLAG_RW, 0, "");
 
 /* timer values */
 static VNET_DEFINE(int, arpt_keep) = (20*60);	/* once resolved, good for 20
@@ -433,7 +433,7 @@ arpintr(struct mbuf *m)
 
 	if (m->m_len < sizeof(struct arphdr) &&
 	    ((m = m_pullup(m, sizeof(struct arphdr))) == NULL)) {
-		log(LOG_ERR, "arp: runt packet -- m_pullup failed\n");
+		log(LOG_NOTICE, "arp: runt packet -- m_pullup failed\n");
 		return;
 	}
 	ar = mtod(m, struct arphdr *);
@@ -443,15 +443,17 @@ arpintr(struct mbuf *m)
 	    ntohs(ar->ar_hrd) != ARPHRD_ARCNET &&
 	    ntohs(ar->ar_hrd) != ARPHRD_IEEE1394 &&
 	    ntohs(ar->ar_hrd) != ARPHRD_INFINIBAND) {
-		log(LOG_ERR, "arp: unknown hardware address format (0x%2D)\n",
-		    (unsigned char *)&ar->ar_hrd, "");
+		log(LOG_NOTICE, "arp: unknown hardware address format (0x%2D)"
+		    " (from %*D to %*D)\n", (unsigned char *)&ar->ar_hrd, "",
+		    ETHER_ADDR_LEN, (u_char *)ar_sha(ar), ":",
+		    ETHER_ADDR_LEN, (u_char *)ar_tha(ar), ":");
 		m_freem(m);
 		return;
 	}
 
 	if (m->m_len < arphdr_len(ar)) {
 		if ((m = m_pullup(m, arphdr_len(ar))) == NULL) {
-			log(LOG_ERR, "arp: runt packet\n");
+			log(LOG_NOTICE, "arp: runt packet\n");
 			m_freem(m);
 			return;
 		}
@@ -527,7 +529,7 @@ in_arpinput(struct mbuf *m)
 
 	req_len = arphdr_len2(ifp->if_addrlen, sizeof(struct in_addr));
 	if (m->m_len < req_len && (m = m_pullup(m, req_len)) == NULL) {
-		log(LOG_ERR, "in_arp: runt packet -- m_pullup failed\n");
+		log(LOG_NOTICE, "in_arp: runt packet -- m_pullup failed\n");
 		return;
 	}
 
@@ -537,13 +539,14 @@ in_arpinput(struct mbuf *m)
 	 * a protocol length not equal to an IPv4 address.
 	 */
 	if (ah->ar_pln != sizeof(struct in_addr)) {
-		log(LOG_ERR, "in_arp: requested protocol length != %zu\n",
+		log(LOG_NOTICE, "in_arp: requested protocol length != %zu\n",
 		    sizeof(struct in_addr));
 		return;
 	}
 
 	if (ETHER_IS_MULTICAST(ar_sha(ah))) {
-		log(LOG_ERR, "in_arp: source hardware address is multicast.");
+		log(LOG_NOTICE, "in_arp: %*D is multicast\n",
+		    ifp->if_addrlen, (u_char *)ar_sha(ah), ":");
 		return;
 	}
 
@@ -645,7 +648,7 @@ match:
 	if (!bcmp(ar_sha(ah), enaddr, ifp->if_addrlen))
 		goto drop;	/* it's from me, ignore it. */
 	if (!bcmp(ar_sha(ah), ifp->if_broadcastaddr, ifp->if_addrlen)) {
-		log(LOG_ERR,
+		log(LOG_NOTICE,
 		    "arp: link address is broadcast for IP address %s!\n",
 		    inet_ntoa(isaddr));
 		goto drop;
@@ -681,7 +684,7 @@ match:
 		/* the following is not an error when doing bridging */
 		if (!bridged && la->lle_tbl->llt_ifp != ifp && !carp_match) {
 			if (log_arp_wrong_iface)
-				log(LOG_ERR, "arp: %s is on %s "
+				log(LOG_WARNING, "arp: %s is on %s "
 				    "but got reply from %*D on %s\n",
 				    inet_ntoa(isaddr),
 				    la->lle_tbl->llt_ifp->if_xname,
@@ -716,10 +719,10 @@ match:
 		    
 		if (ifp->if_addrlen != ah->ar_hln) {
 			LLE_WUNLOCK(la);
-			log(LOG_WARNING,
-			    "arp from %*D: addr len: new %d, i/f %d (ignored)",
-			    ifp->if_addrlen, (u_char *) ar_sha(ah), ":",
-			    ah->ar_hln, ifp->if_addrlen);
+			log(LOG_WARNING, "arp from %*D: addr len: new %d, "
+			    "i/f %d (ignored)\n", ifp->if_addrlen,
+			    (u_char *) ar_sha(ah), ":", ah->ar_hln,
+			    ifp->if_addrlen);
 			goto drop;
 		}
 		(void)memcpy(&la->ll_addr, ar_sha(ah), ifp->if_addrlen);
diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c
index b02d30b..e50d158 100644
--- a/sys/netinet/igmp.c
+++ b/sys/netinet/igmp.c
@@ -185,7 +185,7 @@ static const struct netisr_handler igmp_nh = {
 struct mtx		 igmp_mtx;
 
 struct mbuf		*m_raopt;		 /* Router Alert option */
-MALLOC_DEFINE(M_IGMP, "igmp", "igmp state");
+static MALLOC_DEFINE(M_IGMP, "igmp", "igmp state");
 
 /*
  * VIMAGE-wide globals.
@@ -280,8 +280,9 @@ SYSCTL_VNET_PROC(_net_inet_igmp, OID_AUTO, gsrdelay,
 /*
  * Non-virtualized sysctls.
  */
-SYSCTL_NODE(_net_inet_igmp, OID_AUTO, ifinfo, CTLFLAG_RD | CTLFLAG_MPSAFE,
-    sysctl_igmp_ifinfo, "Per-interface IGMPv3 state");
+static SYSCTL_NODE(_net_inet_igmp, OID_AUTO, ifinfo,
+    CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_igmp_ifinfo,
+    "Per-interface IGMPv3 state");
 
 static __inline void
 igmp_save_context(struct mbuf *m, struct ifnet *ifp)
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index c91df4a..84d6a21 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -234,16 +234,42 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 	 * in_lifaddr_ioctl() and ifp->if_ioctl().
 	 */
 	switch (cmd) {
-	case SIOCAIFADDR:
-	case SIOCDIFADDR:
 	case SIOCGIFADDR:
 	case SIOCGIFBRDADDR:
 	case SIOCGIFDSTADDR:
 	case SIOCGIFNETMASK:
+	case SIOCDIFADDR:
+		break;
+	case SIOCAIFADDR:
+		/*
+		 * ifra_addr must be present and be of INET family.
+		 * ifra_broadaddr and ifra_mask are optional.
+		 */
+		if (ifra->ifra_addr.sin_len != sizeof(struct sockaddr_in) ||
+		    ifra->ifra_addr.sin_family != AF_INET)
+			return (EINVAL);
+		if (ifra->ifra_broadaddr.sin_len != 0 &&
+		    (ifra->ifra_broadaddr.sin_len != sizeof(struct sockaddr_in) ||
+		    ifra->ifra_broadaddr.sin_family != AF_INET))
+			return (EINVAL);
+#if 0
+		/*
+		 * ifconfig(8) historically doesn't set af_family for mask
+		 * for unknown reason.
+		 */
+		if (ifra->ifra_mask.sin_len != 0 &&
+		    (ifra->ifra_mask.sin_len != sizeof(struct sockaddr_in) ||
+		    ifra->ifra_mask.sin_family != AF_INET))
+			return (EINVAL);
+#endif
+		break;
 	case SIOCSIFADDR:
 	case SIOCSIFBRDADDR:
 	case SIOCSIFDSTADDR:
 	case SIOCSIFNETMASK:
+		if (ifr->ifr_addr.sa_family != AF_INET ||
+		    ifr->ifr_addr.sa_len != sizeof(struct sockaddr_in))
+			return (EINVAL);
 		break;
 
 	case SIOCALIFADDR:
@@ -349,7 +375,7 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 	switch (cmd) {
 	case SIOCAIFADDR:
 	case SIOCDIFADDR:
-		if (ifra->ifra_addr.sin_family == AF_INET) {
+		{
 			struct in_ifaddr *oia;
 
 			IN_IFADDR_RLOCK();
@@ -506,7 +532,7 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 		goto out;
 
 	case SIOCSIFNETMASK:
-		ia->ia_sockmask.sin_addr = ifra->ifra_addr.sin_addr;
+		ia->ia_sockmask = *(struct sockaddr_in *)&ifr->ifr_addr;
 		ia->ia_subnetmask = ntohl(ia->ia_sockmask.sin_addr.s_addr);
 		goto out;
 
@@ -514,14 +540,12 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 		maskIsNew = 0;
 		hostIsNew = 1;
 		error = 0;
-		if (ia->ia_addr.sin_family == AF_INET) {
-			if (ifra->ifra_addr.sin_len == 0) {
-				ifra->ifra_addr = ia->ia_addr;
-				hostIsNew = 0;
-			} else if (ifra->ifra_addr.sin_addr.s_addr ==
-					       ia->ia_addr.sin_addr.s_addr)
-				hostIsNew = 0;
-		}
+		if (ifra->ifra_addr.sin_len == 0) {
+			ifra->ifra_addr = ia->ia_addr;
+			hostIsNew = 0;
+		} else if (ifra->ifra_addr.sin_addr.s_addr ==
+			    ia->ia_addr.sin_addr.s_addr)
+			hostIsNew = 0;
 		if (ifra->ifra_mask.sin_len) {
 			/* 
 			 * QL: XXX
@@ -545,14 +569,13 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 			ia->ia_dstaddr = ifra->ifra_dstaddr;
 			maskIsNew  = 1; /* We lie; but the effect's the same */
 		}
-		if (ifra->ifra_addr.sin_family == AF_INET &&
-		    (hostIsNew || maskIsNew))
+		if (hostIsNew || maskIsNew)
 			error = in_ifinit(ifp, ia, &ifra->ifra_addr, 0);
 		if (error != 0 && iaIsNew)
 			break;
 
 		if ((ifp->if_flags & IFF_BROADCAST) &&
-		    (ifra->ifra_broadaddr.sin_family == AF_INET))
+		    ifra->ifra_broadaddr.sin_len)
 			ia->ia_broadaddr = ifra->ifra_broadaddr;
 		if (error == 0) {
 			ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]);
@@ -608,31 +631,26 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 
 	IN_IFADDR_WLOCK();
 	TAILQ_REMOVE(&V_in_ifaddrhead, ia, ia_link);
-	if (ia->ia_addr.sin_family == AF_INET) {
-		struct in_ifaddr *if_ia;
 
-		LIST_REMOVE(ia, ia_hash);
-		IN_IFADDR_WUNLOCK();
-		/*
-		 * If this is the last IPv4 address configured on this
-		 * interface, leave the all-hosts group.
-		 * No state-change report need be transmitted.
-		 */
-		if_ia = NULL;
-		IFP_TO_IA(ifp, if_ia);
-		if (if_ia == NULL) {
-			ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]);
-			IN_MULTI_LOCK();
-			if (ii->ii_allhosts) {
-				(void)in_leavegroup_locked(ii->ii_allhosts,
-				    NULL);
-				ii->ii_allhosts = NULL;
-			}
-			IN_MULTI_UNLOCK();
-		} else
-			ifa_free(&if_ia->ia_ifa);
+	LIST_REMOVE(ia, ia_hash);
+	IN_IFADDR_WUNLOCK();
+	/*
+	 * If this is the last IPv4 address configured on this
+	 * interface, leave the all-hosts group.
+	 * No state-change report need be transmitted.
+	 */
+	IFP_TO_IA(ifp, iap);
+	if (iap == NULL) {
+		ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]);
+		IN_MULTI_LOCK();
+		if (ii->ii_allhosts) {
+			(void)in_leavegroup_locked(ii->ii_allhosts, NULL);
+			ii->ii_allhosts = NULL;
+		}
+		IN_MULTI_UNLOCK();
 	} else
-		IN_IFADDR_WUNLOCK();
+		ifa_free(&iap->ia_ifa);
+
 	ifa_free(&ia->ia_ifa);				/* in_ifaddrhead */
 out:
 	if (ia != NULL)
@@ -828,50 +846,29 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
     int scrub)
 {
 	register u_long i = ntohl(sin->sin_addr.s_addr);
-	struct sockaddr_in oldaddr;
 	int flags = RTF_UP, error = 0;
 
-	oldaddr = ia->ia_addr;
-	if (oldaddr.sin_family == AF_INET)
+	if (scrub)
+		in_scrubprefix(ia, LLE_STATIC);
+
+	IN_IFADDR_WLOCK();
+	if (ia->ia_addr.sin_family == AF_INET)
 		LIST_REMOVE(ia, ia_hash);
 	ia->ia_addr = *sin;
-	if (ia->ia_addr.sin_family == AF_INET) {
-		IN_IFADDR_WLOCK();
-		LIST_INSERT_HEAD(INADDR_HASH(ia->ia_addr.sin_addr.s_addr),
-		    ia, ia_hash);
-		IN_IFADDR_WUNLOCK();
-	}
+	LIST_INSERT_HEAD(INADDR_HASH(ia->ia_addr.sin_addr.s_addr),
+	    ia, ia_hash);
+	IN_IFADDR_WUNLOCK();
+
 	/*
 	 * Give the interface a chance to initialize
 	 * if this is its first address,
 	 * and to validate the address if necessary.
 	 */
-	if (ifp->if_ioctl != NULL) {
-		error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia);
-		if (error) {
+	if (ifp->if_ioctl != NULL &&
+	    (error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia)) != 0)
 			/* LIST_REMOVE(ia, ia_hash) is done in in_control */
-			ia->ia_addr = oldaddr;
-			IN_IFADDR_WLOCK();
-			if (ia->ia_addr.sin_family == AF_INET)
-				LIST_INSERT_HEAD(INADDR_HASH(
-				    ia->ia_addr.sin_addr.s_addr), ia, ia_hash);
-			else 
-				/* 
-				 * If oldaddr family is not AF_INET (e.g. 
-				 * interface has been just created) in_control 
-				 * does not call LIST_REMOVE, and we end up 
-				 * with bogus ia entries in hash
-				 */
-				LIST_REMOVE(ia, ia_hash);
-			IN_IFADDR_WUNLOCK();
 			return (error);
-		}
-	}
-	if (scrub) {
-		ia->ia_ifa.ifa_addr = (struct sockaddr *)&oldaddr;
-		in_ifscrub(ifp, ia, LLE_STATIC);
-		ia->ia_ifa.ifa_addr = (struct sockaddr *)&ia->ia_addr;
-	}
+
 	/*
 	 * Be compatible with network classes, if netmask isn't supplied,
 	 * guess it based on classes.
@@ -916,11 +913,9 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 	if (ia->ia_addr.sin_addr.s_addr == INADDR_ANY)
 		return (0);
 
-	if (ifp->if_flags & IFF_POINTOPOINT) {
-		if (ia->ia_dstaddr.sin_addr.s_addr == ia->ia_addr.sin_addr.s_addr)
+	if (ifp->if_flags & IFF_POINTOPOINT &&
+	    ia->ia_dstaddr.sin_addr.s_addr == ia->ia_addr.sin_addr.s_addr)
 			return (0);
-	}
-
 
 	/*
 	 * add a loopback route to self
diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c
index 4edf309..6c62dca 100644
--- a/sys/netinet/in_mcast.c
+++ b/sys/netinet/in_mcast.c
@@ -155,7 +155,8 @@ static int	inp_set_multicast_if(struct inpcb *, struct sockopt *);
 static int	inp_set_source_filters(struct inpcb *, struct sockopt *);
 static int	sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS);
 
-SYSCTL_NODE(_net_inet_ip, OID_AUTO, mcast, CTLFLAG_RW, 0, "IPv4 multicast");
+static SYSCTL_NODE(_net_inet_ip, OID_AUTO, mcast, CTLFLAG_RW, 0,
+    "IPv4 multicast");
 
 static u_long in_mcast_maxgrpsrc = IP_MAX_GROUP_SRC_FILTER;
 SYSCTL_ULONG(_net_inet_ip_mcast, OID_AUTO, maxgrpsrc,
@@ -174,7 +175,7 @@ SYSCTL_INT(_net_inet_ip_mcast, OID_AUTO, loop, CTLFLAG_RW | CTLFLAG_TUN,
     &in_mcast_loop, 0, "Loopback multicast datagrams by default");
 TUNABLE_INT("net.inet.ip.mcast.loop", &in_mcast_loop);
 
-SYSCTL_NODE(_net_inet_ip_mcast, OID_AUTO, filters,
+static SYSCTL_NODE(_net_inet_ip_mcast, OID_AUTO, filters,
     CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_ip_mcast_filters,
     "Per-interface stack-wide source filters");
 
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 4eb309a..f8f397c 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -162,7 +162,8 @@ sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 
 #undef RANGECHK
 
-SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
+static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
+    "IP Ports");
 
 SYSCTL_VNET_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 	CTLTYPE_INT|CTLFLAG_RW, &VNET_NAME(ipport_lowfirstauto), 0,
@@ -575,8 +576,7 @@ in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 				     ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 				    (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 				     ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
-				     (t->inp_socket->so_options &
-					 SO_REUSEPORT) == 0) &&
+				     (t->inp_flags2 & INP_REUSEPORT) == 0) &&
 				    (inp->inp_cred->cr_uid !=
 				     t->inp_cred->cr_uid))
 					return (EADDRINUSE);
@@ -590,19 +590,19 @@ in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 				 * being in use (for now).  This is better
 				 * than a panic, but not desirable.
 				 */
-				tw = intotw(inp);
+				tw = intotw(t);
 				if (tw == NULL ||
 				    (reuseport & tw->tw_so_options) == 0)
 					return (EADDRINUSE);
-			} else if (t &&
-			    (reuseport & t->inp_socket->so_options) == 0) {
+			} else if (t && (reuseport == 0 ||
+			    (t->inp_flags2 & INP_REUSEPORT) == 0)) {
 #ifdef INET6
 				if (ntohl(sin->sin_addr.s_addr) !=
 				    INADDR_ANY ||
 				    ntohl(t->inp_laddr.s_addr) !=
 				    INADDR_ANY ||
-				    INP_SOCKAF(so) ==
-				    INP_SOCKAF(t->inp_socket))
+				    (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
+				    (t->inp_vflag & INP_IPV6PROTO) == 0)
 #endif
 				return (EADDRINUSE);
 			}
diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
index 55270ec..e10f0b6 100644
--- a/sys/netinet/in_pcb.h
+++ b/sys/netinet/in_pcb.h
@@ -540,6 +540,7 @@ void 	inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
 #define	INP_LLE_VALID		0x00000001 /* cached lle is valid */	
 #define	INP_RT_VALID		0x00000002 /* cached rtentry is valid */
 #define	INP_PCBGROUPWILD	0x00000004 /* in pcbgroup wildcard list */
+#define	INP_REUSEPORT		0x00000008 /* SO_REUSEPORT option is set */
 
 /*
  * Flags passed to in_pcblookup*() functions.
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 34dced8..73024f7 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -472,7 +472,7 @@ carp_clone_destroy(struct ifnet *ifp)
 	mtx_unlock(&carp_mtx);
 	bpfdetach(ifp);
 	if_detach(ifp);
-	if_free_type(ifp, IFT_ETHER);
+	if_free(ifp);
 #ifdef INET
 	free(sc->sc_imo.imo_membership, M_CARP);
 #endif
diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index 234cae2..5892c9d 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -738,7 +738,8 @@ div_pcblist(SYSCTL_HANDLER_ARGS)
 }
 
 #ifdef SYSCTL_NODE
-SYSCTL_NODE(_net_inet, IPPROTO_DIVERT, divert, CTLFLAG_RW, 0, "IPDIVERT");
+static SYSCTL_NODE(_net_inet, IPPROTO_DIVERT, divert, CTLFLAG_RW, 0,
+    "IPDIVERT");
 SYSCTL_PROC(_net_inet_divert, OID_AUTO, pcblist, CTLTYPE_OPAQUE | CTLFLAG_RD,
     NULL, 0, div_pcblist, "S,xinpcb", "List of active divert sockets");
 #endif
diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c
index 8736caf..69a7758 100644
--- a/sys/netinet/ip_mroute.c
+++ b/sys/netinet/ip_mroute.c
@@ -2807,9 +2807,9 @@ out_locked:
 	return (error);
 }
 
-SYSCTL_NODE(_net_inet_ip, OID_AUTO, mfctable, CTLFLAG_RD, sysctl_mfctable,
-    "IPv4 Multicast Forwarding Table (struct *mfc[mfchashsize], "
-    "netinet/ip_mroute.h)");
+static SYSCTL_NODE(_net_inet_ip, OID_AUTO, mfctable, CTLFLAG_RD,
+    sysctl_mfctable, "IPv4 Multicast Forwarding Table "
+    "(struct *mfc[mfchashsize], netinet/ip_mroute.h)");
 
 static void
 vnet_mroute_init(const void *unused __unused)
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index 73e7f36..a9008c5 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -895,12 +895,43 @@ ip_ctloutput(struct socket *so, struct sockopt *sopt)
 
 	error = optval = 0;
 	if (sopt->sopt_level != IPPROTO_IP) {
-		if ((sopt->sopt_level == SOL_SOCKET) &&
-		    (sopt->sopt_name == SO_SETFIB)) {
-			inp->inp_inc.inc_fibnum = so->so_fibnum;
-			return (0);
+		error = EINVAL;
+
+		if (sopt->sopt_level == SOL_SOCKET &&
+		    sopt->sopt_dir == SOPT_SET) {
+			switch (sopt->sopt_name) {
+			case SO_REUSEADDR:
+				INP_WLOCK(inp);
+				if (IN_MULTICAST(ntohl(inp->inp_laddr.s_addr))) {
+					if ((so->so_options &
+					    (SO_REUSEADDR | SO_REUSEPORT)) != 0)
+						inp->inp_flags2 |= INP_REUSEPORT;
+					else
+						inp->inp_flags2 &= ~INP_REUSEPORT;
+				}
+				INP_WUNLOCK(inp);
+				error = 0;
+				break;
+			case SO_REUSEPORT:
+				INP_WLOCK(inp);
+				if ((so->so_options & SO_REUSEPORT) != 0)
+					inp->inp_flags2 |= INP_REUSEPORT;
+				else
+					inp->inp_flags2 &= ~INP_REUSEPORT;
+				INP_WUNLOCK(inp);
+				error = 0;
+				break;
+			case SO_SETFIB:
+				INP_WLOCK(inp);
+				inp->inp_inc.inc_fibnum = so->so_fibnum;
+				INP_WUNLOCK(inp);
+				error = 0;
+				break;
+			default:
+				break;
+			}
 		}
-		return (EINVAL);
+		return (error);
 	}
 
 	switch (sopt->sopt_dir) {
diff --git a/sys/netinet/ipfw/dn_heap.c b/sys/netinet/ipfw/dn_heap.c
index 425d354..3bdfd9d 100644
--- a/sys/netinet/ipfw/dn_heap.c
+++ b/sys/netinet/ipfw/dn_heap.c
@@ -59,7 +59,7 @@ static void my_free(void *p) {	free(p); }
 #define free(p, t)	my_free(p)
 #endif /* !_KERNEL */
 
-MALLOC_DEFINE(M_DN_HEAP, "dummynet", "dummynet heap");
+static MALLOC_DEFINE(M_DN_HEAP, "dummynet", "dummynet heap");
 
 /*
  * Heap management functions.
diff --git a/sys/netinet/ipfw/dummynet.txt b/sys/netinet/ipfw/dummynet.txt
index 3a1efde..e4f3075 100644
--- a/sys/netinet/ipfw/dummynet.txt
+++ b/sys/netinet/ipfw/dummynet.txt
@@ -325,7 +325,7 @@ we do the following:
 
 dummynet_task()
 ===============
-The dummynet_task() is the main dummynet processing function and is
+The dummynet_task() function is the main dummynet processing function and is
 called every tick. This function first calculate the new current time, then
 it checks if it is the time to wake up object from the system_heap comparing
 the current time and the key of the heap. Two types of object (really the
diff --git a/sys/netinet/ipfw/ip_dn_io.c b/sys/netinet/ipfw/ip_dn_io.c
index 6766416..a39f169 100644
--- a/sys/netinet/ipfw/ip_dn_io.c
+++ b/sys/netinet/ipfw/ip_dn_io.c
@@ -102,7 +102,7 @@ SYSBEGIN(f4)
 
 SYSCTL_DECL(_net_inet);
 SYSCTL_DECL(_net_inet_ip);
-SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW, 0, "Dummynet");
+static SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW, 0, "Dummynet");
 
 /* wrapper to pass dn_cfg fields to SYSCTL_* */
 //#define DC(x)	(&(VNET_NAME(_base_dn_cfg).x))
diff --git a/sys/netinet/ipfw/ip_fw2.c b/sys/netinet/ipfw/ip_fw2.c
index 619ce6b..eec67cc 100644
--- a/sys/netinet/ipfw/ip_fw2.c
+++ b/sys/netinet/ipfw/ip_fw2.c
@@ -31,14 +31,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_ipfw.h"
-#if !defined(KLD_MODULE)
 #include "opt_ipdivert.h"
-#include "opt_ipdn.h"
 #include "opt_inet.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
-#endif
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
 
diff --git a/sys/netinet/ipfw/ip_fw_dynamic.c b/sys/netinet/ipfw/ip_fw_dynamic.c
index d1eec89..7239054 100644
--- a/sys/netinet/ipfw/ip_fw_dynamic.c
+++ b/sys/netinet/ipfw/ip_fw_dynamic.c
@@ -34,16 +34,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_ipfw.h"
-#if !defined(KLD_MODULE)
-#include "opt_ipdivert.h"
-#include "opt_ipdn.h"
 #include "opt_inet.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
-#endif
 #include "opt_inet6.h"
-#include "opt_ipsec.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/netinet/ipfw/ip_fw_log.c b/sys/netinet/ipfw/ip_fw_log.c
index 2f6e8b6..b9ed6b3 100644
--- a/sys/netinet/ipfw/ip_fw_log.c
+++ b/sys/netinet/ipfw/ip_fw_log.c
@@ -31,16 +31,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_ipfw.h"
-#if !defined(KLD_MODULE)
-#include "opt_ipdivert.h"
-#include "opt_ipdn.h"
 #include "opt_inet.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
-#endif
 #include "opt_inet6.h"
-#include "opt_ipsec.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/netinet/ipfw/ip_fw_pfil.c b/sys/netinet/ipfw/ip_fw_pfil.c
index a09ca79..695613d 100644
--- a/sys/netinet/ipfw/ip_fw_pfil.c
+++ b/sys/netinet/ipfw/ip_fw_pfil.c
@@ -28,14 +28,11 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_ipfw.h"
-#if !defined(KLD_MODULE)
-#include "opt_ipdn.h"
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
-#endif /* KLD_MODULE */
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/netinet/ipfw/ip_fw_sockopt.c b/sys/netinet/ipfw/ip_fw_sockopt.c
index 610570b..dce3fdc 100644
--- a/sys/netinet/ipfw/ip_fw_sockopt.c
+++ b/sys/netinet/ipfw/ip_fw_sockopt.c
@@ -34,16 +34,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_ipfw.h"
-#if !defined(KLD_MODULE)
-#include "opt_ipdivert.h"
-#include "opt_ipdn.h"
 #include "opt_inet.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
-#endif
 #include "opt_inet6.h"
-#include "opt_ipsec.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
diff --git a/sys/netinet/ipfw/ip_fw_table.c b/sys/netinet/ipfw/ip_fw_table.c
index f724017..117a205 100644
--- a/sys/netinet/ipfw/ip_fw_table.c
+++ b/sys/netinet/ipfw/ip_fw_table.c
@@ -40,16 +40,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_ipfw.h"
-#if !defined(KLD_MODULE)
-#include "opt_ipdivert.h"
-#include "opt_ipdn.h"
 #include "opt_inet.h"
 #ifndef INET
 #error IPFIREWALL requires INET.
 #endif /* INET */
-#endif
 #include "opt_inet6.h"
-#include "opt_ipsec.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -73,7 +68,7 @@ __FBSDID("$FreeBSD$");
 #include <security/mac/mac_framework.h>
 #endif
 
-MALLOC_DEFINE(M_IPFW_TBL, "ipfw_tbl", "IpFw tables");
+static MALLOC_DEFINE(M_IPFW_TBL, "ipfw_tbl", "IpFw tables");
 
 struct table_entry {
 	struct radix_node	rn[2];
diff --git a/sys/netinet/libalias/alias_sctp.c b/sys/netinet/libalias/alias_sctp.c
index bb0f9cc..67e4754 100644
--- a/sys/netinet/libalias/alias_sctp.c
+++ b/sys/netinet/libalias/alias_sctp.c
@@ -181,7 +181,7 @@ void SctpShowAliasStats(struct libalias *la);
 
 #ifdef	_KERNEL
 
-MALLOC_DEFINE(M_SCTPNAT, "sctpnat", "sctp nat dbs");
+static MALLOC_DEFINE(M_SCTPNAT, "sctpnat", "sctp nat dbs");
 /* Use kernel allocator. */
 #ifdef _SYS_MALLOC_H_
 #define	sn_malloc(x)	malloc(x, M_SCTPNAT, M_NOWAIT|M_ZERO)
@@ -364,8 +364,8 @@ SYSCTL_DECL(_net_inet);
 SYSCTL_DECL(_net_inet_ip);
 SYSCTL_DECL(_net_inet_ip_alias);
 
-SYSCTL_NODE(_net_inet_ip_alias, OID_AUTO, sctp, CTLFLAG_RW, NULL, "SCTP NAT");
-
+static SYSCTL_NODE(_net_inet_ip_alias, OID_AUTO, sctp, CTLFLAG_RW, NULL,
+    "SCTP NAT");
 SYSCTL_PROC(_net_inet_ip_alias_sctp, OID_AUTO, log_level, CTLTYPE_UINT | CTLFLAG_RW,
     &sysctl_log_level, 0, sysctl_chg_loglevel, "IU",
     "Level of detail (0 - default, 1 - event, 2 - info, 3 - detail, 4 - debug, 5 - max debug)");
diff --git a/sys/netinet/sctp.h b/sys/netinet/sctp.h
index 394a7cd..5ad5f85 100644
--- a/sys/netinet/sctp.h
+++ b/sys/netinet/sctp.h
@@ -120,6 +120,7 @@ struct sctp_paramhdr {
 #define SCTP_DEFAULT_SNDINFO            0x00000021
 #define SCTP_DEFAULT_PRINFO             0x00000022
 #define SCTP_PEER_ADDR_THLDS            0x00000023
+#define SCTP_REMOTE_UDP_ENCAPS_PORT     0x00000024
 
 /*
  * read-only options
diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c
index 7fd5d4d..081549d 100644
--- a/sys/netinet/sctp_output.c
+++ b/sys/netinet/sctp_output.c
@@ -4062,6 +4062,12 @@ sctp_lowlevel_chunk_output(struct sctp_inpcb *inp,
 				}
 			}
 			if (port) {
+				if (htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port)) == 0) {
+					sctp_handle_no_route(stcb, net, so_locked);
+					SCTP_LTRACE_ERR_RET_PKT(m, inp, stcb, NULL, SCTP_FROM_SCTP_OUTPUT, EHOSTUNREACH);
+					sctp_m_freem(m);
+					return (EHOSTUNREACH);
+				}
 				udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
 				udp->uh_sport = htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port));
 				udp->uh_dport = port;
@@ -4413,6 +4419,12 @@ sctp_lowlevel_chunk_output(struct sctp_inpcb *inp,
 			ip6h->ip6_src = lsa6->sin6_addr;
 
 			if (port) {
+				if (htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port)) == 0) {
+					sctp_handle_no_route(stcb, net, so_locked);
+					SCTP_LTRACE_ERR_RET_PKT(m, inp, stcb, NULL, SCTP_FROM_SCTP_OUTPUT, EHOSTUNREACH);
+					sctp_m_freem(m);
+					return (EHOSTUNREACH);
+				}
 				udp = (struct udphdr *)((caddr_t)ip6h + sizeof(struct ip6_hdr));
 				udp->uh_sport = htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port));
 				udp->uh_dport = port;
@@ -5711,6 +5723,8 @@ do_a_abort:
 			stc.laddress[2] = 0;
 			stc.laddress[3] = 0;
 			stc.laddr_type = SCTP_IPV4_ADDRESS;
+			/* scope_id is only for v6 */
+			stc.scope_id = 0;
 			break;
 #endif
 #ifdef INET6
@@ -5719,6 +5733,7 @@ do_a_abort:
 			memcpy(&stc.address, &sin6->sin6_addr,
 			    sizeof(struct in6_addr));
 			stc.addr_type = SCTP_IPV6_ADDRESS;
+			stc.scope_id = sin6->sin6_scope_id;
 			if (net->src_addr_selected == 0) {
 				/*
 				 * strange case here, the INIT should have
@@ -5746,6 +5761,7 @@ do_a_abort:
 	/* who are we */
 	memcpy(stc.identification, SCTP_VERSION_STRING,
 	    min(strlen(SCTP_VERSION_STRING), sizeof(stc.identification)));
+	memset(stc.reserved, 0, SCTP_RESERVE_SPACE);
 	/* now the chunk header */
 	initack->ch.chunk_type = SCTP_INITIATION_ACK;
 	initack->ch.chunk_flags = 0;
@@ -8003,12 +8019,20 @@ again_one_more_time:
 			if (chk->rec.chunk_id.id != SCTP_ASCONF) {
 				continue;
 			}
-			if (chk->whoTo != net) {
-				/*
-				 * No, not sent to the network we are
-				 * looking at
-				 */
-				break;
+			if (chk->whoTo == NULL) {
+				if (asoc->alternate == NULL) {
+					if (asoc->primary_destination != net) {
+						break;
+					}
+				} else {
+					if (asoc->alternate != net) {
+						break;
+					}
+				}
+			} else {
+				if (chk->whoTo != net) {
+					break;
+				}
 			}
 			if (chk->data == NULL) {
 				break;
@@ -8092,6 +8116,10 @@ again_one_more_time:
 				 */
 				no_data_chunks = 1;
 				chk->sent = SCTP_DATAGRAM_SENT;
+				if (chk->whoTo == NULL) {
+					chk->whoTo = net;
+					atomic_add_int(&net->ref_count, 1);
+				}
 				chk->snd_count++;
 				if (mtu == 0) {
 					/*
@@ -8198,12 +8226,20 @@ again_one_more_time:
 					goto skip_net_check;
 				}
 			}
-			if (chk->whoTo != net) {
-				/*
-				 * No, not sent to the network we are
-				 * looking at
-				 */
-				continue;
+			if (chk->whoTo == NULL) {
+				if (asoc->alternate == NULL) {
+					if (asoc->primary_destination != net) {
+						continue;
+					}
+				} else {
+					if (asoc->alternate != net) {
+						continue;
+					}
+				}
+			} else {
+				if (chk->whoTo != net) {
+					continue;
+				}
 			}
 	skip_net_check:
 			if (chk->data == NULL) {
@@ -8332,6 +8368,10 @@ again_one_more_time:
 						SCTP_STAT_INCR(sctps_sendecne);
 					}
 					chk->sent = SCTP_DATAGRAM_SENT;
+					if (chk->whoTo == NULL) {
+						chk->whoTo = net;
+						atomic_add_int(&net->ref_count, 1);
+					}
 					chk->snd_count++;
 				}
 				if (mtu == 0) {
@@ -10937,6 +10977,10 @@ sctp_send_shutdown_complete2(struct mbuf *m, int iphlen, struct sctphdr *sh,
 		return;
 	}
 	if (port) {
+		if (htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port)) == 0) {
+			sctp_m_freem(mout);
+			return;
+		}
 		udp = (struct udphdr *)comp_cp;
 		udp->uh_sport = htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port));
 		udp->uh_dport = port;
@@ -11897,6 +11941,10 @@ sctp_send_abort(struct mbuf *m, int iphlen, struct sctphdr *sh, uint32_t vtag,
 
 	udp = (struct udphdr *)abm;
 	if (port) {
+		if (htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port)) == 0) {
+			sctp_m_freem(mout);
+			return;
+		}
 		udp->uh_sport = htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port));
 		udp->uh_dport = port;
 		/* set udp->uh_ulen later */
@@ -12158,6 +12206,10 @@ sctp_send_operr_to(struct mbuf *m, int iphlen, struct mbuf *scm, uint32_t vtag,
 
 	udp = (struct udphdr *)sh_out;
 	if (port) {
+		if (htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port)) == 0) {
+			sctp_m_freem(mout);
+			return;
+		}
 		udp->uh_sport = htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port));
 		udp->uh_dport = port;
 		/* set udp->uh_ulen later */
diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c
index 2b36ba4e..a960183 100644
--- a/sys/netinet/sctp_pcb.c
+++ b/sys/netinet/sctp_pcb.c
@@ -331,19 +331,9 @@ sctp_mark_ifa_addr_down(uint32_t vrf_id, struct sockaddr *addr,
 		goto out;
 	}
 	if (if_name) {
-		int len1, len2;
-
-		len1 = strlen(if_name);
-		len2 = strlen(sctp_ifap->ifn_p->ifn_name);
-		if (len1 != len2) {
-			SCTPDBG(SCTP_DEBUG_PCB4, "IFN of ifa names different length %d vs %d - ignored\n",
-			    len1, len2);
-			goto out;
-		}
-		if (strncmp(if_name, sctp_ifap->ifn_p->ifn_name, len1) != 0) {
+		if (strncmp(if_name, sctp_ifap->ifn_p->ifn_name, SCTP_IFNAMSIZ) != 0) {
 			SCTPDBG(SCTP_DEBUG_PCB4, "IFN %s of IFA not the same as %s\n",
-			    sctp_ifap->ifn_p->ifn_name,
-			    if_name);
+			    sctp_ifap->ifn_p->ifn_name, if_name);
 			goto out;
 		}
 	} else {
@@ -384,19 +374,9 @@ sctp_mark_ifa_addr_up(uint32_t vrf_id, struct sockaddr *addr,
 		goto out;
 	}
 	if (if_name) {
-		int len1, len2;
-
-		len1 = strlen(if_name);
-		len2 = strlen(sctp_ifap->ifn_p->ifn_name);
-		if (len1 != len2) {
-			SCTPDBG(SCTP_DEBUG_PCB4, "IFN of ifa names different length %d vs %d - ignored\n",
-			    len1, len2);
-			goto out;
-		}
-		if (strncmp(if_name, sctp_ifap->ifn_p->ifn_name, len1) != 0) {
+		if (strncmp(if_name, sctp_ifap->ifn_p->ifn_name, SCTP_IFNAMSIZ) != 0) {
 			SCTPDBG(SCTP_DEBUG_PCB4, "IFN %s of IFA not the same as %s\n",
-			    sctp_ifap->ifn_p->ifn_name,
-			    if_name);
+			    sctp_ifap->ifn_p->ifn_name, if_name);
 			goto out;
 		}
 	} else {
@@ -559,9 +539,9 @@ sctp_add_addr_to_vrf(uint32_t vrf_id, void *ifn, uint32_t ifn_index,
 		atomic_add_int(&vrf->refcount, 1);
 		sctp_ifnp->ifn_mtu = SCTP_GATHER_MTU_FROM_IFN_INFO(ifn, ifn_index, addr->sa_family);
 		if (if_name != NULL) {
-			memcpy(sctp_ifnp->ifn_name, if_name, SCTP_IFNAMSIZ);
+			snprintf(sctp_ifnp->ifn_name, SCTP_IFNAMSIZ, "%s", if_name);
 		} else {
-			memcpy(sctp_ifnp->ifn_name, "unknown", min(7, SCTP_IFNAMSIZ));
+			snprintf(sctp_ifnp->ifn_name, SCTP_IFNAMSIZ, "%s", "unknown");
 		}
 		hash_ifn_head = &SCTP_BASE_INFO(vrf_ifn_hash)[(ifn_index & SCTP_BASE_INFO(vrf_ifn_hashmark))];
 		LIST_INIT(&sctp_ifnp->ifalist);
@@ -768,19 +748,9 @@ sctp_del_addr_from_vrf(uint32_t vrf_id, struct sockaddr *addr,
 			 * panda who might recycle indexes fast.
 			 */
 			if (if_name) {
-				int len1, len2;
-
-				len1 = min(SCTP_IFNAMSIZ, strlen(if_name));
-				len2 = min(SCTP_IFNAMSIZ, strlen(sctp_ifap->ifn_p->ifn_name));
-				if (len1 && len2 && (len1 == len2)) {
-					/* we can compare them */
-					if (strncmp(if_name, sctp_ifap->ifn_p->ifn_name, len1) == 0) {
-						/*
-						 * They match its a correct
-						 * delete
-						 */
-						valid = 1;
-					}
+				if (strncmp(if_name, sctp_ifap->ifn_p->ifn_name, SCTP_IFNAMSIZ) == 0) {
+					/* They match its a correct delete */
+					valid = 1;
 				}
 			}
 			if (!valid) {
@@ -2680,6 +2650,7 @@ sctp_inpcb_alloc(struct socket *so, uint32_t vrf_id)
 #ifdef INET6
 	m->default_flowlabel = 0;
 #endif
+	m->port = 0;		/* encapsulation disabled by default */
 	sctp_auth_set_default_chunks(m->local_auth_chunks);
 	LIST_INIT(&m->shared_keys);
 	/* add default NULL key as key id 0 */
@@ -4023,13 +3994,9 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 	net->RTO = 0;
 	net->RTO_measured = 0;
 	stcb->asoc.numnets++;
-	*(&net->ref_count) = 1;
+	net->ref_count = 1;
 	net->cwr_window_tsn = net->last_cwr_tsn = stcb->asoc.sending_seq - 1;
-	if (SCTP_BASE_SYSCTL(sctp_udp_tunneling_for_client_enable)) {
-		net->port = htons(SCTP_BASE_SYSCTL(sctp_udp_tunneling_port));
-	} else {
-		net->port = 0;
-	}
+	net->port = stcb->asoc.port;
 	net->dscp = stcb->asoc.default_dscp;
 #ifdef INET6
 	net->flowlabel = stcb->asoc.default_flowlabel;
@@ -4074,13 +4041,8 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 		/* Now get the interface MTU */
 		if (net->ro._s_addr && net->ro._s_addr->ifn_p) {
 			net->mtu = SCTP_GATHER_MTU_FROM_INTFC(net->ro._s_addr->ifn_p);
-		} else {
-			net->mtu = 0;
 		}
-		if (net->mtu == 0) {
-			/* Huh ?? */
-			net->mtu = SCTP_DEFAULT_MTU;
-		} else {
+		if (net->mtu > 0) {
 			uint32_t rmtu;
 
 			rmtu = SCTP_GATHER_MTU_FROM_ROUTE(net->ro._s_addr, &net->ro._l_addr.sa, net->ro.ro_rt);
@@ -4100,11 +4062,31 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 				net->mtu = rmtu;
 			}
 		}
-		if (from == SCTP_ALLOC_ASOC) {
-			stcb->asoc.smallest_mtu = net->mtu;
+	}
+	if (net->mtu == 0) {
+		switch (newaddr->sa_family) {
+#ifdef INET
+		case AF_INET:
+			net->mtu = SCTP_DEFAULT_MTU;
+			break;
+#endif
+#ifdef INET6
+		case AF_INET6:
+			net->mtu = 1280;
+			break;
+#endif
+		default:
+			break;
 		}
-	} else {
-		net->mtu = stcb->asoc.smallest_mtu;
+	}
+	if (net->port) {
+		net->mtu -= (uint32_t) sizeof(struct udphdr);
+	}
+	if (from == SCTP_ALLOC_ASOC) {
+		stcb->asoc.smallest_mtu = net->mtu;
+	}
+	if (stcb->asoc.smallest_mtu > net->mtu) {
+		stcb->asoc.smallest_mtu = net->mtu;
 	}
 #ifdef INET6
 	if (newaddr->sa_family == AF_INET6) {
@@ -4114,12 +4096,7 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 		(void)sa6_recoverscope(sin6);
 	}
 #endif
-	if (net->port) {
-		net->mtu -= sizeof(struct udphdr);
-	}
-	if (stcb->asoc.smallest_mtu > net->mtu) {
-		stcb->asoc.smallest_mtu = net->mtu;
-	}
+
 	/* JRS - Use the congestion control given in the CC module */
 	if (stcb->asoc.cc_functions.sctp_set_initial_cc_param != NULL)
 		(*stcb->asoc.cc_functions.sctp_set_initial_cc_param) (stcb, net);
@@ -4138,6 +4115,9 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 #ifdef INVARIANTS
 	net->flowidset = 1;
 #endif
+	if (netp) {
+		*netp = net;
+	}
 	netfirst = TAILQ_FIRST(&stcb->asoc.nets);
 	if (net->ro.ro_rt == NULL) {
 		/* Since we have no route put it at the back */
@@ -4209,9 +4189,6 @@ sctp_add_remote_addr(struct sctp_tcb *stcb, struct sockaddr *newaddr,
 		TAILQ_INSERT_HEAD(&stcb->asoc.nets,
 		    stcb->asoc.primary_destination, sctp_next);
 	}
-	if (netp) {
-		*netp = net;
-	}
 	return (0);
 }
 
@@ -6181,54 +6158,65 @@ sctp_load_addresses_from_init(struct sctp_tcb *stcb, struct mbuf *m,
 	sin6.sin6_len = sizeof(struct sockaddr_in6);
 	sin6.sin6_port = stcb->rport;
 #endif
-	if (altsa == NULL) {
-		iph = mtod(m, struct ip *);
-		switch (iph->ip_v) {
+	iph = mtod(m, struct ip *);
+	switch (iph->ip_v) {
 #ifdef INET
-		case IPVERSION:
-			{
-				/* its IPv4 */
-				struct sockaddr_in *sin_2;
-
-				sin_2 = (struct sockaddr_in *)(local_sa);
-				memset(sin_2, 0, sizeof(sin));
-				sin_2->sin_family = AF_INET;
-				sin_2->sin_len = sizeof(sin);
-				sin_2->sin_port = sh->dest_port;
-				sin_2->sin_addr.s_addr = iph->ip_dst.s_addr;
+	case IPVERSION:
+		{
+			/* its IPv4 */
+			struct sockaddr_in *sin_2;
+
+			sin_2 = (struct sockaddr_in *)(local_sa);
+			memset(sin_2, 0, sizeof(sin));
+			sin_2->sin_family = AF_INET;
+			sin_2->sin_len = sizeof(sin);
+			sin_2->sin_port = sh->dest_port;
+			sin_2->sin_addr.s_addr = iph->ip_dst.s_addr;
+			if (altsa) {
+				/*
+				 * For cookies we use the src address NOT
+				 * from the packet but from the original
+				 * INIT.
+				 */
+				sa = altsa;
+			} else {
 				sin.sin_addr = iph->ip_src;
 				sa = (struct sockaddr *)&sin;
-				break;
 			}
+			break;
+		}
 #endif
 #ifdef INET6
-		case IPV6_VERSION >> 4:
-			{
-				/* its IPv6 */
-				struct ip6_hdr *ip6;
-				struct sockaddr_in6 *sin6_2;
-
-				ip6 = mtod(m, struct ip6_hdr *);
-				sin6_2 = (struct sockaddr_in6 *)(local_sa);
-				memset(sin6_2, 0, sizeof(sin6));
-				sin6_2->sin6_family = AF_INET6;
-				sin6_2->sin6_len = sizeof(struct sockaddr_in6);
-				sin6_2->sin6_port = sh->dest_port;
+	case IPV6_VERSION >> 4:
+		{
+			/* its IPv6 */
+			struct ip6_hdr *ip6;
+			struct sockaddr_in6 *sin6_2;
+
+			ip6 = mtod(m, struct ip6_hdr *);
+			sin6_2 = (struct sockaddr_in6 *)(local_sa);
+			memset(sin6_2, 0, sizeof(sin6));
+			sin6_2->sin6_family = AF_INET6;
+			sin6_2->sin6_len = sizeof(struct sockaddr_in6);
+			sin6_2->sin6_port = sh->dest_port;
+			sin6_2->sin6_addr = ip6->ip6_dst;
+			if (altsa) {
+				/*
+				 * For cookies we use the src address NOT
+				 * from the packet but from the original
+				 * INIT.
+				 */
+				sa = altsa;
+			} else {
 				sin6.sin6_addr = ip6->ip6_src;
 				sa = (struct sockaddr *)&sin6;
-				break;
 			}
-#endif
-		default:
-			return (-1);
 			break;
 		}
-	} else {
-		/*
-		 * For cookies we use the src address NOT from the packet
-		 * but from the original INIT
-		 */
-		sa = altsa;
+#endif
+	default:
+		return (-1);
+		break;
 	}
 	/* Turn off ECN until we get through all params */
 	ecn_allowed = 0;
diff --git a/sys/netinet/sctp_pcb.h b/sys/netinet/sctp_pcb.h
index e39055c..ce12265 100644
--- a/sys/netinet/sctp_pcb.h
+++ b/sys/netinet/sctp_pcb.h
@@ -328,6 +328,7 @@ struct sctp_pcb {
 	uint8_t default_dscp;
 	char current_secret_number;
 	char last_secret_number;
+	uint16_t port;		/* remote UDP encapsulation port */
 };
 
 #ifndef SCTP_ALIGNMENT
diff --git a/sys/netinet/sctp_structs.h b/sys/netinet/sctp_structs.h
index eec2daa..20ca2c6 100644
--- a/sys/netinet/sctp_structs.h
+++ b/sys/netinet/sctp_structs.h
@@ -1213,6 +1213,7 @@ struct sctp_association {
 	uint8_t sctp_cmt_pf;
 	uint8_t use_precise_time;
 	uint32_t sctp_features;
+	uint16_t port;		/* remote UDP encapsulation port */
 	/*
 	 * The mapping array is used to track out of order sequences above
 	 * last_acked_seq. 0 indicates packet missing 1 indicates packet
diff --git a/sys/netinet/sctp_sysctl.c b/sys/netinet/sctp_sysctl.c
index 25d75f8..aa031dc 100644
--- a/sys/netinet/sctp_sysctl.c
+++ b/sys/netinet/sctp_sysctl.c
@@ -122,7 +122,6 @@ sctp_init_sysctls()
 #if defined(SCTP_LOCAL_TRACE_BUF)
 	memset(&SCTP_BASE_SYSCTL(sctp_log), 0, sizeof(struct sctp_log));
 #endif
-	SCTP_BASE_SYSCTL(sctp_udp_tunneling_for_client_enable) = SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_DEFAULT;
 	SCTP_BASE_SYSCTL(sctp_udp_tunneling_port) = SCTPCTL_UDP_TUNNELING_PORT_DEFAULT;
 	SCTP_BASE_SYSCTL(sctp_enable_sack_immediately) = SCTPCTL_SACK_IMMEDIATELY_ENABLE_DEFAULT;
 	SCTP_BASE_SYSCTL(sctp_inits_include_nat_friendly) = SCTPCTL_NAT_FRIENDLY_INITS_DEFAULT;
@@ -666,10 +665,6 @@ sysctl_sctp_check(SYSCTL_HANDLER_ARGS)
 		RANGECHK(SCTP_BASE_SYSCTL(sctp_use_dccc_ecn), SCTPCTL_RTTVAR_DCCCECN_MIN, SCTPCTL_RTTVAR_DCCCECN_MAX);
 		RANGECHK(SCTP_BASE_SYSCTL(sctp_mobility_base), SCTPCTL_MOBILITY_BASE_MIN, SCTPCTL_MOBILITY_BASE_MAX);
 		RANGECHK(SCTP_BASE_SYSCTL(sctp_mobility_fasthandoff), SCTPCTL_MOBILITY_FASTHANDOFF_MIN, SCTPCTL_MOBILITY_FASTHANDOFF_MAX);
-/* XXX: Remove the #if after tunneling over IPv6 works also on FreeBSD. */
-#if !defined(__FreeBSD__) || defined(INET)
-		RANGECHK(SCTP_BASE_SYSCTL(sctp_udp_tunneling_for_client_enable), SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_MIN, SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_MAX);
-#endif
 		RANGECHK(SCTP_BASE_SYSCTL(sctp_enable_sack_immediately), SCTPCTL_SACK_IMMEDIATELY_ENABLE_MIN, SCTPCTL_SACK_IMMEDIATELY_ENABLE_MAX);
 		RANGECHK(SCTP_BASE_SYSCTL(sctp_inits_include_nat_friendly), SCTPCTL_NAT_FRIENDLY_INITS_MIN, SCTPCTL_NAT_FRIENDLY_INITS_MAX);
 
@@ -1089,10 +1084,6 @@ SYSCTL_VNET_PROC(_net_inet_sctp, OID_AUTO, clear_trace, CTLTYPE_UINT | CTLFLAG_R
 
 /* XXX: Remove the #if after tunneling over IPv6 works also on FreeBSD. */
 #if !defined(__FreeBSD__) || defined(INET)
-SYSCTL_VNET_PROC(_net_inet_sctp, OID_AUTO, udp_tunneling_for_client_enable, CTLTYPE_UINT | CTLFLAG_RW,
-    &SCTP_BASE_SYSCTL(sctp_udp_tunneling_for_client_enable), 0, sysctl_sctp_check, "IU",
-    SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_DESC);
-
 SYSCTL_VNET_PROC(_net_inet_sctp, OID_AUTO, udp_tunneling_port, CTLTYPE_UINT | CTLFLAG_RW,
     &SCTP_BASE_SYSCTL(sctp_udp_tunneling_port), 0, sysctl_sctp_udp_tunneling_check, "IU",
     SCTPCTL_UDP_TUNNELING_PORT_DESC);
diff --git a/sys/netinet/sctp_sysctl.h b/sys/netinet/sctp_sysctl.h
index f62679f..ec87c2a 100644
--- a/sys/netinet/sctp_sysctl.h
+++ b/sys/netinet/sctp_sysctl.h
@@ -110,7 +110,6 @@ struct sctp_sysctl {
 #if defined(SCTP_LOCAL_TRACE_BUF)
 	struct sctp_log sctp_log;
 #endif
-	uint32_t sctp_udp_tunneling_for_client_enable;
 	uint32_t sctp_udp_tunneling_port;
 	uint32_t sctp_enable_sack_immediately;
 	uint32_t sctp_vtag_time_wait;
@@ -464,12 +463,6 @@ struct sctp_sysctl {
 #define SCTPCTL_MOBILITY_FASTHANDOFF_MAX	1
 #define SCTPCTL_MOBILITY_FASTHANDOFF_DEFAULT	SCTP_DEFAULT_MOBILITY_FASTHANDOFF
 
-/* Enable SCTP/UDP tunneling for clients*/
-#define SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_DESC	"Enable SCTP/UDP tunneling for client"
-#define SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_MIN	0
-#define SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_MAX	1
-#define SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_DEFAULT	SCTPCTL_UDP_TUNNELING_FOR_CLIENT_ENABLE_MIN
-
 /* Enable SCTP/UDP tunneling port */
 #define SCTPCTL_UDP_TUNNELING_PORT_DESC		"Set the SCTP/UDP tunneling port"
 #define SCTPCTL_UDP_TUNNELING_PORT_MIN		0
diff --git a/sys/netinet/sctp_uio.h b/sys/netinet/sctp_uio.h
index 37e9b79..6f32d24 100644
--- a/sys/netinet/sctp_uio.h
+++ b/sys/netinet/sctp_uio.h
@@ -671,6 +671,12 @@ struct sctp_timeouts {
 	uint32_t stimo_shutdownack;
 };
 
+struct sctp_udpencaps {
+	sctp_assoc_t sue_assoc_id;
+	struct sockaddr_storage sue_address;
+	uint16_t sue_port;
+};
+
 struct sctp_cwnd_args {
 	struct sctp_nets *net;	/* network to *//* FIXME: LP64 issue */
 	uint32_t cwnd_new_value;/* cwnd in k */
diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c
index c6e77cd..c638e9c 100644
--- a/sys/netinet/sctp_usrreq.c
+++ b/sys/netinet/sctp_usrreq.c
@@ -3208,6 +3208,92 @@ flags_out:
 			}
 			break;
 		}
+	case SCTP_REMOTE_UDP_ENCAPS_PORT:
+		{
+			struct sctp_udpencaps *encaps;
+			struct sctp_nets *net;
+
+			SCTP_CHECK_AND_CAST(encaps, optval, struct sctp_udpencaps, *optsize);
+			SCTP_FIND_STCB(inp, stcb, encaps->sue_assoc_id);
+
+			if (stcb) {
+				net = sctp_findnet(stcb, (struct sockaddr *)&encaps->sue_address);
+			} else {
+				/*
+				 * We increment here since
+				 * sctp_findassociation_ep_addr() wil do a
+				 * decrement if it finds the stcb as long as
+				 * the locked tcb (last argument) is NOT a
+				 * TCB.. aka NULL.
+				 */
+				net = NULL;
+				SCTP_INP_INCR_REF(inp);
+				stcb = sctp_findassociation_ep_addr(&inp, (struct sockaddr *)&encaps->sue_address, &net, NULL, NULL);
+				if (stcb == NULL) {
+					SCTP_INP_DECR_REF(inp);
+				}
+			}
+			if (stcb && (net == NULL)) {
+				struct sockaddr *sa;
+
+				sa = (struct sockaddr *)&encaps->sue_address;
+#ifdef INET
+				if (sa->sa_family == AF_INET) {
+					struct sockaddr_in *sin;
+
+					sin = (struct sockaddr_in *)sa;
+					if (sin->sin_addr.s_addr) {
+						error = EINVAL;
+						SCTP_TCB_UNLOCK(stcb);
+						SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, error);
+						break;
+					}
+				} else
+#endif
+#ifdef INET6
+				if (sa->sa_family == AF_INET6) {
+					struct sockaddr_in6 *sin6;
+
+					sin6 = (struct sockaddr_in6 *)sa;
+					if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
+						error = EINVAL;
+						SCTP_TCB_UNLOCK(stcb);
+						SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, error);
+						break;
+					}
+				} else
+#endif
+				{
+					error = EAFNOSUPPORT;
+					SCTP_TCB_UNLOCK(stcb);
+					SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, error);
+					break;
+				}
+			}
+			if (stcb) {
+				if (net) {
+					encaps->sue_port = net->port;
+				} else {
+					encaps->sue_port = stcb->asoc.port;
+				}
+				SCTP_TCB_UNLOCK(stcb);
+			} else {
+				if ((inp->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) ||
+				    (inp->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) ||
+				    (encaps->sue_assoc_id == SCTP_FUTURE_ASSOC)) {
+					SCTP_INP_RLOCK(inp);
+					encaps->sue_port = inp->sctp_ep.port;
+					SCTP_INP_RUNLOCK(inp);
+				} else {
+					SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, EINVAL);
+					error = EINVAL;
+				}
+			}
+			if (error == 0) {
+				*optsize = sizeof(struct sctp_paddrparams);
+			}
+			break;
+		}
 	default:
 		SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, ENOPROTOOPT);
 		error = ENOPROTOOPT;
@@ -4526,7 +4612,6 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 			break;
 		}
 	case SCTP_PEER_ADDR_PARAMS:
-		/* Applies to the specific association */
 		{
 			struct sctp_paddrparams *paddrp;
 			struct sctp_nets *net;
@@ -5583,6 +5668,89 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 			}
 			break;
 		}
+	case SCTP_REMOTE_UDP_ENCAPS_PORT:
+		{
+			struct sctp_udpencaps *encaps;
+			struct sctp_nets *net;
+
+			SCTP_CHECK_AND_CAST(encaps, optval, struct sctp_udpencaps, optsize);
+			SCTP_FIND_STCB(inp, stcb, encaps->sue_assoc_id);
+			if (stcb) {
+				net = sctp_findnet(stcb, (struct sockaddr *)&encaps->sue_address);
+			} else {
+				/*
+				 * We increment here since
+				 * sctp_findassociation_ep_addr() wil do a
+				 * decrement if it finds the stcb as long as
+				 * the locked tcb (last argument) is NOT a
+				 * TCB.. aka NULL.
+				 */
+				net = NULL;
+				SCTP_INP_INCR_REF(inp);
+				stcb = sctp_findassociation_ep_addr(&inp, (struct sockaddr *)&encaps->sue_address, &net, NULL, NULL);
+				if (stcb == NULL) {
+					SCTP_INP_DECR_REF(inp);
+				}
+			}
+			if (stcb && (net == NULL)) {
+				struct sockaddr *sa;
+
+				sa = (struct sockaddr *)&encaps->sue_address;
+#ifdef INET
+				if (sa->sa_family == AF_INET) {
+
+					struct sockaddr_in *sin;
+
+					sin = (struct sockaddr_in *)sa;
+					if (sin->sin_addr.s_addr) {
+						SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, EINVAL);
+						SCTP_TCB_UNLOCK(stcb);
+						error = EINVAL;
+						break;
+					}
+				} else
+#endif
+#ifdef INET6
+				if (sa->sa_family == AF_INET6) {
+					struct sockaddr_in6 *sin6;
+
+					sin6 = (struct sockaddr_in6 *)sa;
+					if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
+						SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, EINVAL);
+						SCTP_TCB_UNLOCK(stcb);
+						error = EINVAL;
+						break;
+					}
+				} else
+#endif
+				{
+					error = EAFNOSUPPORT;
+					SCTP_TCB_UNLOCK(stcb);
+					SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, error);
+					break;
+				}
+			}
+			if (stcb) {
+				if (net) {
+					net->port = encaps->sue_port;
+				} else {
+					stcb->asoc.port = encaps->sue_port;
+				}
+				SCTP_TCB_UNLOCK(stcb);
+			} else {
+				if ((inp->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) ||
+				    (inp->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL) ||
+				    (encaps->sue_assoc_id == SCTP_FUTURE_ASSOC)) {
+					SCTP_INP_WLOCK(inp);
+					inp->sctp_ep.port = encaps->sue_port;
+					SCTP_INP_WUNLOCK(inp);
+				} else {
+					SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, EINVAL);
+					error = EINVAL;
+				}
+			}
+			break;
+		}
 	default:
 		SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP_USRREQ, ENOPROTOOPT);
 		error = ENOPROTOOPT;
diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c
index 3bd4e6f..a9d120b 100644
--- a/sys/netinet/sctputil.c
+++ b/sys/netinet/sctputil.c
@@ -1117,6 +1117,7 @@ sctp_init_asoc(struct sctp_inpcb *m, struct sctp_tcb *stcb,
 	asoc->authinfo.recv_keyid = 0;
 	LIST_INIT(&asoc->shared_keys);
 	asoc->marked_retrans = 0;
+	asoc->port = m->sctp_ep.port;
 	asoc->timoinit = 0;
 	asoc->timodata = 0;
 	asoc->timosack = 0;
diff --git a/sys/netinet/tcp_hostcache.c b/sys/netinet/tcp_hostcache.c
index 4656204..f9b78e6 100644
--- a/sys/netinet/tcp_hostcache.c
+++ b/sys/netinet/tcp_hostcache.c
@@ -118,7 +118,7 @@ static int sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS);
 static void tcp_hc_purge_internal(int);
 static void tcp_hc_purge(void *);
 
-SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hostcache, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hostcache, CTLFLAG_RW, 0,
     "TCP Host cache");
 
 SYSCTL_VNET_UINT(_net_inet_tcp_hostcache, OID_AUTO, cachelimit, CTLFLAG_RDTUN,
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 128cccd..e9ff355 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -165,7 +165,7 @@ SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, abc_l_var, CTLFLAG_RW,
     &VNET_NAME(tcp_abc_l_var), 2,
     "Cap the max cwnd increment during slow-start to this number of segments");
 
-SYSCTL_NODE(_net_inet_tcp, OID_AUTO, ecn, CTLFLAG_RW, 0, "TCP ECN");
+static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, ecn, CTLFLAG_RW, 0, "TCP ECN");
 
 VNET_DEFINE(int, tcp_do_ecn) = 0;
 SYSCTL_VNET_INT(_net_inet_tcp_ecn, OID_AUTO, enable, CTLFLAG_RW,
@@ -185,7 +185,7 @@ SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, insecure_rst, CTLFLAG_RW,
 
 VNET_DEFINE(int, tcp_recvspace) = 1024*64;
 #define	V_tcp_recvspace	VNET(tcp_recvspace)
-SYSCTL_VNET_INT(_net_inet_tcp, TCPCTL_RECVSPACE, tcp_recvspace, CTLFLAG_RW,
+SYSCTL_VNET_INT(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLFLAG_RW,
     &VNET_NAME(tcp_recvspace), 0, "Initial receive socket buffer size");
 
 VNET_DEFINE(int, tcp_do_autorcvbuf) = 1;
diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c
index 2db178d..21620d3 100644
--- a/sys/netinet/tcp_output.c
+++ b/sys/netinet/tcp_output.c
@@ -97,7 +97,7 @@ SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW,
 
 VNET_DEFINE(int, tcp_sendspace) = 1024*32;
 #define	V_tcp_sendspace	VNET(tcp_sendspace)
-SYSCTL_VNET_INT(_net_inet_tcp, TCPCTL_SENDSPACE, tcp_sendspace, CTLFLAG_RW,
+SYSCTL_VNET_INT(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLFLAG_RW,
 	&VNET_NAME(tcp_sendspace), 0, "Initial send socket buffer size");
 
 VNET_DEFINE(int, tcp_do_autosndbuf) = 1;
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index d130361..b694f00 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -77,7 +77,7 @@ __FBSDID("$FreeBSD$");
 static int tcp_reass_sysctl_maxseg(SYSCTL_HANDLER_ARGS);
 static int tcp_reass_sysctl_qsize(SYSCTL_HANDLER_ARGS);
 
-SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW, 0,
     "TCP Segment Reassembly Queue");
 
 static VNET_DEFINE(int, tcp_reass_maxseg) = 0;
diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c
index 66e4732..2e9c37c 100644
--- a/sys/netinet/tcp_syncache.c
+++ b/sys/netinet/tcp_syncache.c
@@ -147,7 +147,8 @@ static struct syncache
 static VNET_DEFINE(struct tcp_syncache, tcp_syncache);
 #define	V_tcp_syncache			VNET(tcp_syncache)
 
-SYSCTL_NODE(_net_inet_tcp, OID_AUTO, syncache, CTLFLAG_RW, 0, "TCP SYN cache");
+static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, syncache, CTLFLAG_RW, 0,
+    "TCP SYN cache");
 
 SYSCTL_VNET_UINT(_net_inet_tcp_syncache, OID_AUTO, bucketlimit, CTLFLAG_RDTUN,
     &VNET_NAME(tcp_syncache.bucket_limit), 0,
diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index d22bc43..2114a0a 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -2351,8 +2351,6 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	int icmp6len = ntohs(ip6->ip6_plen);
 	char *lladdr = NULL;
 	int lladdrlen = 0;
-	u_char *redirhdr = NULL;
-	int redirhdrlen = 0;
 	struct rtentry *rt = NULL;
 	int is_router;
 	int is_onlink;
@@ -2362,14 +2360,11 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	union nd_opts ndopts;
 	char ip6buf[INET6_ADDRSTRLEN];
 
-	if (!m)
-		return;
+	M_ASSERTPKTHDR(m);
+	KASSERT(m->m_pkthdr.rcvif != NULL, ("%s: no rcvif", __func__));
 
 	ifp = m->m_pkthdr.rcvif;
 
-	if (!ifp)
-		return;
-
 	/* XXX if we are router, we don't update route by icmp6 redirect */
 	if (V_ip6_forwarding)
 		goto freeit;
@@ -2476,9 +2471,8 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	icmp6len -= sizeof(*nd_rd);
 	nd6_option_init(nd_rd + 1, icmp6len, &ndopts);
 	if (nd6_options(&ndopts) < 0) {
-		nd6log((LOG_INFO, "icmp6_redirect_input: "
-		    "invalid ND option, rejected: %s\n",
-		    icmp6_redirect_diag(&src6, &reddst6, &redtgt6)));
+		nd6log((LOG_INFO, "%s: invalid ND option, rejected: %s\n",
+		    __func__, icmp6_redirect_diag(&src6, &reddst6, &redtgt6)));
 		/* nd6_options have incremented stats */
 		goto freeit;
 	}
@@ -2488,16 +2482,10 @@ icmp6_redirect_input(struct mbuf *m, int off)
 		lladdrlen = ndopts.nd_opts_tgt_lladdr->nd_opt_len << 3;
 	}
 
-	if (ndopts.nd_opts_rh) {
-		redirhdrlen = ndopts.nd_opts_rh->nd_opt_rh_len;
-		redirhdr = (u_char *)(ndopts.nd_opts_rh + 1); /* xxx */
-	}
-
 	if (lladdr && ((ifp->if_addrlen + 2 + 7) & ~7) != lladdrlen) {
-		nd6log((LOG_INFO,
-		    "icmp6_redirect_input: lladdrlen mismatch for %s "
+		nd6log((LOG_INFO, "%s: lladdrlen mismatch for %s "
 		    "(if %d, icmp6 packet %d): %s\n",
-		    ip6_sprintf(ip6buf, &redtgt6),
+		    __func__, ip6_sprintf(ip6buf, &redtgt6),
 		    ifp->if_addrlen, lladdrlen - 2,
 		    icmp6_redirect_diag(&src6, &reddst6, &redtgt6)));
 		goto bad;
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index 1e6cb94..8514f73 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -152,7 +152,7 @@ in6_ifaddloop(struct ifaddr *ifa)
 	ia = ifa2ia6(ifa);
 	ifp = ifa->ifa_ifp;
 	IF_AFDATA_LOCK(ifp);
-	ifa->ifa_rtrequest = NULL;
+	ifa->ifa_rtrequest = nd6_rtrequest;
 
 	/* XXX QL
 	 * we need to report rt_newaddrmsg
diff --git a/sys/netinet6/in6_mcast.c b/sys/netinet6/in6_mcast.c
index 1438c32..a1f233a 100644
--- a/sys/netinet6/in6_mcast.c
+++ b/sys/netinet6/in6_mcast.c
@@ -150,7 +150,8 @@ static int	sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS);
 
 SYSCTL_DECL(_net_inet6_ip6);	/* XXX Not in any common header. */
 
-SYSCTL_NODE(_net_inet6_ip6, OID_AUTO, mcast, CTLFLAG_RW, 0, "IPv6 multicast");
+static SYSCTL_NODE(_net_inet6_ip6, OID_AUTO, mcast, CTLFLAG_RW, 0,
+    "IPv6 multicast");
 
 static u_long in6_mcast_maxgrpsrc = IPV6_MAX_GROUP_SRC_FILTER;
 SYSCTL_ULONG(_net_inet6_ip6_mcast, OID_AUTO, maxgrpsrc,
@@ -170,7 +171,7 @@ SYSCTL_INT(_net_inet6_ip6_mcast, OID_AUTO, loop, CTLFLAG_RW | CTLFLAG_TUN,
     &in6_mcast_loop, 0, "Loopback multicast datagrams by default");
 TUNABLE_INT("net.inet6.ip6.mcast.loop", &in6_mcast_loop);
 
-SYSCTL_NODE(_net_inet6_ip6_mcast, OID_AUTO, filters,
+static SYSCTL_NODE(_net_inet6_ip6_mcast, OID_AUTO, filters,
     CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_ip6_mcast_filters,
     "Per-interface stack-wide source filters");
 
@@ -1713,7 +1714,7 @@ ip6_getmoptions(struct inpcb *inp, struct sockopt *sopt)
 		if (im6o == NULL)
 			optval = V_ip6_defmcasthlim;
 		else
-			optval = im6o->im6o_multicast_loop;
+			optval = im6o->im6o_multicast_hlim;
 		INP_WUNLOCK(inp);
 		error = sooptcopyout(sopt, &optval, sizeof(u_int));
 		break;
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index d15c605..8f82963 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -187,6 +187,7 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 		}
 		if (lport) {
 			struct inpcb *t;
+			struct tcptw *tw;
 
 			/* GROSS */
 			if (ntohs(lport) <= V_ipport_reservedhigh &&
@@ -206,8 +207,8 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 				     IN6_IS_ADDR_UNSPECIFIED(&t->in6p_faddr)) &&
 				    (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) ||
 				     !IN6_IS_ADDR_UNSPECIFIED(&t->in6p_laddr) ||
-				     (t->inp_socket->so_options & SO_REUSEPORT)
-				      == 0) && (inp->inp_cred->cr_uid !=
+				     (t->inp_flags2 & INP_REUSEPORT) == 0) &&
+				    (inp->inp_cred->cr_uid !=
 				     t->inp_cred->cr_uid))
 					return (EADDRINUSE);
 #ifdef INET
@@ -233,10 +234,21 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 			}
 			t = in6_pcblookup_local(pcbinfo, &sin6->sin6_addr,
 			    lport, lookupflags, cred);
-			if (t && (reuseport & ((t->inp_flags & INP_TIMEWAIT) ?
-			    intotw(t)->tw_so_options :
-			    t->inp_socket->so_options)) == 0)
+			if (t && (t->inp_flags & INP_TIMEWAIT)) {
+				/*
+				 * XXXRW: If an incpb has had its timewait
+				 * state recycled, we treat the address as
+				 * being in use (for now).  This is better
+				 * than a panic, but not desirable.
+				 */
+				tw = intotw(t);
+				if (tw == NULL ||
+				    (reuseport & tw->tw_so_options) == 0)
+					return (EADDRINUSE);
+			} else if (t && (reuseport & t->inp_socket->so_options)
+			    == 0) {
 				return (EADDRINUSE);
+			}
 #ifdef INET
 			if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
 			    IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
@@ -246,19 +258,19 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 				t = in_pcblookup_local(pcbinfo, sin.sin_addr,
 				    lport, lookupflags, cred);
 				if (t && t->inp_flags & INP_TIMEWAIT) {
-					if ((reuseport &
-					    intotw(t)->tw_so_options) == 0 &&
-					    (ntohl(t->inp_laddr.s_addr) !=
+					tw = intotw(t);
+					if (tw == NULL)
+						return (EADDRINUSE);
+					if ((reuseport & tw->tw_so_options) == 0
+					    && (ntohl(t->inp_laddr.s_addr) !=
 					     INADDR_ANY || ((inp->inp_vflag &
 					     INP_IPV6PROTO) ==
 					     (t->inp_vflag & INP_IPV6PROTO))))
 						return (EADDRINUSE);
-				}
-				else if (t &&
-				    (reuseport & t->inp_socket->so_options)
-				    == 0 && (ntohl(t->inp_laddr.s_addr) !=
-				    INADDR_ANY || INP_SOCKAF(so) ==
-				     INP_SOCKAF(t->inp_socket)))
+				} else if (t && (reuseport == 0 ||
+				    (t->inp_flags2 & INP_REUSEPORT) == 0) &&
+				    (ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
+				    (t->inp_vflag & INP_IPV6PROTO) != 0))
 					return (EADDRINUSE);
 			}
 #endif
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index c802bfc..2b3493e 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -927,7 +927,7 @@ struct walkarg {
 
 static int in6_src_sysctl(SYSCTL_HANDLER_ARGS);
 SYSCTL_DECL(_net_inet6_ip6);
-SYSCTL_NODE(_net_inet6_ip6, IPV6CTL_ADDRCTLPOLICY, addrctlpolicy,
+static SYSCTL_NODE(_net_inet6_ip6, IPV6CTL_ADDRCTLPOLICY, addrctlpolicy,
 	CTLFLAG_RD, in6_src_sysctl, "");
 
 static int
diff --git a/sys/netinet6/ip6_mroute.c b/sys/netinet6/ip6_mroute.c
index 0c1ff78..a4f5044 100644
--- a/sys/netinet6/ip6_mroute.c
+++ b/sys/netinet6/ip6_mroute.c
@@ -156,7 +156,7 @@ static VNET_DEFINE(int, ip6_mrouter_ver) = 0;
 
 SYSCTL_DECL(_net_inet6);
 SYSCTL_DECL(_net_inet6_ip6);
-SYSCTL_NODE(_net_inet6, IPPROTO_PIM, pim, CTLFLAG_RW, 0, "PIM");
+static SYSCTL_NODE(_net_inet6, IPPROTO_PIM, pim, CTLFLAG_RW, 0, "PIM");
 
 static struct mrt6stat mrt6stat;
 SYSCTL_STRUCT(_net_inet6_ip6, OID_AUTO, mrt6stat, CTLFLAG_RW,
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index e4d5172..5372438 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -1421,7 +1421,38 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt)
 	optval = 0;
 	uproto = (int)so->so_proto->pr_protocol;
 
-	if (level == IPPROTO_IPV6) {
+	if (level != IPPROTO_IPV6) {
+		error = EINVAL;
+
+		if (sopt->sopt_level == SOL_SOCKET &&
+		    sopt->sopt_dir == SOPT_SET) {
+			switch (sopt->sopt_name) {
+			case SO_REUSEADDR:
+				INP_WLOCK(in6p);
+				if (IN_MULTICAST(ntohl(in6p->inp_laddr.s_addr))) {
+					if ((so->so_options &
+					    (SO_REUSEADDR | SO_REUSEPORT)) != 0)
+						in6p->inp_flags2 |= INP_REUSEPORT;
+					else
+						in6p->inp_flags2 &= ~INP_REUSEPORT;
+				}
+				INP_WUNLOCK(in6p);
+				error = 0;
+				break;
+			case SO_REUSEPORT:
+				INP_WLOCK(in6p);
+				if ((so->so_options & SO_REUSEPORT) != 0)
+					in6p->inp_flags2 |= INP_REUSEPORT;
+				else
+					in6p->inp_flags2 &= ~INP_REUSEPORT;
+				INP_WUNLOCK(in6p);
+				error = 0;
+				break;
+			default:
+				break;
+			}
+		}
+	} else {		/* level == IPPROTO_IPV6 */
 		switch (op) {
 
 		case SOPT_SET:
@@ -2044,8 +2075,6 @@ do { \
 			}
 			break;
 		}
-	} else {		/* level != IPPROTO_IPV6 */
-		error = EINVAL;
 	}
 	return (error);
 }
diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c
index a56f83d..8f8e326 100644
--- a/sys/netinet6/mld6.c
+++ b/sys/netinet6/mld6.c
@@ -193,7 +193,7 @@ static int	sysctl_mld_ifinfo(SYSCTL_HANDLER_ARGS);
  *    to a vnet in ifp->if_vnet.
  */
 static struct mtx		 mld_mtx;
-MALLOC_DEFINE(M_MLD, "mld", "mld state");
+static MALLOC_DEFINE(M_MLD, "mld", "mld state");
 
 #define	MLD_EMBEDSCOPE(pin6, zoneid)					\
 	if (IN6_IS_SCOPE_LINKLOCAL(pin6) ||				\
@@ -231,8 +231,9 @@ SYSCTL_VNET_PROC(_net_inet6_mld, OID_AUTO, gsrdelay,
 /*
  * Non-virtualized sysctls.
  */
-SYSCTL_NODE(_net_inet6_mld, OID_AUTO, ifinfo, CTLFLAG_RD | CTLFLAG_MPSAFE,
-    sysctl_mld_ifinfo, "Per-interface MLDv2 state");
+static SYSCTL_NODE(_net_inet6_mld, OID_AUTO, ifinfo,
+    CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_mld_ifinfo,
+    "Per-interface MLDv2 state");
 
 static int	mld_v1enable = 1;
 SYSCTL_INT(_net_inet6_mld, OID_AUTO, v1enable, CTLFLAG_RW,
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index ae7cc4a..875daa0 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -1174,6 +1174,46 @@ done:
 }
 
 
+/*
+ * Rejuvenate this function for routing operations related
+ * processing.
+ */
+void
+nd6_rtrequest(int req, struct rtentry *rt, struct rt_addrinfo *info)
+{
+	struct sockaddr_in6 *gateway = (struct sockaddr_in6 *)rt->rt_gateway;
+	struct nd_defrouter *dr;
+	struct ifnet *ifp = rt->rt_ifp;
+
+	RT_LOCK_ASSERT(rt);
+
+	switch (req) {
+	case RTM_ADD:
+		break;
+
+	case RTM_DELETE:
+		if (!ifp)
+			return;
+		/*
+		 * Only indirect routes are interesting.
+		 */
+		if ((rt->rt_flags & RTF_GATEWAY) == 0)
+			return;
+		/*
+		 * check for default route
+		 */
+		if (IN6_ARE_ADDR_EQUAL(&in6addr_any, 
+				       &SIN6(rt_key(rt))->sin6_addr)) {
+
+			dr = defrouter_lookup(&gateway->sin6_addr, ifp);
+			if (dr != NULL)
+				dr->installed = 0;
+		}
+		break;
+	}
+}
+
+
 int
 nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp)
 {
diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h
index 6f63192..ca981d0 100644
--- a/sys/netinet6/nd6.h
+++ b/sys/netinet6/nd6.h
@@ -406,6 +406,7 @@ void nd6_purge __P((struct ifnet *));
 void nd6_nud_hint __P((struct rtentry *, struct in6_addr *, int));
 int nd6_resolve __P((struct ifnet *, struct rtentry *, struct mbuf *,
 	struct sockaddr *, u_char *));
+void nd6_rtrequest __P((int, struct rtentry *, struct rt_addrinfo *));
 int nd6_ioctl __P((u_long, caddr_t, struct ifnet *));
 struct llentry *nd6_cache_lladdr __P((struct ifnet *, struct in6_addr *,
 	char *, int, int, int));
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index 5352dd5..237f14c 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -751,9 +751,10 @@ defrtrlist_update(struct nd_defrouter *new)
 
 			/*
 			 * If the preference does not change, there's no need
-			 * to sort the entries.
+			 * to sort the entries. Also make sure the selected
+			 * router is still installed in the kernel.
 			 */
-			if (rtpref(new) == oldpref) {
+			if (dr->installed && rtpref(new) == oldpref) {
 				splx(s);
 				return (dr);
 			}
diff --git a/sys/netinet6/send.c b/sys/netinet6/send.c
index 71d88d0..5624366 100644
--- a/sys/netinet6/send.c
+++ b/sys/netinet6/send.c
@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/scope6_var.h>
 #include <netinet6/send.h>
 
-MALLOC_DEFINE(M_SEND, "send", "Secure Neighbour Discovery");
+static MALLOC_DEFINE(M_SEND, "send", "Secure Neighbour Discovery");
 
 /*
  * The socket used to communicate with the SeND daemon.
diff --git a/sys/netipx/ipx_proto.c b/sys/netipx/ipx_proto.c
index 56d091a..36215da 100644
--- a/sys/netipx/ipx_proto.c
+++ b/sys/netipx/ipx_proto.c
@@ -156,4 +156,4 @@ SYSCTL_NODE(_net,	PF_IPX,		ipx,	CTLFLAG_RW, 0,
 	"IPX/SPX");
 
 SYSCTL_NODE(_net_ipx,	IPXPROTO_RAW,	ipx,	CTLFLAG_RW, 0, "IPX");
-SYSCTL_NODE(_net_ipx,	IPXPROTO_SPX,	spx,	CTLFLAG_RW, 0, "SPX");
+static SYSCTL_NODE(_net_ipx, IPXPROTO_SPX, spx,	CTLFLAG_RW, 0, "SPX");
diff --git a/sys/netipx/spx_reass.c b/sys/netipx/spx_reass.c
index 4344b29..fea031f 100644
--- a/sys/netipx/spx_reass.c
+++ b/sys/netipx/spx_reass.c
@@ -93,7 +93,7 @@ __FBSDID("$FreeBSD$");
 static int	spx_use_delack = 0;
 static int	spxrexmtthresh = 3;
 
-MALLOC_DEFINE(M_SPXREASSQ, "spxreassq", "SPX reassembly queue entry");
+static MALLOC_DEFINE(M_SPXREASSQ, "spxreassq", "SPX reassembly queue entry");
 
 /*
  * Flesh pending queued segments on SPX close.
diff --git a/sys/netsmb/smb_conn.c b/sys/netsmb/smb_conn.c
index c14cb8b..2ee851b 100644
--- a/sys/netsmb/smb_conn.c
+++ b/sys/netsmb/smb_conn.c
@@ -54,7 +54,7 @@ static int smb_vcnext = 1;	/* next unique id for VC */
 
 SYSCTL_NODE(_net, OID_AUTO, smb, CTLFLAG_RW, NULL, "SMB protocol");
 
-MALLOC_DEFINE(M_SMBCONN, "smb_conn", "SMB connection");
+static MALLOC_DEFINE(M_SMBCONN, "smb_conn", "SMB connection");
 
 static void smb_co_init(struct smb_connobj *cp, int level, char *ilockname,
     char *lockname);
@@ -444,13 +444,29 @@ smb_vc_create(struct smb_vcspec *vcspec,
 		goto fail;
 	if (vcspec->servercs[0]) {
 		error = (int)iconv_open(vcspec->servercs, vcspec->localcs,
-		    &vcp->vc_toserver);
+		    &vcp->vc_cp_toserver);
 		if (error)
 			goto fail;
 		error = (int)iconv_open(vcspec->localcs, vcspec->servercs,
-		    &vcp->vc_tolocal);
+		    &vcp->vc_cp_tolocal);
 		if (error)
 			goto fail;
+		vcp->vc_toserver = vcp->vc_cp_toserver;
+		vcp->vc_tolocal = vcp->vc_cp_tolocal;
+		iconv_add(ENCODING_UNICODE, ENCODING_UNICODE, SMB_UNICODE_NAME);
+		iconv_add(ENCODING_UNICODE, SMB_UNICODE_NAME, ENCODING_UNICODE);
+		error = (int)iconv_open(SMB_UNICODE_NAME, vcspec->localcs,
+		    &vcp->vc_ucs_toserver);
+		if (!error) {
+			error = (int)iconv_open(vcspec->localcs, SMB_UNICODE_NAME,
+			    &vcp->vc_ucs_tolocal);
+		}
+		if (error) {
+			if (vcp->vc_ucs_toserver)
+				iconv_close(vcp->vc_ucs_toserver);
+			vcp->vc_ucs_toserver = NULL;
+			vcp->vc_ucs_tolocal = NULL;
+		}
 	}
 	error = (int)smb_iod_create(vcp);
 	if (error)
@@ -486,9 +502,17 @@ smb_vc_free(struct smb_connobj *cp)
 	if (vcp->vc_toupper)
 		iconv_close(vcp->vc_toupper);
 	if (vcp->vc_tolocal)
-		iconv_close(vcp->vc_tolocal);
+		vcp->vc_tolocal = NULL;
 	if (vcp->vc_toserver)
-		iconv_close(vcp->vc_toserver);
+		vcp->vc_toserver = NULL;
+	if (vcp->vc_cp_tolocal)
+		iconv_close(vcp->vc_cp_tolocal);
+	if (vcp->vc_cp_toserver)
+		iconv_close(vcp->vc_cp_toserver);
+	if (vcp->vc_ucs_tolocal)
+		iconv_close(vcp->vc_ucs_tolocal);
+	if (vcp->vc_ucs_toserver)
+		iconv_close(vcp->vc_ucs_toserver);
 	smb_co_done(VCTOCP(vcp));
 	smb_sl_destroy(&vcp->vc_stlock);
 	free(vcp, M_SMBCONN);
diff --git a/sys/netsmb/smb_conn.h b/sys/netsmb/smb_conn.h
index 370f6cf..94a7a82 100644
--- a/sys/netsmb/smb_conn.h
+++ b/sys/netsmb/smb_conn.h
@@ -242,6 +242,10 @@ struct smb_vc {
 	void *		vc_toupper;	/* local charset */
 	void *		vc_toserver;	/* local charset to server one */
 	void *		vc_tolocal;	/* server charset to local one */
+	void *		vc_cp_toserver;	/* local charset to server one (using CodePage) */
+	void *		vc_cp_tolocal;	/* server charset to local one (using CodePage) */
+	void *		vc_ucs_toserver; /* local charset to server one (using UCS-2) */
+	void *		vc_ucs_tolocal;	/* server charset to local one (using UCS-2) */
 	int		vc_number;	/* number of this VC from the client side */
 	int		vc_genid;
 	uid_t		vc_uid;		/* user id of connection */
@@ -272,6 +276,8 @@ struct smb_vc {
 
 #define SMB_UNICODE_STRINGS(vcp)	((vcp)->vc_hflags2 & SMB_FLAGS2_UNICODE)
 
+#define	SMB_UNICODE_NAME	"UCS-2LE"
+
 /*
  * smb_share structure describes connection to the given SMB share (tree).
  * Connection to share is always built on top of the VC.
diff --git a/sys/netsmb/smb_rq.c b/sys/netsmb/smb_rq.c
index 4f5c5e6..566a252 100644
--- a/sys/netsmb/smb_rq.c
+++ b/sys/netsmb/smb_rq.c
@@ -46,7 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <netsmb/smb_subr.h>
 #include <netsmb/smb_tran.h>
 
-MALLOC_DEFINE(M_SMBRQ, "SMBRQ", "SMB request");
+static MALLOC_DEFINE(M_SMBRQ, "SMBRQ", "SMB request");
 
 MODULE_DEPEND(netsmb, libmchain, 1, 1, 1);
 
diff --git a/sys/netsmb/smb_smb.c b/sys/netsmb/smb_smb.c
index d3fb94a..6c732f9d 100644
--- a/sys/netsmb/smb_smb.c
+++ b/sys/netsmb/smb_smb.c
@@ -121,9 +121,17 @@ smb_smb_negotiate(struct smb_vc *vcp, struct smb_cred *scred)
 	u_int8_t wc, stime[8], sblen;
 	u_int16_t dindex, tw, tw1, swlen, bc;
 	int error, maxqsz;
+	int unicode = SMB_UNICODE_STRINGS(vcp);
+	void * servercharset = vcp->vc_toserver;
+	void * localcharset = vcp->vc_tolocal;
 
 	if (smb_smb_nomux(vcp, scred, __func__) != 0)
 		return EINVAL;
+	/* Disable Unicode for SMB_COM_NEGOTIATE requests */ 
+	if (unicode) {
+		vcp->vc_toserver = vcp->vc_cp_toserver;
+		vcp->vc_tolocal  = vcp->vc_cp_tolocal;
+	}
 	vcp->vc_hflags = 0;
 	vcp->vc_hflags2 = 0;
 	vcp->obj.co_flags &= ~(SMBV_ENCRYPT);
@@ -180,7 +188,7 @@ smb_smb_negotiate(struct smb_vc *vcp, struct smb_cred *scred)
 					SMBERROR("Unexpected length of security blob (%d)\n", sblen);
 					break;
 				}
-				error = md_get_uint16(mdp, &bc);
+				error = md_get_uint16le(mdp, &bc);
 				if (error)
 					break;
 				if (sp->sv_caps & SMB_CAP_EXT_SECURITY)
@@ -193,6 +201,13 @@ smb_smb_negotiate(struct smb_vc *vcp, struct smb_cred *scred)
 			}
 			if (sp->sv_sm & SMB_SM_SIGS_REQUIRE)
 				vcp->vc_hflags2 |= SMB_FLAGS2_SECURITY_SIGNATURE;
+			if (vcp->vc_ucs_toserver &&
+				sp->sv_caps & SMB_CAP_UNICODE) {
+				/*
+				* They do Unicode.
+				*/
+				vcp->obj.co_flags |= SMBV_UNICODE;
+			}
 			vcp->vc_hflags2 |= SMB_FLAGS2_KNOWS_LONG_NAMES;
 			if (dp->d_id == SMB_DIALECT_NTLM0_12 &&
 			    sp->sv_maxtx < 4096 &&
@@ -200,7 +215,13 @@ smb_smb_negotiate(struct smb_vc *vcp, struct smb_cred *scred)
 				vcp->obj.co_flags |= SMBV_WIN95;
 				SMBSDEBUG("Win95 detected\n");
 			}
-		} else if (dp->d_id > SMB_DIALECT_CORE) {
+			error = 0;
+			break;
+		}
+		vcp->vc_hflags2 &= ~(SMB_FLAGS2_EXT_SEC|SMB_FLAGS2_DFS|
+				     SMB_FLAGS2_ERR_STATUS|SMB_FLAGS2_UNICODE);
+		unicode = 0;
+		if (dp->d_id > SMB_DIALECT_CORE) {
 			md_get_uint16le(mdp, &tw);
 			sp->sv_sm = tw;
 			md_get_uint16le(mdp, &tw);
@@ -217,7 +238,7 @@ smb_smb_negotiate(struct smb_vc *vcp, struct smb_cred *scred)
 				if (swlen > SMB_MAXCHALLENGELEN)
 					break;
 				md_get_uint16(mdp, NULL);	/* mbz */
-				if (md_get_uint16(mdp, &bc) != 0)
+				if (md_get_uint16le(mdp, &bc) != 0)
 					break;
 				if (bc < swlen)
 					break;
@@ -259,6 +280,12 @@ smb_smb_negotiate(struct smb_vc *vcp, struct smb_cred *scred)
 		SMBSDEBUG("MAXTX = %d\n", sp->sv_maxtx);
 	}
 bad:
+	/* Restore Unicode conversion state */
+	if (unicode) {
+		vcp->vc_toserver = servercharset;
+		vcp->vc_tolocal  = localcharset;
+		vcp->vc_hflags2 |= SMB_FLAGS2_UNICODE;
+	}
 	smb_rq_done(rqp);
 	return error;
 }
@@ -273,9 +300,13 @@ smb_smb_ssnsetup(struct smb_vc *vcp, struct smb_cred *scred)
 	smb_uniptr unipp, ntencpass = NULL;
 	char *pp, *up, *pbuf, *encpass;
 	int error, plen, uniplen, ulen, upper;
+	u_int32_t caps = 0;
 
 	upper = 0;
 
+	if (vcp->obj.co_flags & SMBV_UNICODE)
+		caps |= SMB_CAP_UNICODE;
+
 again:
 
 	vcp->vc_smbuid = SMB_UID_UNKNOWN;
@@ -374,8 +405,7 @@ again:
 	} else {
 		mb_put_uint16le(mbp, uniplen);
 		mb_put_uint32le(mbp, 0);		/* reserved */
-		mb_put_uint32le(mbp, vcp->obj.co_flags & SMBV_UNICODE ?
-				     SMB_CAP_UNICODE : 0);
+		mb_put_uint32le(mbp, caps);
 		smb_rq_wend(rqp);
 		smb_rq_bstart(rqp);
 		mb_put_mem(mbp, pp, plen, MB_MSYSTEM);
@@ -477,24 +507,13 @@ smb_smb_treeconnect(struct smb_share *ssp, struct smb_cred *scred)
 	upper = 0;
 
 again:
-
-#if 0
 	/* Disable Unicode for SMB_COM_TREE_CONNECT_ANDX requests */
 	if (SSTOVC(ssp)->vc_hflags2 & SMB_FLAGS2_UNICODE) {
 		vcp = SSTOVC(ssp);
-		if (vcp->vc_toserver) {
-			iconv_close(vcp->vc_toserver);
-			/* Use NULL until UTF-8 -> ASCII works */
-			vcp->vc_toserver = NULL;
-		}
-		if (vcp->vc_tolocal) {
-			iconv_close(vcp->vc_tolocal);
-			/* Use NULL until ASCII -> UTF-8 works*/
-			vcp->vc_tolocal = NULL;
-		}
+		vcp->vc_toserver = vcp->vc_cp_toserver;
+		vcp->vc_tolocal = vcp->vc_cp_tolocal;
 		vcp->vc_hflags2 &= ~SMB_FLAGS2_UNICODE;
 	}
-#endif
 
 	ssp->ss_tid = SMB_TID_UNKNOWN;
 	error = smb_rq_alloc(SSTOCP(ssp), SMB_COM_TREE_CONNECT_ANDX, scred, &rqp);
@@ -559,6 +578,15 @@ again:
 	ssp->ss_tid = rqp->sr_rptid;
 	ssp->ss_vcgenid = vcp->vc_genid;
 	ssp->ss_flags |= SMBS_CONNECTED;
+	/*
+	 * If the server can speak Unicode then switch
+	 * our converters to do Unicode <--> Local
+	 */
+	if (vcp->obj.co_flags & SMBV_UNICODE) {
+		vcp->vc_toserver = vcp->vc_ucs_toserver;
+		vcp->vc_tolocal = vcp->vc_ucs_tolocal;
+		vcp->vc_hflags2 |= SMB_FLAGS2_UNICODE;
+	}
 bad:
 	if (encpass)
 		free(encpass, M_SMBTEMP);
diff --git a/sys/netsmb/smb_subr.c b/sys/netsmb/smb_subr.c
index ade9e17..3846b75 100644
--- a/sys/netsmb/smb_subr.c
+++ b/sys/netsmb/smb_subr.c
@@ -46,8 +46,8 @@ __FBSDID("$FreeBSD$");
 #include <netsmb/smb_rq.h>
 #include <netsmb/smb_subr.h>
 
-MALLOC_DEFINE(M_SMBDATA, "SMBDATA", "Misc netsmb data");
-MALLOC_DEFINE(M_SMBSTR, "SMBSTR", "netsmb string data");
+static MALLOC_DEFINE(M_SMBDATA, "SMBDATA", "Misc netsmb data");
+static MALLOC_DEFINE(M_SMBSTR, "SMBSTR", "netsmb string data");
 MALLOC_DEFINE(M_SMBTEMP, "SMBTEMP", "Temp netsmb data");
 
 smb_unichar smb_unieol = 0;
@@ -350,6 +350,8 @@ smb_put_dmem(struct mbchain *mbp, struct smb_vc *vcp, const char *src,
 	}
 	mbp->mb_copy = smb_copy_iconv;
 	mbp->mb_udata = dp;
+	if (SMB_UNICODE_STRINGS(vcp))
+		mb_put_padbyte(mbp);
 	return mb_put_mem(mbp, src, size, MB_MCUSTOM);
 }
 
@@ -362,6 +364,8 @@ smb_put_dstring(struct mbchain *mbp, struct smb_vc *vcp, const char *src,
 	error = smb_put_dmem(mbp, vcp, src, strlen(src), caseopt);
 	if (error)
 		return error;
+	if (SMB_UNICODE_STRINGS(vcp))
+		return mb_put_uint16le(mbp, 0);
 	return mb_put_uint8(mbp, 0);
 }
 
diff --git a/sys/nfs/nfs_lock.c b/sys/nfs/nfs_lock.c
index 0aca8e9..7d11672 100644
--- a/sys/nfs/nfs_lock.c
+++ b/sys/nfs/nfs_lock.c
@@ -65,8 +65,9 @@ extern void (*nlminfo_release_p)(struct proc *p);
 vop_advlock_t	*nfs_advlock_p = nfs_dolock;
 vop_reclaim_t	*nfs_reclaim_p = NULL;
 
-MALLOC_DEFINE(M_NFSLOCK, "nfsclient_lock", "NFS lock request");
-MALLOC_DEFINE(M_NLMINFO, "nfsclient_nlminfo", "NFS lock process structure");
+static MALLOC_DEFINE(M_NFSLOCK, "nfsclient_lock", "NFS lock request");
+static MALLOC_DEFINE(M_NLMINFO, "nfsclient_nlminfo",
+    "NFS lock process structure");
 
 static int nfslockdans(struct thread *td, struct lockd_ans *ansp);
 static void nlminfo_release(struct proc *p);
diff --git a/sys/nfsclient/nfs_krpc.c b/sys/nfsclient/nfs_krpc.c
index 171f7aa..c2406d9 100644
--- a/sys/nfsclient/nfs_krpc.c
+++ b/sys/nfsclient/nfs_krpc.c
@@ -540,6 +540,11 @@ tryagain:
 				    hz);
 			goto tryagain;
 		}
+		/*
+		 * Make sure NFSERR_RETERR isn't bogusly set by a server
+		 * such as amd. (No actual NFS error has bit 31 set.)
+		 */
+		error &= ~NFSERR_RETERR;
 
 		/*
 		 * If the File Handle was stale, invalidate the lookup
diff --git a/sys/nfsclient/nfs_vfsops.c b/sys/nfsclient/nfs_vfsops.c
index 6bcc9b5..42895d0 100644
--- a/sys/nfsclient/nfs_vfsops.c
+++ b/sys/nfsclient/nfs_vfsops.c
@@ -787,7 +787,7 @@ static const char *nfs_opts[] = { "from", "nfs_args",
     "readahead", "readdirsize", "soft", "hard", "mntudp", "tcp", "udp",
     "wsize", "rsize", "retrans", "acregmin", "acregmax", "acdirmin",
     "acdirmax", "deadthresh", "hostname", "timeout", "addr", "fh", "nfsv3",
-    "sec", "maxgroups", "principal", "negnametimeo", "nocto",
+    "sec", "maxgroups", "principal", "negnametimeo", "nocto", "wcommitsize",
     NULL };
 
 /*
@@ -1019,6 +1019,15 @@ nfs_mount(struct mount *mp)
 		}
 		args.flags |= NFSMNT_ACDIRMAX;
 	}
+	if (vfs_getopt(mp->mnt_optnew, "wcommitsize", (void **)&opt, NULL) == 0) {
+		ret = sscanf(opt, "%d", &args.wcommitsize);
+		if (ret != 1 || args.wcommitsize < 0) {
+			vfs_mount_error(mp, "illegal wcommitsize: %s", opt);
+			error = EINVAL;
+			goto out;
+		}
+		args.flags |= NFSMNT_WCOMMITSIZE;
+	}
 	if (vfs_getopt(mp->mnt_optnew, "deadthresh", (void **)&opt, NULL) == 0) {
 		ret = sscanf(opt, "%d", &args.deadthresh);
 		if (ret != 1 || args.deadthresh <= 0) {
diff --git a/sys/nlm/nlm_prot_impl.c b/sys/nlm/nlm_prot_impl.c
index 8a4b3c4..cf7e149 100644
--- a/sys/nlm/nlm_prot_impl.c
+++ b/sys/nlm/nlm_prot_impl.c
@@ -84,8 +84,9 @@ MALLOC_DEFINE(M_NLM, "NLM", "Network Lock Manager");
 /*
  * Support for sysctl vfs.nlm.sysid
  */
-SYSCTL_NODE(_vfs, OID_AUTO, nlm, CTLFLAG_RW, NULL, "Network Lock Manager");
-SYSCTL_NODE(_vfs_nlm, OID_AUTO, sysid, CTLFLAG_RW, NULL, "");
+static SYSCTL_NODE(_vfs, OID_AUTO, nlm, CTLFLAG_RW, NULL,
+    "Network Lock Manager");
+static SYSCTL_NODE(_vfs_nlm, OID_AUTO, sysid, CTLFLAG_RW, NULL, "");
 
 /*
  * Syscall hooks
diff --git a/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c b/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
index fe747af..28b44d9 100644
--- a/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
+++ b/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c
@@ -85,7 +85,7 @@ RW_SYSINIT(sdplockinit, &sdp_lock, "SDP lock");
 #define	SDP_LIST_RLOCK_ASSERT()	rw_assert(&sdp_lock, RW_RLOCKED)
 #define	SDP_LIST_LOCK_ASSERT()	rw_assert(&sdp_lock, RW_LOCKED)
 
-MALLOC_DEFINE(M_SDP, "sdp", "Socket Direct Protocol");
+static MALLOC_DEFINE(M_SDP, "sdp", "Socket Direct Protocol");
 
 static void sdp_stop_keepalive_timer(struct socket *so);
 
@@ -1878,7 +1878,7 @@ next:
 	return (error);
 }
 
-SYSCTL_NODE(_net_inet, -1,  sdp,    CTLFLAG_RW, 0,  "SDP");
+static SYSCTL_NODE(_net_inet, -1,  sdp,    CTLFLAG_RW, 0,  "SDP");
 
 SYSCTL_PROC(_net_inet_sdp, TCPCTL_PCBLIST, pcblist,
     CTLFLAG_RD | CTLTYPE_STRUCT, 0, 0, sdp_pcblist, "S,xtcpcb",
diff --git a/sys/ofed/include/linux/linux_idr.c b/sys/ofed/include/linux/linux_idr.c
index 5cfaff5..b6f5d01 100644
--- a/sys/ofed/include/linux/linux_idr.c
+++ b/sys/ofed/include/linux/linux_idr.c
@@ -49,7 +49,7 @@
  * however it should be fairly fast.  It is basically a radix tree with
  * a builtin bitmap for allocation.
  */
-MALLOC_DEFINE(M_IDR, "idr", "Linux IDR compat");
+static MALLOC_DEFINE(M_IDR, "idr", "Linux IDR compat");
 
 static inline int
 idr_max(struct idr *idr)
diff --git a/sys/ofed/include/linux/linux_radix.c b/sys/ofed/include/linux/linux_radix.c
index e642eae..4503f55 100644
--- a/sys/ofed/include/linux/linux_radix.c
+++ b/sys/ofed/include/linux/linux_radix.c
@@ -37,7 +37,7 @@
 #include <linux/radix-tree.h>
 #include <linux/err.h>
 
-MALLOC_DEFINE(M_RADIX, "radix", "Linux radix compat");
+static MALLOC_DEFINE(M_RADIX, "radix", "Linux radix compat");
 
 static inline int
 radix_max(struct radix_tree_root *root)
diff --git a/sys/pc98/conf/DEFAULTS b/sys/pc98/conf/DEFAULTS
index 6c8b561..342a604 100644
--- a/sys/pc98/conf/DEFAULTS
+++ b/sys/pc98/conf/DEFAULTS
@@ -29,3 +29,6 @@ options 	GEOM_PART_PC98
 device		atpic
 
 options 	NEW_PCIB
+
+# Allow mounting non-MPSAFE filesystems
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/pc98/conf/GENERIC b/sys/pc98/conf/GENERIC
index df52ac1..bc2dadb 100644
--- a/sys/pc98/conf/GENERIC
+++ b/sys/pc98/conf/GENERIC
@@ -66,8 +66,11 @@ options 	AUDIT			# Security event auditing
 options 	MAC			# TrustedBSD MAC Framework
 options 	INCLUDE_CONFIG_FILE     # Include this file in kernel
 
-# Debugging for use in -current
+# Debugging support.  Always need this:
 options 	KDB			# Enable kernel debugger support.
+# For minimum debugger support (stable branch) use:
+#options 	KDB_TRACE		# Print a stack trace for a panic.
+# For full debugger support use this instead:
 options 	DDB			# Support DDB.
 options 	GDB			# Support remote GDB.
 options 	DEADLKRES		# Enable the deadlock resolver
@@ -98,7 +101,7 @@ device		siis		# SiliconImage SiI3124/SiI3132/SiI3531 SATA
 # SCSI Controllers
 device		adv		# Advansys SCSI adapters
 device		ahc		# AHA2940 and onboard AIC7xxx devices
-device		amd		# AMD 53C974 (Tekram DC-390(T))
+device		esp		# AMD Am53C974 (Tekram DC-390(T))
 device		isp		# Qlogic family
 #device		ncr		# NCR/Symbios Logic
 device		sym		# NCR/Symbios Logic (newer chipsets + those of `ncr')
diff --git a/sys/pci/if_rl.c b/sys/pci/if_rl.c
index 87c2fc8..19b84d2 100644
--- a/sys/pci/if_rl.c
+++ b/sys/pci/if_rl.c
@@ -113,6 +113,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 
 #include <dev/mii/mii.h>
+#include <dev/mii/mii_bitbang.h>
 #include <dev/mii/miivar.h>
 
 #include <dev/pci/pcireg.h>
@@ -130,7 +131,7 @@ MODULE_DEPEND(rl, miibus, 1, 1, 1);
 /*
  * Various supported device vendors/types and their names.
  */
-static struct rl_type rl_devs[] = {
+static const struct rl_type const rl_devs[] = {
 	{ RT_VENDORID, RT_DEVICEID_8129, RL_8129,
 		"RealTek 8129 10/100BaseTX" },
 	{ RT_VENDORID, RT_DEVICEID_8139, RL_8139,
@@ -187,10 +188,6 @@ static int rl_ioctl(struct ifnet *, u_long, caddr_t);
 static void rl_intr(void *);
 static void rl_init(void *);
 static void rl_init_locked(struct rl_softc *sc);
-static void rl_mii_send(struct rl_softc *, uint32_t, int);
-static void rl_mii_sync(struct rl_softc *);
-static int rl_mii_readreg(struct rl_softc *, struct rl_mii_frame *);
-static int rl_mii_writereg(struct rl_softc *, struct rl_mii_frame *);
 static int rl_miibus_readreg(device_t, int, int);
 static void rl_miibus_statchg(device_t);
 static int rl_miibus_writereg(device_t, int, int, int);
@@ -215,6 +212,24 @@ static void rl_watchdog(struct rl_softc *);
 static void rl_setwol(struct rl_softc *);
 static void rl_clrwol(struct rl_softc *);
 
+/*
+ * MII bit-bang glue
+ */
+static uint32_t rl_mii_bitbang_read(device_t);
+static void rl_mii_bitbang_write(device_t, uint32_t);
+
+static const struct mii_bitbang_ops rl_mii_bitbang_ops = {
+	rl_mii_bitbang_read,
+	rl_mii_bitbang_write,
+	{
+		RL_MII_DATAOUT,	/* MII_BIT_MDO */
+		RL_MII_DATAIN,	/* MII_BIT_MDI */
+		RL_MII_CLK,	/* MII_BIT_MDC */
+		RL_MII_DIR,	/* MII_BIT_DIR_HOST_PHY */
+		0,		/* MII_BIT_DIR_PHY_HOST */
+	}
+};
+
 static device_method_t rl_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		rl_probe),
@@ -340,181 +355,43 @@ rl_read_eeprom(struct rl_softc *sc, uint8_t *dest, int off, int cnt, int swap)
 }
 
 /*
- * MII access routines are provided for the 8129, which
- * doesn't have a built-in PHY. For the 8139, we fake things
- * up by diverting rl_phy_readreg()/rl_phy_writereg() to the
- * direct access PHY registers.
- */
-#define MII_SET(x)					\
-	CSR_WRITE_1(sc, RL_MII,				\
-		CSR_READ_1(sc, RL_MII) | (x))
-
-#define MII_CLR(x)					\
-	CSR_WRITE_1(sc, RL_MII,				\
-		CSR_READ_1(sc, RL_MII) & ~(x))
-
-/*
- * Sync the PHYs by setting data bit and strobing the clock 32 times.
- */
-static void
-rl_mii_sync(struct rl_softc *sc)
-{
-	register int		i;
-
-	MII_SET(RL_MII_DIR|RL_MII_DATAOUT);
-
-	for (i = 0; i < 32; i++) {
-		MII_SET(RL_MII_CLK);
-		DELAY(1);
-		MII_CLR(RL_MII_CLK);
-		DELAY(1);
-	}
-}
-
-/*
- * Clock a series of bits through the MII.
+ * Read the MII serial port for the MII bit-bang module.
  */
-static void
-rl_mii_send(struct rl_softc *sc, uint32_t bits, int cnt)
+static uint32_t
+rl_mii_bitbang_read(device_t dev)
 {
-	int			i;
-
-	MII_CLR(RL_MII_CLK);
+	struct rl_softc *sc;
+	uint32_t val;
 
-	for (i = (0x1 << (cnt - 1)); i; i >>= 1) {
-		if (bits & i) {
-			MII_SET(RL_MII_DATAOUT);
-		} else {
-			MII_CLR(RL_MII_DATAOUT);
-		}
-		DELAY(1);
-		MII_CLR(RL_MII_CLK);
-		DELAY(1);
-		MII_SET(RL_MII_CLK);
-	}
-}
-
-/*
- * Read an PHY register through the MII.
- */
-static int
-rl_mii_readreg(struct rl_softc *sc, struct rl_mii_frame *frame)
-{
-	int			i, ack;
-
-	/* Set up frame for RX. */
-	frame->mii_stdelim = RL_MII_STARTDELIM;
-	frame->mii_opcode = RL_MII_READOP;
-	frame->mii_turnaround = 0;
-	frame->mii_data = 0;
-
-	CSR_WRITE_2(sc, RL_MII, 0);
-
-	/* Turn on data xmit. */
-	MII_SET(RL_MII_DIR);
-
-	rl_mii_sync(sc);
-
-	/* Send command/address info. */
-	rl_mii_send(sc, frame->mii_stdelim, 2);
-	rl_mii_send(sc, frame->mii_opcode, 2);
-	rl_mii_send(sc, frame->mii_phyaddr, 5);
-	rl_mii_send(sc, frame->mii_regaddr, 5);
-
-	/* Idle bit */
-	MII_CLR((RL_MII_CLK|RL_MII_DATAOUT));
-	DELAY(1);
-	MII_SET(RL_MII_CLK);
-	DELAY(1);
-
-	/* Turn off xmit. */
-	MII_CLR(RL_MII_DIR);
-
-	/* Check for ack */
-	MII_CLR(RL_MII_CLK);
-	DELAY(1);
-	ack = CSR_READ_2(sc, RL_MII) & RL_MII_DATAIN;
-	MII_SET(RL_MII_CLK);
-	DELAY(1);
-
-	/*
-	 * Now try reading data bits. If the ack failed, we still
-	 * need to clock through 16 cycles to keep the PHY(s) in sync.
-	 */
-	if (ack) {
-		for(i = 0; i < 16; i++) {
-			MII_CLR(RL_MII_CLK);
-			DELAY(1);
-			MII_SET(RL_MII_CLK);
-			DELAY(1);
-		}
-		goto fail;
-	}
-
-	for (i = 0x8000; i; i >>= 1) {
-		MII_CLR(RL_MII_CLK);
-		DELAY(1);
-		if (!ack) {
-			if (CSR_READ_2(sc, RL_MII) & RL_MII_DATAIN)
-				frame->mii_data |= i;
-			DELAY(1);
-		}
-		MII_SET(RL_MII_CLK);
-		DELAY(1);
-	}
+	sc = device_get_softc(dev);
 
-fail:
-	MII_CLR(RL_MII_CLK);
-	DELAY(1);
-	MII_SET(RL_MII_CLK);
-	DELAY(1);
+	val = CSR_READ_1(sc, RL_MII);
+	CSR_BARRIER(sc, RL_MII, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 
-	return (ack ? 1 : 0);
+	return (val);
 }
 
 /*
- * Write to a PHY register through the MII.
+ * Write the MII serial port for the MII bit-bang module.
  */
-static int
-rl_mii_writereg(struct rl_softc *sc, struct rl_mii_frame *frame)
+static void
+rl_mii_bitbang_write(device_t dev, uint32_t val)
 {
+	struct rl_softc *sc;
 
-	/* Set up frame for TX. */
-	frame->mii_stdelim = RL_MII_STARTDELIM;
-	frame->mii_opcode = RL_MII_WRITEOP;
-	frame->mii_turnaround = RL_MII_TURNAROUND;
-
-	/* Turn on data output. */
-	MII_SET(RL_MII_DIR);
-
-	rl_mii_sync(sc);
-
-	rl_mii_send(sc, frame->mii_stdelim, 2);
-	rl_mii_send(sc, frame->mii_opcode, 2);
-	rl_mii_send(sc, frame->mii_phyaddr, 5);
-	rl_mii_send(sc, frame->mii_regaddr, 5);
-	rl_mii_send(sc, frame->mii_turnaround, 2);
-	rl_mii_send(sc, frame->mii_data, 16);
-
-	/* Idle bit. */
-	MII_SET(RL_MII_CLK);
-	DELAY(1);
-	MII_CLR(RL_MII_CLK);
-	DELAY(1);
-
-	/* Turn off xmit. */
-	MII_CLR(RL_MII_DIR);
+	sc = device_get_softc(dev);
 
-	return (0);
+	CSR_WRITE_1(sc, RL_MII, val);
+	CSR_BARRIER(sc, RL_MII, 1,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 static int
 rl_miibus_readreg(device_t dev, int phy, int reg)
 {
 	struct rl_softc		*sc;
-	struct rl_mii_frame	frame;
-	uint16_t		rval = 0;
-	uint16_t		rl8139_reg = 0;
+	uint16_t		rl8139_reg;
 
 	sc = device_get_softc(dev);
 
@@ -545,30 +422,22 @@ rl_miibus_readreg(device_t dev, int phy, int reg)
 		 * us the results of parallel detection.
 		 */
 		case RL_MEDIASTAT:
-			rval = CSR_READ_1(sc, RL_MEDIASTAT);
-			return (rval);
+			return (CSR_READ_1(sc, RL_MEDIASTAT));
 		default:
 			device_printf(sc->rl_dev, "bad phy register\n");
 			return (0);
 		}
-		rval = CSR_READ_2(sc, rl8139_reg);
-		return (rval);
+		return (CSR_READ_2(sc, rl8139_reg));
 	}
 
-	bzero((char *)&frame, sizeof(frame));
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	rl_mii_readreg(sc, &frame);
-
-	return (frame.mii_data);
+	return (mii_bitbang_readreg(dev, &rl_mii_bitbang_ops, phy, reg));
 }
 
 static int
 rl_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	struct rl_softc		*sc;
-	struct rl_mii_frame	frame;
-	uint16_t		rl8139_reg = 0;
+	uint16_t		rl8139_reg;
 
 	sc = device_get_softc(dev);
 
@@ -601,11 +470,7 @@ rl_miibus_writereg(device_t dev, int phy, int reg, int data)
 		return (0);
 	}
 
-	bzero((char *)&frame, sizeof(frame));
-	frame.mii_phyaddr = phy;
-	frame.mii_regaddr = reg;
-	frame.mii_data = data;
-	rl_mii_writereg(sc, &frame);
+	mii_bitbang_writereg(dev, &rl_mii_bitbang_ops, phy, reg, data);
 
 	return (0);
 }
@@ -719,7 +584,7 @@ rl_reset(struct rl_softc *sc)
 static int
 rl_probe(device_t dev)
 {
-	struct rl_type		*t;
+	const struct rl_type	*t;
 	uint16_t		devid, revid, vendor;
 	int			i;
 	
@@ -773,7 +638,7 @@ rl_attach(device_t dev)
 	uint16_t		as[3];
 	struct ifnet		*ifp;
 	struct rl_softc		*sc;
-	struct rl_type		*t;
+	const struct rl_type	*t;
 	struct sysctl_ctx_list	*ctx;
 	struct sysctl_oid_list	*children;
 	int			error = 0, hwrev, i, phy, pmc, rid;
diff --git a/sys/pci/if_rlreg.h b/sys/pci/if_rlreg.h
index 3a9da2e..dfad20b 100644
--- a/sys/pci/if_rlreg.h
+++ b/sys/pci/if_rlreg.h
@@ -178,6 +178,10 @@
 #define	RL_HWREV_8168C_SPIN2	0x3C400000
 #define	RL_HWREV_8168CP		0x3C800000
 #define	RL_HWREV_8105E		0x40800000
+#define	RL_HWREV_8105E_SPIN1	0x40C00000
+#define	RL_HWREV_8402		0x44000000
+#define	RL_HWREV_8168F		0x48000000
+#define	RL_HWREV_8411		0x48800000
 #define	RL_HWREV_8139		0x60000000
 #define	RL_HWREV_8139A		0x70000000
 #define	RL_HWREV_8139AG		0x70800000
@@ -597,33 +601,16 @@ struct rl_type {
 	uint16_t		rl_vid;
 	uint16_t		rl_did;
 	int			rl_basetype;
-	char			*rl_name;
+	const char		*rl_name;
 };
 
 struct rl_hwrev {
 	uint32_t		rl_rev;
 	int			rl_type;
-	char			*rl_desc;
+	const char		*rl_desc;
 	int			rl_max_mtu;
 };
 
-struct rl_mii_frame {
-	uint8_t		mii_stdelim;
-	uint8_t		mii_opcode;
-	uint8_t		mii_phyaddr;
-	uint8_t		mii_regaddr;
-	uint8_t		mii_turnaround;
-	uint16_t	mii_data;
-};
-
-/*
- * MII constants
- */
-#define	RL_MII_STARTDELIM	0x01
-#define	RL_MII_READOP		0x02
-#define	RL_MII_WRITEOP		0x01
-#define	RL_MII_TURNAROUND	0x02
-
 #define	RL_8129			1
 #define	RL_8139			2
 #define	RL_8139CPLUS		3
@@ -880,9 +867,10 @@ struct rl_softc {
 	device_t		rl_miibus;
 	bus_dma_tag_t		rl_parent_tag;
 	uint8_t			rl_type;
-	struct rl_hwrev		*rl_hwrev;
+	const struct rl_hwrev	*rl_hwrev;
 	int			rl_eecmd_read;
 	int			rl_eewidth;
+	int			rl_expcap;
 	int			rl_txthresh;
 	struct rl_chain_data	rl_cdata;
 	struct rl_list_data	rl_ldata;
@@ -950,6 +938,9 @@ struct rl_softc {
 #define	CSR_READ_1(sc, reg)		\
 	bus_space_read_1(sc->rl_btag, sc->rl_bhandle, reg)
 
+#define	CSR_BARRIER(sc, reg, length, flags)				\
+	bus_space_barrier(sc->rl_btag, sc->rl_bhandle, reg, length, flags)
+
 #define	CSR_SETBIT_1(sc, offset, val)		\
 	CSR_WRITE_1(sc, offset, CSR_READ_1(sc, offset) | (val))
 
diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index c368c68..2049949 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -303,7 +303,12 @@ powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
 	 */
 	pc = __pcpu;
 	pcpu_init(pc, 0, sizeof(struct pcpu));
-	curthread_reg = pc->pc_curthread = &thread0;
+	pc->pc_curthread = &thread0;
+#ifdef __powerpc64__
+	__asm __volatile("mr 13,%0" :: "r"(pc->pc_curthread));
+#else
+	__asm __volatile("mr 2,%0" :: "r"(pc->pc_curthread));
+#endif
 	pc->pc_cpuid = 0;
 
 	__asm __volatile("mtsprg 0, %0" :: "r"(pc));
diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index 57c35bf..88ca0f7 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -1624,6 +1624,8 @@ moea_pinit(mmu_t mmu, pmap_t pmap)
 			hash &= 0xfffff & ~(VSID_NBPW - 1);
 			hash |= i;
 		}
+		KASSERT(!(moea_vsid_bitmap[n] & mask),
+		    ("Allocating in-use VSID group %#x\n", hash));
 		moea_vsid_bitmap[n] |= mask;
 		for (i = 0; i < 16; i++)
 			pmap->pm_sr[i] = VSID_MAKE(i, hash);
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index 68fe517..607256c 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -88,7 +88,12 @@ cpudep_ap_bootstrap(void)
 	msr = PSL_KERNSET & ~PSL_EE;
 	mtmsr(msr);
 
-	curthread_reg = pcpup->pc_curthread = pcpup->pc_idlethread;
+	pcpup->pc_curthread = pcpup->pc_idlethread;
+#ifdef __powerpc64__
+	__asm __volatile("mr 13,%0" :: "r"(pcpup->pc_curthread));
+#else
+	__asm __volatile("mr 2,%0" :: "r"(pcpup->pc_curthread));
+#endif
 	pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
 	sp = pcpup->pc_curpcb->pcb_sp;
 
diff --git a/sys/powerpc/aim/slb.c b/sys/powerpc/aim/slb.c
index df493b4..1a5ce65 100644
--- a/sys/powerpc/aim/slb.c
+++ b/sys/powerpc/aim/slb.c
@@ -40,7 +40,6 @@
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
-#include <vm/vm_phys.h>
 
 #include <machine/md_var.h>
 #include <machine/platform.h>
@@ -488,15 +487,22 @@ slb_uma_real_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 	static vm_offset_t realmax = 0;
 	void *va;
 	vm_page_t m;
+	int pflags;
 
 	if (realmax == 0)
 		realmax = platform_real_maxaddr();
 
 	*flags = UMA_SLAB_PRIV;
+	if ((wait & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT)
+		pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
+	else
+		pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
+	if (wait & M_ZERO)
+		pflags |= VM_ALLOC_ZERO;
 
 	for (;;) {
-		m = vm_phys_alloc_contig(1, 0, realmax, PAGE_SIZE,
-			    PAGE_SIZE);
+		m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax,
+		    PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT);
 		if (m == NULL) {
 			if (wait & M_NOWAIT)
 				return (NULL);
@@ -513,10 +519,6 @@ slb_uma_real_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 	if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
 		bzero(va, PAGE_SIZE);
 
-	/* vm_phys_alloc_contig does not track wiring */
-	atomic_add_int(&cnt.v_wire_count, 1);
-	m->wire_count = 1;
-
 	return (va);
 }
 
diff --git a/sys/powerpc/aim/trap_subr32.S b/sys/powerpc/aim/trap_subr32.S
index 172150e..cf8d03a 100644
--- a/sys/powerpc/aim/trap_subr32.S
+++ b/sys/powerpc/aim/trap_subr32.S
@@ -303,6 +303,7 @@ cpu_reset:
 
 	lis	%r3,1@l
 	bla	CNAME(cpudep_ap_early_bootstrap)
+	lis	%r3,1@l
 	bla	CNAME(pmap_cpu_bootstrap)
 	bla	CNAME(cpudep_ap_bootstrap)
 	mr	%r1,%r3
@@ -665,8 +666,8 @@ disitrap:
 	GET_CPUINFO(%r1)
 	lwz	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1) /* get DAR */
 	stw	%r30,(PC_DBSAVE  +CPUSAVE_AIM_DAR)(%r1) /* save DAR */
-	lwz	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1) /* get DSISR */
-	stw	%r30,(PC_DBSAVE  +CPUSAVE_AIM_DSISR)(%r1) /* save DSISR */
+	lwz	%r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1) /* get DSISR */
+	stw	%r31,(PC_DBSAVE  +CPUSAVE_AIM_DSISR)(%r1) /* save DSISR */
 	lwz	%r30,(PC_DISISAVE+CPUSAVE_R28)(%r1) /* get  r28 */
 	stw	%r30,(PC_DBSAVE  +CPUSAVE_R28)(%r1) /* save r28 */
 	lwz	%r31,(PC_DISISAVE+CPUSAVE_R29)(%r1) /* get  r29 */
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index 111d516..6f3a2de 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -301,6 +301,7 @@ cpu_reset:
 	lis	%r3,1@l
 	bl	CNAME(cpudep_ap_early_bootstrap) /* Set PCPU */
 	nop
+	lis	%r3,1@l
 	bl	CNAME(pmap_cpu_bootstrap)	/* Turn on virtual memory */
 	nop
 	bl	CNAME(cpudep_ap_bootstrap)	/* Set up PCPU and stack */
diff --git a/sys/powerpc/conf/DEFAULTS b/sys/powerpc/conf/DEFAULTS
index 658c221..54d1940 100644
--- a/sys/powerpc/conf/DEFAULTS
+++ b/sys/powerpc/conf/DEFAULTS
@@ -12,3 +12,6 @@ device		uart_z8530
 
 options 	GEOM_PART_APM
 options 	GEOM_PART_MBR
+
+# Allow mounting non-MPSAFE filesystems
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/powerpc/conf/GENERIC b/sys/powerpc/conf/GENERIC
index 87e8ae2..d0646c3 100644
--- a/sys/powerpc/conf/GENERIC
+++ b/sys/powerpc/conf/GENERIC
@@ -67,8 +67,11 @@ options 	AUDIT			# Security event auditing
 options 	MAC			# TrustedBSD MAC Framework
 options 	INCLUDE_CONFIG_FILE     # Include this file in kernel
 
-# Debugging for use in -current
-options 	KDB			#Enable the kernel debugger
+# Debugging support.  Always need this:
+options 	KDB			# Enable kernel debugger support.
+# For minimum debugger support (stable branch) use:
+#options 	KDB_TRACE		# Print a stack trace for a panic.
+# For full debugger support use this instead:
 options 	DDB			#Support DDB
 #options 	DEADLKRES		#Enable the deadlock resolver
 options 	INVARIANTS		#Enable calls of extra sanity checking
diff --git a/sys/powerpc/fpu/fpu_emu.c b/sys/powerpc/fpu/fpu_emu.c
index 9ac308b..66cc215 100644
--- a/sys/powerpc/fpu/fpu_emu.c
+++ b/sys/powerpc/fpu/fpu_emu.c
@@ -97,7 +97,7 @@ __FBSDID("$FreeBSD$");
 #include <powerpc/fpu/fpu_extern.h>
 #include <powerpc/fpu/fpu_instr.h>
 
-SYSCTL_NODE(_hw, OID_AUTO, fpu_emu, CTLFLAG_RW, 0, "FPU emulator");
+static SYSCTL_NODE(_hw, OID_AUTO, fpu_emu, CTLFLAG_RW, 0, "FPU emulator");
 
 #define	FPU_EMU_EVCNT_DECL(name)					\
 static u_int fpu_emu_evcnt_##name;					\
diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h
index 631abd2..2dac1b4 100644
--- a/sys/powerpc/include/pcpu.h
+++ b/sys/powerpc/include/pcpu.h
@@ -135,13 +135,20 @@ struct pmap;
 #ifdef _KERNEL
 
 #define pcpup	((struct pcpu *) powerpc_get_pcpup())
+
+#ifdef AIM /* Book-E not yet adapted */
+static __inline __pure2 struct thread *
+__curthread(void)
+{
+	struct thread *td;
 #ifdef __powerpc64__
-register struct thread *curthread_reg __asm("%r13");
+	__asm __volatile("mr %0,13" : "=r"(td));
 #else
-register struct thread *curthread_reg __asm("%r2");
+	__asm __volatile("mr %0,2" : "=r"(td));
 #endif
-#ifdef AIM /* Book-E not yet adapted */
-#define curthread	curthread_reg
+	return (td);
+}
+#define curthread (__curthread())
 #endif
 
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)
diff --git a/sys/powerpc/ofw/ofw_real.c b/sys/powerpc/ofw/ofw_real.c
index 1fc2ed1..d265943 100644
--- a/sys/powerpc/ofw/ofw_real.c
+++ b/sys/powerpc/ofw/ofw_real.c
@@ -158,7 +158,8 @@ static ofw_def_t ofw_32bit = {
 };
 OFW_DEF(ofw_32bit);
 
-MALLOC_DEFINE(M_OFWREAL, "ofwreal", "Open Firmware Real Mode Bounce Page");
+static MALLOC_DEFINE(M_OFWREAL, "ofwreal",
+    "Open Firmware Real Mode Bounce Page");
 
 static int (*openfirmware)(void *);
 
diff --git a/sys/powerpc/ofw/ofw_syscons.c b/sys/powerpc/ofw/ofw_syscons.c
index e638e4b..059871c 100644
--- a/sys/powerpc/ofw/ofw_syscons.c
+++ b/sys/powerpc/ofw/ofw_syscons.c
@@ -57,7 +57,7 @@ __FBSDID("$FreeBSD$");
 #include <powerpc/ofw/ofw_syscons.h>
 
 static int ofwfb_ignore_mmap_checks = 1;
-SYSCTL_NODE(_hw, OID_AUTO, ofwfb, CTLFLAG_RD, 0, "ofwfb");
+static SYSCTL_NODE(_hw, OID_AUTO, ofwfb, CTLFLAG_RD, 0, "ofwfb");
 SYSCTL_INT(_hw_ofwfb, OID_AUTO, relax_mmap, CTLFLAG_RW,
     &ofwfb_ignore_mmap_checks, 0, "relaxed mmap bounds checking");
 
diff --git a/sys/powerpc/ofw/rtas.c b/sys/powerpc/ofw/rtas.c
index 66ce12f..c6d8969 100644
--- a/sys/powerpc/ofw/rtas.c
+++ b/sys/powerpc/ofw/rtas.c
@@ -47,7 +47,7 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/ofw/openfirm.h>
 
-MALLOC_DEFINE(M_RTAS, "rtas", "Run Time Abstraction Service");
+static MALLOC_DEFINE(M_RTAS, "rtas", "Run Time Abstraction Service");
 
 static vm_offset_t	rtas_bounce_phys;
 static caddr_t		rtas_bounce_virt;
diff --git a/sys/powerpc/powermac/dbdma.c b/sys/powerpc/powermac/dbdma.c
index 1172606..6712c1a 100644
--- a/sys/powerpc/powermac/dbdma.c
+++ b/sys/powerpc/powermac/dbdma.c
@@ -40,7 +40,7 @@ __FBSDID("$FreeBSD$");
 
 #include "dbdmavar.h"
 
-MALLOC_DEFINE(M_DBDMA, "dbdma", "DBDMA Command List");
+static MALLOC_DEFINE(M_DBDMA, "dbdma", "DBDMA Command List");
 
 static uint32_t dbdma_read_reg(dbdma_channel_t *, u_int);
 static void dbdma_write_reg(dbdma_channel_t *, u_int, uint32_t);
diff --git a/sys/powerpc/powermac/fcu.c b/sys/powerpc/powermac/fcu.c
index cf5b466..109833d 100644
--- a/sys/powerpc/powermac/fcu.c
+++ b/sys/powerpc/powermac/fcu.c
@@ -131,7 +131,7 @@ static driver_t fcu_driver = {
 static devclass_t fcu_devclass;
 
 DRIVER_MODULE(fcu, iicbus, fcu_driver, fcu_devclass, 0, 0);
-MALLOC_DEFINE(M_FCU, "fcu", "FCU Sensor Information");
+static MALLOC_DEFINE(M_FCU, "fcu", "FCU Sensor Information");
 
 static int
 fcu_write(device_t dev, uint32_t addr, uint8_t reg, uint8_t *buff,
diff --git a/sys/powerpc/powermac/powermac_thermal.c b/sys/powerpc/powermac/powermac_thermal.c
index 9c1f59d..db0611a 100644
--- a/sys/powerpc/powermac/powermac_thermal.c
+++ b/sys/powerpc/powermac/powermac_thermal.c
@@ -58,7 +58,7 @@ SYSINIT(pmac_therm_setup, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, kproc_start,
     &pmac_therm_kp);
 SYSCTL_INT(_machdep, OID_AUTO, manage_fans, CTLFLAG_RW | CTLFLAG_TUN,
     &enable_pmac_thermal, 1, "Enable automatic fan management");
-MALLOC_DEFINE(M_PMACTHERM, "pmactherm", "Powermac Thermal Management");
+static MALLOC_DEFINE(M_PMACTHERM, "pmactherm", "Powermac Thermal Management");
 
 struct pmac_fan_le {
 	struct pmac_fan			*fan;
diff --git a/sys/powerpc/powermac/smu.c b/sys/powerpc/powermac/smu.c
index ede97a1..b1b29e1 100644
--- a/sys/powerpc/powermac/smu.c
+++ b/sys/powerpc/powermac/smu.c
@@ -197,7 +197,7 @@ static driver_t smu_driver = {
 static devclass_t smu_devclass;
 
 DRIVER_MODULE(smu, nexus, smu_driver, smu_devclass, 0, 0);
-MALLOC_DEFINE(M_SMU, "smu", "SMU Sensor Information");
+static MALLOC_DEFINE(M_SMU, "smu", "SMU Sensor Information");
 
 #define SMU_MAILBOX		0x8000860c
 #define SMU_FANMGT_INTERVAL	1000 /* ms */
diff --git a/sys/powerpc/powermac/smusat.c b/sys/powerpc/powermac/smusat.c
index fcaa9ed..974e98c 100644
--- a/sys/powerpc/powermac/smusat.c
+++ b/sys/powerpc/powermac/smusat.c
@@ -63,7 +63,7 @@ static int	smusat_attach(device_t);
 static int	smusat_sensor_sysctl(SYSCTL_HANDLER_ARGS);
 static int	smusat_sensor_read(struct smu_sensor *sens);
 
-MALLOC_DEFINE(M_SMUSAT, "smusat", "SMU Sattelite Sensors");
+static MALLOC_DEFINE(M_SMUSAT, "smusat", "SMU Sattelite Sensors");
 
 static device_method_t  smusat_methods[] = {
 	/* Device interface */
diff --git a/sys/powerpc/powerpc/busdma_machdep.c b/sys/powerpc/powerpc/busdma_machdep.c
index 84e3bc6..9ebc097 100644
--- a/sys/powerpc/powerpc/busdma_machdep.c
+++ b/sys/powerpc/powerpc/busdma_machdep.c
@@ -113,7 +113,7 @@ static int total_bpages;
 static int busdma_zonecount;
 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
 
-SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 	   "Total bounce pages");
 
diff --git a/sys/powerpc/powerpc/intr_machdep.c b/sys/powerpc/powerpc/intr_machdep.c
index 1e6342c..abfe905 100644
--- a/sys/powerpc/powerpc/intr_machdep.c
+++ b/sys/powerpc/powerpc/intr_machdep.c
@@ -89,7 +89,7 @@
 
 #define	MAX_STRAY_LOG	5
 
-MALLOC_DEFINE(M_INTR, "intr", "interrupt handler data");
+static MALLOC_DEFINE(M_INTR, "intr", "interrupt handler data");
 
 struct powerpc_intr {
 	struct intr_event *event;
diff --git a/sys/powerpc/powerpc/platform.c b/sys/powerpc/powerpc/platform.c
index 9eb8f30..2877134 100644
--- a/sys/powerpc/powerpc/platform.c
+++ b/sys/powerpc/powerpc/platform.c
@@ -184,7 +184,7 @@ platform_probe_and_attach()
 		 * then statically initialise the MMU object
 		 */
 		kobj_class_compile_static(platp, &plat_kernel_kops);
-		kobj_init((kobj_t)plat_obj, platp);
+		kobj_init_static((kobj_t)plat_obj, platp);
 
 		prio = PLATFORM_PROBE(plat_obj);
 
@@ -223,7 +223,7 @@ platform_probe_and_attach()
 	 */
 
 	kobj_class_compile_static(plat_def_impl, &plat_kernel_kops);
-	kobj_init((kobj_t)plat_obj, plat_def_impl);
+	kobj_init_static((kobj_t)plat_obj, plat_def_impl);
 
 	strlcpy(plat_name,plat_def_impl->name,sizeof(plat_name));
 
diff --git a/sys/powerpc/powerpc/pmap_dispatch.c b/sys/powerpc/powerpc/pmap_dispatch.c
index fe10429..4715a93 100644
--- a/sys/powerpc/powerpc/pmap_dispatch.c
+++ b/sys/powerpc/powerpc/pmap_dispatch.c
@@ -410,7 +410,7 @@ pmap_bootstrap(vm_offset_t start, vm_offset_t end)
 	 * then statically initialise the MMU object
 	 */
 	kobj_class_compile_static(mmu_def_impl, &mmu_kernel_kops);
-	kobj_init((kobj_t)mmu_obj, mmu_def_impl);
+	kobj_init_static((kobj_t)mmu_obj, mmu_def_impl);
 
 	MMU_BOOTSTRAP(mmu_obj, start, end);
 }
diff --git a/sys/powerpc/ps3/ps3disk.c b/sys/powerpc/ps3/ps3disk.c
index 3bab0f2..d4ac7fa 100644
--- a/sys/powerpc/ps3/ps3disk.c
+++ b/sys/powerpc/ps3/ps3disk.c
@@ -67,7 +67,8 @@ __FBSDID("$FreeBSD$");
 
 #define LV1_STORAGE_ATA_HDDOUT 		0x23
 
-SYSCTL_NODE(_hw, OID_AUTO, ps3disk, CTLFLAG_RD, 0, "PS3 Disk driver parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, ps3disk, CTLFLAG_RD, 0,
+    "PS3 Disk driver parameters");
 
 #ifdef PS3DISK_DEBUG
 static int ps3disk_debug = 0;
diff --git a/sys/powerpc/ps3/ps3pic.c b/sys/powerpc/ps3/ps3pic.c
index 3a62e3c..c9b5c0e 100644
--- a/sys/powerpc/ps3/ps3pic.c
+++ b/sys/powerpc/ps3/ps3pic.c
@@ -142,6 +142,8 @@ ps3pic_attach(device_t dev)
 	lv1_construct_event_receive_port(&sc->sc_ipi_outlet[1]);
 	lv1_connect_irq_plug_ext(ppe, !thread, sc->sc_ipi_outlet[0],
 	    sc->sc_ipi_outlet[1], 0);
+#else
+	sc->sc_ipi_outlet[0] = sc->sc_ipi_outlet[1] = 63;
 #endif
 
 	powerpc_register_pic(dev, 0, sc->sc_ipi_outlet[0], 1, FALSE);
diff --git a/sys/rpc/clnt_dg.c b/sys/rpc/clnt_dg.c
index 8a69bf4..c86b18a 100644
--- a/sys/rpc/clnt_dg.c
+++ b/sys/rpc/clnt_dg.c
@@ -1001,12 +1001,12 @@ clnt_dg_destroy(CLIENT *cl)
 	cs = cu->cu_socket->so_rcv.sb_upcallarg;
 	clnt_dg_close(cl);
 
+	SOCKBUF_LOCK(&cu->cu_socket->so_rcv);
 	mtx_lock(&cs->cs_lock);
 
 	cs->cs_refs--;
 	if (cs->cs_refs == 0) {
 		mtx_unlock(&cs->cs_lock);
-		SOCKBUF_LOCK(&cu->cu_socket->so_rcv);
 		soupcall_clear(cu->cu_socket, SO_RCV);
 		clnt_dg_upcallsdone(cu->cu_socket, cs);
 		SOCKBUF_UNLOCK(&cu->cu_socket->so_rcv);
@@ -1015,6 +1015,7 @@ clnt_dg_destroy(CLIENT *cl)
 		lastsocketref = TRUE;
 	} else {
 		mtx_unlock(&cs->cs_lock);
+		SOCKBUF_UNLOCK(&cu->cu_socket->so_rcv);
 		lastsocketref = FALSE;
 	}
 
diff --git a/sys/security/audit/audit.c b/sys/security/audit/audit.c
index 1799f16..74bfde7 100644
--- a/sys/security/audit/audit.c
+++ b/sys/security/audit/audit.c
@@ -81,7 +81,7 @@ MALLOC_DEFINE(M_AUDITPATH, "audit_path", "Audit path storage");
 MALLOC_DEFINE(M_AUDITTEXT, "audit_text", "Audit text storage");
 MALLOC_DEFINE(M_AUDITGIDSET, "audit_gidset", "Audit GID set storage");
 
-SYSCTL_NODE(_security, OID_AUTO, audit, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security, OID_AUTO, audit, CTLFLAG_RW, 0,
     "TrustedBSD audit controls");
 
 /*
diff --git a/sys/security/mac_biba/mac_biba.c b/sys/security/mac_biba/mac_biba.c
index b7ca4e0..3f24ff8 100644
--- a/sys/security/mac_biba/mac_biba.c
+++ b/sys/security/mac_biba/mac_biba.c
@@ -92,7 +92,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, biba, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, biba, CTLFLAG_RW, 0,
     "TrustedBSD mac_biba policy controls");
 
 static int	biba_label_size = sizeof(struct mac_biba);
diff --git a/sys/security/mac_bsdextended/mac_bsdextended.c b/sys/security/mac_bsdextended/mac_bsdextended.c
index c37d02a..ccbc525 100644
--- a/sys/security/mac_bsdextended/mac_bsdextended.c
+++ b/sys/security/mac_bsdextended/mac_bsdextended.c
@@ -72,7 +72,7 @@ static struct mtx ugidfw_mtx;
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, bsdextended, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, bsdextended, CTLFLAG_RW, 0,
     "TrustedBSD extended BSD MAC policy controls");
 
 static int	ugidfw_enabled = 1;
@@ -80,7 +80,8 @@ SYSCTL_INT(_security_mac_bsdextended, OID_AUTO, enabled, CTLFLAG_RW,
     &ugidfw_enabled, 0, "Enforce extended BSD policy");
 TUNABLE_INT("security.mac.bsdextended.enabled", &ugidfw_enabled);
 
-MALLOC_DEFINE(M_MACBSDEXTENDED, "mac_bsdextended", "BSD Extended MAC rule");
+static MALLOC_DEFINE(M_MACBSDEXTENDED, "mac_bsdextended",
+    "BSD Extended MAC rule");
 
 #define	MAC_BSDEXTENDED_MAXRULES	250
 static struct mac_bsdextended_rule *rules[MAC_BSDEXTENDED_MAXRULES];
@@ -202,7 +203,7 @@ out:
 	return (error);
 }
 
-SYSCTL_NODE(_security_mac_bsdextended, OID_AUTO, rules,
+static SYSCTL_NODE(_security_mac_bsdextended, OID_AUTO, rules,
     CTLFLAG_MPSAFE | CTLFLAG_RW, sysctl_rule, "BSD extended MAC rules");
 
 static void
diff --git a/sys/security/mac_ifoff/mac_ifoff.c b/sys/security/mac_ifoff/mac_ifoff.c
index e49e3ad..acfbd25 100644
--- a/sys/security/mac_ifoff/mac_ifoff.c
+++ b/sys/security/mac_ifoff/mac_ifoff.c
@@ -59,7 +59,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, ifoff, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, ifoff, CTLFLAG_RW, 0,
     "TrustedBSD mac_ifoff policy controls");
 
 static int	ifoff_enabled = 1;
diff --git a/sys/security/mac_lomac/mac_lomac.c b/sys/security/mac_lomac/mac_lomac.c
index 5cdfc67..fe71681 100644
--- a/sys/security/mac_lomac/mac_lomac.c
+++ b/sys/security/mac_lomac/mac_lomac.c
@@ -93,7 +93,7 @@ struct mac_lomac_proc {
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, lomac, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, lomac, CTLFLAG_RW, 0,
     "TrustedBSD mac_lomac policy controls");
 
 static int	lomac_label_size = sizeof(struct mac_lomac);
@@ -137,7 +137,7 @@ static int	lomac_slot;
     mac_label_get((l), lomac_slot))
 #define	PSLOT_SET(l, val) mac_label_set((l), lomac_slot, (uintptr_t)(val))
 
-MALLOC_DEFINE(M_LOMAC, "mac_lomac_label", "MAC/LOMAC labels");
+static MALLOC_DEFINE(M_LOMAC, "mac_lomac_label", "MAC/LOMAC labels");
 
 static struct mac_lomac *
 lomac_alloc(int flag)
diff --git a/sys/security/mac_mls/mac_mls.c b/sys/security/mac_mls/mac_mls.c
index 73c5df0..8019469 100644
--- a/sys/security/mac_mls/mac_mls.c
+++ b/sys/security/mac_mls/mac_mls.c
@@ -93,7 +93,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, mls, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, mls, CTLFLAG_RW, 0,
     "TrustedBSD mac_mls policy controls");
 
 static int	mls_label_size = sizeof(struct mac_mls);
diff --git a/sys/security/mac_partition/mac_partition.c b/sys/security/mac_partition/mac_partition.c
index 2fecb27..729413e 100644
--- a/sys/security/mac_partition/mac_partition.c
+++ b/sys/security/mac_partition/mac_partition.c
@@ -65,7 +65,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, partition, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, partition, CTLFLAG_RW, 0,
     "TrustedBSD mac_partition policy controls");
 
 static int	partition_enabled = 1;
diff --git a/sys/security/mac_portacl/mac_portacl.c b/sys/security/mac_portacl/mac_portacl.c
index dbadff8..1dbd199 100644
--- a/sys/security/mac_portacl/mac_portacl.c
+++ b/sys/security/mac_portacl/mac_portacl.c
@@ -83,7 +83,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, portacl, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, portacl, CTLFLAG_RW, 0,
     "TrustedBSD mac_portacl policy controls");
 
 static int	portacl_enabled = 1;
@@ -109,7 +109,7 @@ SYSCTL_INT(_security_mac_portacl, OID_AUTO, port_high, CTLFLAG_RW,
     &portacl_port_high, 0, "Highest port to enforce for");
 TUNABLE_INT("security.mac.portacl.port_high", &portacl_port_high);
 
-MALLOC_DEFINE(M_PORTACL, "portacl_rule", "Rules for mac_portacl");
+static MALLOC_DEFINE(M_PORTACL, "portacl_rule", "Rules for mac_portacl");
 
 #define	MAC_RULE_STRING_LEN	1024
 
diff --git a/sys/security/mac_seeotheruids/mac_seeotheruids.c b/sys/security/mac_seeotheruids/mac_seeotheruids.c
index 872647d..cef0d26 100644
--- a/sys/security/mac_seeotheruids/mac_seeotheruids.c
+++ b/sys/security/mac_seeotheruids/mac_seeotheruids.c
@@ -63,7 +63,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, seeotheruids, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, seeotheruids, CTLFLAG_RW, 0,
     "TrustedBSD mac_seeotheruids policy controls");
 
 static int	seeotheruids_enabled = 1;
diff --git a/sys/security/mac_stub/mac_stub.c b/sys/security/mac_stub/mac_stub.c
index f1d3e10..ed25d23 100644
--- a/sys/security/mac_stub/mac_stub.c
+++ b/sys/security/mac_stub/mac_stub.c
@@ -89,7 +89,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, stub, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, stub, CTLFLAG_RW, 0,
     "TrustedBSD mac_stub policy controls");
 
 static int	stub_enabled = 1;
diff --git a/sys/security/mac_test/mac_test.c b/sys/security/mac_test/mac_test.c
index c92c418..1781798 100644
--- a/sys/security/mac_test/mac_test.c
+++ b/sys/security/mac_test/mac_test.c
@@ -80,7 +80,7 @@
 
 SYSCTL_DECL(_security_mac);
 
-SYSCTL_NODE(_security_mac, OID_AUTO, test, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac, OID_AUTO, test, CTLFLAG_RW, 0,
     "TrustedBSD mac_test policy controls");
 
 #define	MAGIC_BPF	0xfe1ad1b6
@@ -112,7 +112,7 @@ static int	test_slot;
 SYSCTL_INT(_security_mac_test, OID_AUTO, slot, CTLFLAG_RD,
     &test_slot, 0, "Slot allocated by framework");
 
-SYSCTL_NODE(_security_mac_test, OID_AUTO, counter, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_security_mac_test, OID_AUTO, counter, CTLFLAG_RW, 0,
     "TrustedBSD mac_test counters controls");
 
 #define	COUNTER_DECL(variable)						\
diff --git a/sys/sparc64/conf/DEFAULTS b/sys/sparc64/conf/DEFAULTS
index c99480c..f1305a8 100644
--- a/sys/sparc64/conf/DEFAULTS
+++ b/sys/sparc64/conf/DEFAULTS
@@ -21,3 +21,6 @@ options 	GEOM_PART_VTOC8
 options 	SUNKBD_EMULATE_ATKBD
 
 options 	NEW_PCIB
+
+# Allow mounting non-MPSAFE filesystems
+options		VFS_ALLOW_NONMPSAFE
diff --git a/sys/sparc64/conf/GENERIC b/sys/sparc64/conf/GENERIC
index 20ecdf7..a686ca5 100644
--- a/sys/sparc64/conf/GENERIC
+++ b/sys/sparc64/conf/GENERIC
@@ -63,8 +63,11 @@ options 	AUDIT			# Security event auditing
 options 	MAC			# TrustedBSD MAC Framework
 options 	INCLUDE_CONFIG_FILE	# Include this file in kernel
 
-# Debugging for use in -current
+# Debugging support.  Always need this:
 options 	KDB			# Enable kernel debugger support.
+# For minimum debugger support (stable branch) use:
+#options 	KDB_TRACE		# Print a stack trace for a panic.
+# For full debugger support use this instead:
 options 	DDB			# Support DDB.
 options 	GDB			# Support remote GDB.
 options 	DEADLKRES		# Enable the deadlock resolver
@@ -100,11 +103,11 @@ device		ahc		# AHA2940 and onboard AIC7xxx devices
 options 	AHC_ALLOW_MEMIO	# Attempt to use memory mapped I/O
 options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
 					# output.  Adds ~128k to driver.
+device		esp		# AMD Am53C974, Sun ESP and FAS	families
 device		isp		# Qlogic family
 device		ispfw		# Firmware module for Qlogic host adapters
 device		mpt		# LSI-Logic MPT-Fusion
 device		sym		# NCR/Symbios/LSI Logic 53C8XX/53C1010/53C1510D
-device		esp		# NCR53c9x (FEPS/FAS366)
 
 # ATA/SCSI peripherals
 device		scbus		# SCSI bus (required for ATA/SCSI)
diff --git a/sys/sparc64/include/pcpu.h b/sys/sparc64/include/pcpu.h
index 24548d0..033f7e5 100644
--- a/sys/sparc64/include/pcpu.h
+++ b/sys/sparc64/include/pcpu.h
@@ -74,6 +74,16 @@ register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
 
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)
 
+static __inline __pure2 struct thread *
+__curthread(void)
+{
+	struct thread *td;
+
+	__asm("ldx [%" __XSTRING(PCPU_REG) "], %0" : "=r" (td));
+	return (td);
+}
+#define	curthread	(__curthread())
+
 /*
  * XXX The implementation of this operation should be made atomic
  * with respect to preemption.
diff --git a/sys/sparc64/sbus/dma_sbus.c b/sys/sparc64/sbus/dma_sbus.c
index 2594243..8eee8da 100644
--- a/sys/sparc64/sbus/dma_sbus.c
+++ b/sys/sparc64/sbus/dma_sbus.c
@@ -233,9 +233,9 @@ dma_attach(device_t dev)
 	    BUS_SPACE_MAXADDR,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
-	    BUS_SPACE_MAXSIZE_32BIT,	/* maxsize */
-	    0,				/* nsegments */
-	    BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
+	    BUS_SPACE_MAXSIZE,		/* maxsize */
+	    BUS_SPACE_UNRESTRICTED,	/* nsegments */
+	    BUS_SPACE_MAXSIZE,		/* maxsegsize */
 	    0,				/* flags */
 	    NULL, NULL,			/* no locking */
 	    &lsc->sc_parent_dmat);
diff --git a/sys/sparc64/sbus/lsi64854.c b/sys/sparc64/sbus/lsi64854.c
index ba9b5d8..0c0bacf 100644
--- a/sys/sparc64/sbus/lsi64854.c
+++ b/sys/sparc64/sbus/lsi64854.c
@@ -94,7 +94,12 @@ int lsi64854debug = 0;
 #define DPRINTF(a,x)
 #endif
 
-#define MAX_DMA_SZ	(16*1024*1024)
+/*
+ * The rules say we cannot transfer more than the limit of this DMA chip (64k
+ * for old and 16Mb for new), and we cannot cross a 16Mb boundary.
+ */
+#define MAX_DMA_SZ	(64 * 1024)
+#define BOUNDARY	(16 * 1024 * 1024)
 
 static void	lsi64854_reset(struct lsi64854_softc *);
 static void	lsi64854_map_scsi(void *, bus_dma_segment_t *, int, int);
@@ -125,6 +130,7 @@ lsi64854_attach(struct lsi64854_softc *sc)
 
 	lockfunc = NULL;
 	lockfuncarg = NULL;
+	sc->sc_maxdmasize = MAX_DMA_SZ;
 
 	switch (sc->sc_channel) {
 	case L64854_CHANNEL_SCSI:
@@ -135,6 +141,7 @@ lsi64854_attach(struct lsi64854_softc *sc)
 		}
 		lockfunc = busdma_lock_mutex;
 		lockfuncarg = &nsc->sc_lock;
+		sc->sc_maxdmasize = nsc->sc_maxxfer;
 		sc->intr = lsi64854_scsi_intr;
 		sc->setup = lsi64854_setup;
 		break;
@@ -153,13 +160,13 @@ lsi64854_attach(struct lsi64854_softc *sc)
 	if (sc->setup != NULL) {
 		error = bus_dma_tag_create(
 		    sc->sc_parent_dmat,		/* parent */
-		    1, 0,			/* alignment, boundary */
-		    BUS_SPACE_MAXADDR,		/* lowaddr */
+		    1, BOUNDARY,		/* alignment, boundary */
+		    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		    BUS_SPACE_MAXADDR,		/* highaddr */
 		    NULL, NULL,			/* filter, filterarg */
-		    MAX_DMA_SZ,			/* maxsize */
+		    sc->sc_maxdmasize,		/* maxsize */
 		    1,				/* nsegments */
-		    MAX_DMA_SZ,			/* maxsegsize */
+		    sc->sc_maxdmasize,		/* maxsegsize */
 		    BUS_DMA_ALLOCNOW,		/* flags */
 		    lockfunc, lockfuncarg,	/* lockfunc, lockfuncarg */
 		    &sc->sc_buffer_dmat);
@@ -250,24 +257,25 @@ lsi64854_detach(struct lsi64854_softc *sc)
 	 * other revs: D_ESC_R_PEND bit reads as 0			\
 	 */								\
 	DMAWAIT(sc, L64854_GCSR(sc) & D_ESC_R_PEND, "R_PEND", dontpanic);\
-	if (sc->sc_rev != DMAREV_HME) {                                 \
-	        /*							\
-	         * Select drain bit based on revision			\
-	         * also clears errors and D_TC flag			\
-	         */							\
-	        csr = L64854_GCSR(sc);					\
-	        if (sc->sc_rev == DMAREV_1 || sc->sc_rev == DMAREV_0)	\
-		        csr |= D_ESC_DRAIN;				\
-	        else							\
-		        csr |= L64854_INVALIDATE;			\
+	if (sc->sc_rev != DMAREV_HME) {					\
+		/*							\
+		 * Select drain bit based on revision			\
+		 * also clears errors and D_TC flag			\
+		 */							\
+		csr = L64854_GCSR(sc);					\
+		if (sc->sc_rev == DMAREV_1 || sc->sc_rev == DMAREV_0)	\
+			csr |= D_ESC_DRAIN;				\
+		else							\
+			csr |= L64854_INVALIDATE;			\
 									\
-	        L64854_SCSR(sc,csr);					\
+		L64854_SCSR(sc, csr);					\
 	}								\
 	/*								\
 	 * Wait for draining to finish					\
 	 * rev0 & rev1 call this PACKCNT				\
 	 */								\
-	DMAWAIT(sc, L64854_GCSR(sc) & L64854_DRAINING, "DRAINING", dontpanic);\
+	DMAWAIT(sc, L64854_GCSR(sc) & L64854_DRAINING, "DRAINING",	\
+	    dontpanic);							\
 } while (/* CONSTCOND */0)
 
 #define DMA_FLUSH(sc, dontpanic) do {					\
@@ -282,12 +290,14 @@ lsi64854_detach(struct lsi64854_softc *sc)
 	csr = L64854_GCSR(sc);					\
 	csr &= ~(L64854_WRITE|L64854_EN_DMA); /* no-ops on ENET */	\
 	csr |= L64854_INVALIDATE;	 	/* XXX FAS ? */		\
-	L64854_SCSR(sc,csr);						\
+	L64854_SCSR(sc, csr);						\
 } while (/* CONSTCOND */0)
 
 static void
 lsi64854_reset(struct lsi64854_softc *sc)
 {
+	bus_dma_tag_t dmat;
+	bus_dmamap_t dmam;
 	uint32_t csr;
 
 	DMA_FLUSH(sc, 1);
@@ -296,10 +306,11 @@ lsi64854_reset(struct lsi64854_softc *sc)
 	DPRINTF(LDB_ANY, ("%s: csr 0x%x\n", __func__, csr));
 
 	if (sc->sc_dmasize != 0) {
-		bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap,
-		    (csr & D_WRITE) != 0 ? BUS_DMASYNC_PREREAD :
-		    BUS_DMASYNC_PREWRITE);
-		bus_dmamap_unload(sc->sc_buffer_dmat, sc->sc_dmamap);
+		dmat = sc->sc_buffer_dmat;
+		dmam = sc->sc_dmamap;
+		bus_dmamap_sync(dmat, dmam, (csr & D_WRITE) != 0 ?
+		    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
+		bus_dmamap_unload(dmat, dmam);
 	}
 
 	if (sc->sc_rev == DMAREV_HME)
@@ -364,15 +375,16 @@ lsi64854_map_scsi(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 
 	sc = (struct lsi64854_softc *)arg;
 
+	if (error != 0)
+		return;
 	if (nseg != 1)
 		panic("%s: cannot map %d segments\n", __func__, nseg);
 
 	bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap,
-	    sc->sc_datain ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
+	    sc->sc_datain != 0 ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
 	bus_write_4(sc->sc_res, L64854_REG_ADDR, segs[0].ds_addr);
 }
 
-#define	DMAMAX(a)	(MAX_DMA_SZ - ((a) & (MAX_DMA_SZ - 1)))
 /*
  * setup a DMA transfer
  */
@@ -381,6 +393,7 @@ lsi64854_setup(struct lsi64854_softc *sc, void **addr, size_t *len,
     int datain, size_t *dmasize)
 {
 	long bcnt;
+	int error;
 	uint32_t csr;
 
 	DMA_FLUSH(sc, 0);
@@ -392,15 +405,12 @@ lsi64854_setup(struct lsi64854_softc *sc, void **addr, size_t *len,
 	sc->sc_dmalen = len;
 	sc->sc_datain = datain;
 
-	/*
-	 * The rules say we cannot transfer more than the limit
-	 * of this DMA chip (64k for old and 16Mb for new),
-	 * and we cannot cross a 16Mb boundary.
-	 */
-	*dmasize = sc->sc_dmasize =
-	    ulmin(*dmasize, DMAMAX((size_t)*sc->sc_dmaaddr));
+	KASSERT(*dmasize <= sc->sc_maxdmasize,
+	    ("%s: transfer size %ld too large", __func__, (long)*dmasize));
+
+	sc->sc_dmasize = *dmasize;
 
-	DPRINTF(LDB_ANY, ("%s: dmasize=%ld\n", __func__, (long)sc->sc_dmasize));
+	DPRINTF(LDB_ANY, ("%s: dmasize=%ld\n", __func__, (long)*dmasize));
 
 	/*
 	 * XXX what length?
@@ -412,24 +422,31 @@ lsi64854_setup(struct lsi64854_softc *sc, void **addr, size_t *len,
 		bus_write_4(sc->sc_res, L64854_REG_CNT, *dmasize);
 	}
 
-	/* Program the DMA address */
-	if (sc->sc_dmasize != 0)
-		if (bus_dmamap_load(sc->sc_buffer_dmat, sc->sc_dmamap,
-		    *sc->sc_dmaaddr, sc->sc_dmasize, lsi64854_map_scsi, sc, 0))
-			panic("%s: cannot allocate DVMA address", __func__);
+	/*
+	 * Load the transfer buffer and program the DMA address.
+	 * Note that the NCR53C9x core can't handle EINPROGRESS so we set
+	 * BUS_DMA_NOWAIT.
+	 */
+	if (*dmasize != 0) {
+		error = bus_dmamap_load(sc->sc_buffer_dmat, sc->sc_dmamap,
+		    *sc->sc_dmaaddr, *dmasize, lsi64854_map_scsi, sc,
+		    BUS_DMA_NOWAIT);
+		if (error != 0)
+			return (error);
+	}
 
 	if (sc->sc_rev == DMAREV_ESC) {
 		/* DMA ESC chip bug work-around */
-		bcnt = sc->sc_dmasize;
+		bcnt = *dmasize;
 		if (((bcnt + (long)*sc->sc_dmaaddr) & PAGE_MASK_8K) != 0)
 			bcnt = roundup(bcnt, PAGE_SIZE_8K);
 		bus_write_4(sc->sc_res, L64854_REG_CNT, bcnt);
 	}
 
-	/* Setup DMA control register */
+	/* Setup the DMA control register. */
 	csr = L64854_GCSR(sc);
 
-	if (datain)
+	if (datain != 0)
 		csr |= L64854_WRITE;
 	else
 		csr &= ~L64854_WRITE;
@@ -445,7 +462,7 @@ lsi64854_setup(struct lsi64854_softc *sc, void **addr, size_t *len,
 
 /*
  * Pseudo (chained) interrupt from the esp driver to kick the
- * current running DMA transfer. Called from ncr53c9x_intr()
+ * current running DMA transfer.  Called from ncr53c9x_intr()
  * for now.
  *
  * return 1 if it was a DMA continue.
@@ -455,19 +472,23 @@ lsi64854_scsi_intr(void *arg)
 {
 	struct lsi64854_softc *sc = arg;
 	struct ncr53c9x_softc *nsc = sc->sc_client;
-	int trans, resid;
+	bus_dma_tag_t dmat;
+	bus_dmamap_t dmam;
+	size_t dmasize;
+	int lxfer, resid, trans;
 	uint32_t csr;
 
 	csr = L64854_GCSR(sc);
 
 	DPRINTF(LDB_SCSI, ("%s: addr 0x%x, csr %b\n", __func__,
-	     bus_read_4(sc->sc_res, L64854_REG_ADDR), csr, DDMACSR_BITS));
+	    bus_read_4(sc->sc_res, L64854_REG_ADDR), csr, DDMACSR_BITS));
 
-	if (csr & (D_ERR_PEND|D_SLAVE_ERR)) {
-		device_printf(sc->sc_dev, "error: csr=%b\n", csr, DDMACSR_BITS);
-		csr &= ~D_EN_DMA;	/* Stop DMA */
+	if (csr & (D_ERR_PEND | D_SLAVE_ERR)) {
+		device_printf(sc->sc_dev, "error: csr=%b\n", csr,
+		    DDMACSR_BITS);
+		csr &= ~D_EN_DMA;	/* Stop DMA. */
 		/* Invalidate the queue; SLAVE_ERR bit is write-to-clear */
-		csr |= D_INVALIDATE|D_SLAVE_ERR;
+		csr |= D_INVALIDATE | D_SLAVE_ERR;
 		L64854_SCSR(sc, csr);
 		return (-1);
 	}
@@ -483,10 +504,11 @@ lsi64854_scsi_intr(void *arg)
 	L64854_SCSR(sc, csr);
 	sc->sc_active = 0;
 
-	if (sc->sc_dmasize == 0) {
-		/* A "Transfer Pad" operation completed */
-		DPRINTF(LDB_SCSI, ("%s: discarded %d bytes (tcl=%d, tcm=%d)\n",
-		    __func__, NCR_READ_REG(nsc, NCR_TCL) |
+	dmasize = sc->sc_dmasize;
+	if (dmasize == 0) {
+		/* A "Transfer Pad" operation completed. */
+		DPRINTF(LDB_SCSI, ("%s: discarded %d bytes (tcl=%d, "
+		    "tcm=%d)\n", __func__, NCR_READ_REG(nsc, NCR_TCL) |
 		    (NCR_READ_REG(nsc, NCR_TCM) << 8),
 		    NCR_READ_REG(nsc, NCR_TCL), NCR_READ_REG(nsc, NCR_TCM)));
 		return (0);
@@ -499,7 +521,7 @@ lsi64854_scsi_intr(void *arg)
 	 * as residual since the NCR53C9X counter registers get decremented
 	 * as bytes are clocked into the FIFO.
 	 */
-	if (!(csr & D_WRITE) &&
+	if ((csr & D_WRITE) == 0 &&
 	    (resid = (NCR_READ_REG(nsc, NCR_FFLAG) & NCRFIFO_FF)) != 0) {
 		DPRINTF(LDB_SCSI, ("%s: empty esp FIFO of %d ", __func__,
 		    resid));
@@ -509,22 +531,21 @@ lsi64854_scsi_intr(void *arg)
 	}
 
 	if ((nsc->sc_espstat & NCRSTAT_TC) == 0) {
+		lxfer = nsc->sc_features & NCR_F_LARGEXFER;
 		/*
-		 * `Terminal count' is off, so read the residue
+		 * "Terminal count" is off, so read the residue
 		 * out of the NCR53C9X counter registers.
 		 */
 		resid += (NCR_READ_REG(nsc, NCR_TCL) |
 		    (NCR_READ_REG(nsc, NCR_TCM) << 8) |
-		    ((nsc->sc_cfg2 & NCRCFG2_FE) ?
-		    (NCR_READ_REG(nsc, NCR_TCH) << 16) : 0));
+		    (lxfer != 0 ? (NCR_READ_REG(nsc, NCR_TCH) << 16) : 0));
 
-		if (resid == 0 && sc->sc_dmasize == 65536 &&
-		    (nsc->sc_cfg2 & NCRCFG2_FE) == 0)
-			/* A transfer of 64K is encoded as `TCL=TCM=0' */
+		if (resid == 0 && dmasize == 65536 && lxfer == 0)
+			/* A transfer of 64k is encoded as TCL=TCM=0. */
 			resid = 65536;
 	}
 
-	trans = sc->sc_dmasize - resid;
+	trans = dmasize - resid;
 	if (trans < 0) {			/* transferred < 0? */
 #if 0
 		/*
@@ -533,21 +554,22 @@ lsi64854_scsi_intr(void *arg)
 		 * another target.  As such, don't print the warning.
 		 */
 		device_printf(sc->sc_dev, "xfer (%d) > req (%d)\n", trans,
-		    sc->sc_dmasize);
+		    dmasize);
 #endif
-		trans = sc->sc_dmasize;
+		trans = dmasize;
 	}
 
 	DPRINTF(LDB_SCSI, ("%s: tcl=%d, tcm=%d, tch=%d; trans=%d, resid=%d\n",
 	    __func__, NCR_READ_REG(nsc, NCR_TCL), NCR_READ_REG(nsc, NCR_TCM),
-	    (nsc->sc_cfg2 & NCRCFG2_FE) ? NCR_READ_REG(nsc, NCR_TCH) : 0,
-	    trans, resid));
-
-	if (sc->sc_dmasize != 0) {
-		bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap,
-		    (csr & D_WRITE) != 0 ? BUS_DMASYNC_POSTREAD :
-		    BUS_DMASYNC_POSTWRITE);
-		bus_dmamap_unload(sc->sc_buffer_dmat, sc->sc_dmamap);
+	    (nsc->sc_features & NCR_F_LARGEXFER) != 0 ?
+	    NCR_READ_REG(nsc, NCR_TCH) : 0, trans, resid));
+
+	if (dmasize != 0) {
+		dmat = sc->sc_buffer_dmat;
+		dmam = sc->sc_dmamap;
+		bus_dmamap_sync(dmat, dmam, (csr & D_WRITE) != 0 ?
+		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(dmat, dmam);
 	}
 
 	*sc->sc_dmalen -= trans;
@@ -565,7 +587,7 @@ lsi64854_scsi_intr(void *arg)
 }
 
 /*
- * Pseudo (chained) interrupt to le driver to handle DMA errors.
+ * Pseudo (chained) interrupt to le(4) driver to handle DMA errors
  */
 static int
 lsi64854_enet_intr(void *arg)
@@ -579,11 +601,12 @@ lsi64854_enet_intr(void *arg)
 	/* If the DMA logic shows an interrupt, claim it */
 	rv = ((csr & E_INT_PEND) != 0) ? 1 : 0;
 
-	if (csr & (E_ERR_PEND|E_SLAVE_ERR)) {
-		device_printf(sc->sc_dev, "error: csr=%b\n", csr, EDMACSR_BITS);
-		csr &= ~L64854_EN_DMA;	/* Stop DMA */
+	if (csr & (E_ERR_PEND | E_SLAVE_ERR)) {
+		device_printf(sc->sc_dev, "error: csr=%b\n", csr,
+		    EDMACSR_BITS);
+		csr &= ~L64854_EN_DMA;	/* Stop DMA. */
 		/* Invalidate the queue; SLAVE_ERR bit is write-to-clear */
-		csr |= E_INVALIDATE|E_SLAVE_ERR;
+		csr |= E_INVALIDATE | E_SLAVE_ERR;
 		L64854_SCSR(sc, csr);
 		/* Will be drained with the LE_C0_IDON interrupt. */
 		sc->sc_dodrain = 1;
@@ -610,23 +633,26 @@ lsi64854_map_pp(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 
 	sc = (struct lsi64854_softc *)arg;
 
+	if (error != 0)
+		return;
 	if (nsegs != 1)
 		panic("%s: cannot map %d segments\n", __func__, nsegs);
 
-	bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap, sc->sc_datain ?
-	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
+	bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap,
+	    sc->sc_datain != 0 ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
 	bus_write_4(sc->sc_res, L64854_REG_ADDR, segs[0].ds_addr);
 
 	bus_write_4(sc->sc_res, L64854_REG_CNT, sc->sc_dmasize);
 }
 
 /*
- * setup a DMA transfer
+ * Setup a DMA transfer.
  */
 static int
 lsi64854_setup_pp(struct lsi64854_softc *sc, void **addr, size_t *len,
     int datain, size_t *dmasize)
 {
+	int error;
 	uint32_t csr;
 
 	DMA_FLUSH(sc, 0);
@@ -636,25 +662,25 @@ lsi64854_setup_pp(struct lsi64854_softc *sc, void **addr, size_t *len,
 	sc->sc_datain = datain;
 
 	DPRINTF(LDB_PP, ("%s: pp start %ld@%p,%d\n", __func__,
-	    (long)*sc->sc_dmalen, *sc->sc_dmaaddr, datain ? 1 : 0));
+	    (long)*sc->sc_dmalen, *sc->sc_dmaaddr, datain != 0 ? 1 : 0));
 
-	/*
-	 * the rules say we cannot transfer more than the limit
-	 * of this DMA chip (64k for old and 16Mb for new),
-	 * and we cannot cross a 16Mb boundary.
-	 */
-	*dmasize = sc->sc_dmasize =
-	    ulmin(*dmasize, DMAMAX((size_t)*sc->sc_dmaaddr));
+	KASSERT(*dmasize <= sc->sc_maxdmasize,
+	    ("%s: transfer size %ld too large", __func__, (long)*dmasize));
+
+	sc->sc_dmasize = *dmasize;
 
-	DPRINTF(LDB_PP, ("%s: dmasize=%ld\n", __func__, (long)sc->sc_dmasize));
+	DPRINTF(LDB_PP, ("%s: dmasize=%ld\n", __func__, (long)*dmasize));
 
-	/* Program the DMA address */
-	if (sc->sc_dmasize != 0)
-		if (bus_dmamap_load(sc->sc_buffer_dmat, sc->sc_dmamap,
-		    *sc->sc_dmaaddr, sc->sc_dmasize, lsi64854_map_pp, sc, 0))
-			panic("%s: pp cannot allocate DVMA address", __func__);
+	/* Load the transfer buffer and program the DMA address. */
+	if (*dmasize != 0) {
+		error = bus_dmamap_load(sc->sc_buffer_dmat, sc->sc_dmamap,
+		    *sc->sc_dmaaddr, *dmasize, lsi64854_map_pp, sc,
+		    BUS_DMA_NOWAIT);
+		if (error != 0)
+			return (error);
+	}
 
-	/* Setup DMA control register */
+	/* Setup the DMA control register. */
 	csr = L64854_GCSR(sc);
 	csr &= ~L64854_BURST_SIZE;
 	if (sc->sc_burst == 32)
@@ -663,10 +689,10 @@ lsi64854_setup_pp(struct lsi64854_softc *sc, void **addr, size_t *len,
 		csr |= L64854_BURST_16;
 	else
 		csr |= L64854_BURST_0;
-	csr |= P_EN_DMA|P_INT_EN|P_EN_CNT;
+	csr |= P_EN_DMA | P_INT_EN | P_EN_CNT;
 #if 0
-	/* This bit is read-only in PP csr register */
-	if (datain)
+	/* This bit is read-only in PP csr register. */
+	if (datain != 0)
 		csr |= P_WRITE;
 	else
 		csr &= ~P_WRITE;
@@ -677,12 +703,15 @@ lsi64854_setup_pp(struct lsi64854_softc *sc, void **addr, size_t *len,
 }
 
 /*
- * Parallel port DMA interrupt.
+ * Parallel port DMA interrupt
  */
 static int
 lsi64854_pp_intr(void *arg)
 {
 	struct lsi64854_softc *sc = arg;
+	bus_dma_tag_t dmat;
+	bus_dmamap_t dmam;
+	size_t dmasize;
 	int ret, trans, resid = 0;
 	uint32_t csr;
 
@@ -691,13 +720,13 @@ lsi64854_pp_intr(void *arg)
 	DPRINTF(LDB_PP, ("%s: addr 0x%x, csr %b\n", __func__,
 	    bus_read_4(sc->sc_res, L64854_REG_ADDR), csr, PDMACSR_BITS));
 
-	if (csr & (P_ERR_PEND|P_SLAVE_ERR)) {
+	if ((csr & (P_ERR_PEND | P_SLAVE_ERR)) != 0) {
 		resid = bus_read_4(sc->sc_res, L64854_REG_CNT);
 		device_printf(sc->sc_dev, "error: resid %d csr=%b\n", resid,
 		    csr, PDMACSR_BITS);
-		csr &= ~P_EN_DMA;	/* Stop DMA */
+		csr &= ~P_EN_DMA;	/* Stop DMA. */
 		/* Invalidate the queue; SLAVE_ERR bit is write-to-clear */
-		csr |= P_INVALIDATE|P_SLAVE_ERR;
+		csr |= P_INVALIDATE | P_SLAVE_ERR;
 		L64854_SCSR(sc, csr);
 		return (-1);
 	}
@@ -714,17 +743,19 @@ lsi64854_pp_intr(void *arg)
 	L64854_SCSR(sc, csr);
 	sc->sc_active = 0;
 
-	trans = sc->sc_dmasize - resid;
+	dmasize = sc->sc_dmasize;
+	trans = dmasize - resid;
 	if (trans < 0)				/* transferred < 0? */
-		trans = sc->sc_dmasize;
+		trans = dmasize;
 	*sc->sc_dmalen -= trans;
 	*sc->sc_dmaaddr = (char *)*sc->sc_dmaaddr + trans;
 
-	if (sc->sc_dmasize != 0) {
-		bus_dmamap_sync(sc->sc_buffer_dmat, sc->sc_dmamap,
-		    (csr & D_WRITE) != 0 ? BUS_DMASYNC_POSTREAD :
-		    BUS_DMASYNC_POSTWRITE);
-		bus_dmamap_unload(sc->sc_buffer_dmat, sc->sc_dmamap);
+	if (dmasize != 0) {
+		dmat = sc->sc_buffer_dmat;
+		dmam = sc->sc_dmamap;
+		bus_dmamap_sync(dmat, dmam, (csr & D_WRITE) != 0 ?
+		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(dmat, dmam);
 	}
 
 	return (ret != 0);
diff --git a/sys/sparc64/sbus/lsi64854var.h b/sys/sparc64/sbus/lsi64854var.h
index c1a8e99..e6c9edb 100644
--- a/sys/sparc64/sbus/lsi64854var.h
+++ b/sys/sparc64/sbus/lsi64854var.h
@@ -49,6 +49,7 @@ struct lsi64854_softc {
 
 	bus_dma_tag_t		sc_parent_dmat;
 	bus_dma_tag_t		sc_buffer_dmat;
+	bus_size_t		sc_maxdmasize;
 	int			sc_datain;
 	size_t			sc_dmasize;
 	void			**sc_dmaaddr;
diff --git a/sys/sparc64/sparc64/identcpu.c b/sys/sparc64/sparc64/identcpu.c
index 455aa67..592bb34 100644
--- a/sys/sparc64/sparc64/identcpu.c
+++ b/sys/sparc64/sparc64/identcpu.c
@@ -26,7 +26,7 @@ static char cpu_model[128];
 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
     cpu_model, 0, "Machine model");
 
-SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, "");
+static SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, "");
 
 static u_int cpu_count;
 static u_int cpu_freq;
diff --git a/sys/sparc64/sparc64/iommu.c b/sys/sparc64/sparc64/iommu.c
index d151ce7..ace74ec 100644
--- a/sys/sparc64/sparc64/iommu.c
+++ b/sys/sparc64/sparc64/iommu.c
@@ -118,7 +118,7 @@ __FBSDID("$FreeBSD$");
 /* Threshold for using the streaming buffer */
 #define	IOMMU_STREAM_THRESH	128
 
-MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers");
+static MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers");
 
 static	int iommu_strbuf_flush_sync(struct iommu_state *);
 #ifdef IOMMU_DIAG
diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c
index 0c0decc..4950cca 100644
--- a/sys/sparc64/sparc64/tick.c
+++ b/sys/sparc64/sparc64/tick.c
@@ -56,7 +56,7 @@ __FBSDID("$FreeBSD$");
 #define	TICK_QUALITY_MP	10
 #define	TICK_QUALITY_UP	1000
 
-SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
+static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
 
 static int adjust_edges = 0;
 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges,
diff --git a/sys/sys/_ffcounter.h b/sys/sys/_ffcounter.h
new file mode 100644
index 0000000..0d5864a
--- /dev/null
+++ b/sys/sys/_ffcounter.h
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2011 The University of Melbourne
+ * All rights reserved.
+ *
+ * This software was developed by Julien Ridoux at the University of Melbourne
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__FFCOUNTER_H_
+#define _SYS__FFCOUNTER_H_
+
+/*
+ * The feed-forward clock counter. The fundamental element of a feed-forward
+ * clock is a wide monotonically increasing counter that accumulates at the same
+ * rate as the selected timecounter.
+ */
+typedef uint64_t ffcounter;
+
+#endif /* _SYS__FFCOUNTER_H_ */
diff --git a/sys/sys/bus.h b/sys/sys/bus.h
index 5dc92df..25dcbce 100644
--- a/sys/sys/bus.h
+++ b/sys/sys/bus.h
@@ -424,6 +424,7 @@ device_t	device_add_child_ordered(device_t dev, u_int order,
 					 const char *name, int unit);
 void	device_busy(device_t dev);
 int	device_delete_child(device_t dev, device_t child);
+int	device_delete_all_children(device_t dev);
 int	device_attach(device_t dev);
 int	device_detach(device_t dev);
 void	device_disable(device_t dev);
diff --git a/sys/sys/cdefs.h b/sys/sys/cdefs.h
index dbab329..495daf2 100644
--- a/sys/sys/cdefs.h
+++ b/sys/sys/cdefs.h
@@ -253,6 +253,17 @@
 #define	__LONG_LONG_SUPPORTED
 #endif
 
+/* C++11 exposes a load of C99 stuff */
+#if defined(__cplusplus) && __cplusplus >= 201103L
+#define	__LONG_LONG_SUPPORTED
+#ifndef	__STDC_LIMIT_MACROS
+#define	__STDC_LIMIT_MACROS
+#endif
+#ifndef	__STDC_CONSTANT_MACROS
+#define	__STDC_CONSTANT_MACROS
+#endif
+#endif
+
 /*
  * GCC 2.95 provides `__restrict' as an extension to C90 to support the
  * C99-specific `restrict' type qualifier.  We happen to use `__restrict' as
@@ -349,7 +360,8 @@
 #endif
 
 /* Compiler-dependent macros that rely on FreeBSD-specific extensions. */
-#if __FreeBSD_cc_version >= 300001 && defined(__GNUC__) && !defined(__INTEL_COMPILER)
+#if defined(__FreeBSD_cc_version) && __FreeBSD_cc_version >= 300001 && \
+    defined(__GNUC__) && !defined(__INTEL_COMPILER)
 #define	__printf0like(fmtarg, firstvararg) \
 	    __attribute__((__format__ (__printf0__, fmtarg, firstvararg)))
 #else
diff --git a/sys/sys/event.h b/sys/sys/event.h
index d92eb55..08072a7 100644
--- a/sys/sys/event.h
+++ b/sys/sys/event.h
@@ -147,10 +147,6 @@ struct knlist {
 
 #ifdef _KERNEL
 
-#ifdef MALLOC_DECLARE
-MALLOC_DECLARE(M_KQUEUE);
-#endif
-
 /*
  * Flags for knote call
  */
diff --git a/sys/sys/fcntl.h b/sys/sys/fcntl.h
index 6f48ee7..29b2a0c 100644
--- a/sys/sys/fcntl.h
+++ b/sys/sys/fcntl.h
@@ -277,24 +277,33 @@ struct oflock {
 #define	LOCK_UN		0x08		/* unlock file */
 #endif
 
+#if __POSIX_VISIBLE >= 200112
 /*
- * XXX missing posix_fadvise() and POSIX_FADV_* macros.
+ * Advice to posix_fadvise
  */
+#define	POSIX_FADV_NORMAL	0	/* no special treatment */
+#define	POSIX_FADV_RANDOM	1	/* expect random page references */
+#define	POSIX_FADV_SEQUENTIAL	2	/* expect sequential page references */
+#define	POSIX_FADV_WILLNEED	3	/* will need these pages */
+#define	POSIX_FADV_DONTNEED	4	/* dont need these pages */
+#define	POSIX_FADV_NOREUSE	5	/* access data only once */
+#endif
 
 #ifndef _KERNEL
 __BEGIN_DECLS
 int	open(const char *, int, ...);
 int	creat(const char *, mode_t);
 int	fcntl(int, int, ...);
+#if __BSD_VISIBLE
+int	flock(int, int);
+#endif
 #if __BSD_VISIBLE || __POSIX_VISIBLE >= 200809
 int	openat(int, const char *, int, ...);
 #endif
 #if __BSD_VISIBLE || __POSIX_VISIBLE >= 200112
+int	posix_fadvise(int, off_t, off_t, int);
 int	posix_fallocate(int, off_t, off_t);
 #endif
-#if __BSD_VISIBLE
-int	flock(int, int);
-#endif
 __END_DECLS
 #endif
 
diff --git a/sys/sys/file.h b/sys/sys/file.h
index 57e7047..1489ca0 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -122,6 +122,12 @@ struct fileops {
  * none	not locked
  */
 
+struct fadvise_info {
+	int		fa_advice;	/* (f) FADV_* type. */
+	off_t		fa_start;	/* (f) Region start. */
+	off_t		fa_end;		/* (f) Region end. */
+};
+
 struct file {
 	void		*f_data;	/* file descriptor specific data */
 	struct fileops	*f_ops;		/* File operations */
@@ -136,7 +142,11 @@ struct file {
 	 */
 	int		f_seqcount;	/* Count of sequential accesses. */
 	off_t		f_nextoff;	/* next expected read/write offset. */
-	struct cdev_privdata *f_cdevpriv; /* (d) Private data for the cdev. */
+	union {
+		struct cdev_privdata *fvn_cdevpriv;
+					/* (d) Private data for the cdev. */
+		struct fadvise_info *fvn_advice;
+	} f_vnun;
 	/*
 	 *  DFLAG_SEEKABLE specific fields
 	 */
@@ -147,6 +157,9 @@ struct file {
 	void		*f_label;	/* Place-holder for MAC label. */
 };
 
+#define	f_cdevpriv	f_vnun.fvn_cdevpriv
+#define	f_advice	f_vnun.fvn_advice
+
 #define	FOFFSET_LOCKED       0x1
 #define	FOFFSET_LOCK_WAITING 0x2		 
 
@@ -172,10 +185,6 @@ struct xfile {
 
 #ifdef _KERNEL
 
-#ifdef MALLOC_DECLARE
-MALLOC_DECLARE(M_FILE);
-#endif
-
 extern struct fileops vnops;
 extern struct fileops badfileops;
 extern struct fileops socketops;
diff --git a/sys/sys/iconv.h b/sys/sys/iconv.h
index 7ca84b5..671b24e 100644
--- a/sys/sys/iconv.h
+++ b/sys/sys/iconv.h
@@ -47,6 +47,7 @@
 #define	KICONV_FROM_UPPER	8	/* toupper source character, then convert */
 #define	KICONV_WCTYPE		16	/* towlower/towupper characters */
 
+#define	ENCODING_UNICODE	"UTF-16BE"
 #define	KICONV_WCTYPE_NAME	"_wctype"
 
 /*
@@ -85,7 +86,6 @@ struct iconv_add_out {
 
 __BEGIN_DECLS
 
-#define	ENCODING_UNICODE	"UTF-16BE"
 #define	KICONV_VENDOR_MICSFT	1	/* Microsoft Vendor Code for quirk */
 
 int   kiconv_add_xlat_table(const char *, const char *, const u_char *);
@@ -162,6 +162,7 @@ int iconv_convchr(void *handle, const char **inbuf,
 	size_t *inbytesleft, char **outbuf, size_t *outbytesleft);
 int iconv_convchr_case(void *handle, const char **inbuf,
 	size_t *inbytesleft, char **outbuf, size_t *outbytesleft, int casetype);
+int iconv_add(const char *converter, const char *to, const char *from);
 char* iconv_convstr(void *handle, char *dst, const char *src);
 void* iconv_convmem(void *handle, void *dst, const void *src, int size);
 int iconv_vfs_refcount(const char *fsname);
diff --git a/sys/sys/kobj.h b/sys/sys/kobj.h
index 3143bea..0cb25fb 100644
--- a/sys/sys/kobj.h
+++ b/sys/sys/kobj.h
@@ -34,7 +34,7 @@
  */
 typedef struct kobj		*kobj_t;
 typedef struct kobj_class	*kobj_class_t;
-typedef struct kobj_method	kobj_method_t;
+typedef const struct kobj_method kobj_method_t;
 typedef int			(*kobjop_t)(void);
 typedef struct kobj_ops		*kobj_ops_t;
 typedef struct kobjop_desc	*kobjop_desc_t;
@@ -86,7 +86,7 @@ struct kobj_ops {
 
 struct kobjop_desc {
 	unsigned int	id;		/* unique ID */
-	kobj_method_t	*deflt;		/* default implementation */
+	kobj_method_t	deflt;		/* default implementation */
 };
 
 /*
@@ -201,6 +201,7 @@ kobj_t		kobj_create(kobj_class_t cls,
  * Initialise a pre-allocated object.
  */
 void		kobj_init(kobj_t obj, kobj_class_t cls);
+void		kobj_init_static(kobj_t obj, kobj_class_t cls);
 
 /*
  * Delete an object. If mtype is non-zero, free the memory.
diff --git a/sys/sys/lock.h b/sys/sys/lock.h
index 7ad9c58..303717f 100644
--- a/sys/sys/lock.h
+++ b/sys/sys/lock.h
@@ -58,10 +58,10 @@ struct thread;
 struct lock_class {
 	const	char *lc_name;
 	u_int	lc_flags;
-	void	(*lc_assert)(struct lock_object *lock, int what);
-	void	(*lc_ddb_show)(struct lock_object *lock);
+	void	(*lc_assert)(const struct lock_object *lock, int what);
+	void	(*lc_ddb_show)(const struct lock_object *lock);
 	void	(*lc_lock)(struct lock_object *lock, int how);
-	int	(*lc_owner)(struct lock_object *lock, struct thread **owner);
+	int	(*lc_owner)(const struct lock_object *lock, struct thread **owner);
 	int	(*lc_unlock)(struct lock_object *lock);
 };
 
@@ -215,7 +215,7 @@ void	witness_restore(struct lock_object *, const char *, int);
 int	witness_list_locks(struct lock_list_entry **,
 	    int (*)(const char *, ...));
 int	witness_warn(int, struct lock_object *, const char *, ...);
-void	witness_assert(struct lock_object *, int, const char *, int);
+void	witness_assert(const struct lock_object *, int, const char *, int);
 void	witness_display_spinlock(struct lock_object *, struct thread *,
 	    int (*)(const char *, ...));
 int	witness_line(struct lock_object *);
diff --git a/sys/sys/lockmgr.h b/sys/sys/lockmgr.h
index 4c3a272..1a2c92c 100644
--- a/sys/sys/lockmgr.h
+++ b/sys/sys/lockmgr.h
@@ -69,7 +69,7 @@ struct thread;
 int	 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 	    const char *wmesg, int prio, int timo, const char *file, int line);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
-void	 _lockmgr_assert(struct lock *lk, int what, const char *file, int line);
+void	 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line);
 #endif
 void	 _lockmgr_disown(struct lock *lk, const char *file, int line);
 
@@ -82,8 +82,8 @@ void	 lockinit(struct lock *lk, int prio, const char *wmesg, int timo,
 #ifdef DDB
 int	 lockmgr_chain(struct thread *td, struct thread **ownerp);
 #endif
-void	 lockmgr_printinfo(struct lock *lk);
-int	 lockstatus(struct lock *lk);
+void	 lockmgr_printinfo(const struct lock *lk);
+int	 lockstatus(const struct lock *lk);
 
 /*
  * As far as the ilk can be a static NULL pointer these functions need a
diff --git a/sys/sys/mchain.h b/sys/sys/mchain.h
index 4ee32e4..b89cbfd 100644
--- a/sys/sys/mchain.h
+++ b/sys/sys/mchain.h
@@ -67,6 +67,7 @@ struct mbuf *mb_detach(struct mbchain *mbp);
 int  mb_fixhdr(struct mbchain *mbp);
 caddr_t mb_reserve(struct mbchain *mbp, int size);
 
+int  mb_put_padbyte(struct mbchain *mbp);
 int  mb_put_uint8(struct mbchain *mbp, u_int8_t x);
 int  mb_put_uint16be(struct mbchain *mbp, u_int16_t x);
 int  mb_put_uint16le(struct mbchain *mbp, u_int16_t x);
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index e7e0b28..1e88e10 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -81,6 +81,10 @@
  *	 of the kernel via macros, thus allowing us to use the cpp LOCK_FILE
  *	 and LOCK_LINE. These functions should not be called directly by any
  *	 code using the API. Their macros cover their functionality.
+ *	 Functions with a `_' suffix are the entrypoint for the common
+ *	 KPI covering both compat shims and fast path case.  These can be
+ *	 used by consumers willing to pass options, file and line
+ *	 informations, in an option-independent way.
  *
  * [See below for descriptions]
  *
@@ -88,6 +92,8 @@
 void	mtx_init(struct mtx *m, const char *name, const char *type, int opts);
 void	mtx_destroy(struct mtx *m);
 void	mtx_sysinit(void *arg);
+int	mtx_trylock_flags_(struct mtx *m, int opts, const char *file,
+	    int line);
 void	mutex_init(void);
 void	_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts,
 	    const char *file, int line);
@@ -97,7 +103,6 @@ void	_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts,
 	    const char *file, int line);
 #endif
 void	_mtx_unlock_spin(struct mtx *m, int opts, const char *file, int line);
-int	_mtx_trylock(struct mtx *m, int opts, const char *file, int line);
 void	_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line);
 void	_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line);
 void	_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file,
@@ -105,14 +110,14 @@ void	_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file,
 void	_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file,
 	     int line);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
-void	_mtx_assert(struct mtx *m, int what, const char *file, int line);
+void	_mtx_assert(const struct mtx *m, int what, const char *file, int line);
 #endif
-void	_thread_lock_flags(struct thread *, int, const char *, int);
+void	thread_lock_flags_(struct thread *, int, const char *, int);
 
 #define	thread_lock(tdp)						\
-    _thread_lock_flags((tdp), 0, __FILE__, __LINE__)
+	thread_lock_flags_((tdp), 0, __FILE__, __LINE__)
 #define	thread_lock_flags(tdp, opt)					\
-    _thread_lock_flags((tdp), (opt), __FILE__, __LINE__)
+	thread_lock_flags_((tdp), (opt), __FILE__, __LINE__)
 #define	thread_unlock(tdp)						\
        mtx_unlock_spin((tdp)->td_lock)
 
@@ -290,27 +295,48 @@ extern struct mtx_pool *mtxpool_sleep;
 #error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/mutex.h>
 #endif
 #if LOCK_DEBUG > 0 || defined(MUTEX_NOINLINE)
-#define	mtx_lock_flags(m, opts)						\
-	_mtx_lock_flags((m), (opts), LOCK_FILE, LOCK_LINE)
-#define	mtx_unlock_flags(m, opts)					\
-	_mtx_unlock_flags((m), (opts), LOCK_FILE, LOCK_LINE)
-#define	mtx_lock_spin_flags(m, opts)					\
-	_mtx_lock_spin_flags((m), (opts), LOCK_FILE, LOCK_LINE)
-#define	mtx_unlock_spin_flags(m, opts)					\
-	_mtx_unlock_spin_flags((m), (opts), LOCK_FILE, LOCK_LINE)
+#define	mtx_lock_flags_(m, opts, file, line)				\
+	_mtx_lock_flags((m), (opts), (file), (line))
+#define	mtx_unlock_flags_(m, opts, file, line)				\
+	_mtx_unlock_flags((m), (opts), (file), (line))
+#define	mtx_lock_spin_flags_(m, opts, file, line)			\
+	_mtx_lock_spin_flags((m), (opts), (file), (line))
+#define	mtx_unlock_spin_flags_(m, opts, file, line)			\
+	_mtx_unlock_spin_flags((m), (opts), (file), (line))
 #else	/* LOCK_DEBUG == 0 && !MUTEX_NOINLINE */
+#define	mtx_lock_flags_(m, opts, file, line)				\
+	__mtx_lock((m), curthread, (opts), (file), (line))
+#define	mtx_unlock_flags_(m, opts, file, line)				\
+	__mtx_unlock((m), curthread, (opts), (file), (line))
+#define	mtx_lock_spin_flags_(m, opts, file, line)			\
+	__mtx_lock_spin((m), curthread, (opts), (file), (line))
+#define	mtx_unlock_spin_flags_(m, opts, file, line)			\
+	__mtx_unlock_spin((m))
+#endif	/* LOCK_DEBUG > 0 || MUTEX_NOINLINE */
+
+#ifdef INVARIANTS
+#define	mtx_assert_(m, what, file, line)				\
+	_mtx_assert((m), (what), (file), (line))
+
+#define GIANT_REQUIRED	mtx_assert_(&Giant, MA_OWNED, __FILE__, __LINE__)
+
+#else	/* INVARIANTS */
+#define mtx_assert_(m, what, file, line)	(void)0
+#define GIANT_REQUIRED
+#endif	/* INVARIANTS */
+
 #define	mtx_lock_flags(m, opts)						\
-	__mtx_lock((m), curthread, (opts), LOCK_FILE, LOCK_LINE)
+	mtx_lock_flags_((m), (opts), LOCK_FILE, LOCK_LINE)
 #define	mtx_unlock_flags(m, opts)					\
-	__mtx_unlock((m), curthread, (opts), LOCK_FILE, LOCK_LINE)
+	mtx_unlock_flags_((m), (opts), LOCK_FILE, LOCK_LINE)
 #define	mtx_lock_spin_flags(m, opts)					\
-	__mtx_lock_spin((m), curthread, (opts), LOCK_FILE, LOCK_LINE)
+	mtx_lock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE)
 #define	mtx_unlock_spin_flags(m, opts)					\
-	__mtx_unlock_spin((m))
-#endif	/* LOCK_DEBUG > 0 || MUTEX_NOINLINE */
-
+	mtx_unlock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE)
 #define mtx_trylock_flags(m, opts)					\
-	_mtx_trylock((m), (opts), LOCK_FILE, LOCK_LINE)
+	mtx_trylock_flags_((m), (opts), LOCK_FILE, LOCK_LINE)
+#define	mtx_assert(m, what)						\
+	mtx_assert_((m), (what), __FILE__, __LINE__)
 
 #define	mtx_sleep(chan, mtx, pri, wmesg, timo)				\
 	_sleep((chan), &(mtx)->lock_object, (pri), (wmesg), (timo))
@@ -398,17 +424,6 @@ struct mtx_args {
 #define MA_NOTRECURSED	LA_NOTRECURSED
 #endif
 
-#ifdef INVARIANTS
-#define	mtx_assert(m, what)						\
-	_mtx_assert((m), (what), __FILE__, __LINE__)
-
-#define GIANT_REQUIRED	mtx_assert(&Giant, MA_OWNED)
-
-#else	/* INVARIANTS */
-#define mtx_assert(m, what)	(void)0
-#define GIANT_REQUIRED
-#endif	/* INVARIANTS */
-
 /*
  * Common lock type names.
  */
diff --git a/sys/sys/param.h b/sys/sys/param.h
index 57ff3a3..d2ead83 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -58,7 +58,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1000000	/* Master, propagated to newvers */
+#define __FreeBSD_version 1000001	/* Master, propagated to newvers */
 
 #ifdef _KERNEL
 #define	P_OSREL_SIGWAIT		700000
@@ -324,6 +324,6 @@ __END_DECLS
  * Access a variable length array that has been declared as a fixed
  * length array.
  */
-#define __PAST_END(array, offset) (((typeof(*(array)) *)(array))[offset])
+#define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset])
 
 #endif	/* _SYS_PARAM_H_ */
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index fb97913..82cdae1 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -532,7 +532,7 @@ struct proc {
 	struct thread	*p_singlethread;/* (c + j) If single threading this is it */
 	int		p_suspcount;	/* (j) Num threads in suspended mode. */
 	struct thread	*p_xthread;	/* (c) Trap thread */
-	int		p_boundary_count;/* (c) Num threads at user boundary */
+	int		p_boundary_count;/* (j) Num threads at user boundary */
 	int		p_pendingcnt;	/* how many signals are pending */
 	struct itimers	*p_itimers;	/* (c) POSIX interval timers. */
 	struct procdesc	*p_procdesc;	/* (e) Process descriptor, if any. */
@@ -755,6 +755,7 @@ MALLOC_DECLARE(M_SUBPROC);
 } while (0)
 #define	_PRELE(p) do {							\
 	PROC_LOCK_ASSERT((p), MA_OWNED);				\
+	PROC_ASSERT_HELD(p);						\
 	(--(p)->p_lock);						\
 	if (((p)->p_flag & P_WEXIT) && (p)->p_lock == 0)		\
 		wakeup(&(p)->p_lock);					\
diff --git a/sys/sys/rmlock.h b/sys/sys/rmlock.h
index ef5776b..0ae2099 100644
--- a/sys/sys/rmlock.h
+++ b/sys/sys/rmlock.h
@@ -49,7 +49,7 @@
 void	rm_init(struct rmlock *rm, const char *name);
 void	rm_init_flags(struct rmlock *rm, const char *name, int opts);
 void	rm_destroy(struct rmlock *rm);
-int	rm_wowned(struct rmlock *rm);
+int	rm_wowned(const struct rmlock *rm);
 void	rm_sysinit(void *arg);
 void	rm_sysinit_flags(void *arg);
 
diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h
index ad17826..83bf57d 100644
--- a/sys/sys/rwlock.h
+++ b/sys/sys/rwlock.h
@@ -127,7 +127,7 @@ void	rw_init_flags(struct rwlock *rw, const char *name, int opts);
 void	rw_destroy(struct rwlock *rw);
 void	rw_sysinit(void *arg);
 void	rw_sysinit_flags(void *arg);
-int	rw_wowned(struct rwlock *rw);
+int	rw_wowned(const struct rwlock *rw);
 void	_rw_wlock(struct rwlock *rw, const char *file, int line);
 int	_rw_try_wlock(struct rwlock *rw, const char *file, int line);
 void	_rw_wunlock(struct rwlock *rw, const char *file, int line);
@@ -141,7 +141,7 @@ void	_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file,
 int	_rw_try_upgrade(struct rwlock *rw, const char *file, int line);
 void	_rw_downgrade(struct rwlock *rw, const char *file, int line);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
-void	_rw_assert(struct rwlock *rw, int what, const char *file, int line);
+void	_rw_assert(const struct rwlock *rw, int what, const char *file, int line);
 #endif
 
 /*
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
index d903102..3ee18a8 100644
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -190,7 +190,7 @@ struct sdt_provider {
 		    (uintptr_t) arg3, (uintptr_t) arg4)
 
 #define SDT_PROBE_ARGTYPE(prov, mod, func, name, num, type)			\
-	struct sdt_argtype sdt_##prov##_##mod##_##func##_##name##num[1]		\
+	static struct sdt_argtype sdt_##prov##_##mod##_##func##_##name##num[1]	\
 	    = { { num, type, { NULL, NULL },					\
 	    sdt_##prov##_##mod##_##func##_##name }				\
 	};									\
diff --git a/sys/sys/stack.h b/sys/sys/stack.h
index 734b967..5531467 100644
--- a/sys/sys/stack.h
+++ b/sys/sys/stack.h
@@ -37,16 +37,17 @@ struct sbuf;
 struct stack	*stack_create(void);
 void		 stack_destroy(struct stack *);
 int		 stack_put(struct stack *, vm_offset_t);
-void		 stack_copy(struct stack *, struct stack *);
+void		 stack_copy(const struct stack *, struct stack *);
 void		 stack_zero(struct stack *);
-void		 stack_print(struct stack *);
-void		 stack_print_ddb(struct stack *);
-void		 stack_print_short(struct stack *);
-void		 stack_print_short_ddb(struct stack *);
-void		 stack_sbuf_print(struct sbuf *, struct stack *);
-void		 stack_sbuf_print_ddb(struct sbuf *, struct stack *);
+void		 stack_print(const struct stack *);
+void		 stack_print_ddb(const struct stack *);
+void		 stack_print_short(const struct stack *);
+void		 stack_print_short_ddb(const struct stack *);
+void		 stack_sbuf_print(struct sbuf *, const struct stack *);
+void		 stack_sbuf_print_ddb(struct sbuf *, const struct stack *);
 #ifdef KTR
-void		 stack_ktr(u_int, const char *, int, struct stack *, u_int, int);
+void		 stack_ktr(u_int, const char *, int, const struct stack *,
+		    u_int, int);
 #define	CTRSTACK(m, st, depth, cheap) do {				\
 	if (KTR_COMPILE & (m))						\
 		stack_ktr((m), __FILE__, __LINE__, st, depth, cheap);	\
diff --git a/sys/sys/sx.h b/sys/sys/sx.h
index da16b60..8c76a50 100644
--- a/sys/sys/sx.h
+++ b/sys/sys/sx.h
@@ -94,14 +94,14 @@ void	sx_sysinit(void *arg);
 #define	sx_init(sx, desc)	sx_init_flags((sx), (desc), 0)
 void	sx_init_flags(struct sx *sx, const char *description, int opts);
 void	sx_destroy(struct sx *sx);
+int	sx_try_slock_(struct sx *sx, const char *file, int line);
+int	sx_try_xlock_(struct sx *sx, const char *file, int line);
+int	sx_try_upgrade_(struct sx *sx, const char *file, int line);
+void	sx_downgrade_(struct sx *sx, const char *file, int line);
 int	_sx_slock(struct sx *sx, int opts, const char *file, int line);
 int	_sx_xlock(struct sx *sx, int opts, const char *file, int line);
-int	_sx_try_slock(struct sx *sx, const char *file, int line);
-int	_sx_try_xlock(struct sx *sx, const char *file, int line);
 void	_sx_sunlock(struct sx *sx, const char *file, int line);
 void	_sx_xunlock(struct sx *sx, const char *file, int line);
-int	_sx_try_upgrade(struct sx *sx, const char *file, int line);
-void	_sx_downgrade(struct sx *sx, const char *file, int line);
 int	_sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts,
 	    const char *file, int line);
 int	_sx_slock_hard(struct sx *sx, int opts, const char *file, int line);
@@ -109,7 +109,7 @@ void	_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
 	    line);
 void	_sx_sunlock_hard(struct sx *sx, const char *file, int line);
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
-void	_sx_assert(struct sx *sx, int what, const char *file, int line);
+void	_sx_assert(const struct sx *sx, int what, const char *file, int line);
 #endif
 #ifdef DDB
 int	sx_chain(struct thread *td, struct thread **ownerp);
@@ -208,30 +208,50 @@ __sx_sunlock(struct sx *sx, const char *file, int line)
 #error	"LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>"
 #endif
 #if	(LOCK_DEBUG > 0) || defined(SX_NOINLINE)
-#define	sx_xlock(sx)		(void)_sx_xlock((sx), 0, LOCK_FILE, LOCK_LINE)
-#define	sx_xlock_sig(sx)						\
-	_sx_xlock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
-#define	sx_xunlock(sx)		_sx_xunlock((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_slock(sx)		(void)_sx_slock((sx), 0, LOCK_FILE, LOCK_LINE)
-#define	sx_slock_sig(sx)						\
-	_sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
-#define	sx_sunlock(sx)		_sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_xlock_(sx, file, line)					\
+	(void)_sx_xlock((sx), 0, (file), (line))
+#define	sx_xlock_sig_(sx, file, line)					\
+	_sx_xlock((sx), SX_INTERRUPTIBLE, (file), (line))
+#define	sx_xunlock_(sx, file, line)					\
+	_sx_xunlock((sx), (file), (line))
+#define	sx_slock_(sx, file, line)					\
+	(void)_sx_slock((sx), 0, (file), (line))
+#define	sx_slock_sig_(sx, file, line)					\
+	_sx_slock((sx), SX_INTERRUPTIBLE, (file) , (line))
+#define	sx_sunlock_(sx, file, line)					\
+	_sx_sunlock((sx), (file), (line))
 #else
-#define	sx_xlock(sx)							\
-	(void)__sx_xlock((sx), curthread, 0, LOCK_FILE, LOCK_LINE)
-#define	sx_xlock_sig(sx)						\
-	__sx_xlock((sx), curthread, SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
-#define	sx_xunlock(sx)							\
-	__sx_xunlock((sx), curthread, LOCK_FILE, LOCK_LINE)
-#define	sx_slock(sx)		(void)__sx_slock((sx), 0, LOCK_FILE, LOCK_LINE)
-#define	sx_slock_sig(sx)						\
-	__sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
-#define	sx_sunlock(sx)		__sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_xlock_(sx, file, line)					\
+	(void)__sx_xlock((sx), curthread, 0, (file), (line))
+#define	sx_xlock_sig_(sx, file, line)					\
+	__sx_xlock((sx), curthread, SX_INTERRUPTIBLE, (file), (line))
+#define	sx_xunlock_(sx, file, line)					\
+	__sx_xunlock((sx), curthread, (file), (line))
+#define	sx_slock_(sx, file, line)					\
+	(void)__sx_slock((sx), 0, (file), (line))
+#define	sx_slock_sig_(sx, file, line)					\
+	__sx_slock((sx), SX_INTERRUPTIBLE, (file), (line))
+#define	sx_sunlock_(sx, file, line)					\
+	__sx_sunlock((sx), (file), (line))
 #endif	/* LOCK_DEBUG > 0 || SX_NOINLINE */
-#define	sx_try_slock(sx)	_sx_try_slock((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_try_xlock(sx)	_sx_try_xlock((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_try_upgrade(sx)	_sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE)
-#define	sx_downgrade(sx)	_sx_downgrade((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_try_slock(sx)	sx_try_slock_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_try_xlock(sx)	sx_try_xlock_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_try_upgrade(sx)	sx_try_upgrade_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_downgrade(sx)	sx_downgrade_((sx), LOCK_FILE, LOCK_LINE)
+#ifdef INVARIANTS
+#define	sx_assert_(sx, what, file, line)				\
+	_sx_assert((sx), (what), (file), (line))
+#else
+#define	sx_assert_(sx, what, file, line)	(void)0
+#endif
+
+#define	sx_xlock(sx)		sx_xlock_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_xlock_sig(sx)	sx_xlock_sig_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_xunlock(sx)		sx_xunlock_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_slock(sx)		sx_slock_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_slock_sig(sx)	sx_slock_sig_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_sunlock(sx)		sx_sunlock_((sx), LOCK_FILE, LOCK_LINE)
+#define	sx_assert(sx, what)	sx_assert_((sx), (what), __FILE__, __LINE__)
 
 /*
  * Return a pointer to the owning thread if the lock is exclusively
@@ -245,13 +265,15 @@ __sx_sunlock(struct sx *sx, const char *file, int line)
 	(((sx)->sx_lock & ~(SX_LOCK_FLAGMASK & ~SX_LOCK_SHARED)) ==	\
 	    (uintptr_t)curthread)
 
-#define	sx_unlock(sx) do {						\
+#define	sx_unlock_(sx, file, line) do {					\
 	if (sx_xlocked(sx))						\
-		sx_xunlock(sx);						\
+		sx_xunlock_(sx, file, line);				\
 	else								\
-		sx_sunlock(sx);						\
+		sx_sunlock_(sx, file, line);				\
 } while (0)
 
+#define	sx_unlock(sx)	sx_unlock_((sx), LOCK_FILE, LOCK_LINE)
+
 #define	sx_sleep(chan, sx, pri, wmesg, timo)				\
 	_sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo))
 
@@ -287,12 +309,6 @@ __sx_sunlock(struct sx *sx, const char *file, int line)
 #define	SX_NOTRECURSED		LA_NOTRECURSED
 #endif
 
-#ifdef INVARIANTS
-#define	sx_assert(sx, what)	_sx_assert((sx), (what), LOCK_FILE, LOCK_LINE)
-#else
-#define	sx_assert(sx, what)	(void)0
-#endif
-
 #endif /* _KERNEL */
 
 #endif /* !_SYS_SX_H_ */
diff --git a/sys/sys/syscall.h b/sys/sys/syscall.h
index 31252c1..a9a1fb2 100644
--- a/sys/sys/syscall.h
+++ b/sys/sys/syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 224987 2011-08-18 22:51:30Z jonathan 
+ * created from FreeBSD: head/sys/kern/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #define	SYS_syscall	0
@@ -216,6 +216,9 @@
 #define	SYS_ktimer_gettime	238
 #define	SYS_ktimer_getoverrun	239
 #define	SYS_nanosleep	240
+#define	SYS_ffclock_getcounter	241
+#define	SYS_ffclock_setestimate	242
+#define	SYS_ffclock_getestimate	243
 #define	SYS_ntp_gettime	248
 #define	SYS_minherit	250
 #define	SYS_rfork	251
@@ -446,4 +449,5 @@
 #define	SYS_rctl_add_rule	528
 #define	SYS_rctl_remove_rule	529
 #define	SYS_posix_fallocate	530
+#define	SYS_posix_fadvise	531
 #define	SYS_MAXSYSCALL	532
diff --git a/sys/sys/syscall.mk b/sys/sys/syscall.mk
index e448707..1113886 100644
--- a/sys/sys/syscall.mk
+++ b/sys/sys/syscall.mk
@@ -1,7 +1,7 @@
 # FreeBSD system call names.
 # DO NOT EDIT-- this file is automatically generated.
 # $FreeBSD$
-# created from FreeBSD: head/sys/kern/syscalls.master 224987 2011-08-18 22:51:30Z jonathan 
+# created from FreeBSD: head/sys/kern/syscalls.master 227691 2011-11-19 06:35:15Z ed 
 MIASM =  \
 	syscall.o \
 	exit.o \
@@ -168,6 +168,9 @@ MIASM =  \
 	ktimer_gettime.o \
 	ktimer_getoverrun.o \
 	nanosleep.o \
+	ffclock_getcounter.o \
+	ffclock_setestimate.o \
+	ffclock_getestimate.o \
 	ntp_gettime.o \
 	minherit.o \
 	rfork.o \
@@ -394,4 +397,5 @@ MIASM =  \
 	rctl_get_limits.o \
 	rctl_add_rule.o \
 	rctl_remove_rule.o \
-	posix_fallocate.o
+	posix_fallocate.o \
+	posix_fadvise.o
diff --git a/sys/sys/syscallsubr.h b/sys/sys/syscallsubr.h
index 7649f1a..08dba2f 100644
--- a/sys/sys/syscallsubr.h
+++ b/sys/sys/syscallsubr.h
@@ -153,6 +153,10 @@ int	kern_openat(struct thread *td, int fd, char *path,
 int	kern_pathconf(struct thread *td, char *path, enum uio_seg pathseg,
 	    int name, u_long flags);
 int	kern_pipe(struct thread *td, int fildes[2]);
+int	kern_posix_fadvise(struct thread *td, int fd, off_t offset, off_t len,
+	    int advice);
+int	kern_posix_fallocate(struct thread *td, int fd, off_t offset,
+	    off_t len);
 int	kern_preadv(struct thread *td, int fd, struct uio *auio, off_t offset);
 int	kern_pselect(struct thread *td, int nd, fd_set *in, fd_set *ou,
 	    fd_set *ex, struct timeval *tvp, sigset_t *uset, int abi_nfdbits);
diff --git a/sys/sys/sysproto.h b/sys/sys/sysproto.h
index bb692b8..84a0ed7 100644
--- a/sys/sys/sysproto.h
+++ b/sys/sys/sysproto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 224987 2011-08-18 22:51:30Z jonathan 
+ * created from FreeBSD: head/sys/kern/syscalls.master 227691 2011-11-19 06:35:15Z ed 
  */
 
 #ifndef _SYS_SYSPROTO_H_
@@ -12,6 +12,7 @@
 #include <sys/signal.h>
 #include <sys/acl.h>
 #include <sys/cpuset.h>
+#include <sys/_ffcounter.h>
 #include <sys/_semaphore.h>
 #include <sys/ucontext.h>
 
@@ -158,7 +159,7 @@ struct getsockname_args {
 };
 struct access_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char amode_l_[PADL_(int)]; int amode; char amode_r_[PADR_(int)];
 };
 struct chflags_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
@@ -726,6 +727,15 @@ struct nanosleep_args {
 	char rqtp_l_[PADL_(const struct timespec *)]; const struct timespec * rqtp; char rqtp_r_[PADR_(const struct timespec *)];
 	char rmtp_l_[PADL_(struct timespec *)]; struct timespec * rmtp; char rmtp_r_[PADR_(struct timespec *)];
 };
+struct ffclock_getcounter_args {
+	char ffcount_l_[PADL_(ffcounter *)]; ffcounter * ffcount; char ffcount_r_[PADR_(ffcounter *)];
+};
+struct ffclock_setestimate_args {
+	char cest_l_[PADL_(struct ffclock_estimate *)]; struct ffclock_estimate * cest; char cest_r_[PADR_(struct ffclock_estimate *)];
+};
+struct ffclock_getestimate_args {
+	char cest_l_[PADL_(struct ffclock_estimate *)]; struct ffclock_estimate * cest; char cest_r_[PADR_(struct ffclock_estimate *)];
+};
 struct ntp_gettime_args {
 	char ntvp_l_[PADL_(struct ntptimeval *)]; struct ntptimeval * ntvp; char ntvp_r_[PADR_(struct ntptimeval *)];
 };
@@ -1073,7 +1083,7 @@ struct __setugid_args {
 };
 struct eaccess_args {
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char amode_l_[PADL_(int)]; int amode; char amode_r_[PADR_(int)];
 };
 struct afs3_syscall_args {
 	char syscall_l_[PADL_(long)]; long syscall; char syscall_r_[PADR_(long)];
@@ -1532,7 +1542,7 @@ struct cpuset_setaffinity_args {
 struct faccessat_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
 	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
-	char mode_l_[PADL_(int)]; int mode; char mode_r_[PADR_(int)];
+	char amode_l_[PADL_(int)]; int amode; char amode_r_[PADR_(int)];
 	char flag_l_[PADL_(int)]; int flag; char flag_r_[PADR_(int)];
 };
 struct fchmodat_args {
@@ -1733,6 +1743,12 @@ struct posix_fallocate_args {
 	char offset_l_[PADL_(off_t)]; off_t offset; char offset_r_[PADR_(off_t)];
 	char len_l_[PADL_(off_t)]; off_t len; char len_r_[PADR_(off_t)];
 };
+struct posix_fadvise_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char offset_l_[PADL_(off_t)]; off_t offset; char offset_r_[PADR_(off_t)];
+	char len_l_[PADL_(off_t)]; off_t len; char len_r_[PADR_(off_t)];
+	char advice_l_[PADL_(int)]; int advice; char advice_r_[PADR_(int)];
+};
 int	nosys(struct thread *, struct nosys_args *);
 void	sys_sys_exit(struct thread *, struct sys_exit_args *);
 int	sys_fork(struct thread *, struct fork_args *);
@@ -1888,6 +1904,9 @@ int	sys_ktimer_settime(struct thread *, struct ktimer_settime_args *);
 int	sys_ktimer_gettime(struct thread *, struct ktimer_gettime_args *);
 int	sys_ktimer_getoverrun(struct thread *, struct ktimer_getoverrun_args *);
 int	sys_nanosleep(struct thread *, struct nanosleep_args *);
+int	sys_ffclock_getcounter(struct thread *, struct ffclock_getcounter_args *);
+int	sys_ffclock_setestimate(struct thread *, struct ffclock_setestimate_args *);
+int	sys_ffclock_getestimate(struct thread *, struct ffclock_getestimate_args *);
 int	sys_ntp_gettime(struct thread *, struct ntp_gettime_args *);
 int	sys_minherit(struct thread *, struct minherit_args *);
 int	sys_rfork(struct thread *, struct rfork_args *);
@@ -2109,6 +2128,7 @@ int	sys_rctl_get_limits(struct thread *, struct rctl_get_limits_args *);
 int	sys_rctl_add_rule(struct thread *, struct rctl_add_rule_args *);
 int	sys_rctl_remove_rule(struct thread *, struct rctl_remove_rule_args *);
 int	sys_posix_fallocate(struct thread *, struct posix_fallocate_args *);
+int	sys_posix_fadvise(struct thread *, struct posix_fadvise_args *);
 
 #ifdef COMPAT_43
 
@@ -2574,6 +2594,9 @@ int	freebsd7_shmctl(struct thread *, struct freebsd7_shmctl_args *);
 #define	SYS_AUE_ktimer_gettime	AUE_NULL
 #define	SYS_AUE_ktimer_getoverrun	AUE_NULL
 #define	SYS_AUE_nanosleep	AUE_NULL
+#define	SYS_AUE_ffclock_getcounter	AUE_NULL
+#define	SYS_AUE_ffclock_setestimate	AUE_NULL
+#define	SYS_AUE_ffclock_getestimate	AUE_NULL
 #define	SYS_AUE_ntp_gettime	AUE_NULL
 #define	SYS_AUE_minherit	AUE_MINHERIT
 #define	SYS_AUE_rfork	AUE_RFORK
@@ -2799,6 +2822,7 @@ int	freebsd7_shmctl(struct thread *, struct freebsd7_shmctl_args *);
 #define	SYS_AUE_rctl_add_rule	AUE_NULL
 #define	SYS_AUE_rctl_remove_rule	AUE_NULL
 #define	SYS_AUE_posix_fallocate	AUE_NULL
+#define	SYS_AUE_posix_fadvise	AUE_NULL
 
 #undef PAD_
 #undef PADL_
diff --git a/sys/sys/timeffc.h b/sys/sys/timeffc.h
new file mode 100644
index 0000000..0c58c05
--- /dev/null
+++ b/sys/sys/timeffc.h
@@ -0,0 +1,178 @@
+/*-
+ * Copyright (c) 2011 The University of Melbourne
+ * All rights reserved.
+ *
+ * This software was developed by Julien Ridoux at the University of Melbourne
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS_TIMEFF_H_
+#define _SYS_TIMEFF_H_
+
+#include <sys/_ffcounter.h>
+
+/*
+ * Feed-forward clock estimate
+ * Holds time mark as a ffcounter and conversion to bintime based on current
+ * timecounter period and offset estimate passed by the synchronization daemon.
+ * Provides time of last daemon update, clock status and bound on error.
+ */
+struct ffclock_estimate {
+	struct bintime	update_time;	/* Time of last estimates update. */
+	ffcounter	update_ffcount;	/* Counter value at last update. */
+	ffcounter	leapsec_next;	/* Counter value of next leap second. */
+	uint64_t	period;		/* Estimate of counter period. */
+	uint32_t	errb_abs;	/* Bound on absolute clock error [ns]. */
+	uint32_t	errb_rate;	/* Bound on counter rate error [ps/s]. */
+	uint32_t	status;		/* Clock status. */
+	int16_t		leapsec_total;	/* All leap seconds seen so far. */
+	int8_t		leapsec;	/* Next leap second (in {-1,0,1}). */
+};
+
+#if __BSD_VISIBLE
+#ifdef _KERNEL
+
+/*
+ * Index into the sysclocks array for obtaining the ASCII name of a particular
+ * sysclock.
+ */
+#define	SYSCLOCK_FBCK	0
+#define	SYSCLOCK_FFWD	1
+
+/*
+ * Parameters of counter characterisation required by feed-forward algorithms.
+ */
+#define	FFCLOCK_SKM_SCALE	1024
+
+/*
+ * Feed-forward clock status
+ */
+#define	FFCLOCK_STA_UNSYNC	1
+#define	FFCLOCK_STA_WARMUP	2
+
+/*
+ * Clock flags to select how the feed-forward counter is converted to absolute
+ * time by ffclock_convert_abs().
+ * FAST:    do not read the hardware counter, return feed-forward clock time
+ *          at last tick. The time returned has the resolution of the kernel
+ *          tick (1/hz [s]).
+ * LERP:    linear interpolation of ffclock time to guarantee monotonic time.
+ * LEAPSEC: include leap seconds.
+ * UPTIME:  removes time of boot.
+ */
+#define	FFCLOCK_FAST		1
+#define	FFCLOCK_LERP		2
+#define	FFCLOCK_LEAPSEC		4
+#define	FFCLOCK_UPTIME		8
+
+/* Resets feed-forward clock from RTC */
+void ffclock_reset_clock(struct timespec *ts);
+
+/*
+ * Return the current value of the feed-forward clock counter. Essential to
+ * measure time interval in counter units. If a fast timecounter is used by the
+ * system, may also allow fast but accurate timestamping.
+ */
+void ffclock_read_counter(ffcounter *ffcount);
+
+/*
+ * Retrieve feed-forward counter value and time of last kernel tick. This
+ * accepts the FFCLOCK_LERP flag.
+ */
+void ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags);
+
+/*
+ * Low level routines to convert a counter timestamp into absolute time and a
+ * counter timestamp interval into an interval in seconds. The absolute time
+ * conversion accepts the FFCLOCK_LERP flag.
+ */
+void ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags);
+void ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt);
+
+/*
+ * Feed-forward clock routines.
+ *
+ * These functions rely on the timecounters and ffclock_estimates stored in
+ * fftimehands. Note that the error_bound parameter is not the error of the
+ * clock but an upper bound on the error of the absolute time or time interval
+ * returned.
+ *
+ * ffclock_abstime(): retrieves current time as counter value and convert this
+ *     timestamp in seconds. The value (in seconds) of the converted timestamp
+ *     depends on the flags passed: for a given counter value, different
+ *     conversions are possible. Different clock models can be selected by
+ *     combining flags (for example (FFCLOCK_LERP|FFCLOCK_UPTIME) produces
+ *     linearly interpolated uptime).
+ * ffclock_difftime(): computes a time interval in seconds based on an interval
+ *     measured in ffcounter units. This should be the preferred way to measure
+ *     small time intervals very accurately.
+ */
+void ffclock_abstime(ffcounter *ffcount, struct bintime *bt,
+    struct bintime *error_bound, uint32_t flags);
+void ffclock_difftime(ffcounter ffdelta, struct bintime *bt,
+    struct bintime *error_bound);
+
+/*
+ * Wrapper routines to return current absolute time using the feed-forward
+ * clock. These functions are named after those defined in <sys/time.h>, which
+ * contains a description of the original ones.
+ */
+void ffclock_bintime(struct bintime *bt);
+void ffclock_nanotime(struct timespec *tsp);
+void ffclock_microtime(struct timeval *tvp);
+
+void ffclock_getbintime(struct bintime *bt);
+void ffclock_getnanotime(struct timespec *tsp);
+void ffclock_getmicrotime(struct timeval *tvp);
+
+void ffclock_binuptime(struct bintime *bt);
+void ffclock_nanouptime(struct timespec *tsp);
+void ffclock_microuptime(struct timeval *tvp);
+
+void ffclock_getbinuptime(struct bintime *bt);
+void ffclock_getnanouptime(struct timespec *tsp);
+void ffclock_getmicrouptime(struct timeval *tvp);
+
+/*
+ * Wrapper routines to convert a time interval specified in ffcounter units into
+ * seconds using the current feed-forward clock estimates.
+ */
+void ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt);
+void ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp);
+void ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp);
+
+#else /* !_KERNEL */
+
+/* Feed-Forward Clock system calls. */
+__BEGIN_DECLS
+int ffclock_getcounter(ffcounter *ffcount);
+int ffclock_getestimate(struct ffclock_estimate *cest);
+int ffclock_setestimate(struct ffclock_estimate *cest);
+__END_DECLS
+
+#endif /* _KERNEL */
+#endif /* __BSD_VISIBLE */
+#endif /* _SYS_TIMEFF_H_ */
diff --git a/sys/sys/timepps.h b/sys/sys/timepps.h
index cde9396..8083f33 100644
--- a/sys/sys/timepps.h
+++ b/sys/sys/timepps.h
@@ -6,6 +6,12 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
+ * Copyright (c) 2011 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Julien Ridoux at the University
+ * of Melbourne under sponsorship from the FreeBSD Foundation.
+ *
  * $FreeBSD$
  *
  * The is a FreeBSD version of the RFC 2783 API for Pulse Per Second 
@@ -15,6 +21,7 @@
 #ifndef _SYS_TIMEPPS_H_
 #define _SYS_TIMEPPS_H_
 
+#include <sys/_ffcounter.h>
 #include <sys/ioccom.h>
 #include <sys/time.h>
 
@@ -43,6 +50,16 @@ typedef struct {
 	int		current_mode;		/* current mode bits */
 } pps_info_t;
 
+typedef struct {
+	pps_seq_t	assert_sequence;	/* assert event seq # */
+	pps_seq_t	clear_sequence;		/* clear event seq # */
+	pps_timeu_t	assert_tu;
+	pps_timeu_t	clear_tu;
+	ffcounter	assert_ffcount;		/* ffcounter on assert event */
+	ffcounter	clear_ffcount;		/* ffcounter on clear event */
+	int		current_mode;		/* current mode bits */
+} pps_info_ffc_t;
+
 #define assert_timestamp        assert_tu.tspec
 #define clear_timestamp         clear_tu.tspec
 
@@ -79,6 +96,10 @@ typedef struct {
 #define PPS_TSFMT_TSPEC		0x1000
 #define PPS_TSFMT_NTPFP		0x2000
 
+#define	PPS_TSCLK_FBCK		0x10000
+#define	PPS_TSCLK_FFWD		0x20000
+#define	PPS_TSCLK_MASK		0x30000
+
 #define PPS_KC_HARDPPS		0
 #define PPS_KC_HARDPPS_PLL	1
 #define PPS_KC_HARDPPS_FLL	2
@@ -89,6 +110,12 @@ struct pps_fetch_args {
 	struct timespec	timeout;
 };
 
+struct pps_fetch_ffc_args {
+	int		tsformat;
+	pps_info_ffc_t	pps_info_buf_ffc;
+	struct timespec	timeout;
+};
+
 struct pps_kcbind_args {
 	int kernel_consumer;
 	int edge;
@@ -102,18 +129,21 @@ struct pps_kcbind_args {
 #define PPS_IOC_GETCAP		_IOR('1', 5, int)
 #define PPS_IOC_FETCH		_IOWR('1', 6, struct pps_fetch_args)
 #define PPS_IOC_KCBIND		_IOW('1', 7, struct pps_kcbind_args)
+#define	PPS_IOC_FETCH_FFCOUNTER	_IOWR('1', 8, struct pps_fetch_ffc_args)
 
 #ifdef _KERNEL
 
 struct pps_state {
 	/* Capture information. */
 	struct timehands *capth;
+	struct fftimehands *capffth;
 	unsigned	capgen;
 	unsigned	capcount;
 
 	/* State information. */
 	pps_params_t	ppsparam;
 	pps_info_t	ppsinfo;
+	pps_info_ffc_t	ppsinfo_ffc;
 	int		kcmode;
 	int		ppscap;
 	struct timecounter *ppstc;
@@ -184,6 +214,25 @@ time_pps_fetch(pps_handle_t handle, const int tsformat,
 }
 
 static __inline int
+time_pps_fetch_ffc(pps_handle_t handle, const int tsformat,
+	pps_info_ffc_t *ppsinfobuf, const struct timespec *timeout)
+{
+	struct pps_fetch_ffc_args arg;
+	int error;
+
+	arg.tsformat = tsformat;
+	if (timeout == NULL) {
+		arg.timeout.tv_sec = -1;
+		arg.timeout.tv_nsec = -1;
+	} else {
+		arg.timeout = *timeout;
+	}
+	error = ioctl(handle, PPS_IOC_FETCH_FFCOUNTER, &arg);
+	*ppsinfobuf = arg.pps_info_buf_ffc;
+	return (error);
+}
+
+static __inline int
 time_pps_kcbind(pps_handle_t handle, const int kernel_consumer,
 	const int edge, const int tsformat)
 {
diff --git a/sys/sys/uio.h b/sys/sys/uio.h
index 5df279b..f3cc20f 100644
--- a/sys/sys/uio.h
+++ b/sys/sys/uio.h
@@ -89,11 +89,11 @@ struct vm_page;
 struct uio *cloneuio(struct uio *uiop);
 int	copyinfrom(const void * __restrict src, void * __restrict dst,
 	    size_t len, int seg);
-int	copyiniov(struct iovec *iovp, u_int iovcnt, struct iovec **iov,
+int	copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov,
 	    int error);
 int	copyinstrfrom(const void * __restrict src, void * __restrict dst,
 	    size_t len, size_t * __restrict copied, int seg);
-int	copyinuio(struct iovec *iovp, u_int iovcnt, struct uio **uiop);
+int	copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop);
 int	copyout_map(struct thread *td, vm_offset_t *addr, size_t sz);
 int	copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz);
 int	uiomove(void *cp, int n, struct uio *uio);
diff --git a/sys/sys/unistd.h b/sys/sys/unistd.h
index 9e7f7e6..d007836 100644
--- a/sys/sys/unistd.h
+++ b/sys/sys/unistd.h
@@ -49,7 +49,7 @@
  * the POSIX standard; however, if the relevant sysconf() function
  * returns -1, the functions may be stubbed out.
  */
-#define	_POSIX_ADVISORY_INFO		-1
+#define	_POSIX_ADVISORY_INFO		200112L
 #define	_POSIX_ASYNCHRONOUS_IO		0
 #define	_POSIX_CHOWN_RESTRICTED		1
 #define	_POSIX_CLOCK_SELECTION		-1
diff --git a/sys/sys/user.h b/sys/sys/user.h
index a139d4f..2c37e99 100644
--- a/sys/sys/user.h
+++ b/sys/sys/user.h
@@ -412,6 +412,7 @@ struct kinfo_file {
 #define	KVME_FLAG_COW		0x00000001
 #define	KVME_FLAG_NEEDS_COPY	0x00000002
 #define	KVME_FLAG_NOCOREDUMP	0x00000004
+#define	KVME_FLAG_SUPER		0x00000008
 
 #if defined(__amd64__)
 #define	KINFO_OVMENTRY_SIZE	1168
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index 4cb6633..7382336 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -384,6 +384,7 @@ extern int		vttoif_tab[];
 #define	V_SAVE		0x0001	/* vinvalbuf: sync file first */
 #define	V_ALT		0x0002	/* vinvalbuf: invalidate only alternate bufs */
 #define	V_NORMAL	0x0004	/* vinvalbuf: invalidate only regular bufs */
+#define	V_CLEANONLY	0x0008	/* vinvalbuf: invalidate only clean bufs */
 #define	REVOKEALL	0x0001	/* vop_revoke: revoke all aliases */
 #define	V_WAIT		0x0001	/* vn_start_write: sleep for suspend */
 #define	V_NOWAIT	0x0002	/* vn_start_write: don't sleep for suspend */
@@ -685,6 +686,7 @@ int	vop_stdunlock(struct vop_unlock_args *);
 int	vop_nopoll(struct vop_poll_args *);
 int	vop_stdaccess(struct vop_access_args *ap);
 int	vop_stdaccessx(struct vop_accessx_args *ap);
+int	vop_stdadvise(struct vop_advise_args *ap);
 int	vop_stdadvlock(struct vop_advlock_args *ap);
 int	vop_stdadvlockasync(struct vop_advlockasync_args *ap);
 int	vop_stdadvlockpurge(struct vop_advlockpurge_args *ap);
diff --git a/sys/tools/makeobjops.awk b/sys/tools/makeobjops.awk
index 0406b85..8ca99bb 100644
--- a/sys/tools/makeobjops.awk
+++ b/sys/tools/makeobjops.awk
@@ -307,12 +307,8 @@ function handle_method (static, doc)
 	    line_width, length(prototype)));
 
 	# Print out the method desc
-	printc("struct kobj_method " mname "_method_default = {");
-	printc("\t&" mname "_desc, (kobjop_t) " default_function);
-	printc("};\n");
-
 	printc("struct kobjop_desc " mname "_desc = {");
-	printc("\t0, &" mname "_method_default");
+	printc("\t0, { &" mname "_desc, (kobjop_t)" default_function " }");
 	printc("};\n");
 
 	# Print out the method itself
diff --git a/sys/ufs/ffs/ffs_softdep.c b/sys/ufs/ffs/ffs_softdep.c
index 6c0e7d7..5b4b6b9 100644
--- a/sys/ufs/ffs/ffs_softdep.c
+++ b/sys/ufs/ffs/ffs_softdep.c
@@ -658,12 +658,13 @@ unsigned long dep_total[D_LAST + 1];
 unsigned long dep_write[D_LAST + 1];
 
 
-SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0, "soft updates stats");
-SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0,
+    "soft updates stats");
+static SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0,
     "total dependencies allocated");
-SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0,
     "current dependencies allocated");
-SYSCTL_NODE(_debug_softdep, OID_AUTO, write, CTLFLAG_RW, 0,
+static SYSCTL_NODE(_debug_softdep, OID_AUTO, write, CTLFLAG_RW, 0,
     "current dependencies written");
 
 #define	SOFTDEP_TYPE(type, str, long)					\
diff --git a/sys/ufs/ffs/fs.h b/sys/ufs/ffs/fs.h
index b1e2174..3715539 100644
--- a/sys/ufs/ffs/fs.h
+++ b/sys/ufs/ffs/fs.h
@@ -338,7 +338,7 @@ struct fs {
 	ufs2_daddr_t fs_csaddr;		/* blk addr of cyl grp summary area */
 	int64_t	 fs_pendingblocks;	/* (u) blocks being freed */
 	u_int32_t fs_pendinginodes;	/* (u) inodes being freed */
-	ino_t	 fs_snapinum[FSMAXSNAP];/* list of snapshot inode numbers */
+	uint32_t fs_snapinum[FSMAXSNAP];/* list of snapshot inode numbers */
 	u_int32_t fs_avgfilesize;	/* expected average file size */
 	u_int32_t fs_avgfpdir;		/* expected # of files per directory */
 	int32_t	 fs_save_cgsize;	/* save real cg size to use fs_bsize */
@@ -695,11 +695,11 @@ struct jsegrec {
  */
 struct jrefrec {
 	uint32_t	jr_op;
-	ino_t		jr_ino;
-	ino_t		jr_parent;
+	uint32_t	jr_ino;
+	uint32_t	jr_parent;
 	uint16_t	jr_nlink;
 	uint16_t	jr_mode;
-	off_t		jr_diroff;
+	int64_t		jr_diroff;
 	uint64_t	jr_unused;
 };
 
@@ -709,11 +709,11 @@ struct jrefrec {
  */
 struct jmvrec {
 	uint32_t	jm_op;
-	ino_t		jm_ino;
-	ino_t		jm_parent;
+	uint32_t	jm_ino;
+	uint32_t	jm_parent;
 	uint16_t	jm_unused;
-	off_t		jm_oldoff;
-	off_t		jm_newoff;
+	int64_t		jm_oldoff;
+	int64_t		jm_newoff;
 };
 
 /*
@@ -737,7 +737,7 @@ struct jblkrec {
 struct jtrncrec {
 	uint32_t	jt_op;
 	uint32_t	jt_ino;
-	off_t		jt_size;
+	int64_t		jt_size;
 	uint32_t	jt_extsize;
 	uint32_t	jt_pad[3];
 };
diff --git a/sys/ufs/ufs/dinode.h b/sys/ufs/ufs/dinode.h
index c75257c..8aa1fe0 100644
--- a/sys/ufs/ufs/dinode.h
+++ b/sys/ufs/ufs/dinode.h
@@ -146,7 +146,7 @@ struct ufs2_dinode {
 	ufs2_daddr_t	di_db[NDADDR];	/* 112: Direct disk blocks. */
 	ufs2_daddr_t	di_ib[NIADDR];	/* 208: Indirect disk blocks. */
 	u_int64_t	di_modrev;	/* 232: i_modrev for NFSv4 */
-	ino_t		di_freelink;	/* 240: SUJ: Next unlinked inode. */
+	uint32_t	di_freelink;	/* 240: SUJ: Next unlinked inode. */
 	uint32_t	di_spare[3];	/* 244: Reserved; currently unused */
 };
 
@@ -168,7 +168,7 @@ struct ufs2_dinode {
 struct ufs1_dinode {
 	u_int16_t	di_mode;	/*   0: IFMT, permissions; see below. */
 	int16_t		di_nlink;	/*   2: File link count. */
-	ino_t		di_freelink;	/*   4: SUJ: Next unlinked inode. */
+	uint32_t	di_freelink;	/*   4: SUJ: Next unlinked inode. */
 	u_int64_t	di_size;	/*   8: File byte count. */
 	int32_t		di_atime;	/*  16: Last access time. */
 	int32_t		di_atimensec;	/*  20: Last access time. */
diff --git a/sys/ufs/ufs/extattr.h b/sys/ufs/ufs/extattr.h
index 3716ec2..7dd0235 100644
--- a/sys/ufs/ufs/extattr.h
+++ b/sys/ufs/ufs/extattr.h
@@ -115,10 +115,6 @@ struct extattr {
 
 #include <sys/_sx.h>
 
-#ifdef MALLOC_DECLARE
-MALLOC_DECLARE(M_EXTATTR);
-#endif
-
 struct vnode;
 LIST_HEAD(ufs_extattr_list_head, ufs_extattr_list_entry);
 struct ufs_extattr_list_entry {
diff --git a/sys/ufs/ufs/ufs_vnops.c b/sys/ufs/ufs/ufs_vnops.c
index 733413d..171ef75 100644
--- a/sys/ufs/ufs/ufs_vnops.c
+++ b/sys/ufs/ufs/ufs_vnops.c
@@ -1519,6 +1519,15 @@ relock:
 		cache_purge(fdvp);
 	}
 	error = ufs_dirremove(fdvp, fip, fcnp->cn_flags, 0);
+	/*
+	 * The kern_renameat() looks up the fvp using the DELETE flag, which
+	 * causes the removal of the name cache entry for fvp.
+	 * As the relookup of the fvp is done in two steps:
+	 * ufs_lookup_ino() and then VFS_VGET(), another thread might do a
+	 * normal lookup of the from name just before the VFS_VGET() call,
+	 * causing the cache entry to be re-instantiated.
+	 */
+	cache_purge(fvp);
 
 unlockout:
 	vput(fdvp);
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index d46d170..899369c 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -76,6 +76,18 @@ struct pagerops devicepagerops = {
 	.pgo_haspage =	dev_pager_haspage,
 };
 
+static int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
+    vm_ooffset_t foff, struct ucred *cred, u_short *color);
+static void old_dev_pager_dtor(void *handle);
+static int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset,
+    int prot, vm_page_t *mres);
+
+static struct cdev_pager_ops old_dev_pager_ops = {
+	.cdev_pg_ctor =	old_dev_pager_ctor,
+	.cdev_pg_dtor =	old_dev_pager_dtor,
+	.cdev_pg_fault = old_dev_pager_fault
+};
+
 static void
 dev_pager_init()
 {
@@ -83,22 +95,28 @@ dev_pager_init()
 	mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF);
 }
 
-/*
- * MPSAFE
- */
-static vm_object_t
-dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
-    vm_ooffset_t foff, struct ucred *cred)
+vm_object_t
+cdev_pager_lookup(void *handle)
+{
+	vm_object_t object;
+
+	mtx_lock(&dev_pager_mtx);
+	object = vm_pager_object_lookup(&dev_pager_object_list, handle);
+	vm_object_reference(object);
+	mtx_unlock(&dev_pager_mtx);
+	return (object);
+}
+
+vm_object_t
+cdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops,
+    vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred)
 {
-	struct cdev *dev;
 	vm_object_t object, object1;
 	vm_pindex_t pindex;
-	unsigned int npages;
-	vm_paddr_t paddr;
-	vm_ooffset_t off;
-	vm_memattr_t dummy;
-	struct cdevsw *csw;
-	int ref;
+	u_short color;
+
+	if (tp != OBJT_DEVICE)
+		return (NULL);
 
 	/*
 	 * Offset should be page aligned.
@@ -109,27 +127,8 @@ dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
 	size = round_page(size);
 	pindex = OFF_TO_IDX(foff + size);
 
-	/*
-	 * Make sure this device can be mapped.
-	 */
-	dev = handle;
-	csw = dev_refthread(dev, &ref);
-	if (csw == NULL)
+	if (ops->cdev_pg_ctor(handle, size, prot, foff, cred, &color) != 0)
 		return (NULL);
-
-	/*
-	 * Check that the specified range of the device allows the desired
-	 * protection.
-	 *
-	 * XXX assumes VM_PROT_* == PROT_*
-	 */
-	npages = OFF_TO_IDX(size);
-	for (off = foff; npages--; off += PAGE_SIZE)
-		if (csw->d_mmap(dev, off, &paddr, (int)prot, &dummy) != 0) {
-			dev_relthread(dev, ref);
-			return (NULL);
-		}
-
 	mtx_lock(&dev_pager_mtx);
 
 	/*
@@ -144,9 +143,11 @@ dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
 		 * device's memory.
 		 */
 		mtx_unlock(&dev_pager_mtx);
-		object1 = vm_object_allocate(OBJT_DEVICE, pindex);
+		object1 = vm_object_allocate(tp, pindex);
 		object1->flags |= OBJ_COLORED;
-		object1->pg_color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE);
+		object1->pg_color = color;
+		object1->handle = handle;
+		object1->un_pager.devp.ops = ops;
 		TAILQ_INIT(&object1->un_pager.devp.devp_pglist);
 		mtx_lock(&dev_pager_mtx);
 		object = vm_pager_object_lookup(&dev_pager_object_list, handle);
@@ -162,13 +163,14 @@ dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
 			object->handle = handle;
 			TAILQ_INSERT_TAIL(&dev_pager_object_list, object,
 			    pager_object_list);
+			KASSERT(object->type == tp,
+		("Inconsistent device pager type %p %d", object, tp));
 		}
 	} else {
 		if (pindex > object->size)
 			object->size = pindex;
 	}
 	mtx_unlock(&dev_pager_mtx);
-	dev_relthread(dev, ref);
 	if (object1 != NULL) {
 		object1->handle = object1;
 		mtx_lock(&dev_pager_mtx);
@@ -180,6 +182,24 @@ dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
 	return (object);
 }
 
+static vm_object_t
+dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
+    vm_ooffset_t foff, struct ucred *cred)
+{
+
+	return (cdev_pager_allocate(handle, OBJT_DEVICE, &old_dev_pager_ops,
+	    size, prot, foff, cred));
+}
+
+void
+cdev_pager_free_page(vm_object_t object, vm_page_t m)
+{
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+	TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq);
+	vm_page_putfake(m);
+}
+
 static void
 dev_pager_dealloc(object)
 	vm_object_t object;
@@ -187,6 +207,8 @@ dev_pager_dealloc(object)
 	vm_page_t m;
 
 	VM_OBJECT_UNLOCK(object);
+	object->un_pager.devp.ops->cdev_pg_dtor(object->handle);
+
 	mtx_lock(&dev_pager_mtx);
 	TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list);
 	mtx_unlock(&dev_pager_mtx);
@@ -194,35 +216,57 @@ dev_pager_dealloc(object)
 	/*
 	 * Free up our fake pages.
 	 */
-	while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) != NULL) {
-		TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq);
-		vm_page_putfake(m);
+	while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) != NULL)
+		cdev_pager_free_page(object, m);
+}
+
+static int
+dev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage)
+{
+	int error, i;
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+	error = object->un_pager.devp.ops->cdev_pg_fault(object,
+	    IDX_TO_OFF(ma[reqpage]->pindex), PROT_READ, &ma[reqpage]);
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+
+	for (i = 0; i < count; i++) {
+		if (i != reqpage) {
+			vm_page_lock(ma[i]);
+			vm_page_free(ma[i]);
+			vm_page_unlock(ma[i]);
+		}
+	}
+
+	if (error == VM_PAGER_OK) {
+		TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist,
+		    ma[reqpage], pageq);
 	}
+
+	return (error);
 }
 
 static int
-dev_pager_getpages(object, m, count, reqpage)
-	vm_object_t object;
-	vm_page_t *m;
-	int count;
-	int reqpage;
+old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,
+    vm_page_t *mres)
 {
-	vm_pindex_t offset;
+	vm_pindex_t pidx;
 	vm_paddr_t paddr;
 	vm_page_t m_paddr, page;
-	vm_memattr_t memattr;
 	struct cdev *dev;
-	int i, ref, ret;
 	struct cdevsw *csw;
-	struct thread *td;
 	struct file *fpop;
+	struct thread *td;
+	vm_memattr_t memattr;
+	int ref, ret;
 
-	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
-	dev = object->handle;
-	page = m[reqpage];
-	offset = page->pindex;
+	pidx = OFF_TO_IDX(offset);
 	memattr = object->memattr;
+
 	VM_OBJECT_UNLOCK(object);
+
+	dev = object->handle;
 	csw = dev_refthread(dev, &ref);
 	if (csw == NULL) {
 		VM_OBJECT_LOCK(object);
@@ -231,11 +275,16 @@ dev_pager_getpages(object, m, count, reqpage)
 	td = curthread;
 	fpop = td->td_fpop;
 	td->td_fpop = NULL;
-	ret = csw->d_mmap(dev, (vm_ooffset_t)offset << PAGE_SHIFT, &paddr,
-	    PROT_READ, &memattr);
-	KASSERT(ret == 0, ("dev_pager_getpage: map function returns error"));
+	ret = csw->d_mmap(dev, offset, &paddr, prot, &memattr);
 	td->td_fpop = fpop;
 	dev_relthread(dev, ref);
+	if (ret != 0) {
+		printf(
+	    "WARNING: dev_pager_getpage: map function returns error %d", ret);
+		VM_OBJECT_LOCK(object);
+		return (VM_PAGER_FAIL);
+	}
+
 	/* If "paddr" is a real page, perform a sanity check on "memattr". */
 	if ((m_paddr = vm_phys_paddr_to_vm_page(paddr)) != NULL &&
 	    pmap_page_get_memattr(m_paddr) != memattr) {
@@ -243,23 +292,14 @@ dev_pager_getpages(object, m, count, reqpage)
 		printf(
 	    "WARNING: A device driver has set \"memattr\" inconsistently.\n");
 	}
-	if ((page->flags & PG_FICTITIOUS) != 0) {
+	if (((*mres)->flags & PG_FICTITIOUS) != 0) {
 		/*
-		 * If the passed in reqpage page is a fake page, update it with
+		 * If the passed in result page is a fake page, update it with
 		 * the new physical address.
 		 */
+		page = *mres;
 		VM_OBJECT_LOCK(object);
 		vm_page_updatefake(page, paddr, memattr);
-		if (count > 1) {
-
-			for (i = 0; i < count; i++) {
-				if (i != reqpage) {
-					vm_page_lock(m[i]);
-					vm_page_free(m[i]);
-					vm_page_unlock(m[i]);
-				}
-			}
-		}
 	} else {
 		/*
 		 * Replace the passed in reqpage page with our own fake page and
@@ -267,14 +307,11 @@ dev_pager_getpages(object, m, count, reqpage)
 		 */
 		page = vm_page_getfake(paddr, memattr);
 		VM_OBJECT_LOCK(object);
-		TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq);
-		for (i = 0; i < count; i++) {
-			vm_page_lock(m[i]);
-			vm_page_free(m[i]);
-			vm_page_unlock(m[i]);
-		}
-		vm_page_insert(page, object, offset);
-		m[reqpage] = page;
+		vm_page_lock(*mres);
+		vm_page_free(*mres);
+		vm_page_unlock(*mres);
+		*mres = page;
+		vm_page_insert(page, object, pidx);
 	}
 	page->valid = VM_PAGE_BITS_ALL;
 	return (VM_PAGER_OK);
@@ -288,6 +325,7 @@ dev_pager_putpages(object, m, count, sync, rtvals)
 	boolean_t sync;
 	int *rtvals;
 {
+
 	panic("dev_pager_putpage called");
 }
 
@@ -304,3 +342,50 @@ dev_pager_haspage(object, pindex, before, after)
 		*after = 0;
 	return (TRUE);
 }
+
+static int
+old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
+    vm_ooffset_t foff, struct ucred *cred, u_short *color)
+{
+	struct cdev *dev;
+	struct cdevsw *csw;
+	vm_memattr_t dummy;
+	vm_ooffset_t off;
+	vm_paddr_t paddr;
+	unsigned int npages;
+	int ref;
+
+	/*
+	 * Make sure this device can be mapped.
+	 */
+	dev = handle;
+	csw = dev_refthread(dev, &ref);
+	if (csw == NULL)
+		return (ENXIO);
+
+	/*
+	 * Check that the specified range of the device allows the desired
+	 * protection.
+	 *
+	 * XXX assumes VM_PROT_* == PROT_*
+	 */
+	npages = OFF_TO_IDX(size);
+	for (off = foff; npages--; off += PAGE_SIZE) {
+		if (csw->d_mmap(dev, off, &paddr, (int)prot, &dummy) != 0) {
+			dev_relthread(dev, ref);
+			return (EINVAL);
+		}
+	}
+
+	dev_ref(dev);
+	dev_relthread(dev, ref);
+	*color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE);
+	return (0);
+}
+
+static void
+old_dev_pager_dtor(void *handle)
+{
+
+	dev_rel(handle);
+}
diff --git a/sys/vm/memguard.c b/sys/vm/memguard.c
index 25bea1b..5a690e7 100644
--- a/sys/vm/memguard.c
+++ b/sys/vm/memguard.c
@@ -59,7 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/uma_int.h>
 #include <vm/memguard.h>
 
-SYSCTL_NODE(_vm, OID_AUTO, memguard, CTLFLAG_RW, NULL, "MemGuard data");
+static SYSCTL_NODE(_vm, OID_AUTO, memguard, CTLFLAG_RW, NULL, "MemGuard data");
 /*
  * The vm_memguard_divisor variable controls how much of kmem_map should be
  * reserved for MemGuard.
diff --git a/sys/vm/redzone.c b/sys/vm/redzone.c
index fc4d955..e4b5f6c 100644
--- a/sys/vm/redzone.c
+++ b/sys/vm/redzone.c
@@ -36,7 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/redzone.h>
 
 
-SYSCTL_NODE(_vm, OID_AUTO, redzone, CTLFLAG_RW, NULL, "RedZone data");
+static SYSCTL_NODE(_vm, OID_AUTO, redzone, CTLFLAG_RW, NULL, "RedZone data");
 static u_long redzone_extra_mem = 0;
 SYSCTL_ULONG(_vm_redzone, OID_AUTO, extra_mem, CTLFLAG_RD, &redzone_extra_mem,
     0, "Extra memory allocated by redzone");     
diff --git a/sys/vm/vm_contig.c b/sys/vm/vm_contig.c
index dc87799..ea2c904 100644
--- a/sys/vm/vm_contig.c
+++ b/sys/vm/vm_contig.c
@@ -82,7 +82,6 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_pager.h>
-#include <vm/vm_phys.h>
 #include <vm/vm_extern.h>
 
 static int
@@ -185,22 +184,6 @@ vm_contig_launder(int queue, vm_paddr_t low, vm_paddr_t high)
 }
 
 /*
- *	Frees the given physically contiguous pages.
- *
- *	N.B.: Any pages with PG_ZERO set must, in fact, be zero filled.
- */
-static void
-vm_page_release_contig(vm_page_t m, vm_pindex_t count)
-{
-
-	while (count--) {
-		/* Leave PG_ZERO unchanged. */
-		vm_page_free_toq(m);
-		m++;
-	}
-}
-
-/*
  * Increase the number of cached pages.
  */
 void
@@ -238,9 +221,10 @@ kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
     vm_paddr_t high, vm_memattr_t memattr)
 {
 	vm_object_t object = kernel_object;
-	vm_offset_t addr, i, offset;
+	vm_offset_t addr;
+	vm_ooffset_t end_offset, offset;
 	vm_page_t m;
-	int tries;
+	int pflags, tries;
 
 	size = round_page(size);
 	vm_map_lock(map);
@@ -252,14 +236,22 @@ kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
 	vm_object_reference(object);
 	vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL,
 	    VM_PROT_ALL, 0);
+	if ((flags & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT)
+		pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOBUSY;
+	else
+		pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOBUSY;
+	if (flags & M_ZERO)
+		pflags |= VM_ALLOC_ZERO;
 	VM_OBJECT_LOCK(object);
-	for (i = 0; i < size; i += PAGE_SIZE) {
+	end_offset = offset + size;
+	for (; offset < end_offset; offset += PAGE_SIZE) {
 		tries = 0;
 retry:
-		m = vm_phys_alloc_contig(1, low, high, PAGE_SIZE, 0);
+		m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 1,
+		    low, high, PAGE_SIZE, 0, memattr);
 		if (m == NULL) {
+			VM_OBJECT_UNLOCK(object);
 			if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
-				VM_OBJECT_UNLOCK(object);
 				vm_map_unlock(map);
 				vm_contig_grow_cache(tries, low, high);
 				vm_map_lock(map);
@@ -267,20 +259,16 @@ retry:
 				tries++;
 				goto retry;
 			}
-			while (i != 0) {
-				i -= PAGE_SIZE;
-				m = vm_page_lookup(object, OFF_TO_IDX(offset +
-				    i));
-				vm_page_free(m);
-			}
-			VM_OBJECT_UNLOCK(object);
+			/*
+			 * Since the pages that were allocated by any previous
+			 * iterations of this loop are not busy, they can be
+			 * freed by vm_object_page_remove(), which is called
+			 * by vm_map_delete().
+			 */
 			vm_map_delete(map, addr, addr + size);
 			vm_map_unlock(map);
 			return (0);
 		}
-		if (memattr != VM_MEMATTR_DEFAULT)
-			pmap_page_set_memattr(m, memattr);
-		vm_page_insert(m, object, OFF_TO_IDX(offset + i));
 		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
 			pmap_zero_page(m);
 		m->valid = VM_PAGE_BITS_ALL;
@@ -300,65 +288,61 @@ retry:
  *	specified through the given flags, then the pages are zeroed
  *	before they are mapped.
  */
-static vm_offset_t
-contigmapping(vm_map_t map, vm_size_t size, vm_page_t m, vm_memattr_t memattr,
-    int flags)
+vm_offset_t
+kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
+    vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
+    vm_memattr_t memattr)
 {
 	vm_object_t object = kernel_object;
-	vm_offset_t addr, tmp_addr;
+	vm_offset_t addr;
+	vm_ooffset_t offset;
+	vm_page_t end_m, m;
+	int pflags, tries;
  
+	size = round_page(size);
 	vm_map_lock(map);
 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
 		vm_map_unlock(map);
 		return (0);
 	}
+	offset = addr - VM_MIN_KERNEL_ADDRESS;
 	vm_object_reference(object);
-	vm_map_insert(map, object, addr - VM_MIN_KERNEL_ADDRESS,
-	    addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0);
-	vm_map_unlock(map);
+	vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL,
+	    VM_PROT_ALL, 0);
+	if ((flags & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT)
+		pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOBUSY;
+	else
+		pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOBUSY;
+	if (flags & M_ZERO)
+		pflags |= VM_ALLOC_ZERO;
 	VM_OBJECT_LOCK(object);
-	for (tmp_addr = addr; tmp_addr < addr + size; tmp_addr += PAGE_SIZE) {
-		if (memattr != VM_MEMATTR_DEFAULT)
-			pmap_page_set_memattr(m, memattr);
-		vm_page_insert(m, object,
-		    OFF_TO_IDX(tmp_addr - VM_MIN_KERNEL_ADDRESS));
-		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
-			pmap_zero_page(m);
-		m->valid = VM_PAGE_BITS_ALL;
-		m++;
-	}
-	VM_OBJECT_UNLOCK(object);
-	vm_map_wire(map, addr, addr + size,
-	    VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
-	return (addr);
-}
-
-vm_offset_t
-kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
-    vm_paddr_t high, unsigned long alignment, unsigned long boundary,
-    vm_memattr_t memattr)
-{
-	vm_offset_t ret;
-	vm_page_t pages;
-	unsigned long npgs;
-	int tries;
-
-	size = round_page(size);
-	npgs = size >> PAGE_SHIFT;
 	tries = 0;
 retry:
-	pages = vm_phys_alloc_contig(npgs, low, high, alignment, boundary);
-	if (pages == NULL) {
+	m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags,
+	    atop(size), low, high, alignment, boundary, memattr);
+	if (m == NULL) {
+		VM_OBJECT_UNLOCK(object);
 		if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
+			vm_map_unlock(map);
 			vm_contig_grow_cache(tries, low, high);
+			vm_map_lock(map);
+			VM_OBJECT_LOCK(object);
 			tries++;
 			goto retry;
 		}
-		ret = 0;
-	} else {
-		ret = contigmapping(map, size, pages, memattr, flags);
-		if (ret == 0)
-			vm_page_release_contig(pages, npgs);
+		vm_map_delete(map, addr, addr + size);
+		vm_map_unlock(map);
+		return (0);
+	}
+	end_m = m + atop(size);
+	for (; m < end_m; m++) {
+		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
+			pmap_zero_page(m);
+		m->valid = VM_PAGE_BITS_ALL;
 	}
-	return (ret);
+	VM_OBJECT_UNLOCK(object);
+	vm_map_unlock(map);
+	vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM |
+	    VM_MAP_WIRE_NOHOLES);
+	return (addr);
 }
diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h
index 72f0254..74a9b39 100644
--- a/sys/vm/vm_extern.h
+++ b/sys/vm/vm_extern.h
@@ -44,8 +44,8 @@ vm_offset_t kmem_alloc(vm_map_t, vm_size_t);
 vm_offset_t kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags,
     vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr);
 vm_offset_t kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags,
-    vm_paddr_t low, vm_paddr_t high, unsigned long alignment,
-    unsigned long boundary, vm_memattr_t memattr);
+    vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
+    vm_memattr_t memattr);
 vm_offset_t kmem_alloc_nofault(vm_map_t, vm_size_t);
 vm_offset_t kmem_alloc_nofault_space(vm_map_t, vm_size_t, int);
 vm_offset_t kmem_alloc_wait(vm_map_t, vm_size_t);
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index 24c2448..2c23c48 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -543,7 +543,7 @@ kmem_init_zero_region(void)
 	 * zeros, while not using much more physical resources.
 	 */
 	addr = kmem_alloc_nofault(kernel_map, ZERO_REGION_SIZE);
-	m = vm_page_alloc(NULL, OFF_TO_IDX(addr - VM_MIN_KERNEL_ADDRESS),
+	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 	    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 	if ((m->flags & PG_ZERO) == 0)
 		pmap_zero_page(m);
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index 8493478..d62576f 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -464,9 +464,9 @@ _vm_map_lock(vm_map_t map, const char *file, int line)
 {
 
 	if (map->system_map)
-		_mtx_lock_flags(&map->system_mtx, 0, file, line);
+		mtx_lock_flags_(&map->system_mtx, 0, file, line);
 	else
-		(void)_sx_xlock(&map->lock, 0, file, line);
+		sx_xlock_(&map->lock, file, line);
 	map->timestamp++;
 }
 
@@ -489,9 +489,9 @@ _vm_map_unlock(vm_map_t map, const char *file, int line)
 {
 
 	if (map->system_map)
-		_mtx_unlock_flags(&map->system_mtx, 0, file, line);
+		mtx_unlock_flags_(&map->system_mtx, 0, file, line);
 	else {
-		_sx_xunlock(&map->lock, file, line);
+		sx_xunlock_(&map->lock, file, line);
 		vm_map_process_deferred();
 	}
 }
@@ -501,9 +501,9 @@ _vm_map_lock_read(vm_map_t map, const char *file, int line)
 {
 
 	if (map->system_map)
-		_mtx_lock_flags(&map->system_mtx, 0, file, line);
+		mtx_lock_flags_(&map->system_mtx, 0, file, line);
 	else
-		(void)_sx_slock(&map->lock, 0, file, line);
+		sx_slock_(&map->lock, file, line);
 }
 
 void
@@ -511,9 +511,9 @@ _vm_map_unlock_read(vm_map_t map, const char *file, int line)
 {
 
 	if (map->system_map)
-		_mtx_unlock_flags(&map->system_mtx, 0, file, line);
+		mtx_unlock_flags_(&map->system_mtx, 0, file, line);
 	else {
-		_sx_sunlock(&map->lock, file, line);
+		sx_sunlock_(&map->lock, file, line);
 		vm_map_process_deferred();
 	}
 }
@@ -524,8 +524,8 @@ _vm_map_trylock(vm_map_t map, const char *file, int line)
 	int error;
 
 	error = map->system_map ?
-	    !_mtx_trylock(&map->system_mtx, 0, file, line) :
-	    !_sx_try_xlock(&map->lock, file, line);
+	    !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
+	    !sx_try_xlock_(&map->lock, file, line);
 	if (error == 0)
 		map->timestamp++;
 	return (error == 0);
@@ -537,8 +537,8 @@ _vm_map_trylock_read(vm_map_t map, const char *file, int line)
 	int error;
 
 	error = map->system_map ?
-	    !_mtx_trylock(&map->system_mtx, 0, file, line) :
-	    !_sx_try_slock(&map->lock, file, line);
+	    !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
+	    !sx_try_slock_(&map->lock, file, line);
 	return (error == 0);
 }
 
@@ -558,21 +558,19 @@ _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
 	unsigned int last_timestamp;
 
 	if (map->system_map) {
-#ifdef INVARIANTS
-		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
-#endif
+		mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
 	} else {
-		if (!_sx_try_upgrade(&map->lock, file, line)) {
+		if (!sx_try_upgrade_(&map->lock, file, line)) {
 			last_timestamp = map->timestamp;
-			_sx_sunlock(&map->lock, file, line);
+			sx_sunlock_(&map->lock, file, line);
 			vm_map_process_deferred();
 			/*
 			 * If the map's timestamp does not change while the
 			 * map is unlocked, then the upgrade succeeds.
 			 */
-			(void)_sx_xlock(&map->lock, 0, file, line);
+			sx_xlock_(&map->lock, file, line);
 			if (last_timestamp != map->timestamp) {
-				_sx_xunlock(&map->lock, file, line);
+				sx_xunlock_(&map->lock, file, line);
 				return (1);
 			}
 		}
@@ -586,11 +584,9 @@ _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
 {
 
 	if (map->system_map) {
-#ifdef INVARIANTS
-		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
-#endif
+		mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
 	} else
-		_sx_downgrade(&map->lock, file, line);
+		sx_downgrade_(&map->lock, file, line);
 }
 
 /*
@@ -615,30 +611,15 @@ _vm_map_assert_locked(vm_map_t map, const char *file, int line)
 {
 
 	if (map->system_map)
-		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
-	else
-		_sx_assert(&map->lock, SA_XLOCKED, file, line);
-}
-
-#if 0
-static void
-_vm_map_assert_locked_read(vm_map_t map, const char *file, int line)
-{
-
-	if (map->system_map)
-		_mtx_assert(&map->system_mtx, MA_OWNED, file, line);
+		mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
 	else
-		_sx_assert(&map->lock, SA_SLOCKED, file, line);
+		sx_assert_(&map->lock, SA_XLOCKED, file, line);
 }
-#endif
 
 #define	VM_MAP_ASSERT_LOCKED(map) \
     _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
-#define	VM_MAP_ASSERT_LOCKED_READ(map) \
-    _vm_map_assert_locked_read(map, LOCK_FILE, LOCK_LINE)
 #else
 #define	VM_MAP_ASSERT_LOCKED(map)
-#define	VM_MAP_ASSERT_LOCKED_READ(map)
 #endif
 
 /*
@@ -661,9 +642,9 @@ _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line)
 
 	mtx_lock(&map_sleep_mtx);
 	if (map->system_map)
-		_mtx_unlock_flags(&map->system_mtx, 0, file, line);
+		mtx_unlock_flags_(&map->system_mtx, 0, file, line);
 	else
-		_sx_xunlock(&map->lock, file, line);
+		sx_xunlock_(&map->lock, file, line);
 	return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps",
 	    timo));
 }
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 3de793b..250b769 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -139,7 +139,8 @@ struct mtx vm_object_list_mtx;	/* lock for object list and count */
 struct vm_object kernel_object_store;
 struct vm_object kmem_object_store;
 
-SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0, "VM object stats");
+static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0,
+    "VM object stats");
 
 static long object_collapses;
 SYSCTL_LONG(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
@@ -1863,6 +1864,60 @@ skipmemq:
 }
 
 /*
+ *	vm_object_page_cache:
+ *
+ *	For the given object, attempt to move the specified clean
+ *	pages to the cache queue.  If a page is wired for any reason,
+ *	then it will not be changed.  Pages are specified by the given
+ *	range ["start", "end").  As a special case, if "end" is zero,
+ *	then the range extends from "start" to the end of the object.
+ *	Any mappings to the specified pages are removed before the
+ *	pages are moved to the cache queue.
+ *
+ *	This operation should only be performed on objects that
+ *	contain managed pages.
+ *
+ *	The object must be locked.
+ */
+void
+vm_object_page_cache(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
+{
+	struct mtx *mtx, *new_mtx;
+	vm_page_t p, next;
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+	KASSERT((object->type != OBJT_DEVICE && object->type != OBJT_SG &&
+	    object->type != OBJT_PHYS),
+	    ("vm_object_page_cache: illegal object %p", object));
+	if (object->resident_page_count == 0)
+		return;
+	p = vm_page_find_least(object, start);
+
+	/*
+	 * Here, the variable "p" is either (1) the page with the least pindex
+	 * greater than or equal to the parameter "start" or (2) NULL. 
+	 */
+	mtx = NULL;
+	for (; p != NULL && (p->pindex < end || end == 0); p = next) {
+		next = TAILQ_NEXT(p, listq);
+
+		/*
+		 * Avoid releasing and reacquiring the same page lock.
+		 */
+		new_mtx = vm_page_lockptr(p);
+		if (mtx != new_mtx) {
+			if (mtx != NULL)
+				mtx_unlock(mtx);
+			mtx = new_mtx;
+			mtx_lock(mtx);
+		}
+		vm_page_try_to_cache(p);
+	}
+	if (mtx != NULL)
+		mtx_unlock(mtx);
+}
+
+/*
  *	Populate the specified range of the object with valid pages.  Returns
  *	TRUE if the range is successfully populated and FALSE otherwise.
  *
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index a11f144..0c6c875 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -121,6 +121,7 @@ struct vm_object {
 		 */
 		struct {
 			TAILQ_HEAD(, vm_page) devp_pglist;
+			struct cdev_pager_ops *ops;
 		} devp;
 
 		/*
@@ -223,6 +224,8 @@ void vm_object_destroy (vm_object_t);
 void vm_object_terminate (vm_object_t);
 void vm_object_set_writeable_dirty (vm_object_t);
 void vm_object_init (void);
+void vm_object_page_cache(vm_object_t object, vm_pindex_t start,
+    vm_pindex_t end);
 void vm_object_page_clean(vm_object_t object, vm_ooffset_t start,
     vm_ooffset_t end, int flags);
 void vm_object_page_remove(vm_object_t object, vm_pindex_t start,
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index cde1387..d592ac0 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -137,7 +137,8 @@ SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD,
 
 static uma_zone_t fakepg_zone;
 
-static void vm_page_clear_dirty_mask(vm_page_t m, int pagebits);
+static struct vnode *vm_page_alloc_init(vm_page_t m);
+static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
 static void vm_page_queue_remove(int queue, vm_page_t m);
 static void vm_page_enqueue(int queue, vm_page_t m);
 static void vm_page_init_fakepg(void *dummy);
@@ -1286,8 +1287,9 @@ vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
 /*
  *	vm_page_alloc:
  *
- *	Allocate and return a memory cell associated
- *	with this VM object/offset pair.
+ *	Allocate and return a page that is associated with the specified
+ *	object and offset pair.  By default, this page has the flag VPO_BUSY
+ *	set.
  *
  *	The caller must always specify an allocation class.
  *
@@ -1297,13 +1299,16 @@ vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
  *	VM_ALLOC_INTERRUPT	interrupt time request
  *
  *	optional allocation flags:
- *	VM_ALLOC_ZERO		prefer a zeroed page
- *	VM_ALLOC_WIRED		wire the allocated page
- *	VM_ALLOC_NOOBJ		page is not associated with a vm object
- *	VM_ALLOC_NOBUSY		do not set the page busy
+ *	VM_ALLOC_COUNT(number)	the number of additional pages that the caller
+ *				intends to allocate
  *	VM_ALLOC_IFCACHED	return page only if it is cached
  *	VM_ALLOC_IFNOTCACHED	return NULL, do not reactivate if the page
  *				is cached
+ *	VM_ALLOC_NOBUSY		do not set the flag VPO_BUSY on the page
+ *	VM_ALLOC_NOOBJ		page is not associated with an object and
+ *				should not have the flag VPO_BUSY set
+ *	VM_ALLOC_WIRED		wire the allocated page
+ *	VM_ALLOC_ZERO		prefer a zeroed page
  *
  *	This routine may not sleep.
  */
@@ -1313,27 +1318,26 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 	struct vnode *vp = NULL;
 	vm_object_t m_object;
 	vm_page_t m;
-	int flags, page_req;
+	int flags, req_class;
 
-	if ((req & VM_ALLOC_NOOBJ) == 0) {
-		KASSERT(object != NULL,
-		    ("vm_page_alloc: NULL object."));
+	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0),
+	    ("vm_page_alloc: inconsistent object/req"));
+	if (object != NULL)
 		VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
-	}
 
-	page_req = req & VM_ALLOC_CLASS_MASK;
+	req_class = req & VM_ALLOC_CLASS_MASK;
 
 	/*
-	 * The pager is allowed to eat deeper into the free page list.
+	 * The page daemon is allowed to dig deeper into the free page list.
 	 */
-	if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT))
-		page_req = VM_ALLOC_SYSTEM;
+	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
+		req_class = VM_ALLOC_SYSTEM;
 
 	mtx_lock(&vm_page_queue_free_mtx);
 	if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
-	    (page_req == VM_ALLOC_SYSTEM && 
+	    (req_class == VM_ALLOC_SYSTEM &&
 	    cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
-	    (page_req == VM_ALLOC_INTERRUPT &&
+	    (req_class == VM_ALLOC_INTERRUPT &&
 	    cnt.v_free_count + cnt.v_cache_count > 0)) {
 		/*
 		 * Allocate from the free queue if the number of free pages
@@ -1381,7 +1385,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 		 */
 		mtx_unlock(&vm_page_queue_free_mtx);
 		atomic_add_int(&vm_pageout_deficit,
-		    MAX((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
+		    max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
 		pagedaemon_wakeup();
 		return (NULL);
 	}
@@ -1389,7 +1393,6 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 	/*
 	 *  At this point we had better have found a good page.
 	 */
-
 	KASSERT(m != NULL, ("vm_page_alloc: missing page"));
 	KASSERT(m->queue == PQ_NONE,
 	    ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue));
@@ -1401,6 +1404,8 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 	    ("vm_page_alloc: page %p has unexpected memattr %d", m,
 	    pmap_page_get_memattr(m)));
 	if ((m->flags & PG_CACHED) != 0) {
+		KASSERT((m->flags & PG_ZERO) == 0,
+		    ("vm_page_alloc: cached page %p is PG_ZERO", m));
 		KASSERT(m->valid != 0,
 		    ("vm_page_alloc: cached page %p is invalid", m));
 		if (m->object == object && m->pindex == pindex)
@@ -1477,12 +1482,163 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 }
 
 /*
+ *	vm_page_alloc_contig:
+ *
+ *	Allocate a contiguous set of physical pages of the given size "npages"
+ *	from the free lists.  All of the physical pages must be at or above
+ *	the given physical address "low" and below the given physical address
+ *	"high".  The given value "alignment" determines the alignment of the
+ *	first physical page in the set.  If the given value "boundary" is
+ *	non-zero, then the set of physical pages cannot cross any physical
+ *	address boundary that is a multiple of that value.  Both "alignment"
+ *	and "boundary" must be a power of two.
+ *
+ *	If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT,
+ *	then the memory attribute setting for the physical pages is configured
+ *	to the object's memory attribute setting.  Otherwise, the memory
+ *	attribute setting for the physical pages is configured to "memattr",
+ *	overriding the object's memory attribute setting.  However, if the
+ *	object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the
+ *	memory attribute setting for the physical pages cannot be configured
+ *	to VM_MEMATTR_DEFAULT.
+ *
+ *	The caller must always specify an allocation class.
+ *
+ *	allocation classes:
+ *	VM_ALLOC_NORMAL		normal process request
+ *	VM_ALLOC_SYSTEM		system *really* needs a page
+ *	VM_ALLOC_INTERRUPT	interrupt time request
+ *
+ *	optional allocation flags:
+ *	VM_ALLOC_NOBUSY		do not set the flag VPO_BUSY on the page
+ *	VM_ALLOC_NOOBJ		page is not associated with an object and
+ *				should not have the flag VPO_BUSY set
+ *	VM_ALLOC_WIRED		wire the allocated page
+ *	VM_ALLOC_ZERO		prefer a zeroed page
+ *
+ *	This routine may not sleep.
+ */
+vm_page_t
+vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
+    u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
+    vm_paddr_t boundary, vm_memattr_t memattr)
+{
+	struct vnode *drop;
+	vm_page_t deferred_vdrop_list, m, m_ret;
+	u_int flags, oflags;
+	int req_class;
+
+	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0),
+	    ("vm_page_alloc_contig: inconsistent object/req"));
+	if (object != NULL) {
+		VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+		KASSERT(object->type == OBJT_PHYS,
+		    ("vm_page_alloc_contig: object %p isn't OBJT_PHYS",
+		    object));
+	}
+	KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));
+	req_class = req & VM_ALLOC_CLASS_MASK;
+
+	/*
+	 * The page daemon is allowed to dig deeper into the free page list.
+	 */
+	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
+		req_class = VM_ALLOC_SYSTEM;
+
+	deferred_vdrop_list = NULL;
+	mtx_lock(&vm_page_queue_free_mtx);
+	if (cnt.v_free_count + cnt.v_cache_count >= npages +
+	    cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM &&
+	    cnt.v_free_count + cnt.v_cache_count >= npages +
+	    cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT &&
+	    cnt.v_free_count + cnt.v_cache_count >= npages)) {
+#if VM_NRESERVLEVEL > 0
+retry:
+#endif
+		m_ret = vm_phys_alloc_contig(npages, low, high, alignment,
+		    boundary);
+	} else {
+		mtx_unlock(&vm_page_queue_free_mtx);
+		atomic_add_int(&vm_pageout_deficit, npages);
+		pagedaemon_wakeup();
+		return (NULL);
+	}
+	if (m_ret != NULL)
+		for (m = m_ret; m < &m_ret[npages]; m++) {
+			drop = vm_page_alloc_init(m);
+			if (drop != NULL) {
+				/*
+				 * Enqueue the vnode for deferred vdrop().
+				 *
+				 * Once the pages are removed from the free
+				 * page list, "pageq" can be safely abused to
+				 * construct a short-lived list of vnodes.
+				 */
+				m->pageq.tqe_prev = (void *)drop;
+				m->pageq.tqe_next = deferred_vdrop_list;
+				deferred_vdrop_list = m;
+			}
+		}
+	else {
+#if VM_NRESERVLEVEL > 0
+		if (vm_reserv_reclaim_contig(npages << PAGE_SHIFT, low, high,
+		    alignment, boundary))
+			goto retry;
+#endif
+	}
+	mtx_unlock(&vm_page_queue_free_mtx);
+	if (m_ret == NULL)
+		return (NULL);
+
+	/*
+	 * Initialize the pages.  Only the PG_ZERO flag is inherited.
+	 */
+	flags = 0;
+	if ((req & VM_ALLOC_ZERO) != 0)
+		flags = PG_ZERO;
+	if ((req & VM_ALLOC_WIRED) != 0)
+		atomic_add_int(&cnt.v_wire_count, npages);
+	oflags = VPO_UNMANAGED;
+	if (object != NULL) {
+		if ((req & VM_ALLOC_NOBUSY) == 0)
+			oflags |= VPO_BUSY;
+		if (object->memattr != VM_MEMATTR_DEFAULT &&
+		    memattr == VM_MEMATTR_DEFAULT)
+			memattr = object->memattr;
+	}
+	for (m = m_ret; m < &m_ret[npages]; m++) {
+		m->aflags = 0;
+		m->flags &= flags;
+		if ((req & VM_ALLOC_WIRED) != 0)
+			m->wire_count = 1;
+		/* Unmanaged pages don't use "act_count". */
+		m->oflags = oflags;
+		if (memattr != VM_MEMATTR_DEFAULT)
+			pmap_page_set_memattr(m, memattr);
+		if (object != NULL)
+			vm_page_insert(m, object, pindex);
+		else
+			m->pindex = pindex;
+		pindex++;
+	}
+	while (deferred_vdrop_list != NULL) {
+		vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev);
+		deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next;
+	}
+	if (vm_paging_needed())
+		pagedaemon_wakeup();
+	return (m_ret);
+}
+
+/*
  * Initialize a page that has been freshly dequeued from a freelist.
  * The caller has to drop the vnode returned, if it is not NULL.
  *
+ * This function may only be used to initialize unmanaged pages.
+ *
  * To be called with vm_page_queue_free_mtx held.
  */
-struct vnode *
+static struct vnode *
 vm_page_alloc_init(vm_page_t m)
 {
 	struct vnode *drop;
@@ -1505,11 +1661,12 @@ vm_page_alloc_init(vm_page_t m)
 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
 	drop = NULL;
 	if ((m->flags & PG_CACHED) != 0) {
+		KASSERT((m->flags & PG_ZERO) == 0,
+		    ("vm_page_alloc_init: cached page %p is PG_ZERO", m));
 		m->valid = 0;
 		m_object = m->object;
 		vm_page_cache_remove(m);
-		if (m_object->type == OBJT_VNODE &&
-		    m_object->cache == NULL)
+		if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
 			drop = m_object->handle;
 	} else {
 		KASSERT(VM_PAGE_IS_FREE(m),
@@ -1517,43 +1674,66 @@ vm_page_alloc_init(vm_page_t m)
 		KASSERT(m->valid == 0,
 		    ("vm_page_alloc_init: free page %p is valid", m));
 		cnt.v_free_count--;
+		if ((m->flags & PG_ZERO) != 0)
+			vm_page_zero_count--;
 	}
-	if (m->flags & PG_ZERO)
-		vm_page_zero_count--;
 	/* Don't clear the PG_ZERO flag; we'll need it later. */
 	m->flags &= PG_ZERO;
-	m->aflags = 0;
-	m->oflags = VPO_UNMANAGED;
-	/* Unmanaged pages don't use "act_count". */
 	return (drop);
 }
 
 /*
  * 	vm_page_alloc_freelist:
- * 
- *	Allocate a page from the specified freelist.
- *	Only the ALLOC_CLASS values in req are honored, other request flags
- *	are ignored.
+ *
+ *	Allocate a physical page from the specified free page list.
+ *
+ *	The caller must always specify an allocation class.
+ *
+ *	allocation classes:
+ *	VM_ALLOC_NORMAL		normal process request
+ *	VM_ALLOC_SYSTEM		system *really* needs a page
+ *	VM_ALLOC_INTERRUPT	interrupt time request
+ *
+ *	optional allocation flags:
+ *	VM_ALLOC_COUNT(number)	the number of additional pages that the caller
+ *				intends to allocate
+ *	VM_ALLOC_WIRED		wire the allocated page
+ *	VM_ALLOC_ZERO		prefer a zeroed page
+ *
+ *	This routine may not sleep.
  */
 vm_page_t
 vm_page_alloc_freelist(int flind, int req)
 {
 	struct vnode *drop;
 	vm_page_t m;
-	int page_req;
+	u_int flags;
+	int req_class;
+
+	req_class = req & VM_ALLOC_CLASS_MASK;
+
+	/*
+	 * The page daemon is allowed to dig deeper into the free page list.
+	 */
+	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
+		req_class = VM_ALLOC_SYSTEM;
 
-	m = NULL;
-	page_req = req & VM_ALLOC_CLASS_MASK;
-	mtx_lock(&vm_page_queue_free_mtx);
 	/*
 	 * Do not allocate reserved pages unless the req has asked for it.
 	 */
+	mtx_lock(&vm_page_queue_free_mtx);
 	if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
-	    (page_req == VM_ALLOC_SYSTEM && 
+	    (req_class == VM_ALLOC_SYSTEM &&
 	    cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
-	    (page_req == VM_ALLOC_INTERRUPT &&
-	    cnt.v_free_count + cnt.v_cache_count > 0)) {
+	    (req_class == VM_ALLOC_INTERRUPT &&
+	    cnt.v_free_count + cnt.v_cache_count > 0))
 		m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0);
+	else {
+		mtx_unlock(&vm_page_queue_free_mtx);
+		atomic_add_int(&vm_pageout_deficit,
+		    max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
+		pagedaemon_wakeup();
+		return (NULL);
 	}
 	if (m == NULL) {
 		mtx_unlock(&vm_page_queue_free_mtx);
@@ -1561,8 +1741,29 @@ vm_page_alloc_freelist(int flind, int req)
 	}
 	drop = vm_page_alloc_init(m);
 	mtx_unlock(&vm_page_queue_free_mtx);
-	if (drop)
+
+	/*
+	 * Initialize the page.  Only the PG_ZERO flag is inherited.
+	 */
+	m->aflags = 0;
+	flags = 0;
+	if ((req & VM_ALLOC_ZERO) != 0)
+		flags = PG_ZERO;
+	m->flags &= flags;
+	if ((req & VM_ALLOC_WIRED) != 0) {
+		/*
+		 * The page lock is not required for wiring a page that does
+		 * not belong to an object.
+		 */
+		atomic_add_int(&cnt.v_wire_count, 1);
+		m->wire_count = 1;
+	}
+	/* Unmanaged pages don't use "act_count". */
+	m->oflags = VPO_UNMANAGED;
+	if (drop != NULL)
 		vdrop(drop);
+	if (vm_paging_needed())
+		pagedaemon_wakeup();
 	return (m);
 }
 
@@ -2315,7 +2516,7 @@ retrylookup:
  *
  * Inputs are required to range within a page.
  */
-int
+vm_page_bits_t
 vm_page_bits(int base, int size)
 {
 	int first_bit;
@@ -2332,7 +2533,8 @@ vm_page_bits(int base, int size)
 	first_bit = base >> DEV_BSHIFT;
 	last_bit = (base + size - 1) >> DEV_BSHIFT;
 
-	return ((2 << last_bit) - (1 << first_bit));
+	return (((vm_page_bits_t)2 << last_bit) -
+	    ((vm_page_bits_t)1 << first_bit));
 }
 
 /*
@@ -2391,7 +2593,7 @@ vm_page_set_valid(vm_page_t m, int base, int size)
  * Clear the given bits from the specified page's dirty field.
  */
 static __inline void
-vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
+vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits)
 {
 	uintptr_t addr;
 #if PAGE_SIZE < 16384
@@ -2420,7 +2622,6 @@ vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
 		 */
 		addr = (uintptr_t)&m->dirty;
 #if PAGE_SIZE == 32768
-#error pagebits too short
 		atomic_clear_64((uint64_t *)addr, pagebits);
 #elif PAGE_SIZE == 16384
 		atomic_clear_32((uint32_t *)addr, pagebits);
@@ -2457,8 +2658,8 @@ vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
 void
 vm_page_set_validclean(vm_page_t m, int base, int size)
 {
-	u_long oldvalid;
-	int endoff, frag, pagebits;
+	vm_page_bits_t oldvalid, pagebits;
+	int endoff, frag;
 
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 	if (size == 0)	/* handle degenerate case */
@@ -2470,7 +2671,7 @@ vm_page_set_validclean(vm_page_t m, int base, int size)
 	 * first block.
 	 */
 	if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
-	    (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
+	    (m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0)
 		pmap_zero_page_area(m, frag, base - frag);
 
 	/*
@@ -2480,7 +2681,7 @@ vm_page_set_validclean(vm_page_t m, int base, int size)
 	 */
 	endoff = base + size;
 	if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
-	    (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
+	    (m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0)
 		pmap_zero_page_area(m, endoff,
 		    DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
 
@@ -2550,7 +2751,7 @@ vm_page_clear_dirty(vm_page_t m, int base, int size)
 void
 vm_page_set_invalid(vm_page_t m, int base, int size)
 {
-	int bits;
+	vm_page_bits_t bits;
 
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 	KASSERT((m->oflags & VPO_BUSY) == 0,
@@ -2590,8 +2791,7 @@ vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
 	 */
 	for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
 		if (i == (PAGE_SIZE / DEV_BSIZE) || 
-		    (m->valid & (1 << i))
-		) {
+		    (m->valid & ((vm_page_bits_t)1 << i))) {
 			if (i > b) {
 				pmap_zero_page_area(m, 
 				    b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
@@ -2621,9 +2821,10 @@ vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
 int
 vm_page_is_valid(vm_page_t m, int base, int size)
 {
-	int bits = vm_page_bits(base, size);
+	vm_page_bits_t bits;
 
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
+	bits = vm_page_bits(base, size);
 	if (m->valid && ((m->valid & bits) == bits))
 		return 1;
 	else
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 23637bb..151710d 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -113,6 +113,20 @@
 
 TAILQ_HEAD(pglist, vm_page);
 
+#if PAGE_SIZE == 4096
+#define VM_PAGE_BITS_ALL 0xffu
+typedef uint8_t vm_page_bits_t;
+#elif PAGE_SIZE == 8192
+#define VM_PAGE_BITS_ALL 0xffffu
+typedef uint16_t vm_page_bits_t;
+#elif PAGE_SIZE == 16384
+#define VM_PAGE_BITS_ALL 0xffffffffu
+typedef uint32_t vm_page_bits_t;
+#elif PAGE_SIZE == 32768
+#define VM_PAGE_BITS_ALL 0xfffffffffffffffflu
+typedef uint64_t vm_page_bits_t;
+#endif
+
 struct vm_page {
 	TAILQ_ENTRY(vm_page) pageq;	/* queue info for FIFO queue or free list (Q) */
 	TAILQ_ENTRY(vm_page) listq;	/* pages in same object (O) 	*/
@@ -137,20 +151,8 @@ struct vm_page {
 	u_char	busy;			/* page busy count (O) */
 	/* NOTE that these must support one bit per DEV_BSIZE in a page!!! */
 	/* so, on normal X86 kernels, they must be at least 8 bits wide */
-	/* In reality, support for 32KB pages is not fully implemented. */
-#if PAGE_SIZE == 4096
-	uint8_t	valid;			/* map of valid DEV_BSIZE chunks (O) */
-	uint8_t	dirty;			/* map of dirty DEV_BSIZE chunks (M) */
-#elif PAGE_SIZE == 8192
-	uint16_t valid;			/* map of valid DEV_BSIZE chunks (O) */
-	uint16_t dirty;			/* map of dirty DEV_BSIZE chunks (M) */
-#elif PAGE_SIZE == 16384
-	uint32_t valid;			/* map of valid DEV_BSIZE chunks (O) */
-	uint32_t dirty;			/* map of dirty DEV_BSIZE chunks (M) */
-#elif PAGE_SIZE == 32768
-	uint64_t valid;			/* map of valid DEV_BSIZE chunks (O) */
-	uint64_t dirty;			/* map of dirty DEV_BSIZE chunks (M) */
-#endif
+	vm_page_bits_t valid;		/* map of valid DEV_BSIZE chunks (O) */
+	vm_page_bits_t dirty;		/* map of dirty DEV_BSIZE chunks (M) */
 };
 
 /*
@@ -322,16 +324,6 @@ extern struct vpglocks vm_page_queue_lock;
 #define vm_page_lock_queues()   mtx_lock(&vm_page_queue_mtx)
 #define vm_page_unlock_queues() mtx_unlock(&vm_page_queue_mtx)
 
-#if PAGE_SIZE == 4096
-#define VM_PAGE_BITS_ALL 0xffu
-#elif PAGE_SIZE == 8192
-#define VM_PAGE_BITS_ALL 0xffffu
-#elif PAGE_SIZE == 16384
-#define VM_PAGE_BITS_ALL 0xffffffffu
-#elif PAGE_SIZE == 32768
-#define VM_PAGE_BITS_ALL 0xfffffffffffffffflu
-#endif
-
 /* page allocation classes: */
 #define VM_ALLOC_NORMAL		0
 #define VM_ALLOC_INTERRUPT	1
@@ -367,8 +359,10 @@ void vm_pageq_remove(vm_page_t m);
 
 void vm_page_activate (vm_page_t);
 vm_page_t vm_page_alloc (vm_object_t, vm_pindex_t, int);
+vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
+    u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
+    vm_paddr_t boundary, vm_memattr_t memattr);
 vm_page_t vm_page_alloc_freelist(int, int);
-struct vnode *vm_page_alloc_init(vm_page_t);
 vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int);
 void vm_page_cache(vm_page_t);
 void vm_page_cache_free(vm_object_t, vm_pindex_t, vm_pindex_t);
@@ -403,7 +397,7 @@ void vm_page_clear_dirty (vm_page_t, int, int);
 void vm_page_set_invalid (vm_page_t, int, int);
 int vm_page_is_valid (vm_page_t, int, int);
 void vm_page_test_dirty (vm_page_t);
-int vm_page_bits (int, int);
+vm_page_bits_t vm_page_bits(int base, int size);
 void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid);
 void vm_page_free_toq(vm_page_t m);
 void vm_page_zero_idle_wakeup(void);
diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c
index 5713ddb..eba926e 100644
--- a/sys/vm/vm_pager.c
+++ b/sys/vm/vm_pager.c
@@ -188,7 +188,7 @@ vm_pager_init()
 	 * Initialize known pagers
 	 */
 	for (pgops = pagertab; pgops < &pagertab[npagers]; pgops++)
-		if (pgops && ((*pgops)->pgo_init != NULL))
+		if ((*pgops)->pgo_init != NULL)
 			(*(*pgops)->pgo_init) ();
 }
 
diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h
index c18719a..c361be8 100644
--- a/sys/vm/vm_pager.h
+++ b/sys/vm/vm_pager.h
@@ -192,5 +192,19 @@ vm_pager_page_unswapped(vm_page_t m)
 		(*pagertab[m->object->type]->pgo_pageunswapped)(m);
 }
 
+struct cdev_pager_ops {
+	int (*cdev_pg_fault)(vm_object_t vm_obj, vm_ooffset_t offset,
+	    int prot, vm_page_t *mres);
+	int (*cdev_pg_ctor)(void *handle, vm_ooffset_t size, vm_prot_t prot,
+	    vm_ooffset_t foff, struct ucred *cred, u_short *color);
+	void (*cdev_pg_dtor)(void *handle);
+};
+
+vm_object_t cdev_pager_allocate(void *handle, enum obj_type tp,
+    struct cdev_pager_ops *ops, vm_ooffset_t size, vm_prot_t prot,
+    vm_ooffset_t foff, struct ucred *cred);
+vm_object_t cdev_pager_lookup(void *handle);
+void cdev_pager_free_page(vm_object_t object, vm_page_t m);
+
 #endif				/* _KERNEL */
 #endif				/* _VM_PAGER_ */
diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c
index b47455d..8c026e6 100644
--- a/sys/vm/vm_phys.c
+++ b/sys/vm/vm_phys.c
@@ -29,11 +29,17 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+/*
+ *	Physical memory system implementation
+ *
+ * Any external functions defined by this module are only to be used by the
+ * virtual memory system.
+ */
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
-#include "opt_vm.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -45,7 +51,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/sbuf.h>
 #include <sys/sysctl.h>
 #include <sys/vmmeter.h>
-#include <sys/vnode.h>
 
 #include <ddb/ddb.h>
 
@@ -55,7 +60,6 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
 #include <vm/vm_phys.h>
-#include <vm/vm_reserv.h>
 
 /*
  * VM_FREELIST_DEFAULT is split into VM_NDOMAIN lists, one for each
@@ -490,26 +494,6 @@ vm_phys_alloc_freelist_pages(int flind, int pool, int order)
 }
 
 /*
- * Allocate physical memory from phys_avail[].
- */
-vm_paddr_t
-vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment)
-{
-	vm_paddr_t pa;
-	int i;
-
-	size = round_page(size);
-	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
-		if (phys_avail[i + 1] - phys_avail[i] < size)
-			continue;
-		pa = phys_avail[i];
-		phys_avail[i] += size;
-		return (pa);
-	}
-	panic("vm_phys_bootstrap_alloc");
-}
-
-/*
  * Find the vm_page corresponding to the given physical address.
  */
 vm_page_t
@@ -554,7 +538,7 @@ vm_phys_free_pages(vm_page_t m, int order)
 {
 	struct vm_freelist *fl;
 	struct vm_phys_seg *seg;
-	vm_paddr_t pa, pa_buddy;
+	vm_paddr_t pa;
 	vm_page_t m_buddy;
 
 	KASSERT(m->order == VM_NFREEORDER,
@@ -566,25 +550,26 @@ vm_phys_free_pages(vm_page_t m, int order)
 	KASSERT(order < VM_NFREEORDER,
 	    ("vm_phys_free_pages: order %d is out of range", order));
 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
-	pa = VM_PAGE_TO_PHYS(m);
 	seg = &vm_phys_segs[m->segind];
-	while (order < VM_NFREEORDER - 1) {
-		pa_buddy = pa ^ (1 << (PAGE_SHIFT + order));
-		if (pa_buddy < seg->start ||
-		    pa_buddy >= seg->end)
-			break;
-		m_buddy = &seg->first_page[atop(pa_buddy - seg->start)];
-		if (m_buddy->order != order)
-			break;
-		fl = (*seg->free_queues)[m_buddy->pool];
-		TAILQ_REMOVE(&fl[m_buddy->order].pl, m_buddy, pageq);
-		fl[m_buddy->order].lcnt--;
-		m_buddy->order = VM_NFREEORDER;
-		if (m_buddy->pool != m->pool)
-			vm_phys_set_pool(m->pool, m_buddy, order);
-		order++;
-		pa &= ~((1 << (PAGE_SHIFT + order)) - 1);
-		m = &seg->first_page[atop(pa - seg->start)];
+	if (order < VM_NFREEORDER - 1) {
+		pa = VM_PAGE_TO_PHYS(m);
+		do {
+			pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order));
+			if (pa < seg->start || pa >= seg->end)
+				break;
+			m_buddy = &seg->first_page[atop(pa - seg->start)];
+			if (m_buddy->order != order)
+				break;
+			fl = (*seg->free_queues)[m_buddy->pool];
+			TAILQ_REMOVE(&fl[order].pl, m_buddy, pageq);
+			fl[order].lcnt--;
+			m_buddy->order = VM_NFREEORDER;
+			if (m_buddy->pool != m->pool)
+				vm_phys_set_pool(m->pool, m_buddy, order);
+			order++;
+			pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1);
+			m = &seg->first_page[atop(pa - seg->start)];
+		} while (order < VM_NFREEORDER - 1);
 	}
 	m->order = order;
 	fl = (*seg->free_queues)[m->pool];
@@ -593,6 +578,47 @@ vm_phys_free_pages(vm_page_t m, int order)
 }
 
 /*
+ * Free a contiguous, arbitrarily sized set of physical pages.
+ *
+ * The free page queues must be locked.
+ */
+void
+vm_phys_free_contig(vm_page_t m, u_long npages)
+{
+	u_int n;
+	int order;
+
+	/*
+	 * Avoid unnecessary coalescing by freeing the pages in the largest
+	 * possible power-of-two-sized subsets.
+	 */
+	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+	for (;; npages -= n) {
+		/*
+		 * Unsigned "min" is used here so that "order" is assigned
+		 * "VM_NFREEORDER - 1" when "m"'s physical address is zero
+		 * or the low-order bits of its physical address are zero
+		 * because the size of a physical address exceeds the size of
+		 * a long.
+		 */
+		order = min(ffsl(VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) - 1,
+		    VM_NFREEORDER - 1);
+		n = 1 << order;
+		if (npages < n)
+			break;
+		vm_phys_free_pages(m, order);
+		m += n;
+	}
+	/* The residual "npages" is less than "1 << (VM_NFREEORDER - 1)". */
+	for (; npages > 0; npages -= n) {
+		order = flsl(npages) - 1;
+		n = 1 << order;
+		vm_phys_free_pages(m, order);
+		m += n;
+	}
+}
+
+/*
  * Set the pool for a contiguous, power of two-sized set of physical pages. 
  */
 void
@@ -728,16 +754,17 @@ vm_phys_zero_pages_idle(void)
  * "alignment" and "boundary" must be a power of two.
  */
 vm_page_t
-vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high,
-    unsigned long alignment, unsigned long boundary)
+vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high,
+    u_long alignment, vm_paddr_t boundary)
 {
 	struct vm_freelist *fl;
 	struct vm_phys_seg *seg;
-	struct vnode *vp;
 	vm_paddr_t pa, pa_last, size;
-	vm_page_t deferred_vdrop_list, m, m_ret;
-	int domain, flind, i, oind, order, pind;
+	vm_page_t m, m_ret;
+	u_long npages_end;
+	int domain, flind, oind, order, pind;
 
+	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
 #if VM_NDOMAIN > 1
 	domain = PCPU_GET(domain);
 #else
@@ -750,13 +777,8 @@ vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high,
 	    ("vm_phys_alloc_contig: alignment must be a power of 2"));
 	KASSERT((boundary & (boundary - 1)) == 0,
 	    ("vm_phys_alloc_contig: boundary must be a power of 2"));
-	deferred_vdrop_list = NULL;
 	/* Compute the queue that is the best fit for npages. */
 	for (order = 0; (1 << order) < npages; order++);
-	mtx_lock(&vm_page_queue_free_mtx);
-#if VM_NRESERVLEVEL > 0
-retry:
-#endif
 	for (flind = 0; flind < vm_nfreelists; flind++) {
 		for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) {
 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
@@ -815,11 +837,6 @@ retry:
 			}
 		}
 	}
-#if VM_NRESERVLEVEL > 0
-	if (vm_reserv_reclaim_contig(size, low, high, alignment, boundary))
-		goto retry;
-#endif
-	mtx_unlock(&vm_page_queue_free_mtx);
 	return (NULL);
 done:
 	for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) {
@@ -832,34 +849,10 @@ done:
 		vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind);
 	fl = (*seg->free_queues)[m_ret->pool];
 	vm_phys_split_pages(m_ret, oind, fl, order);
-	for (i = 0; i < npages; i++) {
-		m = &m_ret[i];
-		vp = vm_page_alloc_init(m);
-		if (vp != NULL) {
-			/*
-			 * Enqueue the vnode for deferred vdrop().
-			 *
-			 * Unmanaged pages don't use "pageq", so it
-			 * can be safely abused to construct a short-
-			 * lived queue of vnodes.
-			 */
-			m->pageq.tqe_prev = (void *)vp;
-			m->pageq.tqe_next = deferred_vdrop_list;
-			deferred_vdrop_list = m;
-		}
-	}
-	for (; i < roundup2(npages, 1 << imin(oind, order)); i++) {
-		m = &m_ret[i];
-		KASSERT(m->order == VM_NFREEORDER,
-		    ("vm_phys_alloc_contig: page %p has unexpected order %d",
-		    m, m->order));
-		vm_phys_free_pages(m, 0);
-	}
-	mtx_unlock(&vm_page_queue_free_mtx);
-	while (deferred_vdrop_list != NULL) {
-		vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev);
-		deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next;
-	}
+	/* Return excess pages to the free lists. */
+	npages_end = roundup2(npages, 1 << imin(oind, order));
+	if (npages < npages_end)
+		vm_phys_free_contig(&m_ret[npages], npages_end - npages);
 	return (m_ret);
 }
 
diff --git a/sys/vm/vm_phys.h b/sys/vm/vm_phys.h
index a5b9e93..047e4a9 100644
--- a/sys/vm/vm_phys.h
+++ b/sys/vm/vm_phys.h
@@ -49,13 +49,15 @@ struct mem_affinity {
 
 extern struct mem_affinity *mem_affinity;
 
+/*
+ * The following functions are only to be used by the virtual memory system.
+ */
 void vm_phys_add_page(vm_paddr_t pa);
-vm_page_t vm_phys_alloc_contig(unsigned long npages,
-    vm_paddr_t low, vm_paddr_t high,
-    unsigned long alignment, unsigned long boundary);
+vm_page_t vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high,
+    u_long alignment, vm_paddr_t boundary);
 vm_page_t vm_phys_alloc_freelist_pages(int flind, int pool, int order);
 vm_page_t vm_phys_alloc_pages(int pool, int order);
-vm_paddr_t vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment);
+void vm_phys_free_contig(vm_page_t m, u_long npages);
 void vm_phys_free_pages(vm_page_t m, int order);
 void vm_phys_init(void);
 void vm_phys_set_pool(int pool, vm_page_t m, int order);
diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c
index fec0539..12340ff 100644
--- a/sys/vm/vm_reserv.c
+++ b/sys/vm/vm_reserv.c
@@ -628,7 +628,7 @@ vm_reserv_reclaim_inactive(void)
  */
 boolean_t
 vm_reserv_reclaim_contig(vm_paddr_t size, vm_paddr_t low, vm_paddr_t high,
-    unsigned long alignment, unsigned long boundary)
+    u_long alignment, vm_paddr_t boundary)
 {
 	vm_paddr_t pa, pa_length;
 	vm_reserv_t rv;
diff --git a/sys/vm/vm_reserv.h b/sys/vm/vm_reserv.h
index 9eb1d06..9ab9568 100644
--- a/sys/vm/vm_reserv.h
+++ b/sys/vm/vm_reserv.h
@@ -49,8 +49,7 @@ void		vm_reserv_init(void);
 int		vm_reserv_level_iffullpop(vm_page_t m);
 boolean_t	vm_reserv_reactivate_page(vm_page_t m);
 boolean_t	vm_reserv_reclaim_contig(vm_paddr_t size, vm_paddr_t low,
-		    vm_paddr_t high, unsigned long alignment,
-		    unsigned long boundary);
+		    vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
 boolean_t	vm_reserv_reclaim_inactive(void);
 void		vm_reserv_rename(vm_page_t m, vm_object_t new_object,
 		    vm_object_t old_object, vm_pindex_t old_object_offset);
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index e3222cb..cd2658d 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -486,15 +486,16 @@ vnode_pager_input_smlfs(object, m)
 	vm_object_t object;
 	vm_page_t m;
 {
-	int bits, i;
 	struct vnode *vp;
 	struct bufobj *bo;
 	struct buf *bp;
 	struct sf_buf *sf;
 	daddr_t fileaddr;
 	vm_offset_t bsize;
-	int error = 0;
+	vm_page_bits_t bits;
+	int error, i;
 
+	error = 0;
 	vp = object->handle;
 	if (vp->v_iflag & VI_DOOMED)
 		return VM_PAGER_BAD;
diff --git a/sys/x86/acpica/acpi_apm.c b/sys/x86/acpica/acpi_apm.c
index 776b1be..e24e5e1 100644
--- a/sys/x86/acpica/acpi_apm.c
+++ b/sys/x86/acpica/acpi_apm.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 static int apm_active;
 static struct clonedevs *apm_clones;
 
-MALLOC_DEFINE(M_APMDEV, "apmdev", "APM device emulation");
+static MALLOC_DEFINE(M_APMDEV, "apmdev", "APM device emulation");
 
 static d_open_t		apmopen;
 static d_close_t	apmclose;
diff --git a/sys/x86/acpica/madt.c b/sys/x86/acpica/madt.c
index 13bbb55..f464f59 100644
--- a/sys/x86/acpica/madt.c
+++ b/sys/x86/acpica/madt.c
@@ -65,7 +65,7 @@ static ACPI_TABLE_MADT *madt;
 static vm_paddr_t madt_physaddr;
 static vm_offset_t madt_length;
 
-MALLOC_DEFINE(M_MADT, "madt_table", "ACPI MADT Table Items");
+static MALLOC_DEFINE(M_MADT, "madt_table", "ACPI MADT Table Items");
 
 static enum intr_polarity interrupt_polarity(UINT16 IntiFlags, UINT8 Source);
 static enum intr_trigger interrupt_trigger(UINT16 IntiFlags, UINT8 Source);
diff --git a/sys/x86/bios/vpd.c b/sys/x86/bios/vpd.c
index 246b76d..8d03d71 100644
--- a/sys/x86/bios/vpd.c
+++ b/sys/x86/bios/vpd.c
@@ -103,14 +103,14 @@ static int	vpd_modevent	(module_t, int, void *);
 
 static int	vpd_cksum	(struct vpd *);
 
-SYSCTL_NODE(_hw, OID_AUTO, vpd, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd, OID_AUTO, machine, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd_machine, OID_AUTO, type, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd_machine, OID_AUTO, model, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd, OID_AUTO, build_id, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd, OID_AUTO, serial, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd_serial, OID_AUTO, box, CTLFLAG_RD, NULL, NULL);
-SYSCTL_NODE(_hw_vpd_serial, OID_AUTO, planar, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw, OID_AUTO, vpd, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd, OID_AUTO, machine, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd_machine, OID_AUTO, type, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd_machine, OID_AUTO, model, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd, OID_AUTO, build_id, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd, OID_AUTO, serial, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd_serial, OID_AUTO, box, CTLFLAG_RD, NULL, NULL);
+static SYSCTL_NODE(_hw_vpd_serial, OID_AUTO, planar, CTLFLAG_RD, NULL, NULL);
 
 static void
 vpd_identify (driver_t *driver, device_t parent)
diff --git a/sys/x86/x86/busdma_machdep.c b/sys/x86/x86/busdma_machdep.c
index d4508c4..70dd72b 100644
--- a/sys/x86/x86/busdma_machdep.c
+++ b/sys/x86/x86/busdma_machdep.c
@@ -113,7 +113,7 @@ static int total_bpages;
 static int busdma_zonecount;
 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
 
-SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 	   "Total bounce pages");
 
diff --git a/sys/x86/x86/io_apic.c b/sys/x86/x86/io_apic.c
index 772aeaa..e72a636 100644
--- a/sys/x86/x86/io_apic.c
+++ b/sys/x86/x86/io_apic.c
@@ -133,7 +133,7 @@ struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
 static int next_ioapic_base;
 static u_int next_id;
 
-SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
+static SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 static int enable_extint;
 SYSCTL_INT(_hw_apic, OID_AUTO, enable_extint, CTLFLAG_RDTUN, &enable_extint, 0,
     "Enable the ExtINT pin in the first I/O APIC");
diff --git a/sys/x86/x86/mca.c b/sys/x86/x86/mca.c
index 7ae3c78..59de9d8 100644
--- a/sys/x86/x86/mca.c
+++ b/sys/x86/x86/mca.c
@@ -86,7 +86,8 @@ static MALLOC_DEFINE(M_MCA, "MCA", "Machine Check Architecture");
 
 static int mca_count;		/* Number of records stored. */
 
-SYSCTL_NODE(_hw, OID_AUTO, mca, CTLFLAG_RD, NULL, "Machine Check Architecture");
+static SYSCTL_NODE(_hw, OID_AUTO, mca, CTLFLAG_RD, NULL,
+    "Machine Check Architecture");
 
 static int mca_enabled = 1;
 TUNABLE_INT("hw.mca.enabled", &mca_enabled);
diff --git a/tools/regression/bin/sh/builtins/for1.0 b/tools/regression/bin/sh/builtins/for1.0
new file mode 100644
index 0000000..cd55e2c
--- /dev/null
+++ b/tools/regression/bin/sh/builtins/for1.0
@@ -0,0 +1,4 @@
+# $FreeBSD$
+
+false
+for i in `false`; do exit 3; done
diff --git a/tools/regression/bin/sh/parameters/optind1.0 b/tools/regression/bin/sh/parameters/optind1.0
new file mode 100644
index 0000000..33e0288
--- /dev/null
+++ b/tools/regression/bin/sh/parameters/optind1.0
@@ -0,0 +1,3 @@
+# $FreeBSD$
+
+unset OPTIND && [ -z "$OPTIND" ]
diff --git a/tools/regression/fifo/fifo_misc/fifo_misc.c b/tools/regression/fifo/fifo_misc/fifo_misc.c
index d0a2cf7..5e04b06 100644
--- a/tools/regression/fifo/fifo_misc/fifo_misc.c
+++ b/tools/regression/fifo/fifo_misc/fifo_misc.c
@@ -115,7 +115,7 @@ test_lseek(void)
 		exit(-1);
 	}
 
-	if (lseek(reader_fd, SEEK_CUR, 1) >= 0) {
+	if (lseek(reader_fd, 1, SEEK_CUR) >= 0) {
 		warnx("%s: lseek succeeded instead of returning ESPIPE",
 		    __func__);
 		cleanfifo("testfifo", reader_fd, writer_fd);
diff --git a/tools/regression/lib/libc/gen/Makefile b/tools/regression/lib/libc/gen/Makefile
index 72a0750..8517913 100644
--- a/tools/regression/lib/libc/gen/Makefile
+++ b/tools/regression/lib/libc/gen/Makefile
@@ -1,6 +1,7 @@
 # $FreeBSD$
 
-TESTS=	test-fmtcheck test-fnmatch test-fpclassify test-posix_spawn test-wordexp
+TESTS=	test-arc4random test-fmtcheck test-fnmatch \
+	test-fpclassify test-posix_spawn test-wordexp
 
 .PHONY: tests
 tests: ${TESTS}
diff --git a/tools/regression/lib/libc/gen/test-arc4random.c b/tools/regression/lib/libc/gen/test-arc4random.c
new file mode 100644
index 0000000..eb3d88c
--- /dev/null
+++ b/tools/regression/lib/libc/gen/test-arc4random.c
@@ -0,0 +1,89 @@
+/*-
+ * Copyright (c) 2011 David Schultz
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/*
+ * BUFSIZE is the number of bytes of rc4 output to compare.  The probability
+ * that this test fails spuriously is 2**(-BUFSIZE * 8).
+ */
+#define	BUFSIZE		8
+
+/*
+ * Test whether arc4random_buf() returns the same sequence of bytes in both
+ * parent and child processes.  (Hint: It shouldn't.)
+ */
+int main(int argc, char *argv[]) {
+	struct shared_page {
+		char parentbuf[BUFSIZE];
+		char childbuf[BUFSIZE];
+	} *page;
+	pid_t pid;
+	char c;
+
+	printf("1..1\n");
+
+	page = mmap(NULL, sizeof(struct shared_page), PROT_READ | PROT_WRITE,
+		    MAP_ANON | MAP_SHARED, -1, 0);
+	if (page == MAP_FAILED) {
+		printf("fail 1 - mmap\n");
+		exit(1);
+	}
+
+	arc4random_buf(&c, 1);
+
+	pid = fork();
+	if (pid < 0) {
+		printf("fail 1 - fork\n");
+		exit(1);
+	}
+	if (pid == 0) {
+		/* child */
+		arc4random_buf(page->childbuf, BUFSIZE);
+		exit(0);
+	} else {
+		/* parent */
+		int status;
+		arc4random_buf(page->parentbuf, BUFSIZE);
+		wait(&status);
+	}
+	if (memcmp(page->parentbuf, page->childbuf, BUFSIZE) == 0) {
+		printf("fail 1 - sequences are the same\n");
+		exit(1);
+	}
+
+	printf("ok 1 - sequences are different\n");
+	exit(0);
+}
diff --git a/tools/tools/README b/tools/tools/README
index 253b2e0..9c3db2f 100644
--- a/tools/tools/README
+++ b/tools/tools/README
@@ -50,6 +50,7 @@ mfc		Merge a directory from HEAD to a branch where it does not
 mid	 	Create a Message-ID database for mailing lists.
 mwl		Tools specific to the Marvell 88W8363 support
 ncpus		Count the number of processors
+netmap		Test applications for netmap(4)
 notescheck	Check for missing devices and options in NOTES files.
 npe		Tools specific to the Intel IXP4XXX NPE device
 nxge		A diagnostic tool for the nxge(4) driver
diff --git a/tools/tools/netmap/Makefile b/tools/tools/netmap/Makefile
new file mode 100644
index 0000000..4b682e5
--- /dev/null
+++ b/tools/tools/netmap/Makefile
@@ -0,0 +1,25 @@
+#
+# $FreeBSD$
+#
+# For multiple programs using a single source file each,
+# we can just define 'progs' and create custom targets.
+PROGS	=	pkt-gen bridge testpcap libnetmap.so
+
+CLEANFILES = $(PROGS) pcap.o
+NO_MAN=
+CFLAGS += -Werror -Wall -nostdinc -I/usr/include -I../../../sys
+CFLAGS += -Wextra
+
+LDFLAGS += -lpthread -lpcap
+
+.include <bsd.prog.mk>
+.include <bsd.lib.mk>
+
+all: $(PROGS)
+
+testpcap: pcap.c libnetmap.so
+	$(CC) $(CFLAGS) -L. -lnetmap -o ${.TARGET} pcap.c
+	
+libnetmap.so:	pcap.c
+	$(CC) $(CFLAGS) -fpic -c ${.ALLSRC}
+	$(CC) -shared -o ${.TARGET} ${.ALLSRC:.c=.o}
diff --git a/tools/tools/netmap/README b/tools/tools/netmap/README
new file mode 100644
index 0000000..9a1ba60
--- /dev/null
+++ b/tools/tools/netmap/README
@@ -0,0 +1,11 @@
+$FreeBSD$
+
+This directory contains examples that use netmap
+
+	pkt-gen		a packet sink/source using the netmap API
+
+	bridge		a two-port jumper wire, also using the native API
+
+	testpcap	a jumper wire using libnetmap (or libpcap)
+
+	click*		various click examples
diff --git a/tools/tools/netmap/bridge.c b/tools/tools/netmap/bridge.c
new file mode 100644
index 0000000..2385a08
--- /dev/null
+++ b/tools/tools/netmap/bridge.c
@@ -0,0 +1,456 @@
+/*
+ * (C) 2011 Luigi Rizzo, Matteo Landi
+ *
+ * BSD license
+ *
+ * A netmap client to bridge two network interfaces
+ * (or one interface and the host stack).
+ *
+ * $FreeBSD$
+ */
+
+#include <errno.h>
+#include <signal.h> /* signal */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h> /* strcmp */
+#include <fcntl.h> /* open */
+#include <unistd.h> /* close */
+
+#include <sys/endian.h> /* le64toh */
+#include <sys/mman.h> /* PROT_* */
+#include <sys/ioctl.h> /* ioctl */
+#include <machine/param.h>
+#include <sys/poll.h>
+#include <sys/socket.h> /* sockaddr.. */
+#include <arpa/inet.h> /* ntohs */
+
+#include <net/if.h>	/* ifreq */
+#include <net/ethernet.h>
+#include <net/netmap.h>
+#include <net/netmap_user.h>
+
+#include <netinet/in.h> /* sockaddr_in */
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+int verbose = 0;
+
+/* debug support */
+#define ND(format, ...) {}
+#define D(format, ...) do {					\
+	if (!verbose) break;					\
+	struct timeval _xxts;					\
+	gettimeofday(&_xxts, NULL);				\
+        fprintf(stderr, "%03d.%06d %s [%d] " format "\n",	\
+	(int)_xxts.tv_sec %1000, (int)_xxts.tv_usec,		\
+        __FUNCTION__, __LINE__, ##__VA_ARGS__);			\
+	} while (0)
+
+
+char *version = "$Id: bridge.c 9642 2011-11-07 21:39:47Z luigi $";
+
+static int do_abort = 0;
+
+/*
+ * info on a ring we handle
+ */
+struct my_ring {
+	const char *ifname;
+	int fd;
+	char *mem;			/* userspace mmap address */
+	u_int memsize;
+	u_int queueid;
+	u_int begin, end;		/* first..last+1 rings to check */
+	struct netmap_if *nifp;
+	struct netmap_ring *tx, *rx;	/* shortcuts */
+
+	uint32_t if_flags;
+	uint32_t if_reqcap;
+	uint32_t if_curcap;
+};
+
+static void
+sigint_h(__unused int sig)
+{
+	do_abort = 1;
+	signal(SIGINT, SIG_DFL);
+}
+
+
+static int
+do_ioctl(struct my_ring *me, int what)
+{
+	struct ifreq ifr;
+	int error;
+
+	bzero(&ifr, sizeof(ifr));
+	strncpy(ifr.ifr_name, me->ifname, sizeof(ifr.ifr_name));
+	switch (what) {
+	case SIOCSIFFLAGS:
+		ifr.ifr_flagshigh = me->if_flags >> 16;
+		ifr.ifr_flags = me->if_flags & 0xffff;
+		break;
+	case SIOCSIFCAP:
+		ifr.ifr_reqcap = me->if_reqcap;
+		ifr.ifr_curcap = me->if_curcap;
+		break;
+	}
+	error = ioctl(me->fd, what, &ifr);
+	if (error) {
+		D("ioctl error %d", what);
+		return error;
+	}
+	switch (what) {
+	case SIOCGIFFLAGS:
+		me->if_flags = (ifr.ifr_flagshigh << 16) |
+			(0xffff & ifr.ifr_flags);
+		if (verbose)
+			D("flags are 0x%x", me->if_flags);
+		break;
+
+	case SIOCGIFCAP:
+		me->if_reqcap = ifr.ifr_reqcap;
+		me->if_curcap = ifr.ifr_curcap;
+		if (verbose)
+			D("curcap are 0x%x", me->if_curcap);
+		break;
+	}
+	return 0;
+}
+
+/*
+ * open a device. if me->mem is null then do an mmap.
+ */
+static int
+netmap_open(struct my_ring *me, int ringid)
+{
+	int fd, err, l;
+	struct nmreq req;
+
+	me->fd = fd = open("/dev/netmap", O_RDWR);
+	if (fd < 0) {
+		D("Unable to open /dev/netmap");
+		return (-1);
+	}
+	bzero(&req, sizeof(req));
+	strncpy(req.nr_name, me->ifname, sizeof(req.nr_name));
+	req.nr_ringid = ringid;
+	err = ioctl(fd, NIOCGINFO, &req);
+	if (err) {
+		D("cannot get info on %s", me->ifname);
+		goto error;
+	}
+	me->memsize = l = req.nr_memsize;
+	if (verbose)
+		D("memsize is %d MB", l>>20);
+	err = ioctl(fd, NIOCREGIF, &req);
+	if (err) {
+		D("Unable to register %s", me->ifname);
+		goto error;
+	}
+
+	if (me->mem == NULL) {
+		me->mem = mmap(0, l, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
+		if (me->mem == MAP_FAILED) {
+			D("Unable to mmap");
+			me->mem = NULL;
+			goto error;
+		}
+	}
+
+	me->nifp = NETMAP_IF(me->mem, req.nr_offset);
+	me->queueid = ringid;
+	if (ringid & NETMAP_SW_RING) {
+		me->begin = req.nr_numrings;
+		me->end = me->begin + 1;
+	} else if (ringid & NETMAP_HW_RING) {
+		me->begin = ringid & NETMAP_RING_MASK;
+		me->end = me->begin + 1;
+	} else {
+		me->begin = 0;
+		me->end = req.nr_numrings;
+	}
+	me->tx = NETMAP_TXRING(me->nifp, me->begin);
+	me->rx = NETMAP_RXRING(me->nifp, me->begin);
+	return (0);
+error:
+	close(me->fd);
+	return -1;
+}
+
+
+static int
+netmap_close(struct my_ring *me)
+{
+	D("");
+	if (me->mem)
+		munmap(me->mem, me->memsize);
+	ioctl(me->fd, NIOCUNREGIF, NULL);
+	close(me->fd);
+	return (0);
+}
+
+
+/*
+ * move up to 'limit' pkts from rxring to txring swapping buffers.
+ */
+static int
+process_rings(struct netmap_ring *rxring, struct netmap_ring *txring,
+	      u_int limit, const char *msg)
+{
+	u_int j, k, m = 0;
+
+	/* print a warning if any of the ring flags is set (e.g. NM_REINIT) */
+	if (rxring->flags || txring->flags)
+		D("%s rxflags %x txflags %x",
+			msg, rxring->flags, txring->flags);
+	j = rxring->cur; /* RX */
+	k = txring->cur; /* TX */
+	if (rxring->avail < limit)
+		limit = rxring->avail;
+	if (txring->avail < limit)
+		limit = txring->avail;
+	m = limit;
+	while (limit-- > 0) {
+		struct netmap_slot *rs = &rxring->slot[j];
+		struct netmap_slot *ts = &txring->slot[k];
+		uint32_t pkt;
+
+		/* swap packets */
+		if (ts->buf_idx < 2 || rs->buf_idx < 2) {
+			D("wrong index rx[%d] = %d  -> tx[%d] = %d",
+				j, rs->buf_idx, k, ts->buf_idx);
+			sleep(2);
+		}
+		pkt = ts->buf_idx;
+		ts->buf_idx = rs->buf_idx;
+		rs->buf_idx = pkt;
+
+		/* copy the packet lenght. */
+		if (rs->len < 14 || rs->len > 2048)
+			D("wrong len %d rx[%d] -> tx[%d]", rs->len, j, k);
+		else if (verbose > 1)
+			D("send len %d rx[%d] -> tx[%d]", rs->len, j, k);
+		ts->len = rs->len;
+
+		/* report the buffer change. */
+		ts->flags |= NS_BUF_CHANGED;
+		rs->flags |= NS_BUF_CHANGED;
+		j = NETMAP_RING_NEXT(rxring, j);
+		k = NETMAP_RING_NEXT(txring, k);
+	}
+	rxring->avail -= m;
+	txring->avail -= m;
+	rxring->cur = j;
+	txring->cur = k;
+	if (verbose && m > 0)
+		D("sent %d packets to %p", m, txring);
+
+	return (m);
+}
+
+/* move packts from src to destination */
+static int
+move(struct my_ring *src, struct my_ring *dst, u_int limit)
+{
+	struct netmap_ring *txring, *rxring;
+	u_int m = 0, si = src->begin, di = dst->begin;
+	const char *msg = (src->queueid & NETMAP_SW_RING) ?
+		"host->net" : "net->host";
+
+	while (si < src->end && di < dst->end) {
+		rxring = NETMAP_RXRING(src->nifp, si);
+		txring = NETMAP_TXRING(dst->nifp, di);
+		ND("txring %p rxring %p", txring, rxring);
+		if (rxring->avail == 0) {
+			si++;
+			continue;
+		}
+		if (txring->avail == 0) {
+			di++;
+			continue;
+		}
+		m += process_rings(rxring, txring, limit, msg);
+	}
+
+	return (m);
+}
+
+/*
+ * how many packets on this set of queues ?
+ */
+static int
+howmany(struct my_ring *me, int tx)
+{
+	u_int i, tot = 0;
+
+	ND("me %p begin %d end %d", me, me->begin, me->end);
+	for (i = me->begin; i < me->end; i++) {
+		struct netmap_ring *ring = tx ?
+			NETMAP_TXRING(me->nifp, i) : NETMAP_RXRING(me->nifp, i);
+		tot += ring->avail;
+	}
+	if (0 && verbose && tot && !tx)
+		D("ring %s %s %s has %d avail at %d",
+			me->ifname, tx ? "tx": "rx",
+			me->end > me->nifp->ni_num_queues ?
+				"host":"net",
+			tot, NETMAP_TXRING(me->nifp, me->begin)->cur);
+	return tot;
+}
+
+/*
+ * bridge [-v] if1 [if2]
+ *
+ * If only one name, or the two interfaces are the same,
+ * bridges userland and the adapter. Otherwise bridge
+ * two intefaces.
+ */
+int
+main(int argc, char **argv)
+{
+	struct pollfd pollfd[2];
+	int i;
+	u_int burst = 1024;
+	struct my_ring me[2];
+
+	fprintf(stderr, "%s %s built %s %s\n",
+		argv[0], version, __DATE__, __TIME__);
+
+	bzero(me, sizeof(me));
+
+	while (argc > 1 && !strcmp(argv[1], "-v")) {
+		verbose++;
+		argv++;
+		argc--;
+	}
+
+	if (argc < 2 || argc > 4) {
+		D("Usage: %s IFNAME1 [IFNAME2 [BURST]]", argv[0]);
+		return (1);
+	}
+
+	/* setup netmap interface #1. */
+	me[0].ifname = argv[1];
+	if (argc == 2 || !strcmp(argv[1], argv[2])) {
+		D("same interface, endpoint 0 goes to host");
+		i = NETMAP_SW_RING;
+		me[1].ifname = argv[1];
+	} else {
+		/* two different interfaces. Take all rings on if1 */
+		i = 0;	// all hw rings
+		me[1].ifname = argv[2];
+	}
+	if (netmap_open(me, i))
+		return (1);
+	me[1].mem = me[0].mem; /* copy the pointer, so only one mmap */
+	if (netmap_open(me+1, 0))
+		return (1);
+
+	/* if bridging two interfaces, set promisc mode */
+	if (i != NETMAP_SW_RING) {
+		do_ioctl(me, SIOCGIFFLAGS);
+		if ((me[0].if_flags & IFF_UP) == 0) {
+			D("%s is down, bringing up...", me[0].ifname);
+			me[0].if_flags |= IFF_UP;
+		}
+		me[0].if_flags |= IFF_PPROMISC;
+		do_ioctl(me, SIOCSIFFLAGS);
+
+		do_ioctl(me+1, SIOCGIFFLAGS);
+		me[1].if_flags |= IFF_PPROMISC;
+		do_ioctl(me+1, SIOCSIFFLAGS);
+
+		/* also disable checksums etc. */
+		do_ioctl(me, SIOCGIFCAP);
+		me[0].if_reqcap = me[0].if_curcap;
+		me[0].if_reqcap &= ~(IFCAP_HWCSUM | IFCAP_TSO | IFCAP_TOE);
+		do_ioctl(me+0, SIOCSIFCAP);
+	}
+	do_ioctl(me+1, SIOCGIFFLAGS);
+	if ((me[1].if_flags & IFF_UP) == 0) {
+		D("%s is down, bringing up...", me[1].ifname);
+		me[1].if_flags |= IFF_UP;
+	}
+	do_ioctl(me+1, SIOCSIFFLAGS);
+
+	do_ioctl(me+1, SIOCGIFCAP);
+	me[1].if_reqcap = me[1].if_curcap;
+	me[1].if_reqcap &= ~(IFCAP_HWCSUM | IFCAP_TSO | IFCAP_TOE);
+	do_ioctl(me+1, SIOCSIFCAP);
+	if (argc > 3)
+		burst = atoi(argv[3]);	/* packets burst size. */
+
+	/* setup poll(2) variables. */
+	memset(pollfd, 0, sizeof(pollfd));
+	for (i = 0; i < 2; i++) {
+		pollfd[i].fd = me[i].fd;
+		pollfd[i].events = (POLLIN);
+	}
+
+	D("Wait 2 secs for link to come up...");
+	sleep(2);
+	D("Ready to go, %s 0x%x/%d <-> %s 0x%x/%d.",
+		me[0].ifname, me[0].queueid, me[0].nifp->ni_num_queues,
+		me[1].ifname, me[1].queueid, me[1].nifp->ni_num_queues);
+
+	/* main loop */
+	signal(SIGINT, sigint_h);
+	while (!do_abort) {
+		int n0, n1, ret;
+		pollfd[0].events = pollfd[1].events = 0;
+		pollfd[0].revents = pollfd[1].revents = 0;
+		n0 = howmany(me, 0);
+		n1 = howmany(me + 1, 0);
+		if (n0)
+			pollfd[1].events |= POLLOUT;
+		else
+			pollfd[0].events |= POLLIN;
+		if (n1)
+			pollfd[0].events |= POLLOUT;
+		else
+			pollfd[1].events |= POLLIN;
+		ret = poll(pollfd, 2, 2500);
+		if (ret <= 0 || verbose)
+		    D("poll %s [0] ev %x %x rx %d@%d tx %d,"
+			     " [1] ev %x %x rx %d@%d tx %d",
+				ret <= 0 ? "timeout" : "ok",
+				pollfd[0].events,
+				pollfd[0].revents,
+				howmany(me, 0),
+				me[0].rx->cur,
+				howmany(me, 1),
+				pollfd[1].events,
+				pollfd[1].revents,
+				howmany(me+1, 0),
+				me[1].rx->cur,
+				howmany(me+1, 1)
+			);
+		if (ret < 0)
+			continue;
+		if (pollfd[0].revents & POLLERR) {
+			D("error on fd0, rxcur %d@%d",
+				me[0].rx->avail, me[0].rx->cur);
+		}
+		if (pollfd[1].revents & POLLERR) {
+			D("error on fd1, rxcur %d@%d",
+				me[1].rx->avail, me[1].rx->cur);
+		}
+		if (pollfd[0].revents & POLLOUT) {
+			move(me + 1, me, burst);
+			// XXX we don't need the ioctl */
+			// ioctl(me[0].fd, NIOCTXSYNC, NULL);
+		}
+		if (pollfd[1].revents & POLLOUT) {
+			move(me, me + 1, burst);
+			// XXX we don't need the ioctl */
+			// ioctl(me[1].fd, NIOCTXSYNC, NULL);
+		}
+	}
+	D("exiting");
+	netmap_close(me + 1);
+	netmap_close(me + 0);
+
+	return (0);
+}
diff --git a/tools/tools/netmap/click-test.cfg b/tools/tools/netmap/click-test.cfg
new file mode 100644
index 0000000..fc5759f
--- /dev/null
+++ b/tools/tools/netmap/click-test.cfg
@@ -0,0 +1,19 @@
+//
+// $FreeBSD$
+//
+// A sample test configuration for click
+//
+//
+// create a switch
+
+myswitch :: EtherSwitch;
+
+// two input devices
+
+c0 :: FromDevice(ix0, PROMISC true);
+c1 :: FromDevice(ix1, PROMISC true);
+
+// and now pass packets around
+
+c0[0] -> [0]sw[0] -> Queue(10000) -> ToDevice(ix0);
+c1[0] -> [1]sw[1] -> Queue(10000) -> ToDevice(ix1);
diff --git a/tools/tools/netmap/pcap.c b/tools/tools/netmap/pcap.c
new file mode 100644
index 0000000..f010b83
--- /dev/null
+++ b/tools/tools/netmap/pcap.c
@@ -0,0 +1,761 @@
+/*
+ * (C) 2011 Luigi Rizzo
+ *
+ * BSD license
+ *
+ * A simple library that maps some pcap functions onto netmap
+ * This is not 100% complete but enough to let tcpdump, trafshow
+ * and other apps work.
+ *
+ * $FreeBSD$
+ */
+
+#include <errno.h>
+#include <signal.h> /* signal */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h> /* strcmp */
+#include <fcntl.h> /* open */
+#include <unistd.h> /* close */
+
+#include <sys/endian.h> /* le64toh */
+#include <sys/mman.h> /* PROT_* */
+#include <sys/ioctl.h> /* ioctl */
+#include <machine/param.h>
+#include <sys/poll.h>
+#include <sys/socket.h> /* sockaddr.. */
+#include <arpa/inet.h> /* ntohs */
+
+#include <net/if.h>	/* ifreq */
+#include <net/ethernet.h>
+#include <net/netmap.h>
+#include <net/netmap_user.h>
+
+#include <netinet/in.h> /* sockaddr_in */
+
+#include <sys/socket.h>
+#include <ifaddrs.h>
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+char *version = "$Id$";
+int verbose = 0;
+
+/* debug support */
+#define ND(format, ...) do {} while (0)
+#define D(format, ...) do {				\
+    if (verbose)					\
+        fprintf(stderr, "--- %s [%d] " format "\n",	\
+        __FUNCTION__, __LINE__, ##__VA_ARGS__);		\
+	} while (0)
+
+
+/*
+ * We redefine here a number of structures that are in pcap.h
+ * so we can compile this file without the system header.
+ */
+#ifndef PCAP_ERRBUF_SIZE
+#define PCAP_ERRBUF_SIZE 128
+
+/*
+ * Each packet is accompanied by a header including the timestamp,
+ * captured size and actual size.
+ */
+struct pcap_pkthdr {
+	struct timeval ts;	/* time stamp */
+	uint32_t caplen;	/* length of portion present */
+	uint32_t len;		/* length this packet (off wire) */
+};
+
+typedef struct pcap_if pcap_if_t;
+
+/*
+ * Representation of an interface address.
+ */
+struct pcap_addr {
+	struct pcap_addr *next;
+	struct sockaddr *addr;		/* address */
+	struct sockaddr *netmask;	/* netmask for the above */
+	struct sockaddr *broadaddr;	/* broadcast addr for the above */
+	struct sockaddr *dstaddr;	/* P2P dest. address for the above */
+};
+
+struct pcap_if {
+	struct pcap_if *next;
+	char *name;		/* name to hand to "pcap_open_live()" */
+	char *description;	/* textual description of interface, or NULL */
+	struct pcap_addr *addresses;
+	uint32_t flags;      /* PCAP_IF_ interface flags */
+};
+
+/*
+ * We do not support stats (yet)
+ */
+struct pcap_stat {
+	u_int ps_recv;		/* number of packets received */
+	u_int ps_drop;		/* number of packets dropped */
+	u_int ps_ifdrop;	/* drops by interface XXX not yet supported */
+#ifdef WIN32
+	u_int bs_capt;		/* number of packets that reach the app. */
+#endif /* WIN32 */
+};
+
+typedef void	pcap_t;
+typedef enum {
+	PCAP_D_INOUT = 0,
+	PCAP_D_IN,
+	PCAP_D_OUT
+} pcap_direction_t;
+ 
+
+
+typedef void (*pcap_handler)(u_char *user,
+		const struct pcap_pkthdr *h, const u_char *bytes);
+
+char errbuf[PCAP_ERRBUF_SIZE];
+
+pcap_t *pcap_open_live(const char *device, int snaplen,
+               int promisc, int to_ms, char *errbuf);
+
+int pcap_findalldevs(pcap_if_t **alldevsp, char *errbuf);
+void pcap_close(pcap_t *p);
+int pcap_get_selectable_fd(pcap_t *p);
+int pcap_dispatch(pcap_t *p, int cnt, pcap_handler callback, u_char *user);
+int pcap_setnonblock(pcap_t *p, int nonblock, char *errbuf);
+int pcap_setdirection(pcap_t *p, pcap_direction_t d);
+char *pcap_lookupdev(char *errbuf);
+int pcap_inject(pcap_t *p, const void *buf, size_t size);
+int pcap_fileno(pcap_t *p);
+
+struct eproto {
+	const char *s;
+	u_short p;
+};
+#endif /* !PCAP_ERRBUF_SIZE */
+
+#ifdef __PIC__
+/*
+ * build as a shared library
+ */
+
+char pcap_version[] = "libnetmap version 0.3";
+
+/*
+ * Our equivalent of pcap_t
+ */
+struct my_ring {
+	struct nmreq nmr;
+
+	int fd;
+	char *mem;			/* userspace mmap address */
+	u_int memsize;
+	u_int queueid;
+	u_int begin, end;		/* first..last+1 rings to check */
+	struct netmap_if *nifp;
+
+	int snaplen;
+	char *errbuf;
+	int promisc;
+	int to_ms;
+
+	struct pcap_pkthdr hdr;
+
+	uint32_t if_flags;
+	uint32_t if_reqcap;
+	uint32_t if_curcap;
+
+	struct pcap_stat st;
+
+	char msg[PCAP_ERRBUF_SIZE];
+};
+
+
+static int
+do_ioctl(struct my_ring *me, int what)
+{
+	struct ifreq ifr;
+	int error;
+
+	bzero(&ifr, sizeof(ifr));
+	strncpy(ifr.ifr_name, me->nmr.nr_name, sizeof(ifr.ifr_name));
+	switch (what) {
+	case SIOCSIFFLAGS:
+		D("call SIOCSIFFLAGS 0x%x", me->if_flags);
+		ifr.ifr_flagshigh = (me->if_flags >> 16) & 0xffff;
+		ifr.ifr_flags = me->if_flags & 0xffff;
+		break;
+	case SIOCSIFCAP:
+		ifr.ifr_reqcap = me->if_reqcap;
+		ifr.ifr_curcap = me->if_curcap;
+		break;
+	}
+	error = ioctl(me->fd, what, &ifr);
+	if (error) {
+		D("ioctl 0x%x error %d", what, error);
+		return error;
+	}
+	switch (what) {
+	case SIOCSIFFLAGS:
+	case SIOCGIFFLAGS:
+		me->if_flags = (ifr.ifr_flagshigh << 16) |
+			(0xffff & ifr.ifr_flags);
+		D("flags are L 0x%x H 0x%x 0x%x",
+			(uint16_t)ifr.ifr_flags,
+			(uint16_t)ifr.ifr_flagshigh, me->if_flags);
+		break;
+
+	case SIOCGIFCAP:
+		me->if_reqcap = ifr.ifr_reqcap;
+		me->if_curcap = ifr.ifr_curcap;
+		D("curcap are 0x%x", me->if_curcap);
+		break;
+	}
+	return 0;
+}
+
+
+/*
+ * open a device. if me->mem is null then do an mmap.
+ */
+static int
+netmap_open(struct my_ring *me, int ringid)
+{
+	int fd, err, l;
+	u_int i;
+	struct nmreq req;
+
+	me->fd = fd = open("/dev/netmap", O_RDWR);
+	if (fd < 0) {
+		D("Unable to open /dev/netmap");
+		return (-1);
+	}
+	bzero(&req, sizeof(req));
+	strncpy(req.nr_name, me->nmr.nr_name, sizeof(req.nr_name));
+	req.nr_ringid = ringid;
+	err = ioctl(fd, NIOCGINFO, &req);
+	if (err) {
+		D("cannot get info on %s", me->nmr.nr_name);
+		goto error;
+	}
+	me->memsize = l = req.nr_memsize;
+	ND("memsize is %d MB", l>>20);
+	err = ioctl(fd, NIOCREGIF, &req);
+	if (err) {
+		D("Unable to register %s", me->nmr.nr_name);
+		goto error;
+	}
+
+	if (me->mem == NULL) {
+		me->mem = mmap(0, l, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
+		if (me->mem == MAP_FAILED) {
+			D("Unable to mmap");
+			me->mem = NULL;
+			goto error;
+		}
+	}
+
+	me->nifp = NETMAP_IF(me->mem, req.nr_offset);
+	me->queueid = ringid;
+	if (ringid & NETMAP_SW_RING) {
+		me->begin = req.nr_numrings;
+		me->end = me->begin + 1;
+	} else if (ringid & NETMAP_HW_RING) {
+		me->begin = ringid & NETMAP_RING_MASK;
+		me->end = me->begin + 1;
+	} else {
+		me->begin = 0;
+		me->end = req.nr_numrings;
+	}
+	/* request timestamps for packets */
+	for (i = me->begin; i < me->end; i++) {
+		struct netmap_ring *ring = NETMAP_RXRING(me->nifp, i);
+		ring->flags = NR_TIMESTAMP;
+	}
+	//me->tx = NETMAP_TXRING(me->nifp, 0);
+	return (0);
+error:
+	close(me->fd);
+	return -1;
+}
+
+/*
+ * There is a set of functions that tcpdump expects even if probably
+ * not used
+ */
+struct eproto eproto_db[] = {
+	{ "ip", ETHERTYPE_IP },
+	{ "arp", ETHERTYPE_ARP },
+	{ (char *)0, 0 }
+};
+
+
+int
+pcap_findalldevs(pcap_if_t **alldevsp, __unused char *errbuf)
+{
+	struct ifaddrs *i_head, *i;
+	pcap_if_t *top = NULL, *cur;
+	struct pcap_addr *tail = NULL;
+	int l;
+
+	D("listing all devs");
+	*alldevsp = NULL;
+	i_head = NULL;
+
+	if (getifaddrs(&i_head)) {
+		D("cannot get if addresses");
+		return -1;
+	}
+	for (i = i_head; i; i = i->ifa_next) {
+		//struct ifaddrs   *ifa;
+		struct pcap_addr *pca;
+		//struct sockaddr *sa;
+
+		D("got interface %s", i->ifa_name);
+		if (!top || strcmp(top->name, i->ifa_name)) {
+			/* new interface */
+			l = sizeof(*top) + strlen(i->ifa_name) + 1;
+			cur = calloc(1, l);
+			if (cur == NULL) {
+				D("no space for if descriptor");
+				continue;
+			}
+			cur->name = (char *)(cur + 1);
+			//cur->flags = i->ifa_flags;
+			strcpy(cur->name, i->ifa_name);
+			cur->description = NULL;
+			cur->next = top;
+			top = cur;
+			tail = NULL;
+		}
+		/* now deal with addresses */
+		D("%s addr family %d len %d %s %s",
+			top->name,
+			i->ifa_addr->sa_family, i->ifa_addr->sa_len,
+			i->ifa_netmask ? "Netmask" : "",
+			i->ifa_broadaddr ? "Broadcast" : "");
+		l = sizeof(struct pcap_addr) +
+			(i->ifa_addr ? i->ifa_addr->sa_len:0) +
+			(i->ifa_netmask ? i->ifa_netmask->sa_len:0) +
+			(i->ifa_broadaddr? i->ifa_broadaddr->sa_len:0);
+		pca = calloc(1, l);
+		if (pca == NULL) {
+			D("no space for if addr");
+			continue;
+		}
+#define SA_NEXT(x) ((struct sockaddr *)((char *)(x) + (x)->sa_len))
+		pca->addr = (struct sockaddr *)(pca + 1);
+		bcopy(i->ifa_addr, pca->addr, i->ifa_addr->sa_len);
+		if (i->ifa_netmask) {
+			pca->netmask = SA_NEXT(pca->addr);
+			bcopy(i->ifa_netmask, pca->netmask, i->ifa_netmask->sa_len);
+			if (i->ifa_broadaddr) {
+				pca->broadaddr = SA_NEXT(pca->netmask);
+				bcopy(i->ifa_broadaddr, pca->broadaddr, i->ifa_broadaddr->sa_len);
+			}
+		}
+		if (tail == NULL) {
+			top->addresses = pca;
+		} else {
+			tail->next = pca;
+		}
+		tail = pca;
+
+	}
+	freeifaddrs(i_head);
+	*alldevsp = top;
+	return 0;
+}
+
+void pcap_freealldevs(__unused pcap_if_t *alldevs)
+{
+	D("unimplemented");
+}
+
+char *
+pcap_lookupdev(char *buf)
+{
+	D("%s", buf);
+	strcpy(buf, "/dev/netmap");
+	return buf;
+}
+
+pcap_t *
+pcap_create(const char *source, char *errbuf)
+{
+	D("src %s (call open liveted)", source);
+	return pcap_open_live(source, 0, 1, 100, errbuf);
+}
+
+int
+pcap_activate(pcap_t *p)
+{
+	D("pcap %p running", p);
+	return 0;
+}
+
+int
+pcap_can_set_rfmon(__unused pcap_t *p)
+{
+	D("");
+	return 0;	/* no we can't */
+}
+
+int
+pcap_set_snaplen(pcap_t *p, int snaplen)
+{
+	struct my_ring *me = p;
+
+	D("len %d", snaplen);
+	me->snaplen = snaplen;
+	return 0;
+}
+
+int
+pcap_snapshot(pcap_t *p)
+{
+	struct my_ring *me = p;
+
+	D("len %d", me->snaplen);
+	return me->snaplen;
+}
+
+int
+pcap_lookupnet(const char *device, uint32_t *netp,
+	uint32_t *maskp, __unused char *errbuf)
+{
+
+	D("device %s", device);
+	inet_aton("10.0.0.255", (struct in_addr *)netp);
+	inet_aton("255.255.255.0",(struct in_addr *) maskp);
+	return 0;
+}
+
+int
+pcap_set_promisc(pcap_t *p, int promisc)
+{
+	struct my_ring *me = p;
+
+	D("promisc %d", promisc);
+        if (do_ioctl(me, SIOCGIFFLAGS))
+		D("SIOCGIFFLAGS failed");
+	if (promisc) {
+		me->if_flags |= IFF_PPROMISC;
+	} else {
+		me->if_flags &= ~IFF_PPROMISC;
+	}
+	if (do_ioctl(me, SIOCSIFFLAGS))
+		D("SIOCSIFFLAGS failed");
+	return 0;
+}
+
+int
+pcap_set_timeout(pcap_t *p, int to_ms)
+{
+	struct my_ring *me = p;
+
+	D("%d ms", to_ms);
+	me->to_ms = to_ms;
+	return 0;
+}
+
+struct bpf_program;
+
+int
+pcap_compile(__unused pcap_t *p, __unused struct bpf_program *fp,
+	const char *str, __unused int optimize, __unused uint32_t netmask)
+{
+	D("%s", str);
+	return 0;
+}
+
+int
+pcap_setfilter(__unused pcap_t *p, __unused  struct bpf_program *fp)
+{
+	D("");
+	return 0;
+}
+
+int
+pcap_datalink(__unused pcap_t *p)
+{
+	D("");
+	return 1;	// ethernet
+}
+
+const char *
+pcap_datalink_val_to_name(int dlt)
+{
+	D("%d", dlt);
+	return "DLT_EN10MB";
+}
+
+const char *
+pcap_datalink_val_to_description(int dlt)
+{
+	D("%d", dlt);
+	return "Ethernet link";
+}
+
+struct pcap_stat;
+int
+pcap_stats(pcap_t *p, struct pcap_stat *ps)
+{
+	struct my_ring *me = p;
+	ND("");
+
+	me->st.ps_recv += 10;
+	*ps = me->st;
+	sprintf(me->msg, "stats not supported");
+	return -1;
+};
+
+char *
+pcap_geterr(pcap_t *p)
+{
+	struct my_ring *me = p;
+
+	D("");
+	return me->msg;
+}
+
+pcap_t *
+pcap_open_live(const char *device, __unused int snaplen,
+               int promisc, int to_ms, __unused char *errbuf)
+{
+	struct my_ring *me;
+
+	D("request to open %s", device);
+	me = calloc(1, sizeof(*me));
+	if (me == NULL) {
+		D("failed to allocate struct for %s", device);
+		return NULL;
+	}
+	strncpy(me->nmr.nr_name, device, sizeof(me->nmr.nr_name));
+	if (netmap_open(me, 0)) {
+		D("error opening %s", device);
+		free(me);
+		return NULL;
+	}
+	me->to_ms = to_ms;
+        if (do_ioctl(me, SIOCGIFFLAGS))
+		D("SIOCGIFFLAGS failed");
+	if (promisc) {
+		me->if_flags |= IFF_PPROMISC;
+		if (do_ioctl(me, SIOCSIFFLAGS))
+			D("SIOCSIFFLAGS failed");
+	}
+        if (do_ioctl(me, SIOCGIFCAP))
+		D("SIOCGIFCAP failed");
+        me->if_reqcap &= ~(IFCAP_HWCSUM | IFCAP_TSO | IFCAP_TOE);
+        if (do_ioctl(me, SIOCSIFCAP))
+		D("SIOCSIFCAP failed");
+
+	return (pcap_t *)me;
+}
+
+void
+pcap_close(pcap_t *p)
+{
+	struct my_ring *me = p;
+
+	D("");
+	if (!me)
+		return;
+	if (me->mem)
+		munmap(me->mem, me->memsize);
+	/* restore original flags ? */
+	ioctl(me->fd, NIOCUNREGIF, NULL);
+	close(me->fd);
+	bzero(me, sizeof(*me));
+	free(me);
+}
+
+int
+pcap_fileno(pcap_t *p)
+{
+	struct my_ring *me = p;
+	D("returns %d", me->fd);
+	return me->fd;
+}
+
+int
+pcap_get_selectable_fd(pcap_t *p)
+{
+	struct my_ring *me = p;
+
+	ND("");
+	return me->fd;
+}
+
+int
+pcap_setnonblock(__unused pcap_t *p, int nonblock, __unused char *errbuf)
+{
+	D("mode is %d", nonblock);
+	return 0;	/* ignore */
+}
+
+int
+pcap_setdirection(__unused pcap_t *p, __unused pcap_direction_t d)
+{
+	D("");
+	return 0;	/* ignore */
+};
+
+int
+pcap_dispatch(pcap_t *p, int cnt, pcap_handler callback, u_char *user)
+{
+	struct my_ring *me = p;
+	int got = 0;
+	u_int si;
+
+	ND("cnt %d", cnt);
+	/* scan all rings */
+	for (si = me->begin; si < me->end; si++) {
+		struct netmap_ring *ring = NETMAP_RXRING(me->nifp, si);
+		ND("ring has %d pkts", ring->avail);
+		if (ring->avail == 0)
+			continue;
+		me->hdr.ts = ring->ts;
+		while ((cnt == -1 || cnt != got) && ring->avail > 0) {
+			u_int i = ring->cur;
+			u_int idx = ring->slot[i].buf_idx;
+			if (idx < 2) {
+				D("%s bogus RX index %d at offset %d",
+					me->nifp->ni_name, idx, i);
+				sleep(2);
+			}
+			u_char *buf = (u_char *)NETMAP_BUF(ring, idx);
+			me->hdr.len = me->hdr.caplen = ring->slot[i].len;
+			// D("call %p len %d", p, me->hdr.len);
+			callback(user, &me->hdr, buf);
+			ring->cur = NETMAP_RING_NEXT(ring, i);
+			ring->avail--;
+			got++;
+		}
+	}
+	return got;
+}
+
+int
+pcap_inject(pcap_t *p, const void *buf, size_t size)
+{
+        struct my_ring *me = p;
+        u_int si;
+ 
+        ND("cnt %d", cnt);
+        /* scan all rings */
+        for (si = me->begin; si < me->end; si++) {
+                struct netmap_ring *ring = NETMAP_TXRING(me->nifp, si);
+ 
+                ND("ring has %d pkts", ring->avail);
+                if (ring->avail == 0)
+                        continue;
+		u_int i = ring->cur;
+		u_int idx = ring->slot[i].buf_idx;
+		if (idx < 2) {
+			D("%s bogus TX index %d at offset %d",
+				me->nifp->ni_name, idx, i);
+			sleep(2);
+		}
+		u_char *dst = (u_char *)NETMAP_BUF(ring, idx);
+		ring->slot[i].len = size;
+		bcopy(buf, dst, size);
+		ring->cur = NETMAP_RING_NEXT(ring, i);
+		ring->avail--;
+		// if (ring->avail == 0) ioctl(me->fd, NIOCTXSYNC, NULL);
+		return size;
+        }
+	errno = ENOBUFS;
+	return -1;
+}
+
+int
+pcap_loop(pcap_t *p, int cnt, pcap_handler callback, u_char *user)
+{
+	struct my_ring *me = p;
+	struct pollfd fds[1];
+	int i;
+
+	ND("cnt %d", cnt);
+	memset(fds, 0, sizeof(fds));
+	fds[0].fd = me->fd;
+	fds[0].events = (POLLIN);
+
+	while (cnt == -1 || cnt > 0) {
+                if (poll(fds, 1, me->to_ms) <= 0) {
+                        D("poll error/timeout");
+			continue;
+                }
+		i = pcap_dispatch(p, cnt, callback, user);
+		if (cnt > 0)
+			cnt -= i;
+	}
+	return 0;
+}
+
+#endif /* __PIC__ */
+
+#ifndef __PIC__
+void do_send(u_char *user, const struct pcap_pkthdr *h, const u_char *buf)
+{
+	pcap_inject((pcap_t *)user, buf, h->caplen);
+}
+
+/*
+ * a simple pcap test program, bridge between two interfaces.
+ */
+int
+main(int argc, char **argv)
+{
+	pcap_t *p0, *p1;
+	int burst = 1024;
+	struct pollfd pollfd[2];
+
+	fprintf(stderr, "%s %s built %s %s\n",
+		argv[0], version, __DATE__, __TIME__);
+		
+	while (argc > 1 && !strcmp(argv[1], "-v")) {
+		verbose++;
+		argv++;
+		argc--;
+	}
+
+	if (argc < 3 || argc > 4 || !strcmp(argv[1], argv[2])) {
+		D("Usage: %s IFNAME1 IFNAME2 [BURST]", argv[0]);
+		return (1);
+	}
+	if (argc > 3)
+		burst = atoi(argv[3]);
+
+	p0 = pcap_open_live(argv[1], 0, 1, 100, NULL);
+	p1 = pcap_open_live(argv[2], 0, 1, 100, NULL);
+	D("%s", version);
+	D("open returns %p %p", p0, p1);
+	if (!p0 || !p1)
+		return(1);
+	bzero(pollfd, sizeof(pollfd));
+	pollfd[0].fd = pcap_fileno(p0);
+	pollfd[1].fd = pcap_fileno(p1);
+	pollfd[0].events = pollfd[1].events = POLLIN;
+	for (;;) {
+		/* do i need to reset ? */
+		pollfd[0].revents = pollfd[1].revents = 0;
+		int ret = poll(pollfd, 2, 1000);
+		if (ret <= 0 || verbose)
+                   D("poll %s [0] ev %x %x [1] ev %x %x",
+                        ret <= 0 ? "timeout" : "ok",
+                                pollfd[0].events,
+                                pollfd[0].revents,
+                                pollfd[1].events,
+                                pollfd[1].revents);
+		if (ret < 0)
+			continue;
+		if (pollfd[0].revents & POLLIN)
+			pcap_dispatch(p0, burst, do_send, p1);
+		if (pollfd[1].revents & POLLIN)
+			pcap_dispatch(p1, burst, do_send, p0);
+	}
+
+	return (0);
+}
+#endif /* !__PIC__ */
diff --git a/tools/tools/netmap/pkt-gen.c b/tools/tools/netmap/pkt-gen.c
new file mode 100644
index 0000000..747bd9d
--- /dev/null
+++ b/tools/tools/netmap/pkt-gen.c
@@ -0,0 +1,1021 @@
+/*
+ * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ * $Id: pkt-gen.c 9638 2011-11-07 18:07:43Z luigi $
+ *
+ * Example program to show how to build a multithreaded packet
+ * source/sink using the netmap device.
+ *
+ * In this example we create a programmable number of threads
+ * to take care of all the queues of the interface used to
+ * send or receive traffic.
+ *
+ */
+
+const char *default_payload="netmap pkt-gen Luigi Rizzo and Matteo Landi\n"
+	"http://info.iet.unipi.it/~luigi/netmap/ ";
+
+#include <errno.h>
+#include <pthread.h>	/* pthread_* */
+#include <pthread_np.h>	/* pthread w/ affinity */
+#include <signal.h>	/* signal */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>	/* strcmp */
+#include <fcntl.h>	/* open */
+#include <unistd.h>	/* close */
+#include <ifaddrs.h>	/* getifaddrs */
+
+#include <sys/mman.h>	/* PROT_* */
+#include <sys/ioctl.h>	/* ioctl */
+#include <sys/poll.h>
+#include <sys/socket.h>	/* sockaddr.. */
+#include <arpa/inet.h>	/* ntohs */
+#include <sys/param.h>
+#include <sys/cpuset.h>	/* cpu_set */
+#include <sys/sysctl.h>	/* sysctl */
+#include <sys/time.h>	/* timersub */
+
+#include <net/ethernet.h>
+#include <net/if.h>	/* ifreq */
+#include <net/if_dl.h>	/* LLADDR */
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/udp.h>
+
+#include <net/netmap.h>
+#include <net/netmap_user.h>
+#include <pcap/pcap.h>
+
+
+static inline int min(int a, int b) { return a < b ? a : b; }
+
+/* debug support */
+#define D(format, ...)				\
+	fprintf(stderr, "%s [%d] " format "\n", 	\
+	__FUNCTION__, __LINE__, ##__VA_ARGS__)
+
+#ifndef EXPERIMENTAL
+#define EXPERIMENTAL 0
+#endif
+
+int verbose = 0;
+#define MAX_QUEUES 64	/* no need to limit */
+
+#define SKIP_PAYLOAD 1 /* do not check payload. */
+
+#if EXPERIMENTAL
+/* Wrapper around `rdtsc' to take reliable timestamps flushing the pipeline */ 
+#define netmap_rdtsc(t) \
+	do { \
+		u_int __regs[4];					\
+									\
+		do_cpuid(0, __regs);					\
+		(t) = rdtsc();						\
+	} while (0)
+
+static __inline void
+do_cpuid(u_int ax, u_int *p)
+{
+	__asm __volatile("cpuid"
+			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
+			 :  "0" (ax));
+}
+
+static __inline uint64_t
+rdtsc(void)
+{
+	uint64_t rv;
+
+	__asm __volatile("rdtsc" : "=A" (rv));
+	return (rv);
+}
+#define MAX_SAMPLES 100000
+#endif /* EXPERIMENTAL */
+
+
+struct pkt {
+	struct ether_header eh;
+	struct ip ip;
+	struct udphdr udp;
+	uint8_t body[NETMAP_BUF_SIZE];
+} __attribute__((__packed__));
+
+/*
+ * global arguments for all threads
+ */
+struct glob_arg {
+	const char *src_ip;
+	const char *dst_ip;
+	const char *src_mac;
+	const char *dst_mac;
+	int pkt_size;
+	int burst;
+	int npackets;	/* total packets to send */
+	int nthreads;
+	int cpus;
+	int use_pcap;
+	pcap_t *p;
+};
+
+struct mystat {
+	uint64_t containers[8];
+};
+
+/*
+ * Arguments for a new thread. The same structure is used by
+ * the source and the sink
+ */
+struct targ {
+	struct glob_arg *g;
+	int used;
+	int completed;
+	int fd;
+	struct nmreq nmr;
+	struct netmap_if *nifp;
+	uint16_t	qfirst, qlast; /* range of queues to scan */
+	uint64_t count;
+	struct timeval tic, toc;
+	int me;
+	pthread_t thread;
+	int affinity;
+
+	uint8_t	dst_mac[6];
+	uint8_t	src_mac[6];
+	u_int dst_mac_range;
+	u_int src_mac_range;
+	uint32_t dst_ip;
+	uint32_t src_ip;
+	u_int dst_ip_range;
+	u_int src_ip_range;
+
+	struct pkt pkt;
+};
+
+
+static struct targ *targs;
+static int global_nthreads;
+
+/* control-C handler */
+static void
+sigint_h(__unused int sig)
+{
+	for (int i = 0; i < global_nthreads; i++) {
+		/* cancel active threads. */
+		if (targs[i].used == 0)
+			continue;
+
+		D("Cancelling thread #%d\n", i);
+		pthread_cancel(targs[i].thread);
+		targs[i].used = 0;
+	}
+
+	signal(SIGINT, SIG_DFL);
+}
+
+
+/* sysctl wrapper to return the number of active CPUs */
+static int
+system_ncpus(void)
+{
+	int mib[2], ncpus;
+	size_t len;
+
+	mib[0] = CTL_HW;
+	mib[1] = HW_NCPU;
+	len = sizeof(mib);
+	sysctl(mib, 2, &ncpus, &len, NULL, 0);
+
+	return (ncpus);
+}
+
+/*
+ * locate the src mac address for our interface, put it
+ * into the user-supplied buffer. return 0 if ok, -1 on error.
+ */
+static int
+source_hwaddr(const char *ifname, char *buf)
+{
+	struct ifaddrs *ifaphead, *ifap;
+	int l = sizeof(ifap->ifa_name);
+
+	if (getifaddrs(&ifaphead) != 0) {
+		D("getifaddrs %s failed", ifname);
+		return (-1);
+	}
+
+	for (ifap = ifaphead; ifap; ifap = ifap->ifa_next) {
+		struct sockaddr_dl *sdl =
+			(struct sockaddr_dl *)ifap->ifa_addr;
+		uint8_t *mac;
+
+		if (!sdl || sdl->sdl_family != AF_LINK)
+			continue;
+		if (strncmp(ifap->ifa_name, ifname, l) != 0)
+			continue;
+		mac = (uint8_t *)LLADDR(sdl);
+		sprintf(buf, "%02x:%02x:%02x:%02x:%02x:%02x",
+			mac[0], mac[1], mac[2],
+			mac[3], mac[4], mac[5]);
+		if (verbose)
+			D("source hwaddr %s", buf);
+		break;
+	}
+	freeifaddrs(ifaphead);
+	return ifap ? 0 : 1;
+}
+
+
+/* set the thread affinity. */
+static int
+setaffinity(pthread_t me, int i)
+{
+	cpuset_t cpumask;
+
+	if (i == -1)
+		return 0;
+
+	/* Set thread affinity affinity.*/
+	CPU_ZERO(&cpumask);
+	CPU_SET(i, &cpumask);
+
+	if (pthread_setaffinity_np(me, sizeof(cpuset_t), &cpumask) != 0) {
+		D("Unable to set affinity");
+		return 1;
+	}
+	return 0;
+}
+
+/* Compute the checksum of the given ip header. */
+static uint16_t
+checksum(const void *data, uint16_t len)
+{
+        const uint8_t *addr = data;
+        uint32_t sum = 0;
+
+        while (len > 1) {
+                sum += addr[0] * 256 + addr[1];
+                addr += 2;
+                len -= 2;
+        }
+
+        if (len == 1)
+                sum += *addr * 256;
+
+        sum = (sum >> 16) + (sum & 0xffff);
+        sum += (sum >> 16);
+
+        sum = htons(sum);
+
+        return ~sum;
+}
+
+/*
+ * Fill a packet with some payload.
+ */
+static void
+initialize_packet(struct targ *targ)
+{
+	struct pkt *pkt = &targ->pkt;
+	struct ether_header *eh;
+	struct ip *ip;
+	struct udphdr *udp;
+	uint16_t paylen = targ->g->pkt_size - sizeof(*eh) - sizeof(*ip);
+	int i, l, l0 = strlen(default_payload);
+	char *p;
+
+	for (i = 0; i < paylen;) {
+		l = min(l0, paylen - i);
+		bcopy(default_payload, pkt->body + i, l);
+		i += l;
+	}
+	pkt->body[i-1] = '\0';
+
+	udp = &pkt->udp;
+	udp->uh_sport = htons(1234);
+        udp->uh_dport = htons(4321);
+	udp->uh_ulen = htons(paylen);
+	udp->uh_sum = 0; // checksum(udp, sizeof(*udp));
+
+	ip = &pkt->ip;
+        ip->ip_v = IPVERSION;
+        ip->ip_hl = 5;
+        ip->ip_id = 0;
+        ip->ip_tos = IPTOS_LOWDELAY;
+	ip->ip_len = ntohs(targ->g->pkt_size - sizeof(*eh));
+        ip->ip_id = 0;
+        ip->ip_off = htons(IP_DF); /* Don't fragment */
+        ip->ip_ttl = IPDEFTTL;
+	ip->ip_p = IPPROTO_UDP;
+	inet_aton(targ->g->src_ip, (struct in_addr *)&ip->ip_src);
+	inet_aton(targ->g->dst_ip, (struct in_addr *)&ip->ip_dst);
+	targ->dst_ip = ip->ip_dst.s_addr;
+	targ->src_ip = ip->ip_src.s_addr;
+	p = index(targ->g->src_ip, '-');
+	if (p) {
+		targ->dst_ip_range = atoi(p+1);
+		D("dst-ip sweep %d addresses", targ->dst_ip_range);
+	}
+	ip->ip_sum = checksum(ip, sizeof(*ip));
+
+	eh = &pkt->eh;
+	bcopy(ether_aton(targ->g->src_mac), targ->src_mac, 6);
+	bcopy(targ->src_mac, eh->ether_shost, 6);
+	p = index(targ->g->src_mac, '-');
+	if (p)
+		targ->src_mac_range = atoi(p+1);
+
+	bcopy(ether_aton(targ->g->dst_mac), targ->dst_mac, 6);
+	bcopy(targ->dst_mac, eh->ether_dhost, 6);
+	p = index(targ->g->dst_mac, '-');
+	if (p)
+		targ->dst_mac_range = atoi(p+1);
+	eh->ether_type = htons(ETHERTYPE_IP);
+}
+
+/* Check the payload of the packet for errors (use it for debug).
+ * Look for consecutive ascii representations of the size of the packet.
+ */
+static void
+check_payload(char *p, int psize)
+{
+	char temp[64];
+	int n_read, size, sizelen;
+
+	/* get the length in ASCII of the length of the packet. */
+	sizelen = sprintf(temp, "%d", psize) + 1; // include a whitespace
+
+	/* dummy payload. */
+	p += 14; /* skip packet header. */
+	n_read = 14;
+	while (psize - n_read >= sizelen) {
+		sscanf(p, "%d", &size);
+		if (size != psize) {
+			D("Read %d instead of %d", size, psize);
+			break;
+		}
+
+		p += sizelen;
+		n_read += sizelen;
+	}
+}
+
+
+/*
+ * create and enqueue a batch of packets on a ring.
+ * On the last one set NS_REPORT to tell the driver to generate
+ * an interrupt when done.
+ */
+static int
+send_packets(struct netmap_ring *ring, struct pkt *pkt, 
+		int size, u_int count, int fill_all)
+{
+	u_int sent, cur = ring->cur;
+
+	if (ring->avail < count)
+		count = ring->avail;
+
+	for (sent = 0; sent < count; sent++) {
+		struct netmap_slot *slot = &ring->slot[cur];
+		char *p = NETMAP_BUF(ring, slot->buf_idx);
+
+		if (fill_all)
+			memcpy(p, pkt, size);
+
+		slot->len = size;
+		if (sent == count - 1)
+			slot->flags |= NS_REPORT;
+		cur = NETMAP_RING_NEXT(ring, cur);
+	}
+	ring->avail -= sent;
+	ring->cur = cur;
+
+	return (sent);
+}
+
+static void *
+sender_body(void *data)
+{
+	struct targ *targ = (struct targ *) data;
+
+	struct pollfd fds[1];
+	struct netmap_if *nifp = targ->nifp;
+	struct netmap_ring *txring;
+	int i, n = targ->g->npackets / targ->g->nthreads, sent = 0;
+	int fill_all = 1;
+
+	if (setaffinity(targ->thread, targ->affinity))
+		goto quit;
+	/* setup poll(2) machanism. */
+	memset(fds, 0, sizeof(fds));
+	fds[0].fd = targ->fd;
+	fds[0].events = (POLLOUT);
+
+	/* main loop.*/
+	gettimeofday(&targ->tic, NULL);
+    if (targ->g->use_pcap) {
+	int size = targ->g->pkt_size;
+	void *pkt = &targ->pkt;
+	pcap_t *p = targ->g->p;
+
+	for (; sent < n; sent++) {
+		if (pcap_inject(p, pkt, size) == -1)
+			break;
+	}
+    } else {
+	while (sent < n) {
+
+		/*
+		 * wait for available room in the send queue(s)
+		 */
+		if (poll(fds, 1, 2000) <= 0) {
+			D("poll error/timeout on queue %d\n", targ->me);
+			goto quit;
+		}
+		/*
+		 * scan our queues and send on those with room
+		 */
+		if (sent > 100000)
+			fill_all = 0;
+		for (i = targ->qfirst; i < targ->qlast; i++) {
+			int m, limit = MIN(n - sent, targ->g->burst);
+
+			txring = NETMAP_TXRING(nifp, i);
+			if (txring->avail == 0)
+				continue;
+			m = send_packets(txring, &targ->pkt, targ->g->pkt_size,
+					 limit, fill_all);
+			sent += m;
+			targ->count = sent;
+		}
+	}
+	/* Tell the interface that we have new packets. */
+	ioctl(fds[0].fd, NIOCTXSYNC, NULL);
+
+	/* final part: wait all the TX queues to be empty. */
+	for (i = targ->qfirst; i < targ->qlast; i++) {
+		txring = NETMAP_TXRING(nifp, i);
+		while (!NETMAP_TX_RING_EMPTY(txring)) {
+			ioctl(fds[0].fd, NIOCTXSYNC, NULL);
+			usleep(1); /* wait 1 tick */
+		}
+	}
+    }
+
+	gettimeofday(&targ->toc, NULL);
+	targ->completed = 1;
+	targ->count = sent;
+
+quit:
+	/* reset the ``used`` flag. */
+	targ->used = 0;
+
+	return (NULL);
+}
+
+
+static void
+receive_pcap(u_char *user, __unused const struct pcap_pkthdr * h,
+	__unused const u_char * bytes)
+{
+	int *count = (int *)user;
+	(*count)++;
+}
+
+static int
+receive_packets(struct netmap_ring *ring, u_int limit, int skip_payload)
+{
+	u_int cur, rx;
+
+	cur = ring->cur;
+	if (ring->avail < limit)
+		limit = ring->avail;
+	for (rx = 0; rx < limit; rx++) {
+		struct netmap_slot *slot = &ring->slot[cur];
+		char *p = NETMAP_BUF(ring, slot->buf_idx);
+
+		if (!skip_payload)
+			check_payload(p, slot->len);
+
+		cur = NETMAP_RING_NEXT(ring, cur);
+	}
+	ring->avail -= rx;
+	ring->cur = cur;
+
+	return (rx);
+}
+
+static void *
+receiver_body(void *data)
+{
+	struct targ *targ = (struct targ *) data;
+	struct pollfd fds[1];
+	struct netmap_if *nifp = targ->nifp;
+	struct netmap_ring *rxring;
+	int i, received = 0;
+
+	if (setaffinity(targ->thread, targ->affinity))
+		goto quit;
+
+	/* setup poll(2) machanism. */
+	memset(fds, 0, sizeof(fds));
+	fds[0].fd = targ->fd;
+	fds[0].events = (POLLIN);
+
+	/* unbounded wait for the first packet. */
+	for (;;) {
+		i = poll(fds, 1, 1000);
+		if (i > 0 && !(fds[0].revents & POLLERR))
+			break;
+		D("waiting for initial packets, poll returns %d %d", i, fds[0].revents);
+	}
+
+	/* main loop, exit after 1s silence */
+	gettimeofday(&targ->tic, NULL);
+    if (targ->g->use_pcap) {
+	for (;;) {
+		pcap_dispatch(targ->g->p, targ->g->burst, receive_pcap, NULL);
+	}
+    } else {
+	while (1) {
+		/* Once we started to receive packets, wait at most 1 seconds
+		   before quitting. */
+		if (poll(fds, 1, 1 * 1000) <= 0) {
+			gettimeofday(&targ->toc, NULL);
+			targ->toc.tv_sec -= 1; /* Substract timeout time. */
+			break;
+		}
+
+		for (i = targ->qfirst; i < targ->qlast; i++) {
+			int m;
+
+			rxring = NETMAP_RXRING(nifp, i);
+			if (rxring->avail == 0)
+				continue;
+
+			m = receive_packets(rxring, targ->g->burst,
+					SKIP_PAYLOAD);
+			received += m;
+			targ->count = received;
+		}
+
+		// tell the card we have read the data
+		//ioctl(fds[0].fd, NIOCRXSYNC, NULL);
+	}
+    }
+
+	targ->completed = 1;
+	targ->count = received;
+
+quit:
+	/* reset the ``used`` flag. */
+	targ->used = 0;
+
+	return (NULL);
+}
+
+static void
+tx_output(uint64_t sent, int size, double delta)
+{
+	double amount = 8.0 * (1.0 * size * sent) / delta;
+	double pps = sent / delta;
+	char units[4] = { '\0', 'K', 'M', 'G' };
+	int aunit = 0, punit = 0;
+
+	while (amount >= 1000) {
+		amount /= 1000;
+		aunit += 1;
+	}
+	while (pps >= 1000) {
+		pps /= 1000;
+		punit += 1;
+	}
+
+	printf("Sent %llu packets, %d bytes each, in %.2f seconds.\n",
+	       sent, size, delta);
+	printf("Speed: %.2f%cpps. Bandwidth: %.2f%cbps.\n",
+	       pps, units[punit], amount, units[aunit]);
+}
+
+
+static void
+rx_output(uint64_t received, double delta)
+{
+
+	double pps = received / delta;
+	char units[4] = { '\0', 'K', 'M', 'G' };
+	int punit = 0;
+
+	while (pps >= 1000) {
+		pps /= 1000;
+		punit += 1;
+	}
+
+	printf("Received %llu packets, in %.2f seconds.\n", received, delta);
+	printf("Speed: %.2f%cpps.\n", pps, units[punit]);
+}
+
+static void
+usage(void)
+{
+	const char *cmd = "pkt-gen";
+	fprintf(stderr,
+		"Usage:\n"
+		"%s arguments\n"
+		"\t-i interface		interface name\n"
+		"\t-t pkts_to_send	also forces send mode\n"
+		"\t-r pkts_to_receive	also forces receive mode\n"
+		"\t-l pkts_size		in bytes excluding CRC\n"
+		"\t-d dst-ip		end with %%n to sweep n addresses\n"
+		"\t-s src-ip		end with %%n to sweep n addresses\n"
+		"\t-D dst-mac		end with %%n to sweep n addresses\n"
+		"\t-S src-mac		end with %%n to sweep n addresses\n"
+		"\t-b burst size		testing, mostly\n"
+		"\t-c cores		cores to use\n"
+		"\t-p threads		processes/threads to use\n"
+		"\t-T report_ms		milliseconds between reports\n"
+		"\t-w wait_for_link_time	in seconds\n"
+		"",
+		cmd);
+
+	exit(0);
+}
+
+
+int
+main(int arc, char **argv)
+{
+	int i, fd;
+
+	struct glob_arg g;
+
+	struct nmreq nmr;
+	void *mmap_addr;		/* the mmap address */
+	void *(*td_body)(void *) = receiver_body;
+	int ch;
+	int report_interval = 1000;	/* report interval */
+	char *ifname = NULL;
+	int wait_link = 2;
+	int devqueues = 1;	/* how many device queues */
+
+	bzero(&g, sizeof(g));
+
+	g.src_ip = "10.0.0.1";
+	g.dst_ip = "10.1.0.1";
+	g.dst_mac = "ff:ff:ff:ff:ff:ff";
+	g.src_mac = NULL;
+	g.pkt_size = 60;
+	g.burst = 512;		// default
+	g.nthreads = 1;
+	g.cpus = 1;
+
+	while ( (ch = getopt(arc, argv,
+			"i:t:r:l:d:s:D:S:b:c:p:T:w:v")) != -1) {
+		switch(ch) {
+		default:
+			D("bad option %c %s", ch, optarg);
+			usage();
+			break;
+		case 'i':	/* interface */
+			ifname = optarg;
+			break;
+		case 't':	/* send */
+			td_body = sender_body;
+			g.npackets = atoi(optarg);
+			break;
+		case 'r':	/* receive */
+			td_body = receiver_body;
+			g.npackets = atoi(optarg);
+			break;
+		case 'l':	/* pkt_size */
+			g.pkt_size = atoi(optarg);
+			break;
+		case 'd':
+			g.dst_ip = optarg;
+			break;
+		case 's':
+			g.src_ip = optarg;
+			break;
+		case 'T':	/* report interval */
+			report_interval = atoi(optarg);
+			break;
+		case 'w':
+			wait_link = atoi(optarg);
+			break;
+		case 'b':	/* burst */
+			g.burst = atoi(optarg);
+			break;
+		case 'c':
+			g.cpus = atoi(optarg);
+			break;
+		case 'p':
+			g.nthreads = atoi(optarg);
+			break;
+
+		case 'P':
+			g.use_pcap = 1;
+			break;
+
+		case 'D': /* destination mac */
+			g.dst_mac = optarg;
+	{
+		struct ether_addr *mac = ether_aton(g.dst_mac);
+		D("ether_aton(%s) gives %p", g.dst_mac, mac);
+	}
+			break;
+		case 'S': /* source mac */
+			g.src_mac = optarg;
+			break;
+		case 'v':
+			verbose++;
+		}
+	}
+
+	if (ifname == NULL) {
+		D("missing ifname");
+		usage();
+	}
+	{
+		int n = system_ncpus();
+		if (g.cpus < 0 || g.cpus > n) {
+			D("%d cpus is too high, have only %d cpus", g.cpus, n);
+			usage();
+		}
+		if (g.cpus == 0)
+			g.cpus = n;
+	}
+	if (g.pkt_size < 16 || g.pkt_size > 1536) {
+		D("bad pktsize %d\n", g.pkt_size);
+		usage();
+	}
+
+	bzero(&nmr, sizeof(nmr));
+	/*
+	 * Open the netmap device to fetch the number of queues of our
+	 * interface.
+	 *
+	 * The first NIOCREGIF also detaches the card from the
+	 * protocol stack and may cause a reset of the card,
+	 * which in turn may take some time for the PHY to
+	 * reconfigure.
+	 */
+	fd = open("/dev/netmap", O_RDWR);
+	if (fd == -1) {
+		D("Unable to open /dev/netmap");
+		// fail later
+	} else {
+		if ((ioctl(fd, NIOCGINFO, &nmr)) == -1) {
+			D("Unable to get if info without name");
+		} else {
+			D("map size is %d Kb", nmr.nr_memsize >> 10);
+		}
+		bzero(&nmr, sizeof(nmr));
+		strncpy(nmr.nr_name, ifname, sizeof(nmr.nr_name));
+		if ((ioctl(fd, NIOCGINFO, &nmr)) == -1) {
+			D("Unable to get if info for %s", ifname);
+		}
+		devqueues = nmr.nr_numrings;
+	}
+
+	/* validate provided nthreads. */
+	if (g.nthreads < 1 || g.nthreads > devqueues) {
+		D("bad nthreads %d, have %d queues", g.nthreads, devqueues);
+		// continue, fail later
+	}
+
+	if (td_body == sender_body && g.src_mac == NULL) {
+		static char mybuf[20] = "ff:ff:ff:ff:ff:ff";
+		/* retrieve source mac address. */
+		if (source_hwaddr(ifname, mybuf) == -1) {
+			D("Unable to retrieve source mac");
+			// continue, fail later
+		}
+		g.src_mac = mybuf;
+	}
+
+	/*
+	 * Map the netmap shared memory: instead of issuing mmap()
+	 * inside the body of the threads, we prefer to keep this
+	 * operation here to simplify the thread logic.
+	 */
+	D("mmapping %d Kbytes", nmr.nr_memsize>>10);
+	mmap_addr = (struct netmap_d *) mmap(0, nmr.nr_memsize,
+					    PROT_WRITE | PROT_READ,
+					    MAP_SHARED, fd, 0);
+	if (mmap_addr == MAP_FAILED) {
+		D("Unable to mmap %d KB", nmr.nr_memsize >> 10);
+		// continue, fail later
+	}
+
+	/*
+	 * Register the interface on the netmap device: from now on,
+	 * we can operate on the network interface without any
+	 * interference from the legacy network stack.
+	 *
+	 * We decide to put the first interface registration here to
+	 * give time to cards that take a long time to reset the PHY.
+	 */
+	if (ioctl(fd, NIOCREGIF, &nmr) == -1) {
+		D("Unable to register interface %s", ifname);
+		//continue, fail later
+	}
+
+
+	/* Print some debug information. */
+	fprintf(stdout,
+		"%s %s: %d queues, %d threads and %d cpus.\n",
+		(td_body == sender_body) ? "Sending on" : "Receiving from",
+		ifname,
+		devqueues,
+		g.nthreads,
+		g.cpus);
+	if (td_body == sender_body) {
+		fprintf(stdout, "%s -> %s (%s -> %s)\n",
+			g.src_ip, g.dst_ip,
+			g.src_mac, g.dst_mac);
+	}
+			
+	/* Exit if something went wrong. */
+	if (fd < 0) {
+		D("aborting");
+		usage();
+	}
+
+
+	/* Wait for PHY reset. */
+	D("Wait %d secs for phy reset", wait_link);
+	sleep(wait_link);
+	D("Ready...");
+
+	/* Install ^C handler. */
+	global_nthreads = g.nthreads;
+	signal(SIGINT, sigint_h);
+
+	if (g.use_pcap) {
+		// XXX g.p = pcap_open_live(..);
+	}
+
+	targs = calloc(g.nthreads, sizeof(*targs));
+	/*
+	 * Now create the desired number of threads, each one
+	 * using a single descriptor.
+ 	 */
+	for (i = 0; i < g.nthreads; i++) {
+		struct netmap_if *tnifp;
+		struct nmreq tifreq;
+		int tfd;
+
+	    if (g.use_pcap) {
+		tfd = -1;
+		tnifp = NULL;
+	    } else {
+		/* register interface. */
+		tfd = open("/dev/netmap", O_RDWR);
+		if (tfd == -1) {
+			D("Unable to open /dev/netmap");
+			continue;
+		}
+
+		bzero(&tifreq, sizeof(tifreq));
+		strncpy(tifreq.nr_name, ifname, sizeof(tifreq.nr_name));
+		tifreq.nr_ringid = (g.nthreads > 1) ? (i | NETMAP_HW_RING) : 0;
+
+		/*
+		 * if we are acting as a receiver only, do not touch the transmit ring.
+		 * This is not the default because many apps may use the interface
+		 * in both directions, but a pure receiver does not.
+		 */
+		if (td_body == receiver_body) {
+			tifreq.nr_ringid |= NETMAP_NO_TX_POLL;
+		}
+
+		if ((ioctl(tfd, NIOCREGIF, &tifreq)) == -1) {
+			D("Unable to register %s", ifname);
+			continue;
+		}
+		tnifp = NETMAP_IF(mmap_addr, tifreq.nr_offset);
+	    }
+		/* start threads. */
+		bzero(&targs[i], sizeof(targs[i]));
+		targs[i].g = &g;
+		targs[i].used = 1;
+		targs[i].completed = 0;
+		targs[i].fd = tfd;
+		targs[i].nmr = tifreq;
+		targs[i].nifp = tnifp;
+		targs[i].qfirst = (g.nthreads > 1) ? i : 0;
+		targs[i].qlast = (g.nthreads > 1) ? i+1 : tifreq.nr_numrings;
+		targs[i].me = i;
+		targs[i].affinity = g.cpus ? i % g.cpus : -1;
+		if (td_body == sender_body) {
+			/* initialize the packet to send. */
+			initialize_packet(&targs[i]);
+		}
+
+		if (pthread_create(&targs[i].thread, NULL, td_body,
+				   &targs[i]) == -1) {
+			D("Unable to create thread %d", i);
+			targs[i].used = 0;
+		}
+	}
+
+    {
+	uint64_t my_count = 0, prev = 0;
+	uint64_t count = 0;
+	double delta_t;
+	struct timeval tic, toc;
+
+	gettimeofday(&toc, NULL);
+	for (;;) {
+		struct timeval now, delta;
+		uint64_t pps;
+		int done = 0;
+
+		delta.tv_sec = report_interval/1000;
+		delta.tv_usec = (report_interval%1000)*1000;
+		select(0, NULL, NULL, NULL, &delta);
+		gettimeofday(&now, NULL);
+		timersub(&now, &toc, &toc);
+		my_count = 0;
+		for (i = 0; i < g.nthreads; i++) {
+			my_count += targs[i].count;
+			if (targs[i].used == 0)
+				done++;
+		}
+		pps = toc.tv_sec* 1000000 + toc.tv_usec;
+		if (pps < 10000)
+			continue;
+		pps = (my_count - prev)*1000000 / pps;
+		D("%llu pps", pps);
+		prev = my_count;
+		toc = now;
+		if (done == g.nthreads)
+			break;
+	}
+
+	timerclear(&tic);
+	timerclear(&toc);
+	for (i = 0; i < g.nthreads; i++) {
+		/*
+		 * Join active threads, unregister interfaces and close
+		 * file descriptors.
+		 */
+		pthread_join(targs[i].thread, NULL);
+		ioctl(targs[i].fd, NIOCUNREGIF, &targs[i].nmr);
+		close(targs[i].fd);
+
+		if (targs[i].completed == 0)
+			continue;
+
+		/*
+		 * Collect threads o1utput and extract information about
+		 * how log it took to send all the packets.
+		 */
+		count += targs[i].count;
+		if (!timerisset(&tic) || timercmp(&targs[i].tic, &tic, <))
+			tic = targs[i].tic;
+		if (!timerisset(&toc) || timercmp(&targs[i].toc, &toc, >))
+			toc = targs[i].toc;
+	}
+
+	/* print output. */
+	timersub(&toc, &tic, &toc);
+	delta_t = toc.tv_sec + 1e-6* toc.tv_usec;
+	if (td_body == sender_body)
+		tx_output(count, g.pkt_size, delta_t);
+	else
+		rx_output(count, delta_t);
+    }
+
+	ioctl(fd, NIOCUNREGIF, &nmr);
+	munmap(mmap_addr, nmr.nr_memsize);
+	close(fd);
+
+	return (0);
+}
+/* end of file */
diff --git a/tools/tools/netrate/netblast/netblast.c b/tools/tools/netrate/netblast/netblast.c
index f932fca..bf1f3ae 100644
--- a/tools/tools/netrate/netblast/netblast.c
+++ b/tools/tools/netrate/netblast/netblast.c
@@ -32,13 +32,13 @@
 #include <sys/time.h>
 
 #include <netinet/in.h>
-
-#include <arpa/inet.h>
+#include <netdb.h>			/* getaddrinfo */
 
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>			/* close */
 
 static void
 usage(void)
@@ -141,26 +141,25 @@ blast_loop(int s, long duration, u_char *packet, u_int packet_len)
 int
 main(int argc, char *argv[])
 {
-	long payloadsize, port, duration;
-	struct sockaddr_in sin;
+	long payloadsize, duration;
+	struct addrinfo hints, *res, *res0;
 	char *dummy, *packet;
-	int s;
+	int port, s, error;
+	const char *cause = NULL;
 
 	if (argc != 5)
 		usage();
 
-	bzero(&sin, sizeof(sin));
-	sin.sin_len = sizeof(sin);
-	sin.sin_family = AF_INET;
-	if (inet_aton(argv[1], &sin.sin_addr) == 0) {
-		perror(argv[1]);
-		return (-1);
-	}
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = PF_UNSPEC;
+	hints.ai_socktype = SOCK_DGRAM;
 
 	port = strtoul(argv[2], &dummy, 10);
-	if (port < 1 || port > 65535 || *dummy != '\0')
+	if (port < 1 || port > 65535 || *dummy != '\0') {
+		fprintf(stderr, "Invalid port number: %s\n", argv[2]);
 		usage();
-	sin.sin_port = htons(port);
+		/*NOTREACHED*/
+	}
 
 	payloadsize = strtoul(argv[3], &dummy, 10);
 	if (payloadsize < 0 || *dummy != '\0')
@@ -168,29 +167,55 @@ main(int argc, char *argv[])
 	if (payloadsize > 32768) {
 		fprintf(stderr, "payloadsize > 32768\n");
 		return (-1);
+		/*NOTREACHED*/
 	}
 
 	duration = strtoul(argv[4], &dummy, 10);
-	if (duration < 0 || *dummy != '\0')
+	if (duration < 0 || *dummy != '\0') {
+		fprintf(stderr, "Invalid duration time: %s\n", argv[4]);
 		usage();
+		/*NOTREACHED*/
+	}
 
 	packet = malloc(payloadsize);
 	if (packet == NULL) {
 		perror("malloc");
 		return (-1);
+		/*NOTREACHED*/
 	}
-	bzero(packet, payloadsize);
 
-	s = socket(PF_INET, SOCK_DGRAM, 0);
-	if (s == -1) {
-		perror("socket");
+	bzero(packet, payloadsize);
+	error = getaddrinfo(argv[1],argv[2], &hints, &res0);
+	if (error) {
+		perror(gai_strerror(error));
 		return (-1);
+		/*NOTREACHED*/
 	}
+	s = -1;
+	for (res = res0; res; res = res->ai_next) {
+		s = socket(res->ai_family, res->ai_socktype, 0);
+		if (s < 0) {
+			cause = "socket";
+			continue;
+		}
+
+		if (connect(s, res->ai_addr, res->ai_addrlen) < 0) {
+			cause = "connect";
+			close(s);
+			s = -1;
+			continue;
+		}
 
-	if (connect(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
-		perror("connect");
+		break;  /* okay we got one */
+	}
+	if (s < 0) {
+		perror(cause);
 		return (-1);
+		/*NOTREACHED*/
 	}
 
+	freeaddrinfo(res0);
+
 	return (blast_loop(s, duration, packet, payloadsize));
+
 }
diff --git a/tools/tools/netrate/netreceive/netreceive.c b/tools/tools/netrate/netreceive/netreceive.c
index 4a3a6f6..b54ee7f 100644
--- a/tools/tools/netrate/netreceive/netreceive.c
+++ b/tools/tools/netrate/netreceive/netreceive.c
@@ -29,14 +29,19 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/time.h>
+#include <sys/poll.h>
 
 #include <netinet/in.h>
+#include <netdb.h>          /* getaddrinfo */
 
 #include <arpa/inet.h>
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>         /* close */
+
+#define MAXSOCK 20
 
 static void
 usage(void)
@@ -49,23 +54,26 @@ usage(void)
 int
 main(int argc, char *argv[])
 {
-	struct sockaddr_in sin;
+	struct addrinfo hints, *res, *res0;
 	char *dummy, *packet;
-	long port;
-	int s, v;
+	int port;
+	int error, v, i;
+	const char *cause = NULL;
+	int s[MAXSOCK];
+	struct pollfd fds[MAXSOCK];
+	int nsock;
 
 	if (argc != 2)
 		usage();
 
-	bzero(&sin, sizeof(sin));
-	sin.sin_len = sizeof(sin);
-	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = htonl(INADDR_ANY);
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = PF_UNSPEC;
+	hints.ai_socktype = SOCK_DGRAM;
+	hints.ai_flags = AI_PASSIVE;
 
 	port = strtoul(argv[1], &dummy, 10);
 	if (port < 1 || port > 65535 || *dummy != '\0')
 		usage();
-	sin.sin_port = htons(port);
 
 	packet = malloc(65536);
 	if (packet == NULL) {
@@ -74,27 +82,60 @@ main(int argc, char *argv[])
 	}
 	bzero(packet, 65536);
 
-	s = socket(PF_INET, SOCK_DGRAM, 0);
-	if (s == -1) {
-		perror("socket");
+	error = getaddrinfo(NULL, argv[1], &hints, &res0);
+	if (error) {
+		perror(gai_strerror(error));
 		return (-1);
+		/*NOTREACHED*/
 	}
 
-	v = 128 * 1024;
-	if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, &v, sizeof(v)) < 0) {
-		perror("SO_RCVBUF");
-		return (-1);
+	nsock = 0;
+	for (res = res0; res && nsock < MAXSOCK; res = res->ai_next) {
+		s[nsock] = socket(res->ai_family, res->ai_socktype,
+		res->ai_protocol);
+		if (s[nsock] < 0) {
+			cause = "socket";
+			continue;
+		}
+
+		v = 128 * 1024;
+		if (setsockopt(s[nsock], SOL_SOCKET, SO_RCVBUF, &v, sizeof(v)) < 0) {
+			cause = "SO_RCVBUF";
+			close(s[nsock]);
+			continue;
+		}
+		if (bind(s[nsock], res->ai_addr, res->ai_addrlen) < 0) {
+			cause = "bind";
+			close(s[nsock]);
+			continue;
+		}
+		(void) listen(s[nsock], 5);
+		fds[nsock].fd = s[nsock];
+		fds[nsock].events = POLLIN;
+
+		nsock++;
 	}
-
-	if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
-		perror("bind");
+	if (nsock == 0) {
+		perror(cause);
 		return (-1);
+		/*NOTREACHED*/
 	}
 
 	printf("netreceive listening on UDP port %d\n", (u_short)port);
 
 	while (1) {
-		if (recv(s, packet, 65536, 0) < 0)
-			perror("recv");
+		if (poll(fds, nsock, -1) < 0) 
+			perror("poll");
+		for (i = 0; i > nsock; i++) {
+			if (fds[i].revents & POLLIN) {
+				if (recv(s[i], packet, 65536, 0) < 0)
+					perror("recv");
+			}
+			if ((fds[i].revents &~ POLLIN) != 0)
+				perror("poll");
+		}
 	}
+	
+	/*NOTREACHED*/
+	freeaddrinfo(res0);
 }
diff --git a/tools/tools/netrate/netsend/netsend.c b/tools/tools/netrate/netsend/netsend.c
index 49cd343..f97594e 100644
--- a/tools/tools/netrate/netsend/netsend.c
+++ b/tools/tools/netrate/netsend/netsend.c
@@ -29,6 +29,7 @@
 #include <sys/endian.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <net/if.h>		/* if_nametoindex() */
 #include <sys/time.h>
 
 #include <netinet/in.h>
@@ -40,13 +41,17 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <netdb.h>
+
 /* program arguments */
 struct _a {
 	int s;
+	int ipv6;
 	struct timespec interval;
 	int port, port_max;
 	long duration;
 	struct sockaddr_in sin;
+	struct sockaddr_in6 sin6;
 	int packet_len;
 	void *packet;
 };
@@ -179,9 +184,16 @@ timing_loop(struct _a *a)
 	ic = gettimeofday_cycles;
 	cur_port = a->port;
 	if (a->port == a->port_max) {
-		if (connect(a->s, (struct sockaddr *)&a->sin, sizeof(a->sin))) {
-			perror("connect");
-			return (-1);
+		if (a->ipv6) {
+			if (connect(a->s, (struct sockaddr *)&a->sin6, sizeof(a->sin6))) {
+				perror("connect (ipv6)");
+				return (-1);
+			}
+		} else {
+			if (connect(a->s, (struct sockaddr *)&a->sin, sizeof(a->sin))) {
+				perror("connect (ipv4)");
+				return (-1);
+			}
 		}
 	}
 	while (1) {
@@ -215,8 +227,13 @@ timing_loop(struct _a *a)
 			a->sin.sin_port = htons(cur_port++);
 			if (cur_port > a->port_max)
 				cur_port = a->port;
+			if (a->ipv6) {
+			ret = sendto(a->s, a->packet, a->packet_len, 0,
+			    (struct sockaddr *)&a->sin6, sizeof(a->sin6));
+			} else {
 			ret = sendto(a->s, a->packet, a->packet_len, 0,
 				(struct sockaddr *)&a->sin, sizeof(a->sin));
+			}
 		}
 		if (ret < 0)
 			send_errors++;
@@ -254,25 +271,48 @@ main(int argc, char *argv[])
 	long rate, payloadsize, port;
 	char *dummy;
 	struct _a a;	/* arguments */
+	struct addrinfo hints, *res, *ressave;
 
 	bzero(&a, sizeof(a));
 
 	if (argc != 6)
 		usage();
 
-	a.sin.sin_len = sizeof(a.sin);
-	a.sin.sin_family = AF_INET;
-	if (inet_aton(argv[1], &a.sin.sin_addr) == 0) {
-		perror(argv[1]);
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_UNSPEC;
+
+	if (getaddrinfo(argv[1], NULL, &hints, &res) != 0) {
+		fprintf(stderr, "Couldn't resolv %s\n", argv[1]);
 		return (-1);
 	}
+	ressave = res;
+	while (res) {
+		if (res->ai_family == AF_INET) {
+			memcpy(&a.sin, res->ai_addr, res->ai_addrlen);
+			a.ipv6 = 0;
+			break;
+		} else if (res->ai_family == AF_INET6) {
+			memcpy(&a.sin6, res->ai_addr, res->ai_addrlen);
+			a.ipv6 = 1;
+			break;
+		} 
+		res = res->ai_next;
+	}
+	if (!res) {
+		fprintf(stderr, "Couldn't resolv %s\n", argv[1]);
+		exit(1);
+	}
+	freeaddrinfo(ressave);
 
 	port = strtoul(argv[2], &dummy, 10);
 	if (port < 1 || port > 65535)
 		usage();
 	if (*dummy != '\0' && *dummy != '-')
 		usage();
-	a.sin.sin_port = htons(port);
+	if (a.ipv6)
+		a.sin6.sin6_port = htons(port);
+	else
+		a.sin.sin_port = htons(port);
 	a.port = a.port_max = port;
 	if (*dummy == '-') {	/* set high port */
 		port = strtoul(dummy + 1, &dummy, 10);
@@ -328,7 +368,10 @@ main(int argc, char *argv[])
 	    "seconds\n", payloadsize, (intmax_t)a.interval.tv_sec,
 	    a.interval.tv_nsec, a.duration);
 
-	a.s = socket(PF_INET, SOCK_DGRAM, 0);
+	if (a.ipv6)
+		a.s = socket(PF_INET6, SOCK_DGRAM, 0);
+	else
+		a.s = socket(PF_INET, SOCK_DGRAM, 0);
 	if (a.s == -1) {
 		perror("socket");
 		return (-1);
diff --git a/usr.bin/at/at.c b/usr.bin/at/at.c
index c036013..cb2fadb 100644
--- a/usr.bin/at/at.c
+++ b/usr.bin/at/at.c
@@ -90,20 +90,18 @@ enum { ATQ, ATRM, AT, BATCH, CAT };	/* what program we want to run */
 
 /* File scope variables */
 
-const char *no_export[] =
-{
+static const char *no_export[] = {
     "TERM", "TERMCAP", "DISPLAY", "_"
-} ;
+};
 static int send_mail = 0;
+static char *atinput = NULL;	/* where to get input from */
+static char atqueue = 0;	/* which queue to examine for jobs (atq) */
 
 /* External variables */
 
 extern char **environ;
 int fcreated;
 char atfile[] = ATJOB_DIR "12345678901234";
-
-char *atinput = (char*)0;	/* where to get input from */
-char atqueue = 0;		/* which queue to examine for jobs (atq) */
 char atverify = 0;		/* verify time instead of queuing job */
 char *namep;
 
diff --git a/usr.bin/at/parsetime.c b/usr.bin/at/parsetime.c
index 195a38d..8c83bcc 100644
--- a/usr.bin/at/parsetime.c
+++ b/usr.bin/at/parsetime.c
@@ -72,7 +72,7 @@ enum {	/* symbols */
 
 /* parse translation table - table driven parsers can be your FRIEND!
  */
-struct {
+static const struct {
     const char *name;	/* token name */
     int value;	/* token id */
     int plural;	/* is this plural? */
diff --git a/usr.bin/banner/banner.c b/usr.bin/banner/banner.c
index 7115983..b363f72 100644
--- a/usr.bin/banner/banner.c
+++ b/usr.bin/banner/banner.c
@@ -59,7 +59,7 @@ __FBSDID("$FreeBSD$");
 #define NBYTES 9271
 
 /* Pointers into data_table for each ASCII char */
-const int asc_ptr[NCHARS] = {
+static const int asc_ptr[NCHARS] = {
 /* ^@ */   0,      0,      0,      0,      0,      0,      0,      0,
 /* ^H */   0,      0,      0,      0,      0,      0,      0,      0,
 /* ^P */   0,      0,      0,      0,      0,      0,      0,      0,
@@ -86,7 +86,7 @@ const int asc_ptr[NCHARS] = {
  * is the next elt in array) and goto second
  * next element in array.
  */
-const unsigned char data_table[NBYTES] = {
+static const unsigned char data_table[NBYTES] = {
 /*             0     1     2     3     4     5     6     7     8     9 */
 /*    0 */   129,  227,  130,   34,    6,   90,   19,  129,   32,   10,
 /*   10 */    74,   40,  129,   31,   12,   64,   53,  129,   30,   14,
@@ -1018,11 +1018,11 @@ const unsigned char data_table[NBYTES] = {
 /* 9270 */   193
 };
 
-char	line[DWIDTH];
-char	*message;
-char	print[DWIDTH];
-int	debug, i, j, linen, max, nchars, pc, term, trace, x, y;
-int	width = DWIDTH;	/* -w option: scrunch letters to 80 columns */
+static char	line[DWIDTH];
+static char	*message;
+static char	print[DWIDTH];
+static int	debug, i, j, linen, max, nchars, pc, term, trace, x, y;
+static int	width = DWIDTH;	/* -w option: scrunch letters to 80 columns */
 
 static void usage(void);
 
diff --git a/usr.bin/c99/c99.c b/usr.bin/c99/c99.c
index 553099f..f5f273f 100644
--- a/usr.bin/c99/c99.c
+++ b/usr.bin/c99/c99.c
@@ -43,12 +43,12 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-char **args;
-u_int cargs, nargs;
+static char **args;
+static u_int cargs, nargs;
 
-void addarg(const char *);
-void addlib(const char *);
-void usage(void);
+static void addarg(const char *);
+static void addlib(const char *);
+static void usage(void);
 
 int
 main(int argc, char *argv[])
@@ -91,7 +91,7 @@ main(int argc, char *argv[])
 	err(1, "/usr/bin/cc");
 }
 
-void
+static void
 addarg(const char *item)
 {
 	if (nargs + 1 >= cargs) {
@@ -104,7 +104,7 @@ addarg(const char *item)
 	args[nargs] = NULL;
 }
 
-void
+static void
 addlib(const char *lib)
 {
 
@@ -123,7 +123,7 @@ addlib(const char *lib)
 	}
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr, "%s\n%s\n",
diff --git a/usr.bin/calendar/calendar.1 b/usr.bin/calendar/calendar.1
index 42fa68d..60e8955 100644
--- a/usr.bin/calendar/calendar.1
+++ b/usr.bin/calendar/calendar.1
@@ -36,11 +36,14 @@
 .Nd reminder service
 .Sh SYNOPSIS
 .Nm
-.Op Fl a
 .Op Fl A Ar num
+.Op Fl a
 .Op Fl B Ar num
+.Op Fl D Ar moon|sun
+.Op Fl d
 .Op Fl F Ar friday
 .Op Fl f Ar calendarfile
+.Op Fl l Ar longitude
 .Oo
 .Bk -words
 .Fl t Ar dd Ns
@@ -49,16 +52,14 @@
 .Sm on
 .Ek
 .Oc
-.Op Fl W Ar num
 .Op Fl U Ar UTC-offset
-.Op Fl l Ar longitude
+.Op Fl W Ar num
 .Sh DESCRIPTION
 The
 .Nm
 utility checks the current directory for a file named
 .Pa calendar
-and displays lines that begin with either today's date
-or tomorrow's.
+and displays lines that fall into the specified date range.
 On the day before a weekend (normally Friday), events for the next
 three days are displayed.
 .Pp
@@ -76,6 +77,10 @@ This requires super-user privileges.
 Print lines from today and the previous
 .Ar num
 days (backward, past).
+.It Fl D Ar moon|sun
+Print UTC offset, longitude and moon or sun information.
+.It Fl d
+Debug option: print current date information.
 .It Fl F Ar friday
 Specify which day of the week is ``Friday'' (the day before the
 weekend begins).
@@ -84,6 +89,11 @@ Default is 5.
 Use
 .Pa calendarfile
 as the default calendar file.
+.It Fl l Ar longitude
+Perform lunar and solar calculations from this longitude.
+If neither longitude nor UTC offset is specified, the calculations will 
+be based on the difference between UTC time and localtime.
+If both are specified, UTC offset overrides longitude.
 .It Xo Fl t
 .Sm off
 .Ar dd
@@ -91,12 +101,11 @@ as the default calendar file.
 .Sm on
 .Xc
 For test purposes only: set date directly to argument values.
-.It Fl l Ar longitude , Fl U Ar UTC-offset
-Only one is needed:
-Perform lunar and solar calculations from this longitude or from
-this UTC offset.
-If neither is specified, the calculations will be based on the
-difference between UTC time and localtime.
+.It Fl U Ar UTC-offset
+Perform lunar and solar calculations from this UTC offset.
+If neither UTC offset nor longitude is specified, the calculations
+will be based on the difference between UTC time and localtime.
+If both are specified, UTC offset overrides longitude. 
 .It Fl W Ar num
 Print lines from today and the next
 .Ar num
diff --git a/usr.bin/calendar/calendar.c b/usr.bin/calendar/calendar.c
index 29ac174..bd03e02 100644
--- a/usr.bin/calendar/calendar.c
+++ b/usr.bin/calendar/calendar.c
@@ -35,7 +35,7 @@ static const char copyright[] =
 
 #if 0
 #ifndef lint
-static char sccsid[] = "@(#)calendar.c  8.3 (Berkeley) 3/25/94";
+static char sccsid[] = "@(#)calendar.c	8.3 (Berkeley) 3/25/94";
 #endif
 #endif
 
@@ -79,7 +79,7 @@ main(int argc, char *argv[])
 
 	(void)setlocale(LC_ALL, "");
 
-	while ((ch = getopt(argc, argv, "-A:aB:dD:F:f:l:t:U:W:")) != -1)
+	while ((ch = getopt(argc, argv, "-A:aB:D:dF:f:l:t:U:W:?")) != -1)
 		switch (ch) {
 		case '-':		/* backward contemptible */
 		case 'a':
@@ -90,10 +90,6 @@ main(int argc, char *argv[])
 			doall = 1;
 			break;
 
-		case 'f': /* other calendar file */
-			calendarFile = optarg;
-			break;
-
 		case 'W': /* we don't need no steenking Fridays */
 			Friday = -1;
 			/* FALLTHROUGH */
@@ -106,26 +102,34 @@ main(int argc, char *argv[])
 			f_dayBefore = atoi(optarg);
 			break;
 
+		case 'D': /* debug output of sun and moon info */
+			DEBUG = optarg;
+			break;
+
+		case 'd': /* debug output of current date */
+			debug = 1;
+			break;
+
 		case 'F': /* Change the time: When does weekend start? */
 			Friday = atoi(optarg);
 			break;
+
+		case 'f': /* other calendar file */
+			calendarFile = optarg;
+			break;
+
 		case 'l': /* Change longitudal position */
 			EastLongitude = strtol(optarg, NULL, 10);
 			break;
-		case 'U': /* Change UTC offset */
-			UTCOffset = strtod(optarg, NULL);
-			break;
 
-		case 'd':
-			debug = 1;
-			break;
-		case 'D':
-			DEBUG = optarg;
-			break;
-		case 't': /* other date, undocumented, for tests */
+		case 't': /* other date, for tests */
 			f_time = Mktime(optarg);
 			break;
 
+		case 'U': /* Change UTC offset */
+			UTCOffset = strtod(optarg, NULL);
+			break;
+
 		case '?':
 		default:
 			usage();
@@ -216,10 +220,9 @@ usage(void)
 {
 
 	fprintf(stderr, "%s\n%s\n%s\n",
-	    "usage: calendar [-a] [-A days] [-B days] [-F friday] "
-	    "[-f calendarfile]",
-	    "                [-d] [-t dd[.mm[.year]]] [-W days]",
-	    "                [-U utcoffset] [-l longitude]"
+	    "usage: calendar [-A days] [-a] [-B days] [-D sun|moon] [-d]",
+	    "		     [-F friday] [-f calendarfile] [-l longitude]",
+	    "		     [-t dd[.mm[.year]]] [-U utcoffset] [-W days]"
 	    );
 	exit(1);
 }
diff --git a/usr.bin/calendar/calendars/calendar.birthday b/usr.bin/calendar/calendars/calendar.birthday
index e8749bf..c517799 100644
--- a/usr.bin/calendar/calendars/calendar.birthday
+++ b/usr.bin/calendar/calendars/calendar.birthday
@@ -42,6 +42,7 @@
 01/30	Franklin Delano Roosevelt born in Hyde Park, New York, 1882
 01/31	Jackie Robinson born, 1919
 02/03	Gertrude Stein born, 1874
+02/04	Ken Thompson, creator of unix, born, 1943
 02/05	Alex Harvey (SAHB) is born in Glasgow, Scotland, 1935
 02/06	King George VI of UK dies;  his daughter becomes Elizabeth II, 1952
 02/07	Sinclair Lewis born, 1885
@@ -68,6 +69,7 @@
 02/22	Pierre Jules Cesar Janssen born, 1838, found hydrogen in the sun
 02/23	W.E.B. DuBois born, 1868
 02/24	Winslow Homer born, 1836
+02/24	Steve Jobs born, 1955
 02/25	George Harrison born in Liverpool, England, 1943
 02/25	Renoir born, 1841
 02/26	Dominique Francois Jean Arago born, 1786;
@@ -212,7 +214,7 @@
 09/08	Richard ``the Lionheart'', king of England born in Oxford, 1157
 09/08	Peter Sellers born in Southsea, England, 1925
 09/09	Chinese Communist Party Chairman Mao Tse-Tung dies at age 82, 1976
-09/09	Dennis Ritchie born, 1941
+09/09	Dennis MacAlistair Ritchie, creator of C, born, 1941
 09/12	Jesse Owens born, 1913
 09/13	Walter Reed born, 1851
 09/15	Agatha Christie born in Torquay, England, 1890
@@ -239,6 +241,8 @@
 10/02	Mohandas K. Gandhi born at Porbandar, Kathiawad, India, 1869
 10/04	John V. Atanasoff born, 1903
 10/05	Ray Kroc (founder of McDonald's) born, 1902
+10/05	Steve Jobs died at the age of 56, 2011
+10/12	Dennis MacAlistair Ritchie died at the age of 70, 2011
 10/13	Lenny Bruce is born in New York City, 1925
 10/13	Virgil (Publius Vergilius Maro) born near Mantua, Italy, 70 BC
 10/14	Dwight David Eisenhower, 34th President of the United States, born in
diff --git a/usr.bin/calendar/calendars/calendar.freebsd b/usr.bin/calendar/calendars/calendar.freebsd
index 127cb50..b5f86b2 100644
--- a/usr.bin/calendar/calendars/calendar.freebsd
+++ b/usr.bin/calendar/calendars/calendar.freebsd
@@ -109,6 +109,7 @@
 03/28	Sean C. Farley <scf@FreeBSD.org> born in Indianapolis, Indiana, United States, 1970
 03/29	Thierry Thomas <thierry@FreeBSD.org> born in Luxeuil les Bains, France, 1961
 03/30	Po-Chuan Hsieh <sunpoet@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1978
+03/31	First quarter status reports are due on 04/15
 04/01	Matthew Jacob <mjacob@FreeBSD.org> born in San Francisco, California, United States, 1958
 04/01	Bill Fenner <fenner@FreeBSD.org> born in Bellefonte, Pennsylvania, United States, 1971
 04/01	Peter Edwards <peadar@FreeBSD.org> born in Dublin, Ireland, 1973
@@ -199,6 +200,7 @@
 06/29	Daniel Harris <dannyboy@FreeBSD.org> born in Lubbock, Texas, United States, 1985
 06/29	Andrew Pantyukhin <sat@FreeBSD.org> born in Moscow, Russian Federation, 1985
 06/30	Guido van Rooij <guido@FreeBSD.org> born in Best, Noord-Brabant, the Netherlands, 1965
+06/30	Second quarter status reports are due on 07/15
 07/01	Matthew Dillon <dillon@apollo.backplane.net> born in San Francisco, California, United States, 1966
 07/02	Mark Christopher Ovens <marko@FreeBSD.org> born in Preston, Lancashire, United Kingdom, 1958
 07/02	Vasil Venelinov Dimov <vd@FreeBSD.org> born in Shumen, Bulgaria, 1982
@@ -282,6 +284,7 @@
 09/28	Alex Dupre <ale@FreeBSD.org> born in Milano, Italy, 1980
 09/29	Matthew Hunt <mph@FreeBSD.org> born in Johnstown, Pennsylvania, United States, 1976
 09/30	Hiten M. Pandya <hmp@FreeBSD.org> born in Dar-es-Salaam, Tanzania, East Africa, 1986
+09/30	Third quarter status reports are due on 10/15
 10/02	Beat Gaetzi <beat@FreeBSD.org> born in Zurich, Switzerland, 1980
 10/05	Hiroki Sato <hrs@FreeBSD.org> born in Yamagata, Japan, 1977
 10/05	Chris Costello <chris@FreeBSD.org> born in Houston, Texas, United States, 1985
@@ -341,5 +344,6 @@
 12/28	Ade Lovett <ade@FreeBSD.org> born in London, England, 1969
 12/28	Marius Strobl <marius@FreeBSD.org> born in Cham, Bavaria, Germany, 1978
 12/31	Edwin Groothuis <edwin@FreeBSD.org> born in Geldrop, the Netherlands, 1970
+12/31	Fourth quarter status reports are due on 01/15
 
 #endif /* !_calendar_freebsd_ */
diff --git a/usr.bin/calendar/io.c b/usr.bin/calendar/io.c
index eb37eac..e1ed21d 100644
--- a/usr.bin/calendar/io.c
+++ b/usr.bin/calendar/io.c
@@ -176,7 +176,8 @@ cal(void)
 		*pp = p;
 		if (count < 0) {
 			/* Show error status based on return value */
-			fprintf(stderr, "Ignored: %s\n", buf);
+			if (debug)
+				fprintf(stderr, "Ignored: %s\n", buf);
 			if (count == -1)
 				continue;
 			count = -count + 1;
diff --git a/usr.bin/calendar/parsedata.c b/usr.bin/calendar/parsedata.c
index c7e183c..63e4395 100644
--- a/usr.bin/calendar/parsedata.c
+++ b/usr.bin/calendar/parsedata.c
@@ -10,7 +10,7 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -22,7 +22,7 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- * 
+ *
  */
 
 #include <sys/cdefs.h>
@@ -79,7 +79,7 @@ static char *floattotime(double f);
  *				'300' ... '359' | '360' ... '365'
  * ModifierIndex	::=	'Second' | 'Third' | 'Fourth' | 'Fifth' |
  *				'First' | 'Last'
- * 
+ *
  * SpecialDay		::=	'Easter' | 'Paskha' | 'ChineseNewYear'
  *
  */
@@ -313,7 +313,7 @@ fail:
 allfine:
 	*p = pold;
 	return (1);
-	
+
 }
 
 static void
@@ -407,7 +407,7 @@ parsedaymonth(char *date, int *yearp, int *monthp, int *dayp, int *flags,
 	 *
 	 * Month:     1-12
 	 * Monthname: Jan .. Dec
-	 * Day:       1-31
+	 * Day:	      1-31
 	 * Weekday:   Mon .. Sun
 	 *
 	 */
@@ -754,10 +754,13 @@ parsedaymonth(char *date, int *yearp, int *monthp, int *dayp, int *flags,
 			continue;
 		}
 
-		printf("Unprocessed:\n");
-		debug_determinestyle(2, date, lflags, month, imonth,
-		    dayofmonth, idayofmonth, dayofweek, idayofweek,
-		    modifieroffset, modifierindex, specialday, syear, iyear);
+		if (debug) {
+		    printf("Unprocessed:\n");
+		    debug_determinestyle(2, date, lflags, month, imonth,
+			dayofmonth, idayofmonth, dayofweek, idayofweek,
+			modifieroffset, modifierindex, specialday, syear,
+			iyear);
+		}
 		retvalsign = -1;
 	}
 
@@ -1045,7 +1048,7 @@ dodebug(char *what)
 			printf("\n");
 
 		}
-	
+
 		return;
 	}
 
diff --git a/usr.bin/cap_mkdb/cap_mkdb.c b/usr.bin/cap_mkdb/cap_mkdb.c
index 9ebac27..2f8bd96 100644
--- a/usr.bin/cap_mkdb/cap_mkdb.c
+++ b/usr.bin/cap_mkdb/cap_mkdb.c
@@ -53,15 +53,15 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-void	 db_build(char **);
-void	 dounlink(void);
-void	 usage(void);
+static void	 db_build(char **);
+static void	 dounlink(void);
+static void	 usage(void);
 
-DB *capdbp;
-int verbose;
-char *capdb, *capname, buf[8 * 1024];
+static DB	*capdbp;
+static int	 verbose;
+static char	*capname, buf[8 * 1024];
 
-HASHINFO openinfo = {
+static HASHINFO openinfo = {
 	4096,		/* bsize */
 	0,		/* ffactor */
 	0,		/* nelem */
@@ -134,7 +134,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 dounlink(void)
 {
 	if (capname != NULL)
@@ -153,7 +153,7 @@ dounlink(void)
  * Db_build() builds the name and capability databases according to the
  * details above.
  */
-void
+static void
 db_build(char **ifiles)
 {
 	DBT key, data;
@@ -259,7 +259,7 @@ db_build(char **ifiles)
 		(void)printf("cap_mkdb: %d capability records\n", reccnt);
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr,
diff --git a/usr.bin/catman/catman.c b/usr.bin/catman/catman.c
index 0bb6621..b221111 100644
--- a/usr.bin/catman/catman.c
+++ b/usr.bin/catman/catman.c
@@ -94,7 +94,7 @@ enum Ziptype {NONE, BZIP, GZIP};
 static uid_t uid;
 static int starting_dir;
 static char tmp_file[MAXPATHLEN];
-struct stat test_st;
+static struct stat test_st;
 
 /*
  * A hashtable is an array of chains composed of this entry structure.
diff --git a/usr.bin/checknr/checknr.c b/usr.bin/checknr/checknr.c
index e705106..d3e5355 100644
--- a/usr.bin/checknr/checknr.c
+++ b/usr.bin/checknr/checknr.c
@@ -59,34 +59,34 @@ __FBSDID("$FreeBSD$");
 #define MAXBR	100	/* Max number of bracket pairs known */
 #define MAXCMDS	500	/* Max number of commands known */
 
-void addcmd(char *);
-void addmac(const char *);
-int binsrch(const char *);
-void checkknown(const char *);
-void chkcmd(const char *, const char *);
-void complain(int);
-int eq(const char *, const char *);
-void nomatch(const char *);
-void pe(int);
-void process(FILE *);
-void prop(int);
+static void addcmd(char *);
+static void addmac(const char *);
+static int binsrch(const char *);
+static void checkknown(const char *);
+static void chkcmd(const char *, const char *);
+static void complain(int);
+static int eq(const char *, const char *);
+static void nomatch(const char *);
+static void pe(int);
+static void process(FILE *);
+static void prop(int);
 static void usage(void);
 
 /*
  * The stack on which we remember what we've seen so far.
  */
-struct stkstr {
+static struct stkstr {
 	int opno;	/* number of opening bracket */
 	int pl;		/* '+', '-', ' ' for \s, 1 for \f, 0 for .ft */
 	int parm;	/* parm to size, font, etc */
 	int lno;	/* line number */
 } stk[MAXSTK];
-int stktop;
+static int stktop;
 
 /*
  * The kinds of opening and closing brackets.
  */
-struct brstr {
+static struct brstr {
 	const char *opbr;
 	const char *clbr;
 } br[MAXBR] = {
@@ -145,7 +145,7 @@ struct brstr {
  * All commands known to nroff, plus macro packages.
  * Used so we can complain about unrecognized commands.
  */
-const char *knowncmds[MAXCMDS] = {
+static const char *knowncmds[MAXCMDS] = {
 "$c", "$f", "$h", "$p", "$s", "(b", "(c", "(d", "(f", "(l", "(q", "(t",
 "(x", "(z", ")b", ")c", ")d", ")f", ")l", ")q", ")t", ")x", ")z", "++",
 "+c", "1C", "1c", "2C", "2c", "@(", "@)", "@C", "@D", "@F", "@I", "@M",
@@ -179,13 +179,13 @@ const char *knowncmds[MAXCMDS] = {
 "yr", 0
 };
 
-int	lineno;		/* current line number in input file */
-const char *cfilename;	/* name of current file */
-int	nfiles;		/* number of files to process */
-int	fflag;		/* -f: ignore \f */
-int	sflag;		/* -s: ignore \s */
-int	ncmds;		/* size of knowncmds */
-int	slot;		/* slot in knowncmds found by binsrch */
+static int	lineno;		/* current line number in input file */
+static const char *cfilename;	/* name of current file */
+static int	nfiles;		/* number of files to process */
+static int	fflag;		/* -f: ignore \f */
+static int	sflag;		/* -s: ignore \s */
+static int	ncmds;		/* size of knowncmds */
+static int	slot;		/* slot in knowncmds found by binsrch */
 
 int
 main(int argc, char **argv)
@@ -275,7 +275,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 process(FILE *f)
 {
 	int i, n;
@@ -372,7 +372,7 @@ process(FILE *f)
 	}
 }
 
-void
+static void
 complain(int i)
 {
 	pe(stk[i].lno);
@@ -381,7 +381,7 @@ complain(int i)
 	printf("\n");
 }
 
-void
+static void
 prop(int i)
 {
 	if (stk[i].pl == 0)
@@ -399,7 +399,7 @@ prop(int i)
 	}
 }
 
-void
+static void
 chkcmd(const char *line __unused, const char *mac)
 {
 	int i;
@@ -435,7 +435,7 @@ chkcmd(const char *line __unused, const char *mac)
 	}
 }
 
-void
+static void
 nomatch(const char *mac)
 {
 	int i, j;
@@ -480,14 +480,14 @@ nomatch(const char *mac)
 }
 
 /* eq: are two strings equal? */
-int
+static int
 eq(const char *s1, const char *s2)
 {
 	return (strcmp(s1, s2) == 0);
 }
 
 /* print the first part of an error message, given the line number */
-void
+static void
 pe(int linen)
 {
 	if (nfiles > 1)
@@ -495,7 +495,7 @@ pe(int linen)
 	printf("%d: ", linen);
 }
 
-void
+static void
 checkknown(const char *mac)
 {
 
@@ -513,7 +513,7 @@ checkknown(const char *mac)
 /*
  * We have a .de xx line in "line".  Add xx to the list of known commands.
  */
-void
+static void
 addcmd(char *line)
 {
 	char *mac;
@@ -544,7 +544,7 @@ addcmd(char *line)
  * me someday?)  Anyway, I claim that .de is fairly rare in user
  * nroff programs, and the register loop below is pretty fast.
  */
-void
+static void
 addmac(const char *mac)
 {
 	const char **src, **dest, **loc;
@@ -575,7 +575,7 @@ printf("after: %s %s %s %s %s, %d cmds\n", knowncmds[slot-2], knowncmds[slot-1],
  * Do a binary search in knowncmds for mac.
  * If found, return the index.  If not, return -1.
  */
-int
+static int
 binsrch(const char *mac)
 {
 	const char *p;	/* pointer to current cmd in list */
diff --git a/usr.bin/chpass/chpass.h b/usr.bin/chpass/chpass.h
index ed1a586..fd3a839 100644
--- a/usr.bin/chpass/chpass.h
+++ b/usr.bin/chpass/chpass.h
@@ -48,7 +48,7 @@ typedef struct _entry {
 	int (*func)(char *, struct passwd *, struct _entry *);
 	int restricted;
 	size_t len;
-	char *except, *save;
+	const char *except, *save;
 } ENTRY;
 
 /* Field numbers. */
diff --git a/usr.bin/chpass/table.c b/usr.bin/chpass/table.c
index dffa11c6..a4918b4 100644
--- a/usr.bin/chpass/table.c
+++ b/usr.bin/chpass/table.c
@@ -39,27 +39,24 @@ __FBSDID("$FreeBSD$");
 #include <stddef.h>
 #include "chpass.h"
 
-char e1[] = ": ";
-char e2[] = ":,";
-
 ENTRY list[] = {
-	{ "login",		p_login,  1,   5, e1,   NULL },
-	{ "password",		p_passwd, 1,   8, e1,   NULL },
-	{ "uid",		p_uid,    1,   3, e1,   NULL },
-	{ "gid",		p_gid,    1,   3, e1,   NULL },
-	{ "class",		p_class,  1,   5, e1,   NULL },
+	{ "login",		p_login,  1,   5, ": ", NULL },
+	{ "password",		p_passwd, 1,   8, ": ", NULL },
+	{ "uid",		p_uid,    1,   3, ": ", NULL },
+	{ "gid",		p_gid,    1,   3, ": ", NULL },
+	{ "class",		p_class,  1,   5, ": ", NULL },
 	{ "change",		p_change, 1,   6, NULL, NULL },
 	{ "expire",		p_expire, 1,   6, NULL, NULL },
 #ifdef RESTRICT_FULLNAME_CHANGE		/* do not allow fullname changes */
-	{ "full name",		p_gecos,  1,   9, e2,   NULL },
+	{ "full name",		p_gecos,  1,   9, ":,", NULL },
 #else
-	{ "full name",		p_gecos,  0,   9, e2,   NULL },
+	{ "full name",		p_gecos,  0,   9, ":,", NULL },
 #endif
-	{ "office phone",	p_gecos,  0,  12, e2,   NULL },
-	{ "home phone",		p_gecos,  0,  10, e2,   NULL },
-	{ "office location",	p_gecos,  0,  15, e2,   NULL },
-	{ "other information",	p_gecos,  0,  11, e1,   NULL },
-	{ "home directory",	p_hdir,   1,  14, e1,   NULL },
-	{ "shell",		p_shell,  0,   5, e1,   NULL },
+	{ "office phone",	p_gecos,  0,  12, ":,", NULL },
+	{ "home phone",		p_gecos,  0,  10, ":,", NULL },
+	{ "office location",	p_gecos,  0,  15, ":,", NULL },
+	{ "other information",	p_gecos,  0,  11, ": ", NULL },
+	{ "home directory",	p_hdir,   1,  14, ": ", NULL },
+	{ "shell",		p_shell,  0,   5, ": ", NULL },
 	{ NULL, NULL, 0, 0, NULL, NULL },
 };
diff --git a/usr.bin/cmp/extern.h b/usr.bin/cmp/extern.h
index 84ec872..a35cc3d 100644
--- a/usr.bin/cmp/extern.h
+++ b/usr.bin/cmp/extern.h
@@ -42,4 +42,4 @@ void	c_special(int, const char *, off_t, int, const char *, off_t);
 void	diffmsg(const char *, const char *, off_t, off_t);
 void	eofmsg(const char *);
 
-extern int lflag, sflag, xflag;
+extern int lflag, sflag, xflag, zflag;
diff --git a/usr.bin/col/col.c b/usr.bin/col/col.c
index 1544608..78e9780 100644
--- a/usr.bin/col/col.c
+++ b/usr.bin/col/col.c
@@ -92,22 +92,22 @@ struct line_str {
 	int	l_max_col;		/* max column in the line */
 };
 
-LINE   *alloc_line(void);
-void	dowarn(int);
-void	flush_line(LINE *);
-void	flush_lines(int);
-void	flush_blanks(void);
-void	free_line(LINE *);
-void	usage(void);
-
-CSET	last_set;		/* char_set of last char printed */
-LINE   *lines;
-int	compress_spaces;	/* if doing space -> tab conversion */
-int	fine;			/* if `fine' resolution (half lines) */
-int	max_bufd_lines;		/* max # lines to keep in memory */
-int	nblank_lines;		/* # blanks after last flushed line */
-int	no_backspaces;		/* if not to output any backspaces */
-int	pass_unknown_seqs;	/* pass unknown control sequences */
+static LINE   *alloc_line(void);
+static void	dowarn(int);
+static void	flush_line(LINE *);
+static void	flush_lines(int);
+static void	flush_blanks(void);
+static void	free_line(LINE *);
+static void	usage(void);
+
+static CSET	last_set;		/* char_set of last char printed */
+static LINE    *lines;
+static int	compress_spaces;	/* if doing space -> tab conversion */
+static int	fine;			/* if `fine' resolution (half lines) */
+static int	max_bufd_lines;		/* max # lines to keep in memory */
+static int	nblank_lines;		/* # blanks after last flushed line */
+static int	no_backspaces;		/* if not to output any backspaces */
+static int	pass_unknown_seqs;	/* pass unknown control sequences */
 
 #define	PUTC(ch) \
 	do {					\
@@ -334,7 +334,7 @@ main(int argc, char **argv)
 	exit(0);
 }
 
-void
+static void
 flush_lines(int nflush)
 {
 	LINE *l;
@@ -360,7 +360,7 @@ flush_lines(int nflush)
  * is the number of half line feeds, otherwise it is the number of whole line
  * feeds.
  */
-void
+static void
 flush_blanks(void)
 {
 	int half, i, nb;
@@ -389,7 +389,7 @@ flush_blanks(void)
  * Write a line to stdout taking care of space to tab conversion (-h flag)
  * and character set shifts.
  */
-void
+static void
 flush_line(LINE *l)
 {
 	CHAR *c, *endc;
@@ -502,7 +502,7 @@ flush_line(LINE *l)
 
 static LINE *line_freelist;
 
-LINE *
+static LINE *
 alloc_line(void)
 {
 	LINE *l;
@@ -523,7 +523,7 @@ alloc_line(void)
 	return (l);
 }
 
-void
+static void
 free_line(LINE *l)
 {
 
@@ -531,7 +531,7 @@ free_line(LINE *l)
 	line_freelist = l;
 }
 
-void
+static void
 usage(void)
 {
 
@@ -539,7 +539,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 dowarn(int line)
 {
 
diff --git a/usr.bin/colcrt/colcrt.c b/usr.bin/colcrt/colcrt.c
index 8d08cea..7ac274b 100644
--- a/usr.bin/colcrt/colcrt.c
+++ b/usr.bin/colcrt/colcrt.c
@@ -64,13 +64,13 @@ __FBSDID("$FreeBSD$");
  * Option -2 forces printing of all half lines.
  */
 
-wchar_t	page[267][132];
+static wchar_t	page[267][132];
 
-int	outline = 1;
-int	outcol;
+static int	outline = 1;
+static int	outcol;
 
-char	suppresul;
-char	printall;
+static char	suppresul;
+static char	printall;
 
 static void	move(int, int);
 static void	pflush(int);
diff --git a/usr.bin/column/column.c b/usr.bin/column/column.c
index 0d38742..bc7aa32 100644
--- a/usr.bin/column/column.c
+++ b/usr.bin/column/column.c
@@ -58,21 +58,21 @@ __FBSDID("$FreeBSD$");
 
 #define	TAB	8
 
-void  c_columnate(void);
-void  input(FILE *);
-void  maketbl(void);
-void  print(void);
-void  r_columnate(void);
-void  usage(void);
-int   width(const wchar_t *);
-
-int termwidth = 80;		/* default terminal width */
-
-int entries;			/* number of records */
-int eval;			/* exit value */
-int maxlength;			/* longest record */
-wchar_t **list;			/* array of pointers to records */
-const wchar_t *separator = L"\t ";	/* field separator for table option */
+static void	c_columnate(void);
+static void	input(FILE *);
+static void	maketbl(void);
+static void	print(void);
+static void	r_columnate(void);
+static void	usage(void);
+static int	width(const wchar_t *);
+
+static int	termwidth = 80;		/* default terminal width */
+
+static int	entries;		/* number of records */
+static int	eval;			/* exit value */
+static int	maxlength;		/* longest record */
+static wchar_t	**list;			/* array of pointers to records */
+static const wchar_t *separator = L"\t "; /* field separator for table option */
 
 int
 main(int argc, char **argv)
@@ -149,7 +149,7 @@ main(int argc, char **argv)
 	exit(eval);
 }
 
-void
+static void
 c_columnate(void)
 {
 	int chcnt, col, cnt, endcol, numcols;
@@ -178,7 +178,7 @@ c_columnate(void)
 		putwchar('\n');
 }
 
-void
+static void
 r_columnate(void)
 {
 	int base, chcnt, cnt, col, endcol, numcols, numrows, row;
@@ -205,7 +205,7 @@ r_columnate(void)
 	}
 }
 
-void
+static void
 print(void)
 {
 	int cnt;
@@ -221,7 +221,7 @@ typedef struct _tbl {
 } TBL;
 #define	DEFCOLS	25
 
-void
+static void
 maketbl(void)
 {
 	TBL *t;
@@ -274,7 +274,7 @@ maketbl(void)
 #define	DEFNUM		1000
 #define	MAXLINELEN	(LINE_MAX + 1)
 
-void
+static void
 input(FILE *fp)
 {
 	static int maxentry;
@@ -313,7 +313,7 @@ input(FILE *fp)
 }
 
 /* Like wcswidth(), but ignores non-printing characters. */
-int
+static int
 width(const wchar_t *wcs)
 {
 	int w, cw;
@@ -324,7 +324,7 @@ width(const wchar_t *wcs)
 	return (w);
 }
 
-void
+static void
 usage(void)
 {
 
diff --git a/usr.bin/comm/comm.1 b/usr.bin/comm/comm.1
index 0a34160..2ae5269 100644
--- a/usr.bin/comm/comm.1
+++ b/usr.bin/comm/comm.1
@@ -61,11 +61,13 @@ The filename ``-'' means the standard input.
 The following options are available:
 .Bl -tag -width indent
 .It Fl 1
-Suppress printing of column 1.
+Suppress printing of column 1, lines only in 
+.Ar file1 .
 .It Fl 2
-Suppress printing of column 2.
+Suppress printing of column 2, lines only in
+.Ar file2 .
 .It Fl 3
-Suppress printing of column 3.
+Suppress printing of column 3, lines common to both.
 .It Fl i
 Case insensitive comparison of lines.
 .El
diff --git a/usr.bin/comm/comm.c b/usr.bin/comm/comm.c
index c6729e6..54763e1 100644
--- a/usr.bin/comm/comm.c
+++ b/usr.bin/comm/comm.c
@@ -57,12 +57,12 @@ __FBSDID("$FreeBSD$");
 #include <wchar.h>
 #include <wctype.h>
 
-int iflag;
-const char *tabs[] = { "", "\t", "\t\t" };
+static int iflag;
+static const char *tabs[] = { "", "\t", "\t\t" };
 
-FILE   *file(const char *);
-wchar_t	*convert(const char *);
-void	show(FILE *, const char *, const char *, char **, size_t *);
+static FILE	*file(const char *);
+static wchar_t	*convert(const char *);
+static void	show(FILE *, const char *, const char *, char **, size_t *);
 static void	usage(void);
 
 int
@@ -189,7 +189,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-wchar_t *
+static wchar_t *
 convert(const char *str)
 {
 	size_t n;
@@ -212,7 +212,7 @@ convert(const char *str)
 	return (buf);
 }
 
-void
+static void
 show(FILE *fp, const char *fn, const char *offset, char **bufp, size_t *buflenp)
 {
 	ssize_t n;
@@ -228,7 +228,7 @@ show(FILE *fp, const char *fn, const char *offset, char **bufp, size_t *buflenp)
 		err(1, "%s", fn);
 }
 
-FILE *
+static FILE *
 file(const char *name)
 {
 	FILE *fp;
diff --git a/usr.bin/compress/compress.c b/usr.bin/compress/compress.c
index 99eabbb..f73419d 100644
--- a/usr.bin/compress/compress.c
+++ b/usr.bin/compress/compress.c
@@ -56,15 +56,15 @@ __FBSDID("$FreeBSD$");
 
 #include "zopen.h"
 
-void	compress(const char *, const char *, int);
-void	cwarn(const char *, ...) __printflike(1, 2);
-void	cwarnx(const char *, ...) __printflike(1, 2);
-void	decompress(const char *, const char *, int);
-int	permission(const char *);
-void	setfile(const char *, struct stat *);
-void	usage(int);
+static void	compress(const char *, const char *, int);
+static void	cwarn(const char *, ...) __printflike(1, 2);
+static void	cwarnx(const char *, ...) __printflike(1, 2);
+static void	decompress(const char *, const char *, int);
+static int	permission(const char *);
+static void	setfile(const char *, struct stat *);
+static void	usage(int);
 
-int eval, force, verbose;
+static int eval, force, verbose;
 
 int
 main(int argc, char *argv[])
@@ -191,7 +191,7 @@ main(int argc, char *argv[])
 	exit (eval);
 }
 
-void
+static void
 compress(const char *in, const char *out, int bits)
 {
 	size_t nr;
@@ -281,7 +281,7 @@ err:	if (ofp) {
 		(void)fclose(ifp);
 }
 
-void
+static void
 decompress(const char *in, const char *out, int bits)
 {
 	size_t nr;
@@ -357,7 +357,7 @@ err:	if (ofp) {
 		(void)fclose(ifp);
 }
 
-void
+static void
 setfile(const char *name, struct stat *fs)
 {
 	static struct timeval tv[2];
@@ -387,7 +387,7 @@ setfile(const char *name, struct stat *fs)
 		cwarn("chflags: %s", name);
 }
 
-int
+static int
 permission(const char *fname)
 {
 	int ch, first;
@@ -401,7 +401,7 @@ permission(const char *fname)
 	return (first == 'y');
 }
 
-void
+static void
 usage(int iscompress)
 {
 	if (iscompress)
@@ -413,7 +413,7 @@ usage(int iscompress)
 	exit(1);
 }
 
-void
+static void
 cwarnx(const char *fmt, ...)
 {
 	va_list ap;
@@ -424,7 +424,7 @@ cwarnx(const char *fmt, ...)
 	eval = 1;
 }
 
-void
+static void
 cwarn(const char *fmt, ...)
 {
 	va_list ap;
diff --git a/usr.bin/compress/doc/NOTES b/usr.bin/compress/doc/NOTES
index 29ca311..ef3931a 100644
--- a/usr.bin/compress/doc/NOTES
+++ b/usr.bin/compress/doc/NOTES
@@ -51,7 +51,7 @@ comprehensive survey of an area which will remain murky for some time.
 
 Until the dust clears, how you approach ideas which are patented depends
 on how paranoid you are of a legal onslaught.  Arbitrary?  Yes.  But
-the patent bar the CCPA (Court of Customs and Patent Appeals)
+the patent bar of the CCPA (Court of Customs and Patent Appeals)
 thanks you for any uncertainty as they, at least, stand to gain
 from any trouble.
 
diff --git a/usr.bin/cpuset/cpuset.c b/usr.bin/cpuset/cpuset.c
index 0c0a2f2..898c3e5 100644
--- a/usr.bin/cpuset/cpuset.c
+++ b/usr.bin/cpuset/cpuset.c
@@ -46,22 +46,22 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include <string.h>
 
-int Cflag;
-int cflag;
-int gflag;
-int iflag;
-int jflag;
-int lflag;
-int pflag;
-int rflag;
-int sflag;
-int tflag;
-int xflag;
-id_t id;
-cpulevel_t level;
-cpuwhich_t which;
+static int Cflag;
+static int cflag;
+static int gflag;
+static int iflag;
+static int jflag;
+static int lflag;
+static int pflag;
+static int rflag;
+static int sflag;
+static int tflag;
+static int xflag;
+static id_t id;
+static cpulevel_t level;
+static cpuwhich_t which;
 
-void usage(void);
+static void usage(void);
 
 static void printset(cpuset_t *mask);
 
@@ -161,8 +161,8 @@ printset(cpuset_t *mask)
 	printf("\n");
 }
 
-const char *whichnames[] = { NULL, "tid", "pid", "cpuset", "irq", "jail" };
-const char *levelnames[] = { NULL, " root", " cpuset", "" };
+static const char *whichnames[] = { NULL, "tid", "pid", "cpuset", "irq", "jail" };
+static const char *levelnames[] = { NULL, " root", " cpuset", "" };
 
 static void
 printaffinity(void)
@@ -343,7 +343,7 @@ main(int argc, char *argv[])
 	exit(EXIT_SUCCESS);
 }
 
-void
+static void
 usage(void)
 {
 
diff --git a/usr.bin/csplit/csplit.c b/usr.bin/csplit/csplit.c
index 2dff81f..7143d73 100644
--- a/usr.bin/csplit/csplit.c
+++ b/usr.bin/csplit/csplit.c
@@ -60,36 +60,36 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-void	 cleanup(void);
-void	 do_lineno(const char *);
-void	 do_rexp(const char *);
-char	*getline(void);
-void	 handlesig(int);
-FILE	*newfile(void);
-void	 toomuch(FILE *, long);
-void	 usage(void);
+static void	 cleanup(void);
+static void	 do_lineno(const char *);
+static void	 do_rexp(const char *);
+static char	*getline(void);
+static void	 handlesig(int);
+static FILE	*newfile(void);
+static void	 toomuch(FILE *, long);
+static void	 usage(void);
 
 /*
  * Command line options
  */
-const char *prefix;		/* File name prefix */
-long	 sufflen;		/* Number of decimal digits for suffix */
-int	 sflag;			/* Suppress output of file names */
-int	 kflag;			/* Keep output if error occurs */
+static const char *prefix;	/* File name prefix */
+static long	 sufflen;	/* Number of decimal digits for suffix */
+static int	 sflag;		/* Suppress output of file names */
+static int	 kflag;		/* Keep output if error occurs */
 
 /*
  * Other miscellaneous globals (XXX too many)
  */
-long	 lineno;		/* Current line number in input file */
-long	 reps;			/* Number of repetitions for this pattern */
-long	 nfiles;		/* Number of files output so far */
-long	 maxfiles;		/* Maximum number of files we can create */
-char	 currfile[PATH_MAX];	/* Current output file */
-const char *infn;		/* Name of the input file */
-FILE	*infile;		/* Input file handle */
-FILE	*overfile;		/* Overflow file for toomuch() */
-off_t	 truncofs;		/* Offset this file should be truncated at */
-int	 doclean;		/* Should cleanup() remove output? */
+static long	 lineno;	/* Current line number in input file */
+static long	 reps;		/* Number of repetitions for this pattern */
+static long	 nfiles;	/* Number of files output so far */
+static long	 maxfiles;	/* Maximum number of files we can create */
+static char	 currfile[PATH_MAX]; /* Current output file */
+static const char *infn;	/* Name of the input file */
+static FILE	*infile;	/* Input file handle */
+static FILE	*overfile;	/* Overflow file for toomuch() */
+static off_t	 truncofs;	/* Offset this file should be truncated at */
+static int	 doclean;	/* Should cleanup() remove output? */
 
 int
 main(int argc, char *argv[])
@@ -209,7 +209,7 @@ main(int argc, char *argv[])
 	return (0);
 }
 
-void
+static void
 usage(void)
 {
 
@@ -218,7 +218,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 handlesig(int sig __unused)
 {
 	const char msg[] = "csplit: caught signal, cleaning up\n";
@@ -229,7 +229,7 @@ handlesig(int sig __unused)
 }
 
 /* Create a new output file. */
-FILE *
+static FILE *
 newfile(void)
 {
 	FILE *fp;
@@ -245,7 +245,7 @@ newfile(void)
 }
 
 /* Remove partial output, called before exiting. */
-void
+static void
 cleanup(void)
 {
 	char fnbuf[PATH_MAX];
@@ -269,7 +269,7 @@ cleanup(void)
 }
 
 /* Read a line from the input into a static buffer. */
-char *
+static char *
 getline(void)
 {
 	static char lbuf[LINE_MAX];
@@ -292,7 +292,7 @@ again: if (fgets(lbuf, sizeof(lbuf), src) == NULL) {
 }
 
 /* Conceptually rewind the input (as obtained by getline()) back `n' lines. */
-void
+static void
 toomuch(FILE *ofp, long n)
 {
 	char buf[BUFSIZ];
@@ -351,7 +351,7 @@ toomuch(FILE *ofp, long n)
 }
 
 /* Handle splits for /regexp/ and %regexp% patterns. */
-void
+static void
 do_rexp(const char *expr)
 {
 	regex_t cre;
@@ -433,7 +433,7 @@ do_rexp(const char *expr)
 }
 
 /* Handle splits based on line number. */
-void
+static void
 do_lineno(const char *expr)
 {
 	long lastline, tgtline;
diff --git a/usr.bin/cut/cut.c b/usr.bin/cut/cut.c
index 0818e66..d32d4ab 100644
--- a/usr.bin/cut/cut.c
+++ b/usr.bin/cut/cut.c
@@ -50,25 +50,25 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include <wchar.h>
 
-int	bflag;
-int	cflag;
-wchar_t	dchar;
-char	dcharmb[MB_LEN_MAX + 1];
-int	dflag;
-int	fflag;
-int	nflag;
-int	sflag;
+static int	bflag;
+static int	cflag;
+static wchar_t	dchar;
+static char	dcharmb[MB_LEN_MAX + 1];
+static int	dflag;
+static int	fflag;
+static int	nflag;
+static int	sflag;
 
-size_t	autostart, autostop, maxval;
-char *	positions;
+static size_t	autostart, autostop, maxval;
+static char *	positions;
 
-int	b_cut(FILE *, const char *);
-int	b_n_cut(FILE *, const char *);
-int	c_cut(FILE *, const char *);
-int	f_cut(FILE *, const char *);
-void	get_list(char *);
-void	needpos(size_t);
-static 	void usage(void);
+static int	b_cut(FILE *, const char *);
+static int	b_n_cut(FILE *, const char *);
+static int	c_cut(FILE *, const char *);
+static int	f_cut(FILE *, const char *);
+static void	get_list(char *);
+static void	needpos(size_t);
+static void	usage(void);
 
 int
 main(int argc, char *argv[])
@@ -153,7 +153,7 @@ main(int argc, char *argv[])
 	exit(rval);
 }
 
-void
+static void
 get_list(char *list)
 {
 	size_t setautostart, start, stop;
@@ -208,7 +208,7 @@ get_list(char *list)
 		memset(positions + 1, '1', autostart);
 }
 
-void
+static void
 needpos(size_t n)
 {
 	static size_t npos;
@@ -227,7 +227,7 @@ needpos(size_t n)
 	}
 }
 
-int
+static int
 b_cut(FILE *fp, const char *fname __unused)
 {
 	int ch, col;
@@ -261,7 +261,7 @@ b_cut(FILE *fp, const char *fname __unused)
  * Although this function also handles the case where -n is not specified,
  * b_cut() ought to be much faster.
  */
-int
+static int
 b_n_cut(FILE *fp, const char *fname)
 {
 	size_t col, i, lbuflen;
@@ -323,7 +323,7 @@ b_n_cut(FILE *fp, const char *fname)
 	return (warned);
 }
 
-int
+static int
 c_cut(FILE *fp, const char *fname)
 {
 	wint_t ch;
@@ -358,7 +358,7 @@ out:
 	return (0);
 }
 
-int
+static int
 f_cut(FILE *fp, const char *fname)
 {
 	wchar_t ch;
diff --git a/usr.bin/dc/Makefile b/usr.bin/dc/Makefile
index ee91c95..67af146 100644
--- a/usr.bin/dc/Makefile
+++ b/usr.bin/dc/Makefile
@@ -3,7 +3,6 @@
 
 PROG=	dc
 SRCS=	dc.c bcode.c inout.c mem.c stack.c
-CFLAGS+=--param max-inline-insns-single=64
 DPADD=	${LIBCRYPTO}
 LDADD=	-lcrypto
 
diff --git a/usr.bin/dc/dc.c b/usr.bin/dc/dc.c
index 47315cc..d5edadb 100644
--- a/usr.bin/dc/dc.c
+++ b/usr.bin/dc/dc.c
@@ -39,9 +39,9 @@ static void		 usage(void);
 
 extern char		*__progname;
 
-struct source		 src;
+static struct source	 src;
 
-struct option long_options[] =
+static const struct option long_options[] =
 {
 	{"expression",		required_argument,	NULL,	'e'},
 	{"file",		required_argument,	NULL,	'f'},
diff --git a/usr.bin/du/du.c b/usr.bin/du/du.c
index cdaa70d..5c670d4 100644
--- a/usr.bin/du/du.c
+++ b/usr.bin/du/du.c
@@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <sysexits.h>
 #include <unistd.h>
 
-SLIST_HEAD(ignhead, ignentry) ignores;
+static SLIST_HEAD(ignhead, ignentry) ignores;
 struct ignentry {
 	char			*mask;
 	SLIST_ENTRY(ignentry)	next;
@@ -88,7 +88,6 @@ main(int argc, char *argv[])
 	off_t		savednumber, curblocks;
 	off_t		threshold, threshold_sign;
 	int		ftsoptions;
-	int		listall;
 	int		depth;
 	int		Hflag, Lflag, Pflag, aflag, sflag, dflag, cflag;
 	int		hflag, lflag, ch, notused, rval;
@@ -229,17 +228,10 @@ main(int argc, char *argv[])
 	if (!Aflag && (cblocksize % DEV_BSIZE) != 0)
 		cblocksize = howmany(cblocksize, DEV_BSIZE) * DEV_BSIZE;
 
-	listall = 0;
-
-	if (aflag) {
-		if (sflag || dflag)
-			usage();
-		listall = 1;
-	} else if (sflag) {
-		if (dflag)
-			usage();
+	if (aflag + dflag + sflag > 1)
+		usage();
+	if (sflag)
 		depth = 0;
-	}
 
 	if (!*argv) {
 		argv = save;
@@ -320,7 +312,7 @@ main(int argc, char *argv[])
 			    howmany(p->fts_statp->st_size, cblocksize) :
 			    howmany(p->fts_statp->st_blocks, cblocksize);
 
-			if (listall || p->fts_level == 0) {
+			if (aflag || p->fts_level == 0) {
 				if (hflag) {
 					prthumanval(curblocks);
 					(void)printf("\t%s\n", p->fts_path);
diff --git a/usr.bin/enigma/enigma.c b/usr.bin/enigma/enigma.c
index 68fd29d..49cb006 100644
--- a/usr.bin/enigma/enigma.c
+++ b/usr.bin/enigma/enigma.c
@@ -25,16 +25,16 @@ __FBSDID("$FreeBSD$");
 #define ECHO 010
 #define ROTORSZ 256
 #define MASK 0377
-char	t1[ROTORSZ];
-char	t2[ROTORSZ];
-char	t3[ROTORSZ];
-char	deck[ROTORSZ];
-char	buf[13];
+static char	t1[ROTORSZ];
+static char	t2[ROTORSZ];
+static char	t3[ROTORSZ];
+static char	deck[ROTORSZ];
+static char	buf[13];
 
-void	shuffle(char *);
-void	setup(char *);
+static void	shuffle(char *);
+static void	setup(char *);
 
-void
+static void
 setup(char *pw)
 {
 	int ic, i, k, temp;
@@ -128,7 +128,7 @@ main(int argc, char *argv[])
 	return 0;
 }
 
-void
+static void
 shuffle(char deckary[])
 {
 	int i, ic, k, temp;
diff --git a/usr.bin/expand/expand.c b/usr.bin/expand/expand.c
index e8d85b4..1d89089 100644
--- a/usr.bin/expand/expand.c
+++ b/usr.bin/expand/expand.c
@@ -53,8 +53,8 @@ __FBSDID("$FreeBSD$");
 /*
  * expand - expand tabs to equivalent spaces
  */
-int	nstops;
-int	tabstops[100];
+static int	nstops;
+static int	tabstops[100];
 
 static void getstops(char *);
 static void usage(void);
diff --git a/usr.bin/fold/fold.c b/usr.bin/fold/fold.c
index d577713..1ce9b5e 100644
--- a/usr.bin/fold/fold.c
+++ b/usr.bin/fold/fold.c
@@ -61,8 +61,8 @@ void fold(int);
 static int newpos(int, wint_t);
 static void usage(void);
 
-int bflag;			/* Count bytes, not columns */
-int sflag;			/* Split on word boundaries */
+static int bflag;		/* Count bytes, not columns */
+static int sflag;		/* Split on word boundaries */
 
 int
 main(int argc, char **argv)
diff --git a/usr.bin/fstat/fstat.c b/usr.bin/fstat/fstat.c
index ed85351..159ceb9 100644
--- a/usr.bin/fstat/fstat.c
+++ b/usr.bin/fstat/fstat.c
@@ -57,13 +57,13 @@ __FBSDID("$FreeBSD$");
 
 #include "functions.h"
 
-int 	fsflg,	/* show files on same filesystem as file(s) argument */
-	pflg,	/* show files open by a particular pid */
-	uflg;	/* show files open by a particular (effective) user */
-int 	checkfile; /* true if restricting to particular files or filesystems */
-int	nflg;	/* (numerical) display f.s. and rdev as dev_t */
-int	mflg;	/* include memory-mapped files */
-int	vflg;	/* be verbose */
+static int 	fsflg,	/* show files on same filesystem as file(s) argument */
+		pflg,	/* show files open by a particular pid */
+		uflg;	/* show files open by a particular (effective) user */
+static int 	checkfile; /* restrict to particular files or filesystems */
+static int	nflg;	/* (numerical) display f.s. and rdev as dev_t */
+static int	mflg;	/* include memory-mapped files */
+static int	vflg;	/* be verbose */
 
 typedef struct devs {
 	struct devs	*next;
@@ -72,8 +72,8 @@ typedef struct devs {
 	const char	*name;
 } DEVS;
 
-DEVS *devs;
-char *memf, *nlistf;
+static DEVS *devs;
+static char *memf, *nlistf;
 
 static int	getfname(const char *filename);
 static void	dofiles(struct procstat *procstat, struct kinfo_proc *p);
diff --git a/usr.bin/fstat/fuser.c b/usr.bin/fstat/fuser.c
index 364d57c..dbb0d86 100644
--- a/usr.bin/fstat/fuser.c
+++ b/usr.bin/fstat/fuser.c
@@ -52,7 +52,7 @@ __FBSDID("$FreeBSD$");
 /*
  * File access mode flags table.
  */
-struct {
+static const struct {
 	int	flag;
 	char	ch;
 } fflags[] = {
@@ -67,7 +67,7 @@ struct {
 /*
  * Usage flags translation table.
  */
-struct {
+static const struct {
 	int	flag;
 	char	ch;
 } uflags[] = {
diff --git a/usr.bin/getent/getent.c b/usr.bin/getent/getent.c
index 0459cca..85857d8 100644
--- a/usr.bin/getent/getent.c
+++ b/usr.bin/getent/getent.c
@@ -600,13 +600,24 @@ utmpxprint(const struct utmpx *ut)
 		printf("\" pid=\"%d\" user=\"%s\" line=\"%s\" host=\"%s\"\n",
 		    ut->ut_pid, ut->ut_user, ut->ut_line, ut->ut_host);
 		break;
+	case INIT_PROCESS:
+		printf("init process: id=\"");
+		UTMPXPRINTID;
+		printf("\" pid=\"%d\"\n", ut->ut_pid);
+		break;
+	case LOGIN_PROCESS:
+		printf("login process: id=\"");
+		UTMPXPRINTID;
+		printf("\" pid=\"%d\" user=\"%s\" line=\"%s\" host=\"%s\"\n",
+		    ut->ut_pid, ut->ut_user, ut->ut_line, ut->ut_host);
+		break;
 	case DEAD_PROCESS:
 		printf("dead process: id=\"");
 		UTMPXPRINTID;
 		printf("\" pid=\"%d\"\n", ut->ut_pid);
 		break;
 	default:
-		printf("unknown record type\n");
+		printf("unknown record type %hu\n", ut->ut_type);
 		break;
 	}
 }
diff --git a/usr.bin/id/id.c b/usr.bin/id/id.c
index e352712..a49fd75 100644
--- a/usr.bin/id/id.c
+++ b/usr.bin/id/id.c
@@ -57,16 +57,18 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-void	id_print(struct passwd *, int, int, int);
-void	pline(struct passwd *);
-void	pretty(struct passwd *);
-void	auditid(void);
-void	group(struct passwd *, int);
-void	maclabel(void);
-void	usage(void);
-struct passwd *who(char *);
+static void	id_print(struct passwd *, int, int, int);
+static void	pline(struct passwd *);
+static void	pretty(struct passwd *);
+#ifdef USE_BSM_AUDIT
+static void	auditid(void);
+#endif
+static void	group(struct passwd *, int);
+static void	maclabel(void);
+static void	usage(void);
+static struct passwd *who(char *);
 
-int isgroups, iswhoami;
+static int isgroups, iswhoami;
 
 int
 main(int argc, char *argv[])
@@ -220,7 +222,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 pretty(struct passwd *pw)
 {
 	struct group *gr;
@@ -260,7 +262,7 @@ pretty(struct passwd *pw)
 	}
 }
 
-void
+static void
 id_print(struct passwd *pw, int use_ggl, int p_euid, int p_egid)
 {
 	struct group *gr;
@@ -324,7 +326,7 @@ id_print(struct passwd *pw, int use_ggl, int p_euid, int p_egid)
 }
 
 #ifdef USE_BSM_AUDIT
-void
+static void
 auditid(void)
 {
 	auditinfo_t auditinfo;
@@ -371,7 +373,7 @@ auditid(void)
 }
 #endif
 
-void
+static void
 group(struct passwd *pw, int nflag)
 {
 	struct group *gr;
@@ -411,7 +413,7 @@ group(struct passwd *pw, int nflag)
 	free(groups);
 }
 
-void
+static void
 maclabel(void)
 {
 	char *string;
@@ -435,7 +437,7 @@ maclabel(void)
 	free(string);
 }
 
-struct passwd *
+static struct passwd *
 who(char *u)
 {
 	struct passwd *pw;
@@ -455,7 +457,7 @@ who(char *u)
 	/* NOTREACHED */
 }
 
-void
+static void
 pline(struct passwd *pw)
 {
 
@@ -471,7 +473,7 @@ pline(struct passwd *pw)
 }
 
 
-void
+static void
 usage(void)
 {
 
diff --git a/usr.bin/join/join.c b/usr.bin/join/join.c
index 12e9803..97bb4cf 100644
--- a/usr.bin/join/join.c
+++ b/usr.bin/join/join.c
@@ -83,36 +83,36 @@ typedef struct {
 	u_long setcnt;		/* set count */
 	u_long setalloc;	/* set allocated count */
 } INPUT;
-INPUT input1 = { NULL, 0, 0, 1, NULL, 0, 0, 0, 0 },
-      input2 = { NULL, 0, 0, 2, NULL, 0, 0, 0, 0 };
+static INPUT input1 = { NULL, 0, 0, 1, NULL, 0, 0, 0, 0 },
+    input2 = { NULL, 0, 0, 2, NULL, 0, 0, 0, 0 };
 
 typedef struct {
 	u_long	filenum;	/* file number */
 	u_long	fieldno;	/* field number */
 } OLIST;
-OLIST *olist;			/* output field list */
-u_long olistcnt;		/* output field list count */
-u_long olistalloc;		/* output field allocated count */
-
-int joinout = 1;		/* show lines with matched join fields (-v) */
-int needsep;			/* need separator character */
-int spans = 1;			/* span multiple delimiters (-t) */
-char *empty;			/* empty field replacement string (-e) */
+static OLIST *olist;		/* output field list */
+static u_long olistcnt;		/* output field list count */
+static u_long olistalloc;	/* output field allocated count */
+
+static int joinout = 1;		/* show lines with matched join fields (-v) */
+static int needsep;		/* need separator character */
+static int spans = 1;		/* span multiple delimiters (-t) */
+static char *empty;		/* empty field replacement string (-e) */
 static wchar_t default_tabchar[] = L" \t";
-wchar_t *tabchar = default_tabchar;/* delimiter characters (-t) */
-
-int  cmp(LINE *, u_long, LINE *, u_long);
-void fieldarg(char *);
-void joinlines(INPUT *, INPUT *);
-int  mbscoll(const char *, const char *);
-char *mbssep(char **, const wchar_t *);
-void obsolete(char **);
-void outfield(LINE *, u_long, int);
-void outoneline(INPUT *, LINE *);
-void outtwoline(INPUT *, LINE *, INPUT *, LINE *);
-void slurp(INPUT *);
-wchar_t *towcs(const char *);
-void usage(void);
+static wchar_t *tabchar = default_tabchar; /* delimiter characters (-t) */
+
+static int  cmp(LINE *, u_long, LINE *, u_long);
+static void fieldarg(char *);
+static void joinlines(INPUT *, INPUT *);
+static int  mbscoll(const char *, const char *);
+static char *mbssep(char **, const wchar_t *);
+static void obsolete(char **);
+static void outfield(LINE *, u_long, int);
+static void outoneline(INPUT *, LINE *);
+static void outtwoline(INPUT *, LINE *, INPUT *, LINE *);
+static void slurp(INPUT *);
+static wchar_t *towcs(const char *);
+static void usage(void);
 
 int
 main(int argc, char *argv[])
@@ -270,7 +270,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 slurp(INPUT *F)
 {
 	LINE *lp, *lastlp, tmp;
@@ -359,7 +359,7 @@ slurp(INPUT *F)
 	}
 }
 
-char *
+static char *
 mbssep(char **stringp, const wchar_t *delim)
 {
 	char *s, *tok;
@@ -388,7 +388,7 @@ mbssep(char **stringp, const wchar_t *delim)
 	}
 }
 
-int
+static int
 cmp(LINE *lp1, u_long fieldno1, LINE *lp2, u_long fieldno2)
 {
 	if (lp1->fieldcnt <= fieldno1)
@@ -398,7 +398,7 @@ cmp(LINE *lp1, u_long fieldno1, LINE *lp2, u_long fieldno2)
 	return (mbscoll(lp1->fields[fieldno1], lp2->fields[fieldno2]));
 }
 
-int
+static int
 mbscoll(const char *s1, const char *s2)
 {
 	wchar_t *w1, *w2;
@@ -414,7 +414,7 @@ mbscoll(const char *s1, const char *s2)
 	return (ret);
 }
 
-wchar_t *
+static wchar_t *
 towcs(const char *s)
 {
 	wchar_t *wcs;
@@ -428,7 +428,7 @@ towcs(const char *s)
 	return (wcs);
 }
 
-void
+static void
 joinlines(INPUT *F1, INPUT *F2)
 {
 	u_long cnt1, cnt2;
@@ -448,7 +448,7 @@ joinlines(INPUT *F1, INPUT *F2)
 			outtwoline(F1, &F1->set[cnt1], F2, &F2->set[cnt2]);
 }
 
-void
+static void
 outoneline(INPUT *F, LINE *lp)
 {
 	u_long cnt;
@@ -476,7 +476,7 @@ outoneline(INPUT *F, LINE *lp)
 	needsep = 0;
 }
 
-void
+static void
 outtwoline(INPUT *F1, LINE *lp1, INPUT *F2, LINE *lp2)
 {
 	u_long cnt;
@@ -512,7 +512,7 @@ outtwoline(INPUT *F1, LINE *lp1, INPUT *F2, LINE *lp2)
 	needsep = 0;
 }
 
-void
+static void
 outfield(LINE *lp, u_long fieldno, int out_empty)
 {
 	if (needsep++)
@@ -535,7 +535,7 @@ outfield(LINE *lp, u_long fieldno, int out_empty)
  * Convert an output list argument "2.1, 1.3, 2.4" into an array of output
  * fields.
  */
-void
+static void
 fieldarg(char *option)
 {
 	u_long fieldno, filenum;
@@ -569,7 +569,7 @@ fieldarg(char *option)
 	}
 }
 
-void
+static void
 obsolete(char **argv)
 {
 	size_t len;
@@ -654,7 +654,7 @@ jbad:				errx(1, "illegal option -- %s", ap);
 	}
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr, "%s %s\n%s\n",
diff --git a/usr.bin/lam/lam.c b/usr.bin/lam/lam.c
index 9c36a49..60d1c52 100644
--- a/usr.bin/lam/lam.c
+++ b/usr.bin/lam/lam.c
@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
 #define	MAXOFILES	20
 #define	BIGBUFSIZ	5 * BUFSIZ
 
-struct	openfile {		/* open file structure */
+static struct openfile {	/* open file structure */
 	FILE	*fp;		/* file pointer */
 	short	eof;		/* eof flag */
 	short	pad;		/* pad flag for missing columns */
@@ -64,10 +64,10 @@ struct	openfile {		/* open file structure */
 	const char *format;	/* printf(3) style string spec. */
 }	input[MAXOFILES];
 
-int	morefiles;		/* set by getargs(), changed by gatherline() */
-int	nofinalnl;		/* normally append \n to each output line */
-char	line[BIGBUFSIZ];
-char	*linep;
+static int	morefiles;	/* set by getargs(), changed by gatherline() */
+static int	nofinalnl;	/* normally append \n to each output line */
+static char	line[BIGBUFSIZ];
+static char	*linep;
 
 static char    *gatherline(struct openfile *);
 static void	getargs(char *[]);
diff --git a/usr.bin/last/last.c b/usr.bin/last/last.c
index c2175ff..3ac744e 100644
--- a/usr.bin/last/last.c
+++ b/usr.bin/last/last.c
@@ -70,9 +70,9 @@ typedef struct arg {
 	int	type;				/* type of arg */
 	struct arg	*next;			/* linked list pointer */
 } ARG;
-ARG	*arglist;				/* head of linked list */
+static ARG	*arglist;			/* head of linked list */
 
-LIST_HEAD(idlisthead, idtab) idlist;
+static LIST_HEAD(idlisthead, idtab) idlist;
 
 struct idtab {
 	time_t	logout;				/* log out time */
@@ -94,17 +94,17 @@ static time_t	snaptime;			/* if != 0, we will only
 						 * at this snapshot time
 						 */
 
-void	 addarg(int, char *);
-time_t	 dateconv(char *);
-void	 doentry(struct utmpx *);
-void	 hostconv(char *);
-void	 printentry(struct utmpx *, struct idtab *);
-char	*ttyconv(char *);
-int	 want(struct utmpx *);
-void	 usage(void);
-void	 wtmp(void);
-
-void
+static void	 addarg(int, char *);
+static time_t	 dateconv(char *);
+static void	 doentry(struct utmpx *);
+static void	 hostconv(char *);
+static void	 printentry(struct utmpx *, struct idtab *);
+static char	*ttyconv(char *);
+static int	 want(struct utmpx *);
+static void	 usage(void);
+static void	 wtmp(void);
+
+static void
 usage(void)
 {
 	(void)fprintf(stderr,
@@ -196,7 +196,7 @@ main(int argc, char *argv[])
  * wtmp --
  *	read through the wtmp file
  */
-void
+static void
 wtmp(void)
 {
 	struct utmpx *buf = NULL;
@@ -237,7 +237,7 @@ wtmp(void)
  * doentry --
  *	process a single wtmp entry
  */
-void
+static void
 doentry(struct utmpx *bp)
 {
 	struct idtab	*tt, *ttx;		/* idlist entry */
@@ -313,7 +313,7 @@ doentry(struct utmpx *bp)
  * If `tt' is non-NULL, use it and `crmsg' to print the logout time or
  * logout type (crash/shutdown) as appropriate.
  */
-void
+static void
 printentry(struct utmpx *bp, struct idtab *tt)
 {
 	char ct[80];
@@ -378,7 +378,7 @@ printentry(struct utmpx *bp, struct idtab *tt)
  * want --
  *	see if want this entry
  */
-int
+static int
 want(struct utmpx *bp)
 {
 	ARG *step;
@@ -411,7 +411,7 @@ want(struct utmpx *bp)
  * addarg --
  *	add an entry to a linked list of arguments
  */
-void
+static void
 addarg(int type, char *arg)
 {
 	ARG *cur;
@@ -430,7 +430,7 @@ addarg(int type, char *arg)
  *	has a domain attached that is the same as the current domain, rip
  *	off the domain suffix since that's what login(1) does.
  */
-void
+static void
 hostconv(char *arg)
 {
 	static int first = 1;
@@ -453,7 +453,7 @@ hostconv(char *arg)
  * ttyconv --
  *	convert tty to correct name.
  */
-char *
+static char *
 ttyconv(char *arg)
 {
 	char *mval;
@@ -485,7 +485,7 @@ ttyconv(char *arg)
  * 	[[CC]YY]MMDDhhmm[.SS]] to a time_t.
  * 	Derived from atime_arg1() in usr.bin/touch/touch.c
  */
-time_t
+static time_t
 dateconv(char *arg)
 {
         time_t timet;
diff --git a/usr.bin/lock/lock.c b/usr.bin/lock/lock.c
index b8d0111..2f26171 100644
--- a/usr.bin/lock/lock.c
+++ b/usr.bin/lock/lock.c
@@ -72,17 +72,17 @@ __FBSDID("$FreeBSD$");
 
 #define	TIMEOUT	15
 
-void quit(int);
-void bye(int);
-void hi(int);
+static void quit(int);
+static void bye(int);
+static void hi(int);
 static void usage(void);
 
-struct timeval	timeout;
-struct timeval	zerotime;
-struct termios	tty, ntty;
-long	nexttime;			/* keep the timeout time */
-int            no_timeout;                     /* lock terminal forever */
-int	vtyunlock;			/* Unlock flag and code. */
+static struct timeval	timeout;
+static struct timeval	zerotime;
+static struct termios	tty, ntty;
+static long		nexttime;		/* keep the timeout time */
+static int		no_timeout;		/* lock terminal forever */
+static int		vtyunlock;		/* Unlock flag and code. */
 
 /*ARGSUSED*/
 int
@@ -252,7 +252,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 hi(int signo __unused)
 {
 	struct timeval timval;
@@ -269,7 +269,7 @@ hi(int signo __unused)
 	}
 }
 
-void
+static void
 quit(int signo __unused)
 {
 	(void)putchar('\n');
@@ -279,7 +279,7 @@ quit(int signo __unused)
 	exit(0);
 }
 
-void
+static void
 bye(int signo __unused)
 {
 	if (!no_timeout) {
diff --git a/usr.bin/logger/logger.c b/usr.bin/logger/logger.c
index 68828ed..d3c4786 100644
--- a/usr.bin/logger/logger.c
+++ b/usr.bin/logger/logger.c
@@ -57,8 +57,8 @@ __FBSDID("$FreeBSD$");
 #define	SYSLOG_NAMES
 #include <syslog.h>
 
-int	decode(char *, CODE *);
-int	pencode(char *);
+static int	decode(char *, CODE *);
+static int	pencode(char *);
 static void	logmessage(int, const char *, const char *, const char *,
 			   const char *);
 static void	usage(void);
@@ -70,11 +70,11 @@ struct socks {
 };
 
 #ifdef INET6
-int	family = PF_UNSPEC;	/* protocol family (IPv4, IPv6 or both) */
+static int family = PF_UNSPEC;	/* protocol family (IPv4, IPv6 or both) */
 #else
-int	family = PF_INET;	/* protocol family (IPv4 only) */
+static int family = PF_INET;	/* protocol family (IPv4 only) */
 #endif
-int	send_to_all = 0;	/* send message to all IPv4/IPv6 addresses */
+static int send_to_all = 0;	/* send message to all IPv4/IPv6 addresses */
 
 /*
  * logger -- read and log utility
@@ -176,7 +176,7 @@ main(int argc, char *argv[])
 /*
  *  Send the message to syslog, either on the local host, or on a remote host
  */
-void
+static void
 logmessage(int pri, const char *tag, const char *host, const char *svcname,
 	   const char *buf)
 {
@@ -246,7 +246,7 @@ logmessage(int pri, const char *tag, const char *host, const char *svcname,
 /*
  *  Decode a symbolic name to a numeric value
  */
-int
+static int
 pencode(char *s)
 {
 	char *save;
@@ -270,7 +270,7 @@ pencode(char *s)
 	return ((lev & LOG_PRIMASK) | (fac & LOG_FACMASK));
 }
 
-int
+static int
 decode(char *name, CODE *codetab)
 {
 	CODE *c;
diff --git a/usr.bin/look/look.c b/usr.bin/look/look.c
index 4e8cf34..e77a216 100644
--- a/usr.bin/look/look.c
+++ b/usr.bin/look/look.c
@@ -77,14 +77,14 @@ static char _path_words[] = _PATH_WORDS;
 #define	GREATER		1
 #define	LESS		(-1)
 
-int dflag, fflag;
+static int dflag, fflag;
 
-char    *binary_search(wchar_t *, unsigned char *, unsigned char *);
-int      compare(wchar_t *, unsigned char *, unsigned char *);
-char    *linear_search(wchar_t *, unsigned char *, unsigned char *);
-int      look(wchar_t *, unsigned char *, unsigned char *);
-wchar_t	*prepkey(const char *, wchar_t);
-void     print_from(wchar_t *, unsigned char *, unsigned char *);
+static char	*binary_search(wchar_t *, unsigned char *, unsigned char *);
+static int	 compare(wchar_t *, unsigned char *, unsigned char *);
+static char	*linear_search(wchar_t *, unsigned char *, unsigned char *);
+static int	 look(wchar_t *, unsigned char *, unsigned char *);
+static wchar_t	*prepkey(const char *, wchar_t);
+static void	 print_from(wchar_t *, unsigned char *, unsigned char *);
 
 static void usage(void);
 
@@ -151,7 +151,7 @@ main(int argc, char *argv[])
 	exit(match);
 }
 
-wchar_t *
+static wchar_t *
 prepkey(const char *string, wchar_t termchar)
 {
 	const char *readp;
@@ -182,7 +182,7 @@ prepkey(const char *string, wchar_t termchar)
 	return (key);
 }
 
-int
+static int
 look(wchar_t *string, unsigned char *front, unsigned char *back)
 {
 
@@ -236,7 +236,7 @@ look(wchar_t *string, unsigned char *front, unsigned char *back)
 #define	SKIP_PAST_NEWLINE(p, back) \
 	while (p < back && *p++ != '\n');
 
-char *
+static char *
 binary_search(wchar_t *string, unsigned char *front, unsigned char *back)
 {
 	unsigned char *p;
@@ -270,7 +270,7 @@ binary_search(wchar_t *string, unsigned char *front, unsigned char *back)
  * 	o front points at the first character in a line.
  *	o front is before or at the first line to be printed.
  */
-char *
+static char *
 linear_search(wchar_t *string, unsigned char *front, unsigned char *back)
 {
 	while (front < back) {
@@ -290,7 +290,7 @@ linear_search(wchar_t *string, unsigned char *front, unsigned char *back)
 /*
  * Print as many lines as match string, starting at front.
  */
-void
+static void
 print_from(wchar_t *string, unsigned char *front, unsigned char *back)
 {
 	for (; front < back && compare(string, front, back) == EQUAL; ++front) {
@@ -315,7 +315,7 @@ print_from(wchar_t *string, unsigned char *front, unsigned char *back)
  * The string "s1" is null terminated.  The string s2 is '\n' terminated (or
  * "back" terminated).
  */
-int
+static int
 compare(wchar_t *s1, unsigned char *s2, unsigned char *back)
 {
 	wchar_t ch1, ch2;
diff --git a/usr.bin/m4/gnum4.c b/usr.bin/m4/gnum4.c
index edfe296..fce443c 100644
--- a/usr.bin/m4/gnum4.c
+++ b/usr.bin/m4/gnum4.c
@@ -60,7 +60,7 @@ int mimic_gnu = 0;
  * Then M4PATH env variable
  */
 
-struct path_entry {
+static struct path_entry {
 	char *name;
 	struct path_entry *next;
 } *first, *last;
diff --git a/usr.bin/m4/main.c b/usr.bin/m4/main.c
index 0e60fce..f16b4df 100644
--- a/usr.bin/m4/main.c
+++ b/usr.bin/m4/main.c
@@ -99,7 +99,7 @@ char scommt[MAXCCHARS+1] = {SCOMMT};	/* start character for comment */
 char ecommt[MAXCCHARS+1] = {ECOMMT};	/* end character for comment   */
 int synccpp;			/* Line synchronisation for C preprocessor */
 
-struct keyblk keywrds[] = {	/* m4 keywords to be installed */
+static const struct keyblk keywrds[] = { /* m4 keywords to be installed */
 	{ "include",      INCLTYPE },
 	{ "sinclude",     SINCTYPE },
 	{ "define",       DEFITYPE },
diff --git a/usr.bin/make/globals.h b/usr.bin/make/globals.h
index 06c302e..e3ce51b 100644
--- a/usr.bin/make/globals.h
+++ b/usr.bin/make/globals.h
@@ -75,6 +75,7 @@ extern Boolean	beSilent;	/* True if should print no commands */
 extern Boolean	beVerbose;	/* True if should print extra cruft */
 extern Boolean	beQuiet;	/* True if want quiet headers with -j */
 extern Boolean	noExecute;	/* True if should execute nothing */
+extern Boolean	printGraphOnly;	/* -p flag */
 extern Boolean	allPrecious;	/* True if every target is precious */
 extern Boolean	is_posix;	/* .POSIX target seen */
 extern Boolean	mfAutoDeps;	/* .MAKEFILEDEPS target seen */
diff --git a/usr.bin/make/make.1 b/usr.bin/make/make.1
index 93ea46e..d36a43c 100644
--- a/usr.bin/make/make.1
+++ b/usr.bin/make/make.1
@@ -32,7 +32,7 @@
 .\"	@(#)make.1	8.8 (Berkeley) 6/13/95
 .\" $FreeBSD$
 .\"
-.Dd December 29, 2008
+.Dd November 5, 2011
 .Dt MAKE 1
 .Os
 .Sh NAME
@@ -631,7 +631,7 @@ The following directories are tried in order:
 .Pp
 .Bl -enum -compact
 .It
-${MAKEOBJDIRPREFIX}/`pwd`
+${MAKEOBJDIRPREFIX}/`pwd -P`
 .It
 ${MAKEOBJDIR}
 .It
@@ -639,7 +639,7 @@ obj.${MACHINE}
 .It
 obj
 .It
-/usr/obj/`pwd`
+/usr/obj/`pwd -P`
 .El
 .Pp
 The first directory that
diff --git a/usr.bin/ministat/ministat.c b/usr.bin/ministat/ministat.c
index 720922e..3b51e40 100644
--- a/usr.bin/ministat/ministat.c
+++ b/usr.bin/ministat/ministat.c
@@ -23,8 +23,8 @@ __FBSDID("$FreeBSD$");
 
 #define NSTUDENT 100
 #define NCONF 6
-double const studentpct[] = { 80, 90, 95, 98, 99, 99.5 };
-double student [NSTUDENT + 1][NCONF] = {
+static double const studentpct[] = { 80, 90, 95, 98, 99, 99.5 };
+static double student[NSTUDENT + 1][NCONF] = {
 /* inf */	{	1.282,	1.645,	1.960,	2.326,	2.576,	3.090  },
 /* 1. */	{	3.078,	6.314,	12.706,	31.821,	63.657,	318.313  },
 /* 2. */	{	1.886,	2.920,	4.303,	6.965,	9.925,	22.327  },
diff --git a/usr.bin/mt/mt.c b/usr.bin/mt/mt.c
index b6efec2..766ca2f 100644
--- a/usr.bin/mt/mt.c
+++ b/usr.bin/mt/mt.c
@@ -73,7 +73,7 @@ __FBSDID("$FreeBSD$");
 #define FALSE 0
 #endif
 
-struct commands {
+static const struct commands {
 	const char *c_name;
 	int c_code;
 	int c_ronly;
@@ -114,22 +114,22 @@ struct commands {
 	{ NULL, 0, 0, 0 }
 };
 
-const char *getblksiz(int);
-void printreg(const char *, u_int, const char *);
-void status(struct mtget *);
-void usage(void);
-void st_status (struct mtget *);
-int stringtodens (const char *s);
-const char *denstostring (int d);
-int denstobp(int d, int bpi);
-u_int32_t stringtocomp(const char *s);
-const char *comptostring(u_int32_t comp);
-void warn_eof(void);
+static const char *getblksiz(int);
+static void printreg(const char *, u_int, const char *);
+static void status(struct mtget *);
+static void usage(void);
+static void st_status(struct mtget *);
+static int stringtodens(const char *s);
+static const char *denstostring(int d);
+static int denstobp(int d, int bpi);
+static u_int32_t stringtocomp(const char *s);
+static const char *comptostring(u_int32_t comp);
+static void warn_eof(void);
 
 int
 main(int argc, char *argv[])
 {
-	struct commands *comp;
+	const struct commands *comp;
 	struct mtget mt_status;
 	struct mtop mt_com;
 	int ch, len, mtfd;
@@ -303,7 +303,7 @@ main(int argc, char *argv[])
 	/* NOTREACHED */
 }
 
-struct tape_desc {
+static const struct tape_desc {
 	short	t_type;		/* type of magtape device */
 	const char *t_name;	/* printing name */
 	const char *t_dsbits;	/* "drive status" register */
@@ -316,10 +316,10 @@ struct tape_desc {
 /*
  * Interpret the status buffer returned
  */
-void
+static void
 status(struct mtget *bp)
 {
-	struct tape_desc *mt;
+	const struct tape_desc *mt;
 
 	for (mt = tapes;; mt++) {
 		if (mt->t_type == 0) {
@@ -344,7 +344,7 @@ status(struct mtget *bp)
 /*
  * Print a register a la the %b format of the kernel's printf.
  */
-void
+static void
 printreg(const char *s, u_int v, const char *bits)
 {
 	int i, any = 0;
@@ -374,14 +374,14 @@ printreg(const char *s, u_int v, const char *bits)
 	}
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr, "usage: mt [-f device] command [count]\n");
 	exit(1);
 }
 
-struct densities {
+static const struct densities {
 	int dens;
 	int bpmm;
 	int bpi;
@@ -437,7 +437,7 @@ struct densities {
 	{ 0, 0, 0, NULL }
 };
 
-struct compression_types {
+static const struct compression_types {
 	u_int32_t	comp_number;
 	const char 	*name;
 } comp_types[] = {
@@ -450,11 +450,11 @@ struct compression_types {
 	{ 0xf0f0f0f0, NULL}
 };
 
-const char *
+static const char *
 denstostring(int d)
 {
 	static char buf[20];
-	struct densities *sd;
+	const struct densities *sd;
 
 	/* densities 0 and 0x7f are handled as special cases */
 	if (d == 0)
@@ -475,10 +475,10 @@ denstostring(int d)
  * Given a specific density number, return either the bits per inch or bits
  * per millimeter for the given density.
  */
-int
+static int
 denstobp(int d, int bpi)
 {
-	struct densities *sd;
+	const struct densities *sd;
 
 	for (sd = dens; sd->dens; sd++)
 		if (sd->dens == d)
@@ -493,10 +493,10 @@ denstobp(int d, int bpi)
 	}
 }
 
-int
+static int
 stringtodens(const char *s)
 {
-	struct densities *sd;
+	const struct densities *sd;
 	size_t l = strlen(s);
 
 	for (sd = dens; sd->dens; sd++)
@@ -506,7 +506,7 @@ stringtodens(const char *s)
 }
 
 
-const char *
+static const char *
 getblksiz(int bs)
 {
 	static char buf[25];
@@ -518,11 +518,11 @@ getblksiz(int bs)
 	}
 }
 
-const char *
+static const char *
 comptostring(u_int32_t comp)
 {
 	static char buf[20];
-	struct compression_types *ct;
+	const struct compression_types *ct;
 
 	if (comp == MT_COMP_DISABLED)
 		return "disabled";
@@ -540,10 +540,10 @@ comptostring(u_int32_t comp)
 		return(ct->name);
 }
 
-u_int32_t
+static u_int32_t
 stringtocomp(const char *s)
 {
-	struct compression_types *ct;
+	const struct compression_types *ct;
 	size_t l = strlen(s);
 
 	for (ct = comp_types; ct->name; ct++)
@@ -553,7 +553,7 @@ stringtocomp(const char *s)
 	return(ct->comp_number);
 }
 
-void
+static void
 st_status(struct mtget *bp)
 {
 	printf("Mode      Density              Blocksize      bpi      "
@@ -634,7 +634,7 @@ st_status(struct mtget *bp)
 	    bp->mt_fileno, bp->mt_blkno, bp->mt_resid);
 }
 
-void
+static void
 warn_eof(void)
 {
 	fprintf(stderr,
diff --git a/usr.bin/paste/paste.c b/usr.bin/paste/paste.c
index e907d2c..f03f784 100644
--- a/usr.bin/paste/paste.c
+++ b/usr.bin/paste/paste.c
@@ -57,15 +57,15 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include <wchar.h>
 
-wchar_t *delim;
-int delimcnt;
+static wchar_t *delim;
+static int delimcnt;
 
-int parallel(char **);
-int sequential(char **);
-int tr(wchar_t *);
+static int parallel(char **);
+static int sequential(char **);
+static int tr(wchar_t *);
 static void usage(void);
 
-wchar_t tab[] = L"\t";
+static wchar_t tab[] = L"\t";
 
 int
 main(int argc, char *argv[])
@@ -125,7 +125,7 @@ typedef struct _list {
 	char *name;
 } LIST;
 
-int
+static int
 parallel(char **argv)
 {
 	LIST *lp;
@@ -195,7 +195,7 @@ parallel(char **argv)
 	return (0);
 }
 
-int
+static int
 sequential(char **argv)
 {
 	FILE *fp;
@@ -235,7 +235,7 @@ sequential(char **argv)
 	return (failed != 0);
 }
 
-int
+static int
 tr(wchar_t *arg)
 {
 	int cnt;
diff --git a/usr.bin/pr/extern.h b/usr.bin/pr/extern.h
index da552c0..c39e18f 100644
--- a/usr.bin/pr/extern.h
+++ b/usr.bin/pr/extern.h
@@ -38,7 +38,9 @@
  * $FreeBSD$
  */
 
+extern int eopterr;
 extern int eoptind;
+extern int eoptopt;
 extern char *eoptarg;
 
 void	 addnum(char *, int, int);
diff --git a/usr.bin/pr/pr.c b/usr.bin/pr/pr.c
index ae7ae07..510f9e8 100644
--- a/usr.bin/pr/pr.c
+++ b/usr.bin/pr/pr.c
@@ -80,40 +80,40 @@ __FBSDID("$FreeBSD$");
 /*
  * parameter variables
  */
-int	pgnm;			/* starting page number */
-int	clcnt;			/* number of columns */
-int	colwd;			/* column data width - multiple columns */
-int	across;			/* mult col flag; write across page */
-int	dspace;			/* double space flag */
-char	inchar;			/* expand input char */
-int	ingap;			/* expand input gap */
-int	pausefst;		/* Pause before first page */
-int	pauseall;		/* Pause before each page */
-int	formfeed;		/* use formfeed as trailer */
-char	*header;		/* header name instead of file name */
-char	ochar;			/* contract output char */
-int	ogap;			/* contract output gap */
-int	lines;			/* number of lines per page */
-int	merge;			/* merge multiple files in output */
-char	nmchar;			/* line numbering append char */
-int	nmwd;			/* width of line number field */
-int	offst;			/* number of page offset spaces */
-int	nodiag;			/* do not report file open errors */
-char	schar;			/* text column separation character */
-int	sflag;			/* -s option for multiple columns */
-int	nohead;			/* do not write head and trailer */
-int	pgwd;			/* page width with multiple col output */
-const char *timefrmt;		/* time conversion string */
+static int	pgnm;		/* starting page number */
+static int	clcnt;		/* number of columns */
+static int	colwd;		/* column data width - multiple columns */
+static int	across;		/* mult col flag; write across page */
+static int	dspace;		/* double space flag */
+static char	inchar;		/* expand input char */
+static int	ingap;		/* expand input gap */
+static int	pausefst;	/* Pause before first page */
+static int	pauseall;	/* Pause before each page */
+static int	formfeed;	/* use formfeed as trailer */
+static char	*header;	/* header name instead of file name */
+static char	ochar;		/* contract output char */
+static int	ogap;		/* contract output gap */
+static int	lines;		/* number of lines per page */
+static int	merge;		/* merge multiple files in output */
+static char	nmchar;		/* line numbering append char */
+static int	nmwd;		/* width of line number field */
+static int	offst;		/* number of page offset spaces */
+static int	nodiag;		/* do not report file open errors */
+static char	schar;		/* text column separation character */
+static int	sflag;		/* -s option for multiple columns */
+static int	nohead;		/* do not write head and trailer */
+static int	pgwd;		/* page width with multiple col output */
+static const char *timefrmt;	/* time conversion string */
 
 /*
  * misc globals
  */
-FILE	*err;			/* error message file pointer */
-int	addone;			/* page length is odd with double space */
-int	errcnt;			/* error count on file processing */
-char	digs[] = "0123456789";	/* page number translation map */
+static FILE	*err;		/* error message file pointer */
+static int	addone;		/* page length is odd with double space */
+static int	errcnt;		/* error count on file processing */
+static char	digs[] = "0123456789"; /* page number translation map */
 
-char	fnamedefault[] = FNAME;
+static char	fnamedefault[] = FNAME;
 
 int
 main(int argc, char *argv[])
diff --git a/usr.bin/procstat/procstat.1 b/usr.bin/procstat/procstat.1
index 35fab1f..5ed28dd 100644
--- a/usr.bin/procstat/procstat.1
+++ b/usr.bin/procstat/procstat.1
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 14, 2011
+.Dd November 7, 2011
 .Dt PROCSTAT 1
 .Os
 .Sh NAME
@@ -421,6 +421,8 @@ The following mapping flags may be displayed:
 copy-on-write
 .It N
 needs copy
+.It S
+one or more superpage mappings are used
 .El
 .Sh EXIT STATUS
 .Ex -std
diff --git a/usr.bin/procstat/procstat_vm.c b/usr.bin/procstat/procstat_vm.c
index 2eada92..33d0759 100644
--- a/usr.bin/procstat/procstat_vm.c
+++ b/usr.bin/procstat/procstat_vm.c
@@ -50,7 +50,7 @@ procstat_vm(struct kinfo_proc *kipp)
 
 	ptrwidth = 2*sizeof(void *) + 2;
 	if (!hflag)
-		printf("%5s %*s %*s %3s %4s %4s %3s %3s %2s %-2s %-s\n",
+		printf("%5s %*s %*s %3s %4s %4s %3s %3s %3s %-2s %-s\n",
 		    "PID", ptrwidth, "START", ptrwidth, "END", "PRT", "RES",
 		    "PRES", "REF", "SHD", "FL", "TP", "PATH");
 
@@ -70,8 +70,9 @@ procstat_vm(struct kinfo_proc *kipp)
 		printf("%3d ", kve->kve_ref_count);
 		printf("%3d ", kve->kve_shadow_count);
 		printf("%-1s", kve->kve_flags & KVME_FLAG_COW ? "C" : "-");
-		printf("%-1s ", kve->kve_flags & KVME_FLAG_NEEDS_COPY ? "N" :
+		printf("%-1s", kve->kve_flags & KVME_FLAG_NEEDS_COPY ? "N" :
 		    "-");
+		printf("%-1s ", kve->kve_flags & KVME_FLAG_SUPER ? "S" : "-");
 		switch (kve->kve_type) {
 		case KVME_TYPE_NONE:
 			str = "--";
diff --git a/usr.bin/quota/quota.c b/usr.bin/quota/quota.c
index 67d9c5a..ddfcea0 100644
--- a/usr.bin/quota/quota.c
+++ b/usr.bin/quota/quota.c
@@ -73,8 +73,7 @@ __FBSDID("$FreeBSD$");
 #include <time.h>
 #include <unistd.h>
 
-const char *qfname = QUOTAFILENAME;
-const char *qfextension[] = INITQFNAMES;
+static const char *qfextension[] = INITQFNAMES;
 
 struct quotause {
 	struct	quotause *next;
@@ -101,12 +100,12 @@ static int callaurpc(char *host, int prognum, int versnum, int procnum,
 	xdrproc_t inproc, char *in, xdrproc_t outproc, char *out);
 static int alldigits(char *s);
 
-int	hflag;
-int	lflag;
-int	rflag;
-int	qflag;
-int	vflag;
-char	*filename = NULL;
+static int	hflag;
+static int	lflag;
+static int	rflag;
+static int	qflag;
+static int	vflag;
+static char	*filename = NULL;
 
 int
 main(int argc, char *argv[])
diff --git a/usr.bin/rs/rs.c b/usr.bin/rs/rs.c
index 46de07e..ba29ef2 100644
--- a/usr.bin/rs/rs.c
+++ b/usr.bin/rs/rs.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <string.h>
 
-long	flags;
+static long	flags;
 #define	TRANSPOSE	000001
 #define	MTRANSPOSE	000002
 #define	ONEPERLINE	000004
@@ -72,34 +72,34 @@ long	flags;
 #define ONEPERCHAR	0100000
 #define NOARGS		0200000
 
-short	*colwidths;
-short	*cord;
-short	*icbd;
-short	*ocbd;
-int	nelem;
-char	**elem;
-char	**endelem;
-char	*curline;
-int	allocsize = BUFSIZ;
-int	curlen;
-int	irows, icols;
-int	orows = 0, ocols = 0;
-int	maxlen;
-int	skip;
-int	propgutter;
-char	isep = ' ', osep = ' ';
-char	blank[] = "";
-int	owidth = 80, gutter = 2;
+static short	*colwidths;
+static short	*cord;
+static short	*icbd;
+static short	*ocbd;
+static int	nelem;
+static char	**elem;
+static char	**endelem;
+static char	*curline;
+static int	allocsize = BUFSIZ;
+static int	curlen;
+static int	irows, icols;
+static int	orows = 0, ocols = 0;
+static int	maxlen;
+static int	skip;
+static int	propgutter;
+static char	isep = ' ', osep = ' ';
+static char	blank[] = "";
+static int	owidth = 80, gutter = 2;
 
-void	  getargs(int, char *[]);
-void	  getfile(void);
-int	  getline(void);
-char	 *getlist(short **, char *);
-char	 *getnum(int *, char *, int);
-char	**getptrs(char **);
-void	  prepfile(void);
-void	  prints(char *, int);
-void	  putfile(void);
+static void	  getargs(int, char *[]);
+static void	  getfile(void);
+static int	  getline(void);
+static char	 *getlist(short **, char *);
+static char	 *getnum(int *, char *, int);
+static char	**getptrs(char **);
+static void	  prepfile(void);
+static void	  prints(char *, int);
+static void	  putfile(void);
 static void usage(void);
 
 #define	INCR(ep) do {			\
@@ -121,7 +121,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-void
+static void
 getfile(void)
 {
 	char *p;
@@ -189,7 +189,7 @@ getfile(void)
 	nelem = ep - elem;
 }
 
-void
+static void
 putfile(void)
 {
 	char **ep;
@@ -211,7 +211,7 @@ putfile(void)
 		}
 }
 
-void
+static void
 prints(char *s, int col)
 {
 	int n;
@@ -237,7 +237,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 prepfile(void)
 {
 	char **ep;
@@ -330,9 +330,9 @@ prepfile(void)
 }
 
 #define	BSIZE	(LINE_MAX * 2)
-char	ibuf[BSIZE];
+static char	ibuf[BSIZE];
 
-int
+static int
 getline(void)	/* get line; maintain curline, curlen; manage storage */
 {
 	static	int putlength;
@@ -372,7 +372,7 @@ getline(void)	/* get line; maintain curline, curlen; manage storage */
 	return(c);
 }
 
-char **
+static char **
 getptrs(char **sp)
 {
 	char **p;
@@ -387,7 +387,7 @@ getptrs(char **sp)
 	return(sp);
 }
 
-void
+static void
 getargs(int ac, char *av[])
 {
 	char *p;
@@ -498,7 +498,7 @@ getargs(int ac, char *av[])
 	}
 }
 
-char *
+static char *
 getlist(short **list, char *p)
 {
 	int count = 1;
@@ -534,7 +534,7 @@ getlist(short **list, char *p)
  * num = number p points to; if (strict) complain
  * returns pointer to end of num
  */
-char *
+static char *
 getnum(int *num, char *p, int strict)
 {
 	char *t = p;
diff --git a/usr.bin/ruptime/ruptime.c b/usr.bin/ruptime/ruptime.c
index d156b6d..2f791be 100644
--- a/usr.bin/ruptime/ruptime.c
+++ b/usr.bin/ruptime/ruptime.c
@@ -54,7 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <time.h>
 #include <unistd.h>
 
-struct hs {
+static struct hs {
 	struct	whod hs_wd;
 	int	hs_nusers;
 } *hs;
@@ -62,19 +62,18 @@ struct hs {
 #define	ISDOWN(h)	(now - (h)->hs_wd.wd_recvtime > 11 * 60)
 #define	WHDRSIZE	__offsetof(struct whod, wd_we)
 
-size_t nhosts;
-time_t now;
-int rflg = 1;
-DIR *dirp;
+static size_t nhosts;
+static time_t now;
+static int rflg = 1;
+static DIR *dirp;
 
-int	 hscmp(const void *, const void *);
-char	*interval(time_t, const char *);
-int	 lcmp(const void *, const void *);
-void	 morehosts(void);
-void	 ruptime(const char *, int, int (*)(const void *, const void *));
-int	 tcmp(const void *, const void *);
-int	 ucmp(const void *, const void *);
-void	 usage(void);
+static int	 hscmp(const void *, const void *);
+static char	*interval(time_t, const char *);
+static int	 lcmp(const void *, const void *);
+static void	 ruptime(const char *, int, int (*)(const void *, const void *));
+static int	 tcmp(const void *, const void *);
+static int	 ucmp(const void *, const void *);
+static void	 usage(void);
 
 int
 main(int argc, char *argv[])
@@ -119,7 +118,7 @@ main(int argc, char *argv[])
 	exit(0);
 }
 
-char *
+static char *
 interval(time_t tval, const char *updown)
 {
 	static char resbuf[32];
@@ -147,7 +146,7 @@ interval(time_t tval, const char *updown)
 #define	HS(a)	((const struct hs *)(a))
 
 /* Alphabetical comparison. */
-int
+static int
 hscmp(const void *a1, const void *a2)
 {
 	return (rflg *
@@ -155,7 +154,7 @@ hscmp(const void *a1, const void *a2)
 }
 
 /* Load average comparison. */
-int
+static int
 lcmp(const void *a1, const void *a2)
 {
 	if (ISDOWN(HS(a1)))
@@ -170,7 +169,7 @@ lcmp(const void *a1, const void *a2)
 		   (HS(a2)->hs_wd.wd_loadav[0] - HS(a1)->hs_wd.wd_loadav[0]));
 }
 
-void
+static void
 ruptime(const char *host, int aflg, int (*cmp)(const void *, const void *))
 {
 	struct hs *hsp;
@@ -261,7 +260,7 @@ ruptime(const char *host, int aflg, int (*cmp)(const void *, const void *))
 }
 
 /* Number of users comparison. */
-int
+static int
 ucmp(const void *a1, const void *a2)
 {
 	if (ISDOWN(HS(a1)))
@@ -276,7 +275,7 @@ ucmp(const void *a1, const void *a2)
 }
 
 /* Uptime comparison. */
-int
+static int
 tcmp(const void *a1, const void *a2)
 {
 	return (rflg * (
@@ -288,7 +287,7 @@ tcmp(const void *a1, const void *a2)
 	));
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr, "usage: ruptime [-alrtu] [host ...]\n");
diff --git a/usr.bin/rusers/rusers.c b/usr.bin/rusers/rusers.c
index 705c887..4d641dc 100644
--- a/usr.bin/rusers/rusers.c
+++ b/usr.bin/rusers/rusers.c
@@ -56,10 +56,10 @@ __FBSDID("$FreeBSD$");
 #define HOST_WIDTH	20
 #define LINE_WIDTH	15
 
-int longopt;
-int allopt;
+static int longopt;
+static int allopt;
 
-struct host_list {
+static struct host_list {
 	struct	host_list *next;
 	struct	in_addr addr;
 } *hosts;
diff --git a/usr.bin/rwall/rwall.c b/usr.bin/rwall/rwall.c
index 042b72f..c235ea7 100644
--- a/usr.bin/rwall/rwall.c
+++ b/usr.bin/rwall/rwall.c
@@ -64,11 +64,11 @@ __FBSDID("$FreeBSD$");
 #include <time.h>
 #include <unistd.h>
 
-char *mbuf;
+static char *mbuf;
 
 static char notty[] = "no tty";
 
-void	makemsg(const char *);
+static void	makemsg(const char *);
 static void usage(void);
 
 /* ARGSUSED */
@@ -121,7 +121,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 makemsg(const char *fname)
 {
 	struct tm *lt;
diff --git a/usr.bin/rwho/rwho.c b/usr.bin/rwho/rwho.c
index f134c18..5f1334f 100644
--- a/usr.bin/rwho/rwho.c
+++ b/usr.bin/rwho/rwho.c
@@ -58,16 +58,16 @@ __FBSDID("$FreeBSD$");
 #include <timeconv.h>
 #include <unistd.h>
 
-DIR	*dirp;
+static DIR	*dirp;
 
-struct	whod wd;
+static struct	whod wd;
 #define	NUSERS	1000
-struct	myutmp {
+static struct	myutmp {
 	char    myhost[sizeof(wd.wd_hostname)];
 	int	myidle;
 	struct	outmp myutmp;
 } myutmp[NUSERS];
-int	nusers;
+static int	nusers;
 
 #define	WHDRSIZE	(ssize_t)(sizeof (wd) - sizeof (wd.wd_we))
 /*
@@ -75,11 +75,11 @@ int	nusers;
  */
 #define	down(w,now)	((now) - (w)->wd_recvtime > 11 * 60)
 
-time_t	now;
-int	aflg;
+static time_t	now;
+static int	aflg;
 
 static void usage(void);
-int utmpcmp(const void *, const void *);
+static int utmpcmp(const void *, const void *);
 
 int
 main(int argc, char *argv[])
@@ -202,7 +202,7 @@ usage(void)
 
 #define MYUTMP(a) ((const struct myutmp *)(a))
 
-int
+static int
 utmpcmp(const void *u1, const void *u2)
 {
 	int rc;
diff --git a/usr.bin/sed/sed.1 b/usr.bin/sed/sed.1
index 0744630..a6b9ea2 100644
--- a/usr.bin/sed/sed.1
+++ b/usr.bin/sed/sed.1
@@ -343,7 +343,7 @@ can be preceded by white space and can be followed by white space.
 The function can be preceded by white space.
 The terminating
 .Dq }
-must be preceded by a newline or optional white space.
+must be preceded by a newline, and may also be preceded by white space.
 .Pp
 .Bl -tag -width "XXXXXX" -compact
 .It [2addr] function-list
diff --git a/usr.bin/seq/seq.c b/usr.bin/seq/seq.c
index c18c76c..e077743 100644
--- a/usr.bin/seq/seq.c
+++ b/usr.bin/seq/seq.c
@@ -51,20 +51,19 @@ __FBSDID("$FreeBSD$");
 
 /* Globals */
 
-const char *decimal_point = ".";	/* default */
-char default_format[] = { "%g" };	/* default */
+static const char *decimal_point = ".";	/* default */
+static char default_format[] = { "%g" };	/* default */
 
 /* Prototypes */
 
-double e_atof(const char *);
+static double e_atof(const char *);
 
-int decimal_places(const char *);
-int main(int, char *[]);
-int numeric(const char *);
-int valid_format(const char *);
+static int decimal_places(const char *);
+static int numeric(const char *);
+static int valid_format(const char *);
 
-char *generate_format(double, double, double, int, char);
-char *unescape(char *);
+static char *generate_format(double, double, double, int, char);
+static char *unescape(char *);
 
 /*
  * The seq command will print out a numeric sequence from 1, the default,
@@ -186,7 +185,7 @@ main(int argc, char *argv[])
 /*
  * numeric - verify that string is numeric
  */
-int
+static int
 numeric(const char *s)
 {
 	int seen_decimal_pt, decimal_pt_len;
@@ -223,7 +222,7 @@ numeric(const char *s)
 /*
  * valid_format - validate user specified format string
  */
-int
+static int
 valid_format(const char *fmt)
 {
 	int conversions = 0;
@@ -268,7 +267,7 @@ valid_format(const char *fmt)
 /*
  * unescape - handle C escapes in a string
  */
-char *
+static char *
 unescape(char *orig)
 {
 	char c, *cp, *new = orig;
@@ -358,7 +357,7 @@ unescape(char *orig)
  *	exit if string is not a valid double, or if converted value would
  *	cause overflow or underflow
  */
-double
+static double
 e_atof(const char *num)
 {
 	char *endp;
@@ -383,7 +382,7 @@ e_atof(const char *num)
 /*
  * decimal_places - count decimal places in a number (string)
  */
-int
+static int
 decimal_places(const char *number)
 {
 	int places = 0;
@@ -405,7 +404,7 @@ decimal_places(const char *number)
  * XXX to be bug for bug compatible with Plan9 and GNU return "%g"
  * when "%g" prints as "%e" (this way no width adjustments are made)
  */
-char *
+static char *
 generate_format(double first, double incr, double last, int equalize, char pad)
 {
 	static char buf[256];
diff --git a/usr.bin/split/split.c b/usr.bin/split/split.c
index 061cd6c..0258c8c 100644
--- a/usr.bin/split/split.c
+++ b/usr.bin/split/split.c
@@ -61,16 +61,16 @@ static const char sccsid[] = "@(#)split.c	8.2 (Berkeley) 4/16/94";
 
 #define DEFLINE	1000			/* Default num lines per file. */
 
-off_t	 bytecnt;			/* Byte count to split on. */
-off_t	 chunks = 0;			/* Chunks count to split into. */
-long	 numlines;			/* Line count to split on. */
-int	 file_open;			/* If a file open. */
-int	 ifd = -1, ofd = -1;		/* Input/output file descriptors. */
-char	 bfr[MAXBSIZE];			/* I/O buffer. */
-char	 fname[MAXPATHLEN];		/* File name prefix. */
-regex_t	 rgx;
-int	 pflag;
-long	 sufflen = 2;			/* File name suffix length. */
+static off_t	 bytecnt;		/* Byte count to split on. */
+static off_t	 chunks = 0;		/* Chunks count to split into. */
+static long	 numlines;		/* Line count to split on. */
+static int	 file_open;		/* If a file open. */
+static int	 ifd = -1, ofd = -1;	/* Input/output file descriptors. */
+static char	 bfr[MAXBSIZE];		/* I/O buffer. */
+static char	 fname[MAXPATHLEN];	/* File name prefix. */
+static regex_t	 rgx;
+static int	 pflag;
+static long	 sufflen = 2;		/* File name suffix length. */
 
 static void newfile(void);
 static void split1(void);
diff --git a/usr.bin/tail/forward.c b/usr.bin/tail/forward.c
index e1e0638..3c605bf 100644
--- a/usr.bin/tail/forward.c
+++ b/usr.bin/tail/forward.c
@@ -66,9 +66,9 @@ static void set_events(file_info_t *files);
 #define USE_KQUEUE	1
 #define ADD_EVENTS	2
 
-struct kevent *ev;
-int action = USE_SLEEP;
-int kq;
+static struct kevent *ev;
+static int action = USE_SLEEP;
+static int kq;
 
 static const file_info_t *last;
 
diff --git a/usr.bin/tail/tail.c b/usr.bin/tail/tail.c
index a40646a..a53a568 100644
--- a/usr.bin/tail/tail.c
+++ b/usr.bin/tail/tail.c
@@ -58,7 +58,7 @@ static const char sccsid[] = "@(#)tail.c	8.1 (Berkeley) 6/6/93";
 
 int Fflag, fflag, qflag, rflag, rval, no_files;
 
-file_info_t *files;
+static file_info_t *files;
 
 static void obsolete(char **);
 static void usage(void);
diff --git a/usr.bin/talk/ctl.c b/usr.bin/talk/ctl.c
index 4a3937b..0704b28 100644
--- a/usr.bin/talk/ctl.c
+++ b/usr.bin/talk/ctl.c
@@ -47,6 +47,7 @@ static const char sccsid[] = "@(#)ctl.c	8.1 (Berkeley) 6/6/93";
 #include <string.h>
 
 #include "talk.h"
+#include "talk_ctl.h"
 
 struct	sockaddr_in daemon_addr = { .sin_len = sizeof(daemon_addr), .sin_family = AF_INET };
 struct	sockaddr_in ctl_addr = { .sin_len = sizeof(ctl_addr), .sin_family = AF_INET };
diff --git a/usr.bin/talk/init_disp.c b/usr.bin/talk/init_disp.c
index ae162ef..47556c9 100644
--- a/usr.bin/talk/init_disp.c
+++ b/usr.bin/talk/init_disp.c
@@ -50,8 +50,6 @@ static const char sccsid[] = "@(#)init_disp.c	8.2 (Berkeley) 2/16/94";
 
 #include "talk.h"
 
-extern volatile sig_atomic_t gotwinch;
-
 /*
  * Make sure the callee can write to the screen
  */
diff --git a/usr.bin/talk/invite.c b/usr.bin/talk/invite.c
index b22caaf..12b69b5 100644
--- a/usr.bin/talk/invite.c
+++ b/usr.bin/talk/invite.c
@@ -60,8 +60,8 @@ static const char sccsid[] = "@(#)invite.c	8.1 (Berkeley) 6/6/93";
  * These are used to delete the
  * invitations.
  */
-int	local_id, remote_id;
-jmp_buf invitebuf;
+static int	local_id, remote_id;
+static jmp_buf invitebuf;
 
 void
 invite_remote(void)
diff --git a/usr.bin/talk/talk.h b/usr.bin/talk/talk.h
index 85cdf11..5cf1d56 100644
--- a/usr.bin/talk/talk.h
+++ b/usr.bin/talk/talk.h
@@ -38,6 +38,7 @@
 #include <arpa/inet.h>
 #include <protocols/talkd.h>
 #include <curses.h>
+#include <signal.h>
 
 extern	int sockt;
 extern	int curses_initialized;
@@ -46,6 +47,8 @@ extern	int invitation_waiting;
 extern	const char *current_state;
 extern	int current_line;
 
+extern volatile sig_atomic_t gotwinch;
+
 typedef struct xwin {
 	WINDOW	*x_win;
 	int	x_nlines;
diff --git a/usr.bin/tcopy/tcopy.c b/usr.bin/tcopy/tcopy.c
index b090eb0..4fa0a33 100644
--- a/usr.bin/tcopy/tcopy.c
+++ b/usr.bin/tcopy/tcopy.c
@@ -60,16 +60,16 @@ static const char sccsid[] = "@(#)tcopy.c	8.2 (Berkeley) 4/17/94";
 #define	MAXREC	(64 * 1024)
 #define	NOCOUNT	(-2)
 
-int	filen, guesslen, maxblk = MAXREC;
-u_int64_t	lastrec, record, size, tsize;
-FILE	*msg;
+static int	filen, guesslen, maxblk = MAXREC;
+static uint64_t	lastrec, record, size, tsize;
+static FILE	*msg;
 
-void	*getspace(int);
-void	 intr(int);
+static void	*getspace(int);
+static void	 intr(int);
 static void	 usage(void) __dead2;
-void	 verify(int, int, char *);
-void	 writeop(int, int);
-void	rewind_tape(int);
+static void	 verify(int, int, char *);
+static void	 writeop(int, int);
+static void	 rewind_tape(int);
 
 int
 main(int argc, char *argv[])
@@ -223,7 +223,7 @@ r1:		guesslen = 0;
 	exit(0);
 }
 
-void
+static void
 verify(int inp, int outp, char *outb)
 {
 	int eot, inmaxblk, inn, outmaxblk, outn;
@@ -275,7 +275,7 @@ r2:		if (inn != outn) {
 	exit(1);
 }
 
-void
+static void
 intr(int signo __unused)
 {
 	if (record) {
@@ -289,7 +289,7 @@ intr(int signo __unused)
 	exit(1);
 }
 
-void *
+static void *
 getspace(int blk)
 {
 	void *bp;
@@ -299,7 +299,7 @@ getspace(int blk)
 	return (bp);
 }
 
-void
+static void
 writeop(int fd, int type)
 {
 	struct mtop op;
@@ -317,7 +317,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 rewind_tape(int fd)
 {
 	struct stat sp;
diff --git a/usr.bin/tee/tee.c b/usr.bin/tee/tee.c
index dfcdd66..6b5c00c 100644
--- a/usr.bin/tee/tee.c
+++ b/usr.bin/tee/tee.c
@@ -56,9 +56,9 @@ typedef struct _list {
 	int fd;
 	const char *name;
 } LIST;
-LIST *head;
+static LIST *head;
 
-void add(int, const char *);
+static void add(int, const char *);
 static void usage(void);
 
 int
@@ -125,7 +125,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 add(int fd, const char *name)
 {
 	LIST *p;
diff --git a/usr.bin/tr/tr.c b/usr.bin/tr/tr.c
index aac0c2f..c2b6bf8 100644
--- a/usr.bin/tr/tr.c
+++ b/usr.bin/tr/tr.c
@@ -58,8 +58,8 @@ static const char sccsid[] = "@(#)tr.c	8.2 (Berkeley) 5/4/95";
 #include "cset.h"
 #include "extern.h"
 
-STR s1 = { STRING1, NORMAL, 0, OOBCH, 0, { 0, OOBCH }, NULL, NULL };
-STR s2 = { STRING2, NORMAL, 0, OOBCH, 0, { 0, OOBCH }, NULL, NULL };
+static STR s1 = { STRING1, NORMAL, 0, OOBCH, 0, { 0, OOBCH }, NULL, NULL };
+static STR s2 = { STRING2, NORMAL, 0, OOBCH, 0, { 0, OOBCH }, NULL, NULL };
 
 static struct cset *setup(char *, STR *, int, int);
 static void usage(void);
diff --git a/usr.bin/tsort/tsort.c b/usr.bin/tsort/tsort.c
index def1624..1511b36 100644
--- a/usr.bin/tsort/tsort.c
+++ b/usr.bin/tsort/tsort.c
@@ -96,18 +96,18 @@ typedef struct _buf {
 	int b_bsize;
 } BUF;
 
-DB *db;
-NODE *graph, **cycle_buf, **longest_cycle;
-int debug, longest, quiet;
-
-void	 add_arc(char *, char *);
-int	 find_cycle(NODE *, NODE *, int, int);
-NODE	*get_node(char *);
-void	*grow_buf(void *, size_t);
-void	 remove_node(NODE *);
-void	 clear_cycle(void);
-void	 tsort(void);
-void	 usage(void);
+static DB *db;
+static NODE *graph, **cycle_buf, **longest_cycle;
+static int debug, longest, quiet;
+
+static void	 add_arc(char *, char *);
+static int	 find_cycle(NODE *, NODE *, int, int);
+static NODE	*get_node(char *);
+static void	*grow_buf(void *, size_t);
+static void	 remove_node(NODE *);
+static void	 clear_cycle(void);
+static void	 tsort(void);
+static void	 usage(void);
 
 int
 main(int argc, char *argv[])
@@ -185,7 +185,7 @@ main(int argc, char *argv[])
 }
 
 /* double the size of oldbuf and return a pointer to the new buffer. */
-void *
+static void *
 grow_buf(void *bp, size_t size)
 {
 	if ((bp = realloc(bp, size)) == NULL)
@@ -197,7 +197,7 @@ grow_buf(void *bp, size_t size)
  * add an arc from node s1 to node s2 in the graph.  If s1 or s2 are not in
  * the graph, then add them.
  */
-void
+static void
 add_arc(char *s1, char *s2)
 {
 	NODE *n1;
@@ -232,7 +232,7 @@ add_arc(char *s1, char *s2)
 }
 
 /* Find a node in the graph (insert if not found) and return a pointer to it. */
-NODE *
+static NODE *
 get_node(char *name)
 {
 	DBT data, key;
@@ -284,7 +284,7 @@ get_node(char *name)
 /*
  * Clear the NODEST flag from all nodes.
  */
-void
+static void
 clear_cycle(void)
 {
 	NODE *n;
@@ -294,7 +294,7 @@ clear_cycle(void)
 }
 
 /* do topological sort on graph */
-void
+static void
 tsort(void)
 {
 	NODE *n, *next;
@@ -357,7 +357,7 @@ tsort(void)
 }
 
 /* print node and remove from graph (does not actually free node) */
-void
+static void
 remove_node(NODE *n)
 {
 	NODE **np;
@@ -374,7 +374,7 @@ remove_node(NODE *n)
 
 
 /* look for the longest? cycle from node from to node to. */
-int
+static int
 find_cycle(NODE *from, NODE *to, int longest_len, int depth)
 {
 	NODE **np;
@@ -420,7 +420,7 @@ find_cycle(NODE *from, NODE *to, int longest_len, int depth)
 	return (longest_len);
 }
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr, "usage: tsort [-dlq] [file]\n");
diff --git a/usr.bin/ul/ul.c b/usr.bin/ul/ul.c
index 3baece8..4796956 100644
--- a/usr.bin/ul/ul.c
+++ b/usr.bin/ul/ul.c
@@ -66,8 +66,8 @@ static const char rcsid[] =
 #define	UNDERL	010	/* Ul */
 #define	BOLD	020	/* Bold */
 
-int	must_use_uc, must_overstrike;
-const char
+static int	must_use_uc, must_overstrike;
+static const char
 	*CURS_UP, *CURS_RIGHT, *CURS_LEFT,
 	*ENTER_STANDOUT, *EXIT_STANDOUT, *ENTER_UNDERLINE, *EXIT_UNDERLINE,
 	*ENTER_DIM, *ENTER_BOLD, *ENTER_REVERSE, *UNDER_CHAR, *EXIT_ATTRIBUTES;
@@ -78,25 +78,25 @@ struct	CHAR	{
 	int	c_width;	/* width or -1 if multi-column char. filler */
 } ;
 
-struct	CHAR	obuf[MAXBUF];
-int	col, maxcol;
-int	mode;
-int	halfpos;
-int	upln;
-int	iflag;
+static struct	CHAR	obuf[MAXBUF];
+static int	col, maxcol;
+static int	mode;
+static int	halfpos;
+static int	upln;
+static int	iflag;
 
 static void usage(void);
-void setnewmode(int);
-void initcap(void);
-void reverse(void);
-int outchar(int);
-void fwd(void);
-void initbuf(void);
-void iattr(void);
-void overstrike(void);
-void flushln(void);
-void filter(FILE *);
-void outc(wint_t, int);
+static void setnewmode(int);
+static void initcap(void);
+static void reverse(void);
+static int outchar(int);
+static void fwd(void);
+static void initbuf(void);
+static void iattr(void);
+static void overstrike(void);
+static void flushln(void);
+static void filter(FILE *);
+static void outc(wint_t, int);
 
 #define	PRINT(s)	if (s == NULL) /* void */; else tputs(s, 1, outchar)
 
@@ -165,7 +165,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 filter(FILE *f)
 {
 	wint_t c;
@@ -299,7 +299,7 @@ filter(FILE *f)
 		flushln();
 }
 
-void
+static void
 flushln(void)
 {
 	int lastmode;
@@ -341,7 +341,7 @@ flushln(void)
  * For terminals that can overstrike, overstrike underlines and bolds.
  * We don't do anything with halfline ups and downs, or Greek.
  */
-void
+static void
 overstrike(void)
 {
 	int i;
@@ -381,7 +381,7 @@ overstrike(void)
 	}
 }
 
-void
+static void
 iattr(void)
 {
 	int i;
@@ -405,7 +405,7 @@ iattr(void)
 	putwchar('\n');
 }
 
-void
+static void
 initbuf(void)
 {
 
@@ -415,7 +415,7 @@ initbuf(void)
 	mode &= ALTSET;
 }
 
-void
+static void
 fwd(void)
 {
 	int oldcol, oldmax;
@@ -427,7 +427,7 @@ fwd(void)
 	maxcol = oldmax;
 }
 
-void
+static void
 reverse(void)
 {
 	upln++;
@@ -437,7 +437,7 @@ reverse(void)
 	upln++;
 }
 
-void
+static void
 initcap(void)
 {
 	static char tcapbuf[512];
@@ -490,7 +490,7 @@ initcap(void)
 	must_use_uc = (UNDER_CHAR && !ENTER_UNDERLINE);
 }
 
-int
+static int
 outchar(int c)
 {
 	return (putwchar(c) != WEOF ? c : EOF);
@@ -498,7 +498,7 @@ outchar(int c)
 
 static int curmode = 0;
 
-void
+static void
 outc(wint_t c, int width)
 {
 	int i;
@@ -512,7 +512,7 @@ outc(wint_t c, int width)
 	}
 }
 
-void
+static void
 setnewmode(int newmode)
 {
 	if (!iflag) {
diff --git a/usr.bin/uname/uname.c b/usr.bin/uname/uname.c
index 0e120b9..810b218 100644
--- a/usr.bin/uname/uname.c
+++ b/usr.bin/uname/uname.c
@@ -63,21 +63,22 @@ static const char sccsid[] = "@(#)uname.c	8.2 (Berkeley) 5/4/95";
 #define	IFLAG	0x40
 
 typedef void (*get_t)(void);
-get_t get_ident, get_platform, get_hostname, get_arch, get_release, get_sysname, get_version;
-
-void native_ident(void);
-void native_platform(void);
-void native_hostname(void);
-void native_arch(void);
-void native_release(void);
-void native_sysname(void);
-void native_version(void);
-void print_uname(u_int);
-void setup_get(void);
-void usage(void);
-
-char *ident, *platform, *hostname, *arch, *release, *sysname, *version;
-int space;
+static get_t get_ident, get_platform, get_hostname, get_arch,
+    get_release, get_sysname, get_version;
+
+static void native_ident(void);
+static void native_platform(void);
+static void native_hostname(void);
+static void native_arch(void);
+static void native_release(void);
+static void native_sysname(void);
+static void native_version(void);
+static void print_uname(u_int);
+static void setup_get(void);
+static void usage(void);
+
+static char *ident, *platform, *hostname, *arch, *release, *sysname, *version;
+static int space;
 
 int
 main(int argc, char *argv[])
@@ -142,7 +143,7 @@ do {							\
 	}						\
 } while (0)
 
-void
+static void
 setup_get(void)
 {
 	CHECK_ENV("s", sysname);
@@ -165,7 +166,7 @@ setup_get(void)
 		printf("%s", var);		\
 	}
 
-void
+static void
 print_uname(u_int flags)
 {
 	PRINT_FLAG(flags, SFLAG, sysname);
@@ -179,7 +180,7 @@ print_uname(u_int flags)
 }
 
 #define	NATIVE_SYSCTL2_GET(var,mib0,mib1)	\
-void						\
+static void					\
 native_##var(void)				\
 {						\
 	int mib[] = { (mib0), (mib1) };		\
@@ -193,7 +194,7 @@ native_##var(void)				\
 		err(1, "sysctl");
 
 #define	NATIVE_SYSCTLNAME_GET(var,name)		\
-void						\
+static void					\
 native_##var(void)				\
 {						\
 	size_t len;				\
@@ -242,7 +243,7 @@ NATIVE_SYSCTL2_GET(arch, CTL_HW, HW_MACHINE_ARCH) {
 NATIVE_SYSCTLNAME_GET(ident, "kern.ident") {
 } NATIVE_SET;
 
-void
+static void
 usage(void)
 {
 	fprintf(stderr, "usage: uname [-aimnoprsv]\n");
diff --git a/usr.bin/unexpand/unexpand.c b/usr.bin/unexpand/unexpand.c
index 828ab10..1ef5db1 100644
--- a/usr.bin/unexpand/unexpand.c
+++ b/usr.bin/unexpand/unexpand.c
@@ -55,9 +55,9 @@ static const char sccsid[] = "@(#)unexpand.c	8.1 (Berkeley) 6/6/93";
 #include <wchar.h>
 #include <wctype.h>
 
-int	all;
-int	nstops;
-int	tabstops[100];
+static int	all;
+static int	nstops;
+static int	tabstops[100];
 
 static void getstops(const char *);
 static void usage(void);
diff --git a/usr.bin/unifdef/unifdef.c b/usr.bin/unifdef/unifdef.c
index 521b698..8f7ed0f 100644
--- a/usr.bin/unifdef/unifdef.c
+++ b/usr.bin/unifdef/unifdef.c
@@ -56,7 +56,7 @@
 #include <string.h>
 #include <unistd.h>
 
-const char copyright[] =
+static const char copyright[] =
     "@(#) $Version: unifdef-2.5.6.21f1388 $\n"
     "@(#) $FreeBSD$\n"
     "@(#) $Author: Tony Finch (dot@dotat.at) $\n"
diff --git a/usr.bin/uniq/uniq.c b/usr.bin/uniq/uniq.c
index 0a8f4f0..1077307 100644
--- a/usr.bin/uniq/uniq.c
+++ b/usr.bin/uniq/uniq.c
@@ -57,15 +57,15 @@ static const char rcsid[] =
 #include <wchar.h>
 #include <wctype.h>
 
-int cflag, dflag, uflag, iflag;
-int numchars, numfields, repeats;
-
-FILE	*file(const char *, const char *);
-wchar_t	*convert(const char *);
-int	 inlcmp(const char *, const char *);
-void	 show(FILE *, const char *);
-wchar_t	*skip(wchar_t *);
-void	 obsolete(char *[]);
+static int cflag, dflag, uflag, iflag;
+static int numchars, numfields, repeats;
+
+static FILE	*file(const char *, const char *);
+static wchar_t	*convert(const char *);
+static int	 inlcmp(const char *, const char *);
+static void	 show(FILE *, const char *);
+static wchar_t	*skip(wchar_t *);
+static void	 obsolete(char *[]);
 static void	 usage(void);
 
 int
@@ -184,7 +184,7 @@ main (int argc, char *argv[])
 	exit(0);
 }
 
-wchar_t *
+static wchar_t *
 convert(const char *str)
 {
 	size_t n;
@@ -218,7 +218,7 @@ convert(const char *str)
 	return (ret);
 }
 
-int
+static int
 inlcmp(const char *s1, const char *s2)
 {
 	int c1, c2;
@@ -241,7 +241,7 @@ inlcmp(const char *s1, const char *s2)
  *	Output a line depending on the flags and number of repetitions
  *	of the line.
  */
-void
+static void
 show(FILE *ofp, const char *str)
 {
 
@@ -251,7 +251,7 @@ show(FILE *ofp, const char *str)
 		(void)fprintf(ofp, "%s", str);
 }
 
-wchar_t *
+static wchar_t *
 skip(wchar_t *str)
 {
 	int nchars, nfields;
@@ -267,7 +267,7 @@ skip(wchar_t *str)
 	return(str);
 }
 
-FILE *
+static FILE *
 file(const char *name, const char *mode)
 {
 	FILE *fp;
@@ -277,7 +277,7 @@ file(const char *name, const char *mode)
 	return(fp);
 }
 
-void
+static void
 obsolete(char *argv[])
 {
 	int len;
diff --git a/usr.bin/units/units.c b/usr.bin/units/units.c
index e700c94..3f4dfd1 100644
--- a/usr.bin/units/units.c
+++ b/usr.bin/units/units.c
@@ -42,9 +42,9 @@ static const char rcsid[] =
 
 #define PRIMITIVECHAR '!'
 
-const char *powerstring = "^";
+static const char *powerstring = "^";
 
-struct {
+static struct {
 	char *uname;
 	char *uval;
 }      unittable[MAXUNITS];
@@ -57,13 +57,13 @@ struct unittype {
 	int quantity;
 };
 
-struct {
+static struct {
 	char *prefixname;
 	char *prefixval;
 }      prefixtable[MAXPREFIXES];
 
 
-char NULLUNIT[] = "";
+static char NULLUNIT[] = "";
 
 #ifdef MSDOS
 #define SEPARATOR      ";"
@@ -71,8 +71,8 @@ char NULLUNIT[] = "";
 #define SEPARATOR      ":"
 #endif
 
-int unitcount;
-int prefixcount;
+static int unitcount;
+static int prefixcount;
 
 char	*dupstr(const char *str);
 void	 readunits(const char *userfile);
diff --git a/usr.bin/usbhidaction/usbhidaction.c b/usr.bin/usbhidaction/usbhidaction.c
index fd66b64..876dbca 100644
--- a/usr.bin/usbhidaction/usbhidaction.c
+++ b/usr.bin/usbhidaction/usbhidaction.c
@@ -64,7 +64,7 @@ struct command {
 	char *name;
 	char *action;
 };
-struct command *commands;
+static struct command *commands;
 
 #define SIZE 4000
 
diff --git a/usr.bin/usbhidctl/usbhid.c b/usr.bin/usbhidctl/usbhid.c
index bf3e348..2a28e3e 100644
--- a/usr.bin/usbhidctl/usbhid.c
+++ b/usr.bin/usbhidctl/usbhid.c
@@ -42,7 +42,7 @@
 #include <usbhid.h>
 #include <dev/usb/usbhid.h>
 
-struct variable {
+static struct variable {
 	char *name;
 	int instance;
 	int val;
@@ -50,11 +50,11 @@ struct variable {
 	struct variable *next;
 } *vars;
 
-int verbose = 0;
-int noname = 0;
-int hexdump = 0;
-int wflag = 0;
-int zflag = 0;
+static int verbose = 0;
+static int noname = 0;
+static int hexdump = 0;
+static int wflag = 0;
+static int zflag = 0;
 
 static void usage(void);
 static void dumpitem(const char *label, struct hid_item *h);
diff --git a/usr.bin/uuencode/uuencode.c b/usr.bin/uuencode/uuencode.c
index 44db968..def8bcc 100644
--- a/usr.bin/uuencode/uuencode.c
+++ b/usr.bin/uuencode/uuencode.c
@@ -60,13 +60,13 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-void encode(void);
-void base64_encode(void);
+static void encode(void);
+static void base64_encode(void);
 static void usage(void);
 
-FILE *output;
-int mode;
-char **av;
+static FILE *output;
+static int mode;
+static char **av;
 
 int
 main(int argc, char *argv[])
@@ -138,7 +138,7 @@ main(int argc, char *argv[])
 /*
  * Copy from in to out, encoding in base64 as you go along.
  */
-void
+static void
 base64_encode(void)
 {
 	/*
@@ -168,7 +168,7 @@ base64_encode(void)
 /*
  * Copy from in to out, encoding as you go along.
  */
-void
+static void
 encode(void)
 {
 	register int ch, n;
diff --git a/usr.bin/vis/vis.c b/usr.bin/vis/vis.c
index 6344a91..71b6404 100644
--- a/usr.bin/vis/vis.c
+++ b/usr.bin/vis/vis.c
@@ -50,9 +50,12 @@ static const char sccsid[] = "@(#)vis.c	8.1 (Berkeley) 6/6/93";
 
 #include "extern.h"
 
-int eflags, fold, foldwidth=80, none, markeol, debug;
+static int eflags, fold, foldwidth = 80, none, markeol;
+#ifdef DEBUG
+static int debug;
+#endif
 
-void process(FILE *);
+static void process(FILE *);
 static void usage(void);
 
 int
@@ -133,7 +136,7 @@ usage(void)
 	exit(1);
 }
 
-void
+static void
 process(FILE *fp)
 {
 	static int col = 0;
diff --git a/usr.bin/w/w.c b/usr.bin/w/w.c
index 0331287..8441ce5 100644
--- a/usr.bin/w/w.c
+++ b/usr.bin/w/w.c
@@ -85,25 +85,24 @@ static const char sccsid[] = "@(#)w.c	8.4 (Berkeley) 4/16/94";
 
 #include "extern.h"
 
-struct timeval	boottime;
-struct utmpx   *utmp;
-struct winsize	ws;
-kvm_t	       *kd;
-time_t		now;		/* the current time of day */
-int		ttywidth;	/* width of tty */
-int		argwidth;	/* width of tty */
-int		header = 1;	/* true if -h flag: don't print heading */
-int		nflag;		/* true if -n flag: don't convert addrs */
-int		dflag;		/* true if -d flag: output debug info */
-int		sortidle;	/* sort by idle time */
+static struct utmpx *utmp;
+static struct winsize ws;
+static kvm_t   *kd;
+static time_t	now;		/* the current time of day */
+static int	ttywidth;	/* width of tty */
+static int	argwidth;	/* width of tty */
+static int	header = 1;	/* true if -h flag: don't print heading */
+static int	nflag;		/* true if -n flag: don't convert addrs */
+static int	dflag;		/* true if -d flag: output debug info */
+static int	sortidle;	/* sort by idle time */
 int		use_ampm;	/* use AM/PM time */
-int             use_comma;      /* use comma as floats separator */
-char	      **sel_users;	/* login array of particular users selected */
+static int	use_comma;      /* use comma as floats separator */
+static char   **sel_users;	/* login array of particular users selected */
 
 /*
  * One of these per active utmp entry.
  */
-struct	entry {
+static struct entry {
 	struct	entry *next;
 	struct	utmpx utmp;
 	dev_t	tdev;			/* dev_t of terminal */
diff --git a/usr.bin/wall/wall.c b/usr.bin/wall/wall.c
index eea253f..c3aa323 100644
--- a/usr.bin/wall/wall.c
+++ b/usr.bin/wall/wall.c
@@ -68,14 +68,14 @@ static const char sccsid[] = "@(#)wall.c	8.2 (Berkeley) 11/16/93";
 static void makemsg(char *);
 static void usage(void);
 
-struct wallgroup {
+static struct wallgroup {
 	struct wallgroup *next;
 	char		*name;
 	gid_t		gid;
 } *grouplist;
-int nobanner;
-int mbufsize;
-char *mbuf;
+static int nobanner;
+static int mbufsize;
+static char *mbuf;
 
 static int
 ttystat(char *line)
diff --git a/usr.bin/wc/wc.c b/usr.bin/wc/wc.c
index 270a23b..08823a1 100644
--- a/usr.bin/wc/wc.c
+++ b/usr.bin/wc/wc.c
@@ -58,8 +58,8 @@ __FBSDID("$FreeBSD$");
 #include <wchar.h>
 #include <wctype.h>
 
-uintmax_t tlinect, twordct, tcharct, tlongline;
-int doline, doword, dochar, domulti, dolongline;
+static uintmax_t tlinect, twordct, tcharct, tlongline;
+static int doline, doword, dochar, domulti, dolongline;
 static volatile sig_atomic_t siginfo;
 
 static void	show_cnt(const char *file, uintmax_t linect, uintmax_t wordct,
diff --git a/usr.bin/whereis/whereis.c b/usr.bin/whereis/whereis.c
index 0829f77..91df6dc 100644
--- a/usr.bin/whereis/whereis.c
+++ b/usr.bin/whereis/whereis.c
@@ -56,18 +56,18 @@ __FBSDID("$FreeBSD$");
 
 typedef const char *ccharp;
 
-int opt_a, opt_b, opt_m, opt_q, opt_s, opt_u, opt_x;
-ccharp *bindirs, *mandirs, *sourcedirs;
-char **query;
+static int opt_a, opt_b, opt_m, opt_q, opt_s, opt_u, opt_x;
+static ccharp *bindirs, *mandirs, *sourcedirs;
+static char **query;
 
-const char *sourcepath = PATH_SOURCES;
+static const char *sourcepath = PATH_SOURCES;
 
-char	*colonify(ccharp *);
-int	 contains(ccharp *, const char *);
-void	 decolonify(char *, ccharp **, int *);
-void	 defaults(void);
-void	 scanopts(int, char **);
-void	 usage(void);
+static char	*colonify(ccharp *);
+static int	 contains(ccharp *, const char *);
+static void	 decolonify(char *, ccharp **, int *);
+static void	 defaults(void);
+static void	 scanopts(int, char **);
+static void	 usage(void);
 
 /*
  * Throughout this program, a number of strings are dynamically
@@ -85,7 +85,7 @@ void	 usage(void);
  * abort(3) in case of an allocation failure.
  */
 
-void
+static void
 usage(void)
 {
 	(void)fprintf(stderr,
@@ -99,7 +99,7 @@ usage(void)
  * Note that the -B/-M/-S options expect a list of directory
  * names that must be terminated with -f.
  */
-void
+static void
 scanopts(int argc, char **argv)
 {
 	int c, i;
@@ -174,7 +174,7 @@ scanopts(int argc, char **argv)
 /*
  * Find out whether string `s' is contained in list `cpp'.
  */
-int
+static int
 contains(ccharp *cpp, const char *s)
 {
 	ccharp cp;
@@ -197,7 +197,7 @@ contains(ccharp *cpp, const char *s)
  * partial string is only added if it has a length greater than 0, and
  * if it's not already contained in the string list.
  */
-void
+static void
 decolonify(char *s, ccharp **cppp, int *ip)
 {
 	char *cp;
@@ -223,7 +223,7 @@ decolonify(char *s, ccharp **cppp, int *ip)
 /*
  * Join string list `cpp' into a colon-separated string.
  */
-char *
+static char *
 colonify(ccharp *cpp)
 {
 	size_t s;
@@ -249,7 +249,7 @@ colonify(ccharp *cpp)
 /*
  * Provide defaults for all options and directory lists.
  */
-void
+static void
 defaults(void)
 {
 	size_t s;
diff --git a/usr.bin/which/which.c b/usr.bin/which/which.c
index 3b8224da..2c11b80 100644
--- a/usr.bin/which/which.c
+++ b/usr.bin/which/which.c
@@ -40,8 +40,8 @@ __FBSDID("$FreeBSD$");
 static void	 usage(void);
 static int	 print_matches(char *, char *);
 	
-int 	silent;
-int 	allpaths;
+static int 	 silent;
+static int 	 allpaths;
 
 int
 main(int argc, char **argv)
diff --git a/usr.bin/who/who.1 b/usr.bin/who/who.1
index 8a40396..aadee18 100644
--- a/usr.bin/who/who.1
+++ b/usr.bin/who/who.1
@@ -28,7 +28,7 @@
 .\"     @(#)who.1	8.2 (Berkeley) 12/30/93
 .\" $FreeBSD$
 .\"
-.Dd May 8, 2002
+.Dd Oct 28, 2011
 .Dt WHO 1
 .Os
 .Sh NAME
@@ -36,7 +36,7 @@
 .Nd display who is on the system
 .Sh SYNOPSIS
 .Nm
-.Op Fl HmqsTu
+.Op Fl bHmqsTu
 .Op Cm am I
 .Op Ar file
 .Sh DESCRIPTION
@@ -48,6 +48,8 @@ remote hostname if not local.
 .Pp
 The options are as follows:
 .Bl -tag -width indent
+.It Fl b
+Write the time and date of the last system reboot.
 .It Fl H
 Write column headings above the output.
 .It Fl m
diff --git a/usr.bin/who/who.c b/usr.bin/who/who.c
index d6f38dd..18467f3 100644
--- a/usr.bin/who/who.c
+++ b/usr.bin/who/who.c
@@ -48,14 +48,16 @@ __FBSDID("$FreeBSD$");
 #include <utmpx.h>
 
 static void	heading(void);
+static void	boottime(void);
 static void	process_utmp(void);
 static void	quick(void);
-static void	row(struct utmpx *);
+static void	row(const struct utmpx *);
 static int	ttywidth(void);
 static void	usage(void);
 static void	whoami(void);
 
 static int	Hflag;			/* Write column headings */
+static int	bflag;			/* Show date of the last reboot */
 static int	mflag;			/* Show info about current terminal */
 static int	qflag;			/* "Quick" mode */
 static int	sflag;			/* Show name, line, time */
@@ -69,7 +71,7 @@ main(int argc, char *argv[])
 
 	setlocale(LC_TIME, "");
 
-	while ((ch = getopt(argc, argv, "HTmqsu")) != -1) {
+	while ((ch = getopt(argc, argv, "HTbmqsu")) != -1) {
 		switch (ch) {
 		case 'H':		/* Write column headings */
 			Hflag = 1;
@@ -77,6 +79,9 @@ main(int argc, char *argv[])
 		case 'T':		/* Show terminal state */
 			Tflag = 1;
 			break;
+		case 'b':		/* Show date of the last reboot */
+			bflag = 1;
+			break;
 		case 'm':		/* Show info about current terminal */
 			mflag = 1;
 			break;
@@ -121,6 +126,8 @@ main(int argc, char *argv[])
 			heading();
 		if (mflag)
 			whoami();
+		else if (bflag)
+			boottime();
 		else
 			process_utmp();
 	}
@@ -134,7 +141,7 @@ static void
 usage(void)
 {
 
-	fprintf(stderr, "usage: who [-HmqsTu] [am I] [file]\n");
+	fprintf(stderr, "usage: who [-bHmqsTu] [am I] [file]\n");
 	exit(1);
 }
 
@@ -145,14 +152,14 @@ heading(void)
 	printf("%-16s ", "NAME");
 	if (Tflag)
 		printf("S ");
-	printf("%-8s %-12s ", "LINE", "TIME");
+	printf("%-12s %-12s ", "LINE", "TIME");
 	if (uflag)
 		printf("IDLE  ");
 	printf("%-16s\n", "FROM");
 }
 
 static void
-row(struct utmpx *ut)
+row(const struct utmpx *ut)
 {
 	char buf[80], tty[PATH_MAX];
 	struct stat sb;
@@ -178,7 +185,10 @@ row(struct utmpx *ut)
 	printf("%-16s ", ut->ut_user);
 	if (Tflag)
 		printf("%c ", state);
-	printf("%-8s ", ut->ut_line);
+	if (ut->ut_type == BOOT_TIME)
+		printf("%-12s ", "system boot");
+	else
+		printf("%-12s ", ut->ut_line);
 	t = ut->ut_tv.tv_sec;
 	tm = localtime(&t);
 	strftime(buf, sizeof(buf), d_first ? "%e %b %R" : "%b %e %R", tm);
@@ -225,6 +235,17 @@ process_utmp(void)
 }
 
 static void
+boottime(void)
+{
+	struct utmpx u1, *u2;
+
+	u1.ut_type = BOOT_TIME;
+	if ((u2 = getutxid(&u1)) == NULL)
+		return;
+	row(u2);
+}
+
+static void
 quick(void)
 {
 	struct utmpx *utx;
diff --git a/usr.bin/whois/whois.c b/usr.bin/whois/whois.c
index b49daf0..d76d05a 100644
--- a/usr.bin/whois/whois.c
+++ b/usr.bin/whois/whois.c
@@ -81,9 +81,9 @@ __FBSDID("$FreeBSD$");
 
 #define ishost(h) (isalnum((unsigned char)h) || h == '.' || h == '-')
 
-const char *ip_whois[] = { LNICHOST, RNICHOST, PNICHOST, BNICHOST,
-			   FNICHOST, NULL };
-const char *port = DEFAULT_PORT;
+static const char *ip_whois[] = { LNICHOST, RNICHOST, PNICHOST, BNICHOST,
+				  FNICHOST, NULL };
+static const char *port = DEFAULT_PORT;
 
 static char *choose_server(char *);
 static struct addrinfo *gethostinfo(char const *host, int exit_on_error);
diff --git a/usr.bin/xinstall/xinstall.c b/usr.bin/xinstall/xinstall.c
index b1a565c..a920f85 100644
--- a/usr.bin/xinstall/xinstall.c
+++ b/usr.bin/xinstall/xinstall.c
@@ -73,13 +73,14 @@ __FBSDID("$FreeBSD$");
 #define	NOCHANGEBITS	(UF_IMMUTABLE | UF_APPEND | SF_IMMUTABLE | SF_APPEND)
 #define	BACKUP_SUFFIX	".old"
 
-struct passwd *pp;
-struct group *gp;
-gid_t gid;
-uid_t uid;
-int dobackup, docompare, dodir, dopreserve, dostrip, nommap, safecopy, verbose;
-mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
-const char *suffix = BACKUP_SUFFIX;
+static struct passwd *pp;
+static struct group *gp;
+static gid_t gid;
+static uid_t uid;
+static int dobackup, docompare, dodir, dopreserve, dostrip, nommap, safecopy,
+    verbose;
+static mode_t mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
+static const char *suffix = BACKUP_SUFFIX;
 
 static int	compare(int, const char *, size_t, int, const char *, size_t);
 static void	copy(int, const char *, int, const char *, off_t);
diff --git a/usr.bin/xlint/xlint/xlint.c b/usr.bin/xlint/xlint/xlint.c
index 553ac19..de3b527 100644
--- a/usr.bin/xlint/xlint/xlint.c
+++ b/usr.bin/xlint/xlint/xlint.c
@@ -656,7 +656,7 @@ fname(const char *name)
 	appcstrg(&args, name);
 
 	/* we reuse the same tmp file for cpp output, so rewind and truncate */
-	if (lseek(cppoutfd, SEEK_SET, (off_t)0) != 0) {
+	if (lseek(cppoutfd, (off_t)0, SEEK_SET) != 0) {
 		warn("lseek");
 		terminate(-1);
 	}
diff --git a/usr.bin/xstr/xstr.c b/usr.bin/xstr/xstr.c
index 1636b79..670e09e 100644
--- a/usr.bin/xstr/xstr.c
+++ b/usr.bin/xstr/xstr.c
@@ -62,35 +62,35 @@ static const char sccsid[] = "@(#)xstr.c	8.1 (Berkeley) 6/9/93";
 
 #define	ignore(a)	((void) a)
 
-off_t	tellpt;
+static off_t	tellpt;
 
-off_t	mesgpt;
-char	cstrings[] =	"strings";
-char	*strings =	cstrings;
+static off_t	mesgpt;
+static char	cstrings[] =	"strings";
+static char	*strings =	cstrings;
 
-int	cflg;
-int	vflg;
-int	readstd;
+static int	cflg;
+static int	vflg;
+static int	readstd;
 
-char lastchr(char *);
+static char lastchr(char *);
 
-int fgetNUL(char *, int, FILE *);
-int istail(char *, char *);
-int octdigit(char);
-int xgetc(FILE *);
+static int fgetNUL(char *, int, FILE *);
+static int istail(char *, char *);
+static int octdigit(char);
+static int xgetc(FILE *);
 
-off_t hashit(char *, int);
-off_t yankstr(char **);
+static off_t hashit(char *, int);
+static off_t yankstr(char **);
 
 static void usage(void);
 
-void flushsh(void);
-void found(int, off_t, char *);
-void inithash(void);
-void onintr(int);
-void process(const char *);
-void prstr(char *);
-void xsdotc(void);
+static void flushsh(void);
+static void found(int, off_t, char *);
+static void inithash(void);
+static void onintr(int);
+static void process(const char *);
+static void prstr(char *);
+static void xsdotc(void);
 
 int
 main(int argc, char *argv[])
@@ -155,9 +155,9 @@ usage(void)
 	exit (1);
 }
 
-char linebuf[BUFSIZ];
+static char linebuf[BUFSIZ];
 
-void
+static void
 process(const char *name)
 {
 	char *cp;
@@ -225,7 +225,7 @@ out:
 		warn("x.c"), onintr(0);
 }
 
-off_t
+static off_t
 yankstr(char **cpp)
 {
 	char *cp = *cpp;
@@ -285,13 +285,13 @@ out:
 	return (hashit(dbuf, 1));
 }
 
-int
+static int
 octdigit(char c)
 {
 	return (isdigit(c) && c != '8' && c != '9');
 }
 
-void
+static void
 inithash(void)
 {
 	char buf[BUFSIZ];
@@ -308,7 +308,7 @@ inithash(void)
 	ignore(fclose(mesgread));
 }
 
-int
+static int
 fgetNUL(char *obuf, int rmdr, FILE *file)
 {
 	int c;
@@ -320,7 +320,7 @@ fgetNUL(char *obuf, int rmdr, FILE *file)
 	return ((feof(file) || ferror(file)) ? 0 : 1);
 }
 
-int
+static int
 xgetc(FILE *file)
 {
 
@@ -330,14 +330,14 @@ xgetc(FILE *file)
 
 #define	BUCKETS	128
 
-struct	hash {
+static struct hash {
 	off_t	hpt;
 	char	*hstr;
 	struct	hash *hnext;
 	short	hnew;
 } bucket[BUCKETS];
 
-off_t
+static off_t
 hashit(char *str, int new)
 {
 	int i;
@@ -362,7 +362,7 @@ hashit(char *str, int new)
 	return (hp->hpt);
 }
 
-void
+static void
 flushsh(void)
 {
 	int i;
@@ -395,7 +395,7 @@ flushsh(void)
 		err(4, "%s", strings);
 }
 
-void
+static void
 found(int new, off_t off, char *str)
 {
 	if (vflg == 0)
@@ -408,7 +408,7 @@ found(int new, off_t off, char *str)
 	fprintf(stderr, "\n");
 }
 
-void
+static void
 prstr(char *cp)
 {
 	int c;
@@ -424,7 +424,7 @@ prstr(char *cp)
 			fprintf(stderr, "%c", c);
 }
 
-void
+static void
 xsdotc(void)
 {
 	FILE *strf = fopen(strings, "r");
@@ -459,7 +459,7 @@ out:
 	ignore(fclose(strf));
 }
 
-char
+static char
 lastchr(char *cp)
 {
 
@@ -468,7 +468,7 @@ lastchr(char *cp)
 	return (*cp);
 }
 
-int
+static int
 istail(char *str, char *of)
 {
 	int d = strlen(of) - strlen(str);
@@ -478,7 +478,7 @@ istail(char *str, char *of)
 	return (d);
 }
 
-void
+static void
 onintr(int dummy __unused)
 {
 
diff --git a/usr.sbin/apm/apm.c b/usr.sbin/apm/apm.c
index b4c34ab..05652da 100644
--- a/usr.sbin/apm/apm.c
+++ b/usr.sbin/apm/apm.c
@@ -38,7 +38,7 @@ __FBSDID("$FreeBSD$");
 #define xl(a)	((a) & 0xff)
 #define APMERR(a) xh(a)
 
-int cmos_wall = 0;	/* True when wall time is in cmos clock, else UTC */
+static int cmos_wall = 0; /* True when wall time is in cmos clock, else UTC */
 
 static void
 usage(void)
diff --git a/usr.sbin/boot0cfg/boot0cfg.8 b/usr.sbin/boot0cfg/boot0cfg.8
index c7e664c..a359ffa 100644
--- a/usr.sbin/boot0cfg/boot0cfg.8
+++ b/usr.sbin/boot0cfg/boot0cfg.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 13, 2009
+.Dd November 7, 2011
 .Dt BOOT0CFG 8
 .Os
 .Sh NAME
@@ -153,21 +153,6 @@ Set the timeout value to
 .It Fl v
 Verbose: display information about the slices defined, etc.
 .El
-.Sh NOTE
-Protection mechanisms in the
-.Xr geom 4
-subsystem might prevent
-.Nm
-from being able to update the MBR on a mounted disk.
-Instructions for temporarily disabling these protection mechanisms
-can be found in the
-.Xr geom 4
-manpage. Specifically, do a
-.Pp
-.Dl sysctl kern.geom.debugflags=0x10
-.Pp
-to allow writing to the MBR, and restore it to 0 afterwards.
-.Pp
 .Sh FILES
 .Bl -tag -width /boot/boot0sio -compact
 .It Pa /boot/boot0
@@ -197,7 +182,8 @@ to install the default MBR:
 .Sh SEE ALSO
 .Xr geom 4 ,
 .Xr boot 8 ,
-.Xr fdisk 8
+.Xr fdisk 8 ,
+.Xr gpart 8
 .Sh AUTHORS
 .An Robert Nordier Aq rnordier@FreeBSD.org .
 .Sh BUGS
diff --git a/usr.sbin/boot0cfg/boot0cfg.c b/usr.sbin/boot0cfg/boot0cfg.c
index 37ca140..72baf46 100644
--- a/usr.sbin/boot0cfg/boot0cfg.c
+++ b/usr.sbin/boot0cfg/boot0cfg.c
@@ -60,13 +60,13 @@ struct opt_offsets {
 	int ticks;
 };
 
-struct opt_offsets b0_ofs[] = {
+static struct opt_offsets b0_ofs[] = {
 	{ 0x0, 0x0, 0x0, 0x0 },		/* no boot block */
 	{ 0x1b9, 0x1ba, 0x1bb, 0x1bc },	/* original block */
 	{ 0x1b5, 0x1b6, 0x1b7, 0x1bc },	/* NT_SERIAL block */
 };
 
-int b0_ver;		/* boot block version set by boot0bs */
+static int b0_ver;	/* boot block version set by boot0bs */
 
 #define OFF_OPT		(b0_ofs[b0_ver].opt)	/* default boot option */
 #define OFF_DRIVE	(b0_ofs[b0_ver].drive)	/* setdrv drive */
@@ -96,6 +96,7 @@ static const char fmt0[] = "#   flag     start chs   type"
 static const char fmt1[] = "%d   0x%02x   %4u:%3u:%2u   0x%02x"
     "   %4u:%3u:%2u   %10u   %10u\n";
 
+static int geom_class_available(const char *);
 static int read_mbr(const char *, u_int8_t **, int);
 static void write_mbr(const char *, int, u_int8_t *, int);
 static void display_mbr(u_int8_t *);
@@ -106,9 +107,9 @@ static int argtoi(const char *, int, int, int);
 static int set_bell(u_int8_t *, int, int);
 static void usage(void);
 
-unsigned vol_id[5];	/* 4 plus 1 for flag */
+static unsigned vol_id[5];	/* 4 plus 1 for flag */
 
-int v_flag;
+static int v_flag;
 /*
  * Boot manager installation/configuration utility.
  */
@@ -337,67 +338,95 @@ read_mbr(const char *disk, u_int8_t **mbr, int check_version)
     return sizeof(buf);
 }
 
+static int
+geom_class_available(const char *name)
+{
+	struct gclass *class;
+	struct gmesh mesh;
+	int error;
+
+	error = geom_gettree(&mesh);
+	if (error != 0)
+		errc(1, error, "Cannot get GEOM tree");
+
+	LIST_FOREACH(class, &mesh.lg_class, lg_class) {
+		if (strcmp(class->lg_name, name) == 0) {
+			geom_deletetree(&mesh);
+			return (1);
+		}
+	}
+
+	geom_deletetree(&mesh);
+	return (0);
+}
+
 /*
  * Write out the mbr to the specified file.
  */
 static void
 write_mbr(const char *fname, int flags, u_int8_t *mbr, int mbr_size)
 {
-    int fd;
-    ssize_t n;
-    const char *errmsg;
-    char *pname;
-    struct gctl_req *grq;
-   
-    fd = open(fname, O_WRONLY | flags, 0666);
-    if (fd != -1) {
-	n = write(fd, mbr, mbr_size);
-	close(fd);
-	if (n != mbr_size)
-	   errx(1, "%s: short write", fname);
-	return;
-    }
+	struct gctl_req *grq;
+	const char *errmsg;
+	char *pname;
+	ssize_t n;
+	int fd;
+
+	fd = open(fname, O_WRONLY | flags, 0666);
+	if (fd != -1) {
+		n = write(fd, mbr, mbr_size);
+		close(fd);
+		if (n != mbr_size)
+			errx(1, "%s: short write", fname);
+		return;
+	}
 
-    /*
-     * If we're called to write to a backup file, don't try to
-     * write through GEOM. It only generates additional errors.
-     */
-    if (flags != 0)
-	return;
-
-    /* Try open it read only. */
-    fd = open(fname, O_RDONLY);
-    if (fd == -1) {
-	warn("error opening %s", fname);
-	return;
-    }
-    pname = g_providername(fd);
-    if (pname == NULL) {
-	warn("error getting providername for %s", fname);
-	return;
-    }
-    grq = gctl_get_handle();
-    gctl_ro_param(grq, "class", -1, "PART");
-    gctl_ro_param(grq, "arg0", -1, pname);
-    gctl_ro_param(grq, "verb", -1, "bootcode");
-    gctl_ro_param(grq, "bootcode", mbr_size, mbr);
-    gctl_ro_param(grq, "flags", -1, "C");
-    errmsg = gctl_issue(grq);
-    if (errmsg == NULL)
-	goto out;
-
-    grq = gctl_get_handle();
-    gctl_ro_param(grq, "verb", -1, "write MBR");
-    gctl_ro_param(grq, "class", -1, "MBR");
-    gctl_ro_param(grq, "geom", -1, pname);
-    gctl_ro_param(grq, "data", mbr_size, mbr);
-    errmsg = gctl_issue(grq);
-    if (errmsg != NULL)
-	err(1, "write_mbr: %s", fname);
-
-out:
-    free(pname);
-    gctl_free(grq);
+	/*
+	 * If we're called to write to a backup file, don't try to
+	 * write through GEOM.
+	 */
+	if (flags != 0)
+		err(1, "can't open file %s to write backup", fname);
+
+	/* Try open it read only. */
+	fd = open(fname, O_RDONLY);
+	if (fd == -1) {
+		warn("error opening %s", fname);
+		return;
+	}
+
+	pname = g_providername(fd);
+	if (pname == NULL) {
+		warn("error getting providername for %s", fname);
+		return;
+	}
+
+	/* First check that GEOM_PART is available */
+	if (geom_class_available("PART") != 0) {
+		grq = gctl_get_handle();
+		gctl_ro_param(grq, "class", -1, "PART");
+		gctl_ro_param(grq, "arg0", -1, pname);
+		gctl_ro_param(grq, "verb", -1, "bootcode");
+		gctl_ro_param(grq, "bootcode", mbr_size, mbr);
+		gctl_ro_param(grq, "flags", -1, "C");
+		errmsg = gctl_issue(grq);
+		if (errmsg != NULL && errmsg[0] != '\0')
+			errx(1, "GEOM_PART: write bootcode to %s failed: %s",
+			    fname, errmsg);
+		gctl_free(grq);
+	} else if (geom_class_available("MBR") != 0) {
+		grq = gctl_get_handle();
+		gctl_ro_param(grq, "verb", -1, "write MBR");
+		gctl_ro_param(grq, "class", -1, "MBR");
+		gctl_ro_param(grq, "geom", -1, pname);
+		gctl_ro_param(grq, "data", mbr_size, mbr);
+		errmsg = gctl_issue(grq);
+		if (errmsg != NULL)
+			err(1, "GEOM_MBR: write MBR to %s failed", fname);
+		gctl_free(grq);
+	} else
+		errx(1, "can't write MBR to %s", fname);
+	free(pname);
 }
 
 /*
diff --git a/usr.sbin/cdcontrol/cdcontrol.c b/usr.sbin/cdcontrol/cdcontrol.c
index 526f599..70ef2f5 100644
--- a/usr.sbin/cdcontrol/cdcontrol.c
+++ b/usr.sbin/cdcontrol/cdcontrol.c
@@ -87,7 +87,7 @@ __FBSDID("$FreeBSD$");
 #define STATUS_MEDIA	0x2
 #define STATUS_VOLUME	0x4
 
-struct cmdtab {
+static struct cmdtab {
 	int command;
 	const char *name;
 	unsigned min;
@@ -119,40 +119,40 @@ struct cmdtab {
 { 0,		NULL,		0, NULL }
 };
 
-struct cd_toc_entry	toc_buffer[100];
-
-const char	*cdname;
-int		fd = -1;
-int		verbose = 1;
-int		msf = 1;
-
-int		 setvol(int, int);
-int		 read_toc_entrys(int);
-int		 play_msf(int, int, int, int, int, int);
-int		 play_track(int, int, int, int);
-int		 get_vol(int *, int *);
-int		 status(int *, int *, int *, int *);
-int		 open_cd(void);
-int		 next_prev(char *arg, int);
-int		 play(char *arg);
-int		 info(char *arg);
-int		 cdid(void);
-int		 pstatus(char *arg);
-char		*input(int *);
-void		 prtrack(struct cd_toc_entry *e, int lastflag);
-void		 lba2msf(unsigned long lba, u_char *m, u_char *s, u_char *f);
-unsigned int	 msf2lba(u_char m, u_char s, u_char f);
-int		 play_blocks(int blk, int len);
-int		 run(int cmd, char *arg);
-char		*parse(char *buf, int *cmd);
-void		 help(void);
-void		 usage(void);
-char		*use_cdrom_instead(const char *);
-__const char	*strstatus(int);
+static struct cd_toc_entry toc_buffer[100];
+
+static const char *cdname;
+static int	fd = -1;
+static int	verbose = 1;
+static int	msf = 1;
+
+static int	 setvol(int, int);
+static int	 read_toc_entrys(int);
+static int	 play_msf(int, int, int, int, int, int);
+static int	 play_track(int, int, int, int);
+static int	 status(int *, int *, int *, int *);
+static int	 open_cd(void);
+static int	 next_prev(char *arg, int);
+static int	 play(char *arg);
+static int	 info(char *arg);
+static int	 cdid(void);
+static int	 pstatus(char *arg);
+static char	*input(int *);
+static void	 prtrack(struct cd_toc_entry *e, int lastflag);
+static void	 lba2msf(unsigned long lba, u_char *m, u_char *s, u_char *f);
+static unsigned int msf2lba(u_char m, u_char s, u_char f);
+static int	 play_blocks(int blk, int len);
+static int	 run(int cmd, char *arg);
+static char	*parse(char *buf, int *cmd);
+static void	 help(void);
+static void	 usage(void);
+static char	*use_cdrom_instead(const char *);
+static const char *strstatus(int);
 static u_int	 dbprog_discid(void);
-__const char	*cdcontrol_prompt(void);
+static const char *cdcontrol_prompt(void);
 
-void help (void)
+static void
+help(void)
 {
 	struct cmdtab *c;
 	const char *s;
@@ -178,13 +178,15 @@ void help (void)
 	printf ("\tThe plain target address is taken as a synonym for play.\n");
 }
 
-void usage (void)
+static void
+usage(void)
 {
 	fprintf (stderr, "usage: cdcontrol [-sv] [-f device] [command ...]\n");
 	exit (1);
 }
 
-char *use_cdrom_instead(const char *old_envvar)
+static char *
+use_cdrom_instead(const char *old_envvar)
 {
 	char *device;
 
@@ -196,7 +198,8 @@ char *use_cdrom_instead(const char *old_envvar)
 }
 
 
-int main (int argc, char **argv)
+int
+main(int argc, char **argv)
 {
 	int cmd;
 	char *arg;
@@ -284,7 +287,8 @@ int main (int argc, char **argv)
 	}
 }
 
-int run (int cmd, char *arg)
+static int
+run(int cmd, char *arg)
 {
 	long speed;
 	int l, r, rc, count;
@@ -465,7 +469,8 @@ int run (int cmd, char *arg)
 	}
 }
 
-int play (char *arg)
+static int
+play(char *arg)
 {
 	struct ioc_toc_header h;
 	unsigned int n;
@@ -748,7 +753,8 @@ Clean_up:
 	return (0);
 }
 
-int next_prev (char *arg, int cmd)
+static int
+next_prev(char *arg, int cmd)
 {
 	struct ioc_toc_header h;
 	int dir, junk, n, off, rc, trk;
@@ -778,7 +784,8 @@ int next_prev (char *arg, int cmd)
 	return (play_track (trk, 1, n, 1));
 }
 
-const char *strstatus (int sts)
+static const char *
+strstatus(int sts)
 {
 	switch (sts) {
 	case ASTS_INVALID:	return ("invalid");
@@ -791,7 +798,8 @@ const char *strstatus (int sts)
 	}
 }
 
-int pstatus (char *arg)
+static int
+pstatus(char *arg)
 {
 	struct ioc_vol v;
 	struct ioc_read_subchannel ss;
@@ -933,7 +941,8 @@ dbprog_discid(void)
 	return((n % 0xff) << 24 | t << 8 | ntr);
 }
 
-int cdid (void)
+static int
+cdid(void)
 {
 	u_int	id;
 
@@ -947,7 +956,8 @@ int cdid (void)
 	return id ? 0 : 1;
 }
 
-int info (char *arg __unused)
+static int
+info(char *arg __unused)
 {
 	struct ioc_toc_header h;
 	int rc, i, n;
@@ -984,7 +994,8 @@ int info (char *arg __unused)
 	return (0);
 }
 
-void lba2msf (unsigned long lba, u_char *m, u_char *s, u_char *f)
+static void
+lba2msf(unsigned long lba, u_char *m, u_char *s, u_char *f)
 {
 	lba += 150;			/* block start offset */
 	lba &= 0xffffff;		/* negative lbas use only 24 bits */
@@ -994,12 +1005,14 @@ void lba2msf (unsigned long lba, u_char *m, u_char *s, u_char *f)
 	*f = lba % 75;
 }
 
-unsigned int msf2lba (u_char m, u_char s, u_char f)
+static unsigned int
+msf2lba(u_char m, u_char s, u_char f)
 {
 	return (((m * 60) + s) * 75 + f) - 150;
 }
 
-void prtrack (struct cd_toc_entry *e, int lastflag)
+static void
+prtrack(struct cd_toc_entry *e, int lastflag)
 {
 	int block, next, len;
 	u_char m, s, f;
@@ -1037,7 +1050,8 @@ void prtrack (struct cd_toc_entry *e, int lastflag)
 		(e->control & 4) ? "data" : "audio");
 }
 
-int play_track (int tstart, int istart, int tend, int iend)
+static int
+play_track(int tstart, int istart, int tend, int iend)
 {
 	struct ioc_play_track t;
 
@@ -1049,7 +1063,8 @@ int play_track (int tstart, int istart, int tend, int iend)
 	return ioctl (fd, CDIOCPLAYTRACKS, &t);
 }
 
-int play_blocks (int blk, int len)
+static int
+play_blocks(int blk, int len)
 {
 	struct ioc_play_blocks  t;
 
@@ -1059,7 +1074,8 @@ int play_blocks (int blk, int len)
 	return ioctl (fd, CDIOCPLAYBLOCKS, &t);
 }
 
-int setvol (int left, int right)
+static int
+setvol(int left, int right)
 {
 	struct ioc_vol  v;
 
@@ -1074,7 +1090,8 @@ int setvol (int left, int right)
 	return ioctl (fd, CDIOCSETVOL, &v);
 }
 
-int read_toc_entrys (int len)
+static int
+read_toc_entrys(int len)
 {
 	struct ioc_read_toc_entry t;
 
@@ -1086,7 +1103,8 @@ int read_toc_entrys (int len)
 	return (ioctl (fd, CDIOREADTOCENTRYS, (char *) &t));
 }
 
-int play_msf (int start_m, int start_s, int start_f,
+static int
+play_msf(int start_m, int start_s, int start_f,
 	int end_m, int end_s, int end_f)
 {
 	struct ioc_play_msf a;
@@ -1101,7 +1119,8 @@ int play_msf (int start_m, int start_s, int start_f,
 	return ioctl (fd, CDIOCPLAYMSF, (char *) &a);
 }
 
-int status (int *trk, int *min, int *sec, int *frame)
+static int
+status(int *trk, int *min, int *sec, int *frame)
 {
 	struct ioc_read_subchannel s;
 	struct cd_sub_channel_info data;
@@ -1132,14 +1151,14 @@ int status (int *trk, int *min, int *sec, int *frame)
 	return s.data->header.audio_status;
 }
 
-const char *
+static const char *
 cdcontrol_prompt(void)
 {
 	return ("cdcontrol> ");
 }
 
-char *
-input (int *cmd)
+static char *
+input(int *cmd)
 {
 #define MAXLINE 80
 	static EditLine *el = NULL;
@@ -1188,7 +1207,8 @@ input (int *cmd)
 	return (p);
 }
 
-char *parse (char *buf, int *cmd)
+static char *
+parse(char *buf, int *cmd)
 {
 	struct cmdtab *c;
 	char *p;
@@ -1252,7 +1272,8 @@ char *parse (char *buf, int *cmd)
 	return p;
 }
 
-int open_cd (void)
+static int
+open_cd(void)
 {
 	char devbuf[MAXPATHLEN];
 	const char *dev;
diff --git a/usr.sbin/chown/chown.c b/usr.sbin/chown/chown.c
index e13235c..9780f02 100644
--- a/usr.sbin/chown/chown.c
+++ b/usr.sbin/chown/chown.c
@@ -57,16 +57,16 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-void	a_gid(const char *);
-void	a_uid(const char *);
-void	chownerr(const char *);
-uid_t	id(const char *, const char *);
-void	usage(void);
+static void	a_gid(const char *);
+static void	a_uid(const char *);
+static void	chownerr(const char *);
+static uid_t	id(const char *, const char *);
+static void	usage(void);
 
-uid_t uid;
-gid_t gid;
-int ischown;
-const char *gname;
+static uid_t uid;
+static gid_t gid;
+static int ischown;
+static const char *gname;
 
 int
 main(int argc, char **argv)
@@ -229,7 +229,7 @@ main(int argc, char **argv)
 	exit(rval);
 }
 
-void
+static void
 a_gid(const char *s)
 {
 	struct group *gr;
@@ -240,7 +240,7 @@ a_gid(const char *s)
 	gid = ((gr = getgrnam(s)) != NULL) ? gr->gr_gid : id(s, "group");
 }
 
-void
+static void
 a_uid(const char *s)
 {
 	struct passwd *pw;
@@ -250,7 +250,7 @@ a_uid(const char *s)
 	uid = ((pw = getpwnam(s)) != NULL) ? pw->pw_uid : id(s, "user");
 }
 
-uid_t
+static uid_t
 id(const char *name, const char *type)
 {
 	uid_t val;
@@ -267,7 +267,7 @@ id(const char *name, const char *type)
 	return (val);
 }
 
-void
+static void
 chownerr(const char *file)
 {
 	static uid_t euid = -1;
@@ -299,7 +299,7 @@ chownerr(const char *file)
 	warn("%s", file);
 }
 
-void
+static void
 usage(void)
 {
 
diff --git a/usr.sbin/chroot/chroot.c b/usr.sbin/chroot/chroot.c
index a0122f7..9db0192 100644
--- a/usr.sbin/chroot/chroot.c
+++ b/usr.sbin/chroot/chroot.c
@@ -56,16 +56,12 @@ __FBSDID("$FreeBSD$");
 
 static void usage(void);
 
-char	*user;		/* user to switch to before running program */
-char	*group;		/* group to switch to ... */
-char	*grouplist;	/* group list to switch to ... */
-
 int
 main(int argc, char *argv[])
 {
 	struct group	*gp;
 	struct passwd	*pw;
-	char		*endp, *p;
+	char		*endp, *p, *user, *group, *grouplist;
 	const char	*shell;
 	gid_t		gid, *gidlist;
 	uid_t		uid;
@@ -74,6 +70,7 @@ main(int argc, char *argv[])
 
 	gid = 0;
 	uid = 0;
+	user = group = grouplist = NULL;
 	while ((ch = getopt(argc, argv, "G:g:u:")) != -1) {
 		switch(ch) {
 		case 'u':
diff --git a/usr.sbin/config/mkmakefile.c b/usr.sbin/config/mkmakefile.c
index 2372839..28edb1c 100644
--- a/usr.sbin/config/mkmakefile.c
+++ b/usr.sbin/config/mkmakefile.c
@@ -762,16 +762,21 @@ do_rules(FILE *f)
 				break;
 			}
 			snprintf(cmd, sizeof(cmd),
-			    "${%s_%c%s}\n\t@${NORMAL_CTFCONVERT}", ftype,
+			    "${%s_%c%s}\n", ftype,
 			    toupper(och),
 			    ftp->f_flags & NOWERROR ? "_NOWERROR" : "");
 			compilewith = cmd;
 		}
 		*cp = och;
 		if (strlen(ftp->f_objprefix))
-			fprintf(f, "\t%s $S/%s\n\n", compilewith, np);
+			fprintf(f, "\t%s $S/%s\n", compilewith, np);
 		else
-			fprintf(f, "\t%s\n\n", compilewith);
+			fprintf(f, "\t%s\n", compilewith);
+
+		if (!(ftp->f_flags & NO_OBJ))
+			fprintf(f, "\t@${NORMAL_CTFCONVERT}\n\n");
+		else
+			fprintf(f, "\n");
 	}
 }
 
diff --git a/usr.sbin/devinfo/devinfo.c b/usr.sbin/devinfo/devinfo.c
index 73dcfd5..32d2932 100644
--- a/usr.sbin/devinfo/devinfo.c
+++ b/usr.sbin/devinfo/devinfo.c
@@ -39,8 +39,8 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include "devinfo.h"
 
-int	rflag;
-int	vflag;
+static int	rflag;
+static int	vflag;
 
 static void	print_resource(struct devinfo_res *);
 static int	print_device_matching_resource(struct devinfo_res *, void *);
diff --git a/usr.sbin/fdread/fdread.c b/usr.sbin/fdread/fdread.c
index ba9c758..770f92d 100644
--- a/usr.sbin/fdread/fdread.c
+++ b/usr.sbin/fdread/fdread.c
@@ -44,14 +44,14 @@
 
 #include "fdutil.h"
 
-int	quiet, recover;
-unsigned char fillbyte = 0xf0;	/* "foo" */
+static int	quiet, recover;
+static unsigned char fillbyte = 0xf0;	/* "foo" */
 
-int	doread(int fd, FILE *of, const char *_devname);
-int	doreadid(int fd, unsigned int numids, unsigned int trackno);
-void	usage(void);
+static int	doread(int fd, FILE *of, const char *_devname);
+static int	doreadid(int fd, unsigned int numids, unsigned int trackno);
+static void	usage(void);
 
-void
+static void
 usage(void)
 {
 
@@ -155,7 +155,7 @@ main(int argc, char **argv)
 	return (numids? doreadid(fd, numids, trackno): doread(fd, of, _devname));
 }
 
-int
+static int
 doread(int fd, FILE *of, const char *_devname)
 {
 	char *trackbuf;
@@ -294,7 +294,7 @@ doread(int fd, FILE *of, const char *_devname)
 	return (nerrs? EX_IOERR: EX_OK);
 }
 
-int
+static int
 doreadid(int fd, unsigned int numids, unsigned int trackno)
 {
 	int rv = 0;
diff --git a/usr.sbin/memcontrol/memcontrol.c b/usr.sbin/memcontrol/memcontrol.c
index b6cb25f..542bc3a 100644
--- a/usr.sbin/memcontrol/memcontrol.c
+++ b/usr.sbin/memcontrol/memcontrol.c
@@ -39,8 +39,7 @@
 #include <string.h>
 #include <unistd.h>
 
-struct
-{
+static struct {
     const char	*name;
     int		val;
     int		kind;
@@ -67,8 +66,7 @@ static void	clearfunc(int memfd, int argc, char *argv[]);
 static void	helpfunc(int memfd, int argc, char *argv[]);
 static void	help(const char *what);
 
-struct 
-{
+static struct {
     const char	*cmd;
     const char	*desc;
     void	(*func)(int memfd, int argc, char *argv[]);
diff --git a/usr.sbin/mergemaster/mergemaster.8 b/usr.sbin/mergemaster/mergemaster.8
index 4dcad39..36eba1b 100644
--- a/usr.sbin/mergemaster/mergemaster.8
+++ b/usr.sbin/mergemaster/mergemaster.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 9, 2011
+.Dd November 1, 2011
 .Dt MERGEMASTER 8
 .Os
 .Sh NAME
@@ -34,6 +34,7 @@
 .Nm
 .Op Fl scrvhpCP
 .Op Fl a|iFU
+.Op Fl -run-updates=[always|never]
 .Op Fl m Ar /path/to/sources
 .Op Fl t Ar /path/to/temp/root
 .Op Fl d
@@ -246,6 +247,11 @@ rc file.
 Attempt to auto upgrade files that have not been user modified.
 This option can be dangerous when there are critical changes
 in the new versions that affect your running system.
+.It Fl -run-updates=[always|never]
+Specify always or never to run newaliases, pwd_mkdb, etc.
+at the end of the comparison run.
+If this option is omitted the default is to prompt the user
+for each update as necessary.
 .It Fl m Ar /path/to/sources
 Specify the path to the directory where you want to do the
 .Xr make 1 .
@@ -365,6 +371,9 @@ with all values commented out:
 # ***DANGEROUS***
 #AUTO_UPGRADE=
 #
+# Either always or never run newaliases, pwd_mkdb at the end (--run-updates)
+#RUN_UPDATES=
+#
 # Compare /etc/rc.conf[.local] to /etc/defaults/rc.conf (-C)
 #COMP_CONFS=
 #
diff --git a/usr.sbin/mergemaster/mergemaster.sh b/usr.sbin/mergemaster/mergemaster.sh
index 78d5c08..6b5003e 100755
--- a/usr.sbin/mergemaster/mergemaster.sh
+++ b/usr.sbin/mergemaster/mergemaster.sh
@@ -15,7 +15,7 @@ PATH=/bin:/usr/bin:/usr/sbin
 display_usage () {
   VERSION_NUMBER=`grep "[$]FreeBSD:" $0 | cut -d ' ' -f 4`
   echo "mergemaster version ${VERSION_NUMBER}"
-  echo 'Usage: mergemaster [-scrvhpCP] [-a|[-iFU]]'
+  echo 'Usage: mergemaster [-scrvhpCP] [-a|[-iFU]] [--run-updates=always|never]'
   echo '    [-m /path] [-t /path] [-d] [-u N] [-w N] [-A arch] [-D /path]'
   echo "Options:"
   echo "  -s  Strict comparison (diff every pair of files)"
@@ -31,6 +31,7 @@ display_usage () {
   echo '  -P  Preserve files that are overwritten'
   echo "  -U  Attempt to auto upgrade files that have not been user modified"
   echo '      ***DANGEROUS***'
+  echo '  --run-updates=  Specify always or never to run newalises, pwd_mkdb, etc.'
   echo ''
   echo "  -m /path/directory  Specify location of source to do the make in"
   echo "  -t /path/directory  Specify temp root directory"
@@ -262,6 +263,20 @@ if [ -r "$HOME/.mergemasterrc" ]; then
   . "$HOME/.mergemasterrc"
 fi
 
+for var in "$@" ; do
+  case "$var" in
+  --run-updates*)
+    RUN_UPDATES=`echo ${var#--run-updates=} | tr [:upper:] [:lower:]`
+    ;;
+  *)
+    newopts="$newopts $var"
+    ;;
+  esac
+done
+
+set -- $newopts
+unset var newopts
+
 # Check the command line options
 #
 while getopts ":ascrvhipCPm:t:du:w:D:A:FU" COMMAND_LINE_ARGUMENT ; do
@@ -1224,34 +1239,43 @@ case "${AUTO_UPGRADED_FILES}" in
 esac
 
 run_it_now () {
-  case "${AUTO_RUN}" in
-  '')
-    unset YES_OR_NO
-    echo ''
-    echo -n '    Would you like to run it now? y or n [n] '
-    read YES_OR_NO
+  [ -n "$AUTO_RUN" ] && return
+
+  local answer
+
+  echo ''
+  while : ; do
+    if [ "$RUN_UPDATES" = always ]; then
+      answer=y
+    elif [ "$RUN_UPDATES" = never ]; then
+      answer=n
+    else
+      echo -n '    Would you like to run it now? y or n [n] '
+      read answer
+    fi
 
-    case "${YES_OR_NO}" in
+    case "$answer" in
     y)
       echo "    Running ${1}"
       echo ''
       eval "${1}"
+      return
       ;;
     ''|n)
-      echo ''
-      echo "       *** Cancelled"
-      echo ''
+      if [ ! "$RUN_UPDATES" = never ]; then
+        echo ''
+        echo "       *** Cancelled"
+        echo ''
+      fi
       echo "    Make sure to run ${1} yourself"
+      return
       ;;
     *)
       echo ''
-      echo "       *** Sorry, I do not understand your answer (${YES_OR_NO})"
+      echo "       *** Sorry, I do not understand your answer (${answer})"
       echo ''
-      echo "    Make sure to run ${1} yourself"
     esac
-    ;;
-  *) ;;
-  esac
+  done
 }
 
 case "${NEED_NEWALIASES}" in
@@ -1310,6 +1334,19 @@ case "${NEED_PWD_MKDB}" in
   ;;
 esac
 
+if [ -e "${DESTDIR}/etc/localtime" ]; then	# Ignore if TZ == UTC
+  echo ''
+  if [ -f "${DESTDIR}/var/db/zoneinfo" ]; then
+    echo "*** Reinstalling `cat ${DESTDIR}/var/db/zoneinfo` as ${DESTDIR}/etc/localtime"
+    [ -n "${DESTDIR}" ] && tzs_args="-C ${DESTDIR}"
+    tzsetup $tzs_args -r
+  else
+    echo "*** There is no ${DESTDIR}/var/db/zoneinfo file to update ${DESTDIR}/etc/localtime."
+    echo '    You should run tzsetup'
+    run_it_now tzsetup
+  fi
+fi
+
 echo ''
 
 if [ -r "${MM_EXIT_SCRIPT}" ]; then
diff --git a/usr.sbin/mixer/mixer.c b/usr.sbin/mixer/mixer.c
index 52997bc..245fb06 100644
--- a/usr.sbin/mixer/mixer.c
+++ b/usr.sbin/mixer/mixer.c
@@ -24,7 +24,7 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include <sys/soundcard.h>
 
-const char	*names[SOUND_MIXER_NRDEVICES] = SOUND_DEVICE_NAMES;
+static const char *names[SOUND_MIXER_NRDEVICES] = SOUND_DEVICE_NAMES;
 
 static void	usage(int devmask, int recmask);
 static int	res_name(const char *name, int mask);
diff --git a/usr.sbin/moused/moused.c b/usr.sbin/moused/moused.c
index f598910..84e5c0b 100644
--- a/usr.sbin/moused/moused.c
+++ b/usr.sbin/moused/moused.c
@@ -174,14 +174,14 @@ typedef struct {
 
 /* global variables */
 
-int	debug = 0;
-int	nodaemon = FALSE;
-int	background = FALSE;
-int	paused = FALSE;
-int	identify = ID_NONE;
-int	extioctl = FALSE;
-const char *pidfile = "/var/run/moused.pid";
-struct pidfh *pfh;
+static int	debug = 0;
+static int	nodaemon = FALSE;
+static int	background = FALSE;
+static int	paused = FALSE;
+static int	identify = ID_NONE;
+static int	extioctl = FALSE;
+static const char *pidfile = "/var/run/moused.pid";
+static struct pidfh *pfh;
 
 #define SCROLL_NOTSCROLLING	0
 #define SCROLL_PREPARE		1
diff --git a/usr.sbin/mptable/mptable.c b/usr.sbin/mptable/mptable.c
index a16a1b9..92279bc 100644
--- a/usr.sbin/mptable/mptable.c
+++ b/usr.sbin/mptable/mptable.c
@@ -96,7 +96,7 @@ typedef struct BUSTYPENAME {
     char	name[ 7 ];
 } busTypeName;
 
-static busTypeName busTypeTable[] =
+static const busTypeName busTypeTable[] =
 {
     { CBUS,		"CBUS"   },
     { CBUSII,		"CBUSII" },
@@ -119,7 +119,7 @@ static busTypeName busTypeTable[] =
     { UNKNOWN_BUSTYPE,	"---"    }
 };
 
-const char* whereStrings[] = {
+static const char *whereStrings[] = {
     "Extended BIOS Data Area",
     "BIOS top of memory",
     "Default top of memory",
@@ -135,16 +135,7 @@ typedef struct TABLE_ENTRY {
     char	name[ 32 ];
 } tableEntry;
 
-tableEntry basetableEntryTypes[] =
-{
-    { 0, 20, "Processor" },
-    { 1,  8, "Bus" },
-    { 2,  8, "I/O APIC" },
-    { 3,  8, "I/O INT" },
-    { 4,  8, "Local INT" }
-};
-
-tableEntry extendedtableEntryTypes[] =
+static const tableEntry extendedtableEntryTypes[] =
 {
     { 128, 20, "System Address Space" },
     { 129,  8, "Bus Hierarchy" },
@@ -277,19 +268,19 @@ static void doDmesg( void );
 static void pnstr( char* s, int c );
 
 /* global data */
-int	pfd;		/* physical /dev/mem fd */
+static int	pfd;		/* physical /dev/mem fd */
 
-int	busses[ 16 ];
-int	apics[ 16 ];
+static int	busses[16];
+static int	apics[16];
 
-int	ncpu;
-int	nbus;
-int	napic;
-int	nintr;
+static int	ncpu;
+static int	nbus;
+static int	napic;
+static int	nintr;
 
-int	dmesg;
-int	grope;
-int	verbose;
+static int	dmesg;
+static int	grope;
+static int	verbose;
 
 static void
 usage( void )
@@ -978,14 +969,14 @@ ioApicEntry( void )
 }
 
 
-const char* intTypes[] = {
+static const char *intTypes[] = {
     "INT", "NMI", "SMI", "ExtINT"
 };
 
-const char* polarityMode[] = {
+static const char *polarityMode[] = {
     "conforms", "active-hi", "reserved", "active-lo"
 };
-const char* triggerMode[] = {
+static const char *triggerMode[] = {
     "conforms", "edge", "reserved", "level"
 };
 
diff --git a/usr.sbin/mtree/mtree.5 b/usr.sbin/mtree/mtree.5
index 8962e5f..9d3405e 100644
--- a/usr.sbin/mtree/mtree.5
+++ b/usr.sbin/mtree/mtree.5
@@ -48,7 +48,7 @@ Leading whitespace is always ignored.
 .Pp
 When encoding file or pathnames, any backslash character or
 character outside of the 95 printable ASCII characters must be
-encoded as a a backslash followed by three
+encoded as a backslash followed by three
 octal digits.
 When reading mtree files, any appearance of a backslash
 followed by three octal digits should be converted into the
diff --git a/usr.sbin/pc-sysinstall/backend-query/sys-mem.sh b/usr.sbin/pc-sysinstall/backend-query/sys-mem.sh
index bc0e3a2..3109aa1 100755
--- a/usr.sbin/pc-sysinstall/backend-query/sys-mem.sh
+++ b/usr.sbin/pc-sysinstall/backend-query/sys-mem.sh
@@ -25,4 +25,8 @@
 #
 # $FreeBSD$
 
-expr $(sysctl -n hw.realmem) / 1048576
+if smbios_mem=$(kenv -q smbios.memory.enabled); then
+	expr $smbios_mem / 1024
+else
+	expr $(sysctl -n hw.realmem) / 1048576
+fi
diff --git a/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh b/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh
index 5b30695..904d3ee 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-bsdlabel.sh
@@ -179,7 +179,9 @@ setup_gpart_partitions()
   else
     PARTLETTER="a"
     CURPART="1"
-    rc_halt "gpart create -s BSD ${_wSlice}"
+    if [ "${_pType}" = "mbr" ] ; then
+      rc_halt "gpart create -s BSD ${_wSlice}"
+    fi
   fi
 
   while read line
@@ -255,6 +257,9 @@ setup_gpart_partitions()
         if [ "${CURPART}" = "1" -a "$_pType" = "mbr" ] ; then
           export FOUNDROOT="0"
         fi
+        if [ "${CURPART}" = "1" -a "$_pType" = "gptslice" ] ; then
+          export FOUNDROOT="0"
+        fi
       fi
 
       check_for_mount "${MNT}" "/boot"
@@ -266,6 +271,9 @@ setup_gpart_partitions()
         if [ "${CURPART}" != "1" -a "${_pType}" = "mbr" ] ; then
             exit_err "/boot partition must be first partition"
         fi
+        if [ "${CURPART}" != "1" -a "${_pType}" = "gptslice" ] ; then
+            exit_err "/boot partition must be first partition"
+        fi
 
         if [ "${FS}" != "UFS" -a "${FS}" != "UFS+S" -a "${FS}" != "UFS+J" -a "${FS}" != "UFS+SUJ" ] ; then
           exit_err "/boot partition must be formatted with UFS"
@@ -280,14 +288,14 @@ setup_gpart_partitions()
       if [ "${_pType}" = "gpt" ] ; then
         get_fs_line_xvars "${_pDisk}p${CURPART}" "${STRING}"
       else
-        get_fs_line_xvars "${_wSlice}" "${STRING}"
+        get_fs_line_xvars "${_wSlice}${PARTLETTER}" "${STRING}"
       fi
       XTRAOPTS="${VAR}"
 
       # Check if using zfs mirror
       echo ${XTRAOPTS} | grep -q "mirror" 2>/dev/null
       if [ $? -eq 0 -a "$FS" = "ZFS" ] ; then
-        if [ "${_pType}" = "gpt" ] ; then
+        if [ "${_pType}" = "gpt" -o "${_pType}" = "gptslice" ] ; then
        	  XTRAOPTS=$(setup_zfs_mirror_parts "$XTRAOPTS" "${_pDisk}p${CURPART}")
         else
        	  XTRAOPTS=$(setup_zfs_mirror_parts "$XTRAOPTS" "${_wSlice}")
@@ -305,11 +313,17 @@ setup_gpart_partitions()
       if [ "${_pType}" = "gpt" ] ; then
 	if [ "$CURPART" = "2" ] ; then
 	  # If this is GPT, make sure first partition is aligned to 4k
+          sleep 2
           rc_halt "gpart add -b 2016 ${SOUT} -t ${PARTYPE} ${_pDisk}"
 	else
+          sleep 2
           rc_halt "gpart add ${SOUT} -t ${PARTYPE} ${_pDisk}"
 	fi
+      elif [ "${_pType}" = "gptslice" ]; then
+        sleep 2
+        rc_halt "gpart add ${SOUT} -t ${PARTYPE} ${_wSlice}"
       else
+        sleep 2
         rc_halt "gpart add ${SOUT} -t ${PARTYPE} -i ${CURPART} ${_wSlice}"
       fi
 
@@ -338,7 +352,7 @@ setup_gpart_partitions()
           echo "${ENCPASS}" >${PARTDIR}-enc/${_dFile}p${CURPART}-encpass
         fi
       else
-	# MBR Partition
+	# MBR Partition or GPT slice
 	_dFile="`echo $_wSlice | sed 's|/|-|g'`"
         echo "${FS}:${MNT}:${ENC}:${PLABEL}:MBR:${XTRAOPTS}:${IMAGE}" >${PARTDIR}/${_dFile}${PARTLETTER}
         # Clear out any headers
@@ -422,7 +436,7 @@ populate_disk_label()
   if [ "$type" = "mbr" ] ; then
     wrkslice="${diskid}s${slicenum}"
   fi
-  if [ "$type" = "gpt" ] ; then
+  if [ "$type" = "gpt" -o "$type" = "gptslice" ] ; then
     wrkslice="${diskid}p${slicenum}"
   fi
 
@@ -459,6 +473,9 @@ setup_disk_label()
     if [ "$type" = "gpt" -a ! -e "${disk}p${pnum}" ] ; then
       exit_err "ERROR: The partition ${i} doesn't exist! gpart failure!"
     fi
+    if [ "$type" = "gptslice" -a ! -e "${disk}p${pnum}" ] ; then
+      exit_err "ERROR: The partition ${i} doesn't exist! gpart failure!"
+    fi
   done
 
   # Setup some files which we'll be referring to
diff --git a/usr.sbin/pc-sysinstall/backend/functions-disk.sh b/usr.sbin/pc-sysinstall/backend/functions-disk.sh
index 73c3eb4..51a659e 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-disk.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-disk.sh
@@ -470,7 +470,8 @@ setup_disk_slice()
               # Default to round-robin if the user didn't specify
               if [ -z "$MIRRORBAL" ]; then MIRRORBAL="round-robin" ; fi
 
-              echo "$MIRRORDISK:$MIRRORBAL:gm${gmnum}" >${MIRRORCFGDIR}/$DISK
+	      _mFile=`echo $DISK | sed 's|/|%|g'`
+              echo "$MIRRORDISK:$MIRRORBAL:gm${gmnum}" >${MIRRORCFGDIR}/$_mFile
 	      init_gmirror "$gmnum" "$MIRRORBAL" "$DISK" "$MIRRORDISK"
 
 	      # Reset DISK to the gmirror device
@@ -495,6 +496,13 @@ setup_disk_slice()
             run_gpart_slice "${DISK}" "${BMANAGER}" "${s}"
             ;;
 
+          p1|p2|p3|p4|p5|p6|p7|p8|p9|p10|p11|p12|p13|p14|p15|p16|p17|p18|p19|p20)
+            tmpSLICE="${DISK}${PTYPE}" 
+            # Get the number of the gpt partition we are working on
+            s="`echo ${PTYPE} | awk '{print substr($0,length,1)}'`" 
+            run_gpart_gpt_part "${DISK}" "${BMANAGER}" "${s}"
+            ;;
+
           free)
             tmpSLICE="${DISK}s${LASTSLICE}"
             run_gpart_free "${DISK}" "${LASTSLICE}" "${BMANAGER}"
@@ -703,6 +711,58 @@ run_gpart_full()
   fi
 };
 
+# Function which runs gpart on a specified gpt partition
+run_gpart_gpt_part()
+{
+  DISK=$1
+
+  # Set the slice we will use later
+  slice="${1}p${3}"
+ 
+  # Set our sysctl so we can overwrite any geom using drives
+  sysctl kern.geom.debugflags=16 >>${LOGOUT} 2>>${LOGOUT}
+
+  # Get the number of the slice we are working on
+  slicenum="$3"
+
+  # Stop any journaling
+  stop_gjournal "${slice}"
+
+  # Make sure we have disabled swap on this drive
+  if [ -e "${slice}b" ]
+  then
+   swapoff ${slice}b >/dev/null 2>/dev/null
+   swapoff ${slice}b.eli >/dev/null 2>/dev/null
+  fi
+
+  # Modify partition type
+  echo_log "Running gpart modify on ${DISK}"
+  rc_halt "gpart modify -t freebsd -i ${slicenum} ${DISK}"
+  sleep 2
+
+  # Clean up old partition
+  echo_log "Cleaning up $slice"
+  rc_halt "dd if=/dev/zero of=${DISK}p${slicenum} count=1024"
+
+  sleep 4
+
+  # Init the MBR partition
+  rc_halt "gpart create -s BSD ${DISK}p${slicenum}"
+
+  # Set the slice to the format we'll be using for gpart later
+  slice=`echo "${1}:${3}:gptslice" | sed 's|/|-|g'`
+
+  # Lets save our slice, so we know what to look for in the config file later on
+  if [ -z "$WORKINGSLICES" ]
+  then
+    WORKINGSLICES="${slice}"
+    export WORKINGSLICES
+  else
+    WORKINGSLICES="${WORKINGSLICES} ${slice}"
+    export WORKINGSLICES
+  fi
+};
+
 # Function which runs gpart on a specified s1-4 slice
 run_gpart_slice()
 {
diff --git a/usr.sbin/pc-sysinstall/backend/functions-networking.sh b/usr.sbin/pc-sysinstall/backend/functions-networking.sh
index bdd5a1a..6b8cb1e 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-networking.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-networking.sh
@@ -431,6 +431,10 @@ start_networking()
   elif [ "$NETDEV" = "IPv6-SLAAC" ]
   then
     enable_auto_slaac
+  elif [ "$NETDEV" = "AUTO-DHCP-SLAAC" ]
+  then
+    enable_auto_dhcp
+    enable_auto_slaac
   else
     enable_manual_nic ${NETDEV}
   fi
@@ -457,6 +461,10 @@ save_networking_install()
   elif [ "$NETDEV" = "IPv6-SLAAC" ]
   then
     save_auto_slaac
+  elif [ "$NETDEV" = "AUTO-DHCP-SLAAC" ]
+  then
+    save_auto_dhcp
+    save_auto_slaac
   else
     save_manual_nic ${NETDEV}
   fi
diff --git a/usr.sbin/pc-sysinstall/backend/functions-unmount.sh b/usr.sbin/pc-sysinstall/backend/functions-unmount.sh
index 23630f7..57a417e 100755
--- a/usr.sbin/pc-sysinstall/backend/functions-unmount.sh
+++ b/usr.sbin/pc-sysinstall/backend/functions-unmount.sh
@@ -43,7 +43,7 @@ start_gmirror_sync()
 {
 
   cd ${MIRRORCFGDIR}
-  for DISK in `ls *`
+  for DISK in `ls ${MIRRORCFGDIR}`
   do
     MIRRORDISK="`cat ${DISK} | cut -d ':' -f 1`"
     MIRRORBAL="`cat ${DISK} | cut -d ':' -f 2`"
@@ -51,7 +51,7 @@ start_gmirror_sync()
    
     # Start the mirroring service
     rc_nohalt "gmirror forget ${MIRRORNAME}"
-    rc_halt "gmirror insert ${MIRRORNAME} /dev/${MIRRORDISK}"
+    rc_halt "gmirror insert ${MIRRORNAME} ${MIRRORDISK}"
 
   done
 
diff --git a/usr.sbin/pc-sysinstall/backend/parseconfig.sh b/usr.sbin/pc-sysinstall/backend/parseconfig.sh
index 2e16751..c852cea 100755
--- a/usr.sbin/pc-sysinstall/backend/parseconfig.sh
+++ b/usr.sbin/pc-sysinstall/backend/parseconfig.sh
@@ -70,7 +70,6 @@ check_value installMode "fresh upgrade extract"
 check_value installType "PCBSD FreeBSD"
 check_value installMedium "dvd usb ftp rsync image local"
 check_value packageType "uzip tar rsync split"
-if_check_value_exists partition "all s1 s2 s3 s4 free image"
 if_check_value_exists mirrorbal "load prefer round-robin split"
 
 # We passed all sanity checks! Yay, lets start the install
diff --git a/usr.sbin/pkg_install/lib/version.c b/usr.sbin/pkg_install/lib/version.c
index 0852ba1..c25d272 100644
--- a/usr.sbin/pkg_install/lib/version.c
+++ b/usr.sbin/pkg_install/lib/version.c
@@ -66,7 +66,7 @@ split_version(const char *pkgname, const char **endname, unsigned long *epoch, u
     if (pkgname == NULL)
 	errx(2, "%s: Passed NULL pkgname.", __func__);
 
-    /* Look for the last '-' the pkgname */
+    /* Look for the last '-' in the pkgname */
     ch = strrchr(pkgname, '-');
     /* Cheat if we are just passed a version, not a valid package name */
     versionstr = ch ? ch + 1 : pkgname;
diff --git a/usr.sbin/pmccontrol/pmccontrol.c b/usr.sbin/pmccontrol/pmccontrol.c
index 2a680ae..f0bd31f 100644
--- a/usr.sbin/pmccontrol/pmccontrol.c
+++ b/usr.sbin/pmccontrol/pmccontrol.c
@@ -147,8 +147,9 @@ pmcc_do_enable_disable(struct pmcc_op_list *op_list)
 	npmc = 0;
 	for (c = 0; c < ncpu; c++) {
 		if ((t = pmc_npmc(c)) < 0)
-			err(EX_OSERR, "Unable to determine the number of "
-			    "PMCs in CPU %d", c);
+			err(EX_OSERR,
+			    "Unable to determine the number of PMCs in CPU %d",
+			    c);
 		npmc = t > npmc ? t : npmc;
 	}
 
@@ -211,8 +212,8 @@ pmcc_do_enable_disable(struct pmcc_op_list *op_list)
 
 			if (error < 0)
 				err(EX_OSERR, "%s of PMC %d on CPU %d failed",
-				    b == PMCC_OP_ENABLE ? "Enable" :
-				    "Disable", j, i);
+				    b == PMCC_OP_ENABLE ? "Enable" : "Disable",
+				    j, i);
 		}
 
 	return error;
@@ -308,8 +309,9 @@ pmcc_do_list_events(void)
 
 		printf("%s\n", pmc_name_of_class(c));
 		if (pmc_event_names_of_class(c, &eventnamelist, &nevents) < 0)
-			err(EX_OSERR, "ERROR: Cannot find information for "
-			    "event class \"%s\"", pmc_name_of_class(c));
+			err(EX_OSERR,
+"ERROR: Cannot find information for event class \"%s\"",
+			    pmc_name_of_class(c));
 
 		for (j = 0; j < nevents; j++)
 			printf("\t%s\n", eventnamelist[j]);
@@ -481,7 +483,8 @@ main(int argc, char **argv)
 		break;
 	case PMCC_ENABLE_DISABLE:
 		if (STAILQ_EMPTY(&head))
-			errx(EX_USAGE, "No PMCs specified to enable or disable");
+			errx(EX_USAGE,
+			    "No PMCs specified to enable or disable");
 		error = pmcc_do_enable_disable(&head);
 		break;
 	default:
diff --git a/usr.sbin/pmcstat/pmcpl_calltree.c b/usr.sbin/pmcstat/pmcpl_calltree.c
index af3317a..3f3f717 100644
--- a/usr.sbin/pmcstat/pmcpl_calltree.c
+++ b/usr.sbin/pmcstat/pmcpl_calltree.c
@@ -575,7 +575,8 @@ pmcpl_ct_topkeypress(int c, WINDOW *w)
 	switch (c) {
 	case 'f':
 		pmcstat_skiplink = !pmcstat_skiplink;
-		wprintw(w, "skip empty link %s", pmcstat_skiplink ? "on" : "off");
+		wprintw(w, "skip empty link %s",
+		    pmcstat_skiplink ? "on" : "off");
 		break;
 	}
 
diff --git a/usr.sbin/pmcstat/pmcpl_gprof.c b/usr.sbin/pmcstat/pmcpl_gprof.c
index 2027ecf..8e4cc79 100644
--- a/usr.sbin/pmcstat/pmcpl_gprof.c
+++ b/usr.sbin/pmcstat/pmcpl_gprof.c
@@ -333,8 +333,9 @@ pmcpl_gmon_initimage(struct pmcstat_image *pi)
 		count = 0;
 		do {
 			if (++count > 999)
-				errx(EX_CANTCREAT, "ERROR: cannot create a "
-				    "gmon file for \"%s\"", name);
+				errx(EX_CANTCREAT,
+				    "ERROR: cannot create a gmon file for"
+				    " \"%s\"", name);
 			snprintf(name, sizeof(name), "%.*s~%3.3d.gmon",
 			    nlen, sn, count);
 			if (pmcstat_string_lookup(name) == NULL) {
@@ -516,8 +517,8 @@ pmcpl_gmon_shutdown(FILE *mf)
 					    pgf->pgf_pmcid),
 					    pgf->pgf_nsamples);
 				if (pgf->pgf_overflow && args.pa_verbosity >= 1)
-					warnx("WARNING: profile \"%s\" "
-					    "overflowed.",
+					warnx(
+"WARNING: profile \"%s\" overflowed.",
 					    pmcstat_string_unintern(
 					        pgf->pgf_name));
 			}
diff --git a/usr.sbin/pmcstat/pmcstat.c b/usr.sbin/pmcstat/pmcstat.c
index 5d9db61..8c54a29 100644
--- a/usr.sbin/pmcstat/pmcstat.c
+++ b/usr.sbin/pmcstat/pmcstat.c
@@ -156,8 +156,9 @@ pmcstat_get_cpumask(const char *cpuspec, cpuset_t *cpumask)
 	do {
 		cpu = strtol(s, &end, 0);
 		if (cpu < 0 || end == s)
-			errx(EX_USAGE, "ERROR: Illegal CPU specification "
-			    "\"%s\".", cpuspec);
+			errx(EX_USAGE,
+			    "ERROR: Illegal CPU specification \"%s\".",
+			    cpuspec);
 		CPU_SET(cpu, cpumask);
 		s = end + strspn(end, ", \t");
 	} while (*s);
@@ -179,9 +180,9 @@ pmcstat_attach_pmcs(void)
 			if (pmc_attach(ev->ev_pmcid, pt->pt_pid) == 0)
 				count++;
 			else if (errno != ESRCH)
-				err(EX_OSERR, "ERROR: cannot attach pmc "
-				    "\"%s\" to process %d", ev->ev_name,
-				    (int) pt->pt_pid);
+				err(EX_OSERR,
+"ERROR: cannot attach pmc \"%s\" to process %d",
+				    ev->ev_name, (int)pt->pt_pid);
 	}
 
 	if (count == 0)
@@ -198,11 +199,11 @@ pmcstat_cleanup(void)
 	STAILQ_FOREACH_SAFE(ev, &args.pa_events, ev_next, tmp)
 	    if (ev->ev_pmcid != PMC_ID_INVALID) {
 		if (pmc_stop(ev->ev_pmcid) < 0)
-			err(EX_OSERR, "ERROR: cannot stop pmc 0x%x "
-			    "\"%s\"", ev->ev_pmcid, ev->ev_name);
+			err(EX_OSERR, "ERROR: cannot stop pmc 0x%x \"%s\"",
+			    ev->ev_pmcid, ev->ev_name);
 		if (pmc_release(ev->ev_pmcid) < 0)
-			err(EX_OSERR, "ERROR: cannot release pmc "
-			    "0x%x \"%s\"", ev->ev_pmcid, ev->ev_name);
+			err(EX_OSERR, "ERROR: cannot release pmc 0x%x \"%s\"",
+			    ev->ev_pmcid, ev->ev_name);
 		free(ev->ev_name);
 		free(ev->ev_spec);
 		STAILQ_REMOVE(&args.pa_events, ev, pmcstat_ev, ev_next);
@@ -424,8 +425,8 @@ pmcstat_print_counters(void)
 			continue;
 
 		if (pmc_read(ev->ev_pmcid, &value) < 0)
-			err(EX_OSERR, "ERROR: Cannot read pmc "
-			    "\"%s\"", ev->ev_name);
+			err(EX_OSERR, "ERROR: Cannot read pmc \"%s\"",
+			    ev->ev_name);
 
 		(void) fprintf(args.pa_printfile, "%*ju ",
 		    ev->ev_fieldwidth + extra_width,
@@ -641,8 +642,9 @@ main(int argc, char **argv)
 				err(EX_OSERR, "ERROR: Cannot stat \"%s\"",
 				    optarg);
 			if (!S_ISDIR(sb.st_mode))
-				errx(EX_USAGE, "ERROR: \"%s\" is not a "
-				    "directory.", optarg);
+				errx(EX_USAGE,
+				    "ERROR: \"%s\" is not a directory.",
+				    optarg);
 			args.pa_samplesdir = optarg;
 			args.pa_flags     |= FLAG_HAS_SAMPLESDIR;
 			args.pa_required  |= FLAG_DO_GPROF;
@@ -796,18 +798,21 @@ main(int argc, char **argv)
 			break;
 
 		case 'o':	/* outputfile */
-			if (args.pa_printfile != NULL)
+			if (args.pa_printfile != NULL &&
+			    args.pa_printfile != stdout &&
+			    args.pa_printfile != stderr)
 				(void) fclose(args.pa_printfile);
 			if ((args.pa_printfile = fopen(optarg, "w")) == NULL)
-				errx(EX_OSERR, "ERROR: cannot open \"%s\" for "
-				    "writing.", optarg);
+				errx(EX_OSERR,
+				    "ERROR: cannot open \"%s\" for writing.",
+				    optarg);
 			args.pa_flags |= FLAG_DO_PRINT;
 			break;
 
 		case 'O':	/* sampling output */
 			if (args.pa_outputpath)
-				errx(EX_USAGE, "ERROR: option -O may only be "
-				    "specified once.");
+				errx(EX_USAGE,
+"ERROR: option -O may only be specified once.");
 			args.pa_outputpath = optarg;
 			args.pa_flags |= FLAG_HAS_OUTPUT_LOGFILE;
 			break;
@@ -822,8 +827,8 @@ main(int argc, char **argv)
 
 		case 'R':	/* read an existing log file */
 			if (args.pa_inputpath != NULL)
-				errx(EX_USAGE, "ERROR: option -R may only be "
-				    "specified once.");
+				errx(EX_USAGE,
+"ERROR: option -R may only be specified once.");
 			args.pa_inputpath = optarg;
 			if (args.pa_printfile == stderr)
 				args.pa_printfile = stdout;
@@ -853,8 +858,9 @@ main(int argc, char **argv)
 		case 'w':	/* wait interval */
 			interval = strtod(optarg, &end);
 			if (*end != '\0' || interval <= 0)
-				errx(EX_USAGE, "ERROR: Illegal wait interval "
-				    "value \"%s\".", optarg);
+				errx(EX_USAGE,
+"ERROR: Illegal wait interval value \"%s\".",
+				    optarg);
 			args.pa_flags |= FLAG_HAS_WAIT_INTERVAL;
 			args.pa_interval = interval;
 			break;
@@ -868,8 +874,9 @@ main(int argc, char **argv)
 		case 'z':
 			graphdepth = strtod(optarg, &end);
 			if (*end != '\0' || graphdepth <= 0)
-				errx(EX_USAGE, "ERROR: Illegal callchain "
-				    "depth \"%s\".", optarg);
+				errx(EX_USAGE,
+				    "ERROR: Illegal callchain depth \"%s\".",
+				    optarg);
 			args.pa_graphdepth = graphdepth;
 			args.pa_required |= FLAG_DO_CALLGRAPHS;
 			break;
@@ -906,8 +913,8 @@ main(int argc, char **argv)
 
 	/* disallow -O and -R together */
 	if (args.pa_outputpath && args.pa_inputpath)
-		errx(EX_USAGE, "ERROR: options -O and -R are mutually "
-		    "exclusive.");
+		errx(EX_USAGE,
+		    "ERROR: options -O and -R are mutually exclusive.");
 
 	/* -m option is allowed with -R only. */
 	if (args.pa_flags & FLAG_DO_ANNOTATE && args.pa_inputpath == NULL)
@@ -916,8 +923,8 @@ main(int argc, char **argv)
 	/* -m option is not allowed combined with -g or -G. */
 	if (args.pa_flags & FLAG_DO_ANNOTATE &&
 	    args.pa_flags & (FLAG_DO_GPROF | FLAG_DO_CALLGRAPHS))
-		errx(EX_USAGE, "ERROR: option -m and -g | -G are mutually "
-		    "exclusive");
+		errx(EX_USAGE,
+		    "ERROR: option -m and -g | -G are mutually exclusive");
 
 	if (args.pa_flags & FLAG_READ_LOGFILE) {
 		errmsg = NULL;
@@ -928,8 +935,9 @@ main(int argc, char **argv)
 		else if (!STAILQ_EMPTY(&args.pa_events))
 			errmsg = "a PMC event specification";
 		if (errmsg)
-			errx(EX_USAGE, "ERROR: option -R may not be used with "
-			    "%s.", errmsg);
+			errx(EX_USAGE,
+			    "ERROR: option -R may not be used with %s.",
+			    errmsg);
 	} else if (STAILQ_EMPTY(&args.pa_events))
 		/* All other uses require a PMC spec. */
 		pmcstat_show_usage();
@@ -937,58 +945,66 @@ main(int argc, char **argv)
 	/* check for -t pid without a process PMC spec */
 	if ((args.pa_required & FLAG_HAS_TARGET) &&
 	    (args.pa_flags & FLAG_HAS_PROCESS_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: option -t requires a process mode PMC "
-		    "to be specified.");
+		errx(EX_USAGE,
+"ERROR: option -t requires a process mode PMC to be specified."
+		    );
 
 	/* check for process-mode options without a command or -t pid */
 	if ((args.pa_required & FLAG_HAS_PROCESS_PMCS) &&
 	    (args.pa_flags & (FLAG_HAS_COMMANDLINE | FLAG_HAS_TARGET)) == 0)
-		errx(EX_USAGE, "ERROR: options -d, -E, -p, -P, and -W require "
-		    "a command line or target process.");
+		errx(EX_USAGE,
+"ERROR: options -d, -E, -p, -P, and -W require a command line or target process."
+		    );
 
 	/* check for -p | -P without a target process of some sort */
 	if ((args.pa_required & (FLAG_HAS_COMMANDLINE | FLAG_HAS_TARGET)) &&
 	    (args.pa_flags & (FLAG_HAS_COMMANDLINE | FLAG_HAS_TARGET)) == 0)
-		errx(EX_USAGE, "ERROR: options -P and -p require a "
-		    "target process or a command line.");
+		errx(EX_USAGE,
+"ERROR: options -P and -p require a target process or a command line."
+		    );
 
 	/* check for process-mode options without a process-mode PMC */
 	if ((args.pa_required & FLAG_HAS_PROCESS_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_PROCESS_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -d, -E, and -W require a "
-		    "process mode PMC to be specified.");
+		errx(EX_USAGE,
+"ERROR: options -d, -E, and -W require a process mode PMC to be specified."
+		    );
 
 	/* check for -c cpu with no system mode PMCs or logfile. */
 	if ((args.pa_required & FLAG_HAS_SYSTEM_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_SYSTEM_PMCS) == 0 &&
 	    (args.pa_flags & FLAG_READ_LOGFILE) == 0)
-		errx(EX_USAGE, "ERROR: option -c requires at least one "
-		    "system mode PMC to be specified.");
+		errx(EX_USAGE,
+"ERROR: option -c requires at least one system mode PMC to be specified."
+		    );
 
 	/* check for counting mode options without a counting PMC */
 	if ((args.pa_required & FLAG_HAS_COUNTING_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_COUNTING_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -C, -W and -o require at "
-		    "least one counting mode PMC to be specified.");
+		errx(EX_USAGE,
+"ERROR: options -C, -W and -o require at least one counting mode PMC to be specified."
+		    );
 
 	/* check for sampling mode options without a sampling PMC spec */
 	if ((args.pa_required & FLAG_HAS_SAMPLING_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) == 0)
-		errx(EX_USAGE, "ERROR: options -N, -n and -O require at "
-		    "least one sampling mode PMC to be specified.");
+		errx(EX_USAGE,
+"ERROR: options -N, -n and -O require at least one sampling mode PMC to be specified."
+		    );
 
 	/* check if -g/-G/-m/-T are being used correctly */
 	if ((args.pa_flags & FLAG_DO_ANALYSIS) &&
 	    !(args.pa_flags & (FLAG_HAS_SAMPLING_PMCS|FLAG_READ_LOGFILE)))
-		errx(EX_USAGE, "ERROR: options -g/-G/-m/-T require sampling PMCs "
-		    "or -R to be specified.");
+		errx(EX_USAGE,
+"ERROR: options -g/-G/-m/-T require sampling PMCs or -R to be specified."
+		    );
 
 	/* check if -O was spuriously specified */
 	if ((args.pa_flags & FLAG_HAS_OUTPUT_LOGFILE) &&
 	    (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0)
 		errx(EX_USAGE,
-		    "ERROR: option -O is used only with options "
-		    "-E, -P, -S and -W.");
+"ERROR: option -O is used only with options -E, -P, -S and -W."
+		    );
 
 	/* -k kernel path require -g/-G/-m/-T or -R */
 	if ((args.pa_flags & FLAG_HAS_KERNELPATH) &&
@@ -1015,12 +1031,13 @@ main(int argc, char **argv)
 	if ((args.pa_flags & FLAG_HAS_COUNTING_PMCS) &&
 	    (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) &&
 	    ((args.pa_flags & FLAG_HAS_OUTPUT_LOGFILE) == 0))
-		errx(EX_USAGE, "ERROR: option -O is required if counting and "
-		    "sampling PMCs are specified together.");
+		errx(EX_USAGE,
+"ERROR: option -O is required if counting and sampling PMCs are specified together."
+		    );
 
 	/*
 	 * Check if "-k kerneldir" was specified, and if whether
-	 * 'kerneldir' actually refers to a a file.  If so, use
+	 * 'kerneldir' actually refers to a file.  If so, use
 	 * `dirname path` to determine the kernel directory.
 	 */
 	if (args.pa_flags & FLAG_HAS_KERNELPATH) {
@@ -1042,8 +1059,9 @@ main(int argc, char **argv)
 				err(EX_OSERR, "ERROR: Cannot stat \"%s\"",
 				    buffer);
 			if (!S_ISDIR(sb.st_mode))
-				errx(EX_USAGE, "ERROR: \"%s\" is not a "
-				    "directory.", buffer);
+				errx(EX_USAGE,
+				    "ERROR: \"%s\" is not a directory.",
+				    buffer);
 		}
 	}
 
@@ -1056,8 +1074,9 @@ main(int argc, char **argv)
 		else {
 			args.pa_graphfile = fopen(graphfilename, "w");
 			if (args.pa_graphfile == NULL)
-				err(EX_OSERR, "ERROR: cannot open \"%s\" "
-				    "for writing", graphfilename);
+				err(EX_OSERR,
+				    "ERROR: cannot open \"%s\" for writing",
+				    graphfilename);
 		}
 	}
 	if (args.pa_flags & FLAG_DO_ANNOTATE) {
@@ -1071,11 +1090,13 @@ main(int argc, char **argv)
 	if ((args.pa_flags & FLAG_READ_LOGFILE) == 0) {
 		if (pmc_init() < 0)
 			err(EX_UNAVAILABLE,
-			    "ERROR: Initialization of the pmc(3) library failed");
+			    "ERROR: Initialization of the pmc(3) library failed"
+			    );
 
 		if ((npmc = pmc_npmc(0)) < 0) /* assume all CPUs are identical */
-			err(EX_OSERR, "ERROR: Cannot determine the number of PMCs "
-			    "on CPU %d", 0);
+			err(EX_OSERR,
+"ERROR: Cannot determine the number of PMCs on CPU %d",
+			    0);
 	}
 
 	/* Allocate a kqueue */
@@ -1150,17 +1171,18 @@ main(int argc, char **argv)
 	 */
 
 	STAILQ_FOREACH(ev, &args.pa_events, ev_next) {
-	    if (pmc_allocate(ev->ev_spec, ev->ev_mode,
+		if (pmc_allocate(ev->ev_spec, ev->ev_mode,
 		    ev->ev_flags, ev->ev_cpu, &ev->ev_pmcid) < 0)
-		    err(EX_OSERR, "ERROR: Cannot allocate %s-mode pmc with "
-			"specification \"%s\"",
-			PMC_IS_SYSTEM_MODE(ev->ev_mode) ? "system" : "process",
-			ev->ev_spec);
-
-	    if (PMC_IS_SAMPLING_MODE(ev->ev_mode) &&
-		pmc_set(ev->ev_pmcid, ev->ev_count) < 0)
-		    err(EX_OSERR, "ERROR: Cannot set sampling count "
-			"for PMC \"%s\"", ev->ev_name);
+			err(EX_OSERR,
+"ERROR: Cannot allocate %s-mode pmc with specification \"%s\"",
+			    PMC_IS_SYSTEM_MODE(ev->ev_mode) ?
+			    "system" : "process", ev->ev_spec);
+
+		if (PMC_IS_SAMPLING_MODE(ev->ev_mode) &&
+		    pmc_set(ev->ev_pmcid, ev->ev_count) < 0)
+			err(EX_OSERR,
+			    "ERROR: Cannot set sampling count for PMC \"%s\"",
+			    ev->ev_name);
 	}
 
 	/* compute printout widths */
@@ -1202,8 +1224,8 @@ main(int argc, char **argv)
 		EV_SET(&kev, SIGWINCH, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL);
 
 		if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
-			err(EX_OSERR, "ERROR: Cannot register kevent for "
-			    "SIGWINCH");
+			err(EX_OSERR,
+			    "ERROR: Cannot register kevent for SIGWINCH");
 
 		args.pa_toptty = 1;
 	}
@@ -1246,8 +1268,8 @@ main(int argc, char **argv)
 		    args.pa_interval * 1000, NULL);
 
 		if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0)
-			err(EX_OSERR, "ERROR: Cannot register kevent for "
-			    "timer");
+			err(EX_OSERR,
+			    "ERROR: Cannot register kevent for timer");
 	}
 
 	/* attach PMCs to the target process, starting it if specified */
@@ -1260,8 +1282,8 @@ main(int argc, char **argv)
 	/* Attach process pmcs to the target process. */
 	if (args.pa_flags & (FLAG_HAS_TARGET | FLAG_HAS_COMMANDLINE)) {
 		if (SLIST_EMPTY(&args.pa_targets))
-			errx(EX_DATAERR, "ERROR: No matching target "
-			    "processes.");
+			errx(EX_DATAERR,
+			    "ERROR: No matching target processes.");
 		if (args.pa_flags & FLAG_HAS_PROCESS_PMCS)
 			pmcstat_attach_pmcs();
 
@@ -1317,7 +1339,8 @@ main(int argc, char **argv)
 			keypad(stdscr, TRUE);
 			clear();
 			/* Get terminal width / height with ncurses. */
-			getmaxyx(stdscr, pmcstat_displayheight, pmcstat_displaywidth);
+			getmaxyx(stdscr,
+			    pmcstat_displayheight, pmcstat_displaywidth);
 			pmcstat_displayheight--; pmcstat_displaywidth--;
 			atexit(pmcstat_topexit);
 		}
@@ -1383,8 +1406,8 @@ main(int argc, char **argv)
 			} else if (kev.ident == SIGWINCH) {
 				if (ioctl(fileno(args.pa_printfile),
 					TIOCGWINSZ, &ws) < 0)
-				    err(EX_OSERR, "ERROR: Cannot determine "
-					"window size");
+				    err(EX_OSERR,
+				        "ERROR: Cannot determine window size");
 				pmcstat_displayheight = ws.ws_row - 1;
 				pmcstat_displaywidth  = ws.ws_col - 1;
 			} else
@@ -1394,7 +1417,7 @@ main(int argc, char **argv)
 
 		case EVFILT_TIMER: /* print out counting PMCs */
 			if ((args.pa_flags & FLAG_DO_TOP) &&
-			     pmc_flush_logfile() != ENOBUFS)
+			     pmc_flush_logfile() == 0)
 				do_read = 1;
 			do_print = 1;
 			break;
@@ -1404,7 +1427,8 @@ main(int argc, char **argv)
 		if (do_print && !do_read) {
 			if ((args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) {
 				pmcstat_print_pmcs();
-				if (runstate == PMCSTAT_FINISHED && /* final newline */
+				if (runstate == PMCSTAT_FINISHED &&
+				    /* final newline */
 				    (args.pa_flags & FLAG_DO_PRINT) == 0)
 					(void) fprintf(args.pa_printfile, "\n");
 			}
@@ -1431,19 +1455,19 @@ main(int argc, char **argv)
 	/* check if the driver lost any samples or events */
 	if (check_driver_stats) {
 		if (pmc_get_driver_stats(&ds_end) < 0)
-			err(EX_OSERR, "ERROR: Cannot retrieve driver "
-			    "statistics");
+			err(EX_OSERR,
+			    "ERROR: Cannot retrieve driver statistics");
 		if (ds_start.pm_intr_bufferfull != ds_end.pm_intr_bufferfull &&
 		    args.pa_verbosity > 0)
-			warnx("WARNING: some samples were dropped.  Please "
-			    "consider tuning the \"kern.hwpmc.nsamples\" "
-			    "tunable.");
+			warnx("WARNING: some samples were dropped.\n"
+"Please consider tuning the \"kern.hwpmc.nsamples\" tunable."
+			    );
 		if (ds_start.pm_buffer_requests_failed !=
 		    ds_end.pm_buffer_requests_failed &&
 		    args.pa_verbosity > 0)
-			warnx("WARNING: some events were discarded.  Please "
-			    "consider tuning the \"kern.hwpmc.nbuffers\" "
-			    "tunable.");
+			warnx("WARNING: some events were discarded.\n"
+"Please consider tuning the \"kern.hwpmc.nbuffers\" tunable."
+			    );
 	}
 
 	exit(EX_OK);
diff --git a/usr.sbin/pmcstat/pmcstat_log.c b/usr.sbin/pmcstat/pmcstat_log.c
index 6d0403f..6c0eb14 100644
--- a/usr.sbin/pmcstat/pmcstat_log.c
+++ b/usr.sbin/pmcstat/pmcstat_log.c
@@ -421,8 +421,8 @@ pmcstat_image_get_aout_params(struct pmcstat_image *image)
 	assert(path != NULL);
 
 	if (image->pi_iskernelmodule)
-		errx(EX_SOFTWARE, "ERROR: a.out kernel modules are "
-		    "unsupported \"%s\"", path);
+		errx(EX_SOFTWARE,
+		    "ERROR: a.out kernel modules are unsupported \"%s\"", path);
 
 	(void) snprintf(buffer, sizeof(buffer), "%s%s",
 	    args.pa_fsroot, path);
@@ -645,8 +645,9 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image)
 	}
 
 	if (gelf_getehdr(e, &eh) != &eh) {
-		warnx("WARNING: Cannot retrieve the ELF Header for "
-		    "\"%s\": %s.", buffer, elf_errmsg(-1));
+		warnx(
+		    "WARNING: Cannot retrieve the ELF Header for \"%s\": %s.",
+		    buffer, elf_errmsg(-1));
 		goto done;
 	}
 
@@ -667,16 +668,17 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image)
 	 */
 	if (eh.e_type == ET_EXEC) {
 		if (elf_getphnum(e, &nph) == 0) {
-			warnx("WARNING: Could not determine the number of "
-			    "program headers in \"%s\": %s.", buffer,
+			warnx(
+"WARNING: Could not determine the number of program headers in \"%s\": %s.",
+			    buffer,
 			    elf_errmsg(-1));
 			goto done;
 		}
 		for (i = 0; i < eh.e_phnum; i++) {
 			if (gelf_getphdr(e, i, &ph) != &ph) {
-				warnx("WARNING: Retrieval of PHDR entry #%ju "
-				    "in \"%s\" failed: %s.", (uintmax_t) i,
-				    buffer, elf_errmsg(-1));
+				warnx(
+"WARNING: Retrieval of PHDR entry #%ju in \"%s\" failed: %s.",
+				    (uintmax_t) i, buffer, elf_errmsg(-1));
 				goto done;
 			}
 			switch (ph.p_type) {
@@ -685,8 +687,8 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image)
 				break;
 			case PT_INTERP:
 				if ((elfbase = elf_rawfile(e, NULL)) == NULL) {
-					warnx("WARNING: Cannot retrieve the "
-					    "interpreter for \"%s\": %s.",
+					warnx(
+"WARNING: Cannot retrieve the interpreter for \"%s\": %s.",
 					    buffer, elf_errmsg(-1));
 					goto done;
 				}
@@ -706,17 +708,18 @@ pmcstat_image_get_elf_params(struct pmcstat_image *image)
 	 * Get the min and max VA associated with this ELF object.
 	 */
 	if (elf_getshnum(e, &nsh) == 0) {
-		warnx("WARNING: Could not determine the number of sections "
-		    "for \"%s\": %s.", buffer, elf_errmsg(-1));
+		warnx(
+"WARNING: Could not determine the number of sections for \"%s\": %s.",
+		    buffer, elf_errmsg(-1));
 		goto done;
 	}
 
 	for (i = 0; i < nsh; i++) {
 		if ((scn = elf_getscn(e, i)) == NULL ||
 		    gelf_getshdr(scn, &sh) != &sh) {
-			warnx("WARNING: Could not retrieve section header "
-			    "#%ju in \"%s\": %s.", (uintmax_t) i, buffer,
-			    elf_errmsg(-1));
+			warnx(
+"WARNING: Could not retrieve section header #%ju in \"%s\": %s.",
+			    (uintmax_t) i, buffer, elf_errmsg(-1));
 			goto done;
 		}
 		if (sh.sh_flags & SHF_EXECINSTR) {
@@ -912,8 +915,8 @@ pmcstat_image_unmap(struct pmcstat_process *pp, uintfptr_t start,
 			 * the new one at [end].
 			 */
 			if ((pcmnew = malloc(sizeof(*pcmnew))) == NULL)
-				err(EX_OSERR, "ERROR: Cannot split a map "
-				    "entry");
+				err(EX_OSERR,
+				    "ERROR: Cannot split a map entry");
 
 			pcmnew->ppm_image = pcm->ppm_image;
 
@@ -964,8 +967,9 @@ pmcstat_image_addr2line(struct pmcstat_image *image, uintfptr_t addr,
 		if (image->pi_addr2line == NULL) {
 			if (!addr2line_warn) {
 				addr2line_warn = 1;
-				warnx("WARNING: addr2line is needed"
-				    "for source code information.");
+				warnx(
+"WARNING: addr2line is needed for source code information."
+				    );
 			}
 			return (0);
 		}
@@ -1214,22 +1218,23 @@ pmcstat_process_lookup(pid_t pid, int allocate)
 	hash = (uint32_t) pid & PMCSTAT_HASH_MASK;	/* simplicity wins */
 
 	LIST_FOREACH_SAFE(pp, &pmcstat_process_hash[hash], pp_next, pptmp)
-	    if (pp->pp_pid == pid) {
-		    /* Found a descriptor, check and process zombies */
-		    if (allocate && pp->pp_isactive == 0) {
-			    /* remove maps */
-			    TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next,
-				ppmtmp) {
-				    TAILQ_REMOVE(&pp->pp_map, ppm, ppm_next);
-				    free(ppm);
-			    }
-			    /* remove process entry */
-			    LIST_REMOVE(pp, pp_next);
-			    free(pp);
-			    break;
-		    }
-		    return (pp);
-	    }
+		if (pp->pp_pid == pid) {
+			/* Found a descriptor, check and process zombies */
+			if (allocate && pp->pp_isactive == 0) {
+				/* remove maps */
+				TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next,
+				    ppmtmp) {
+					TAILQ_REMOVE(&pp->pp_map, ppm,
+					    ppm_next);
+					free(ppm);
+				}
+				/* remove process entry */
+				LIST_REMOVE(pp, pp_next);
+				free(pp);
+				break;
+			}
+			return (pp);
+		}
 
 	if (!allocate)
 		return (NULL);
@@ -1283,8 +1288,9 @@ pmcstat_process_exec(struct pmcstat_process *pp,
 		break;
 
 	default:
-		err(EX_SOFTWARE, "ERROR: Unsupported executable type for "
-		    "\"%s\"", pmcstat_string_unintern(path));
+		err(EX_SOFTWARE,
+		    "ERROR: Unsupported executable type for \"%s\"",
+		    pmcstat_string_unintern(path));
 	}
 }
 
@@ -1338,10 +1344,9 @@ pmcstat_analyze_log(void)
 		case PMCLOG_TYPE_INITIALIZE:
 			if ((ev.pl_u.pl_i.pl_version & 0xFF000000) !=
 			    PMC_VERSION_MAJOR << 24 && args.pa_verbosity > 0)
-				warnx("WARNING: Log version 0x%x does not "
-				    "match compiled version 0x%x.",
-				    ev.pl_u.pl_i.pl_version,
-				    PMC_VERSION_MAJOR);
+				warnx(
+"WARNING: Log version 0x%x does not match compiled version 0x%x.",
+				    ev.pl_u.pl_i.pl_version, PMC_VERSION_MAJOR);
 			break;
 
 		case PMCLOG_TYPE_MAP_IN:
@@ -1556,8 +1561,9 @@ pmcstat_analyze_log(void)
 	else if (ev.pl_state == PMCLOG_REQUIRE_DATA)
 		return (PMCSTAT_RUNNING);
 
-	err(EX_DATAERR, "ERROR: event parsing failed (record %jd, "
-	    "offset 0x%jx)", (uintmax_t) ev.pl_count + 1, ev.pl_offset);
+	err(EX_DATAERR,
+	    "ERROR: event parsing failed (record %jd, offset 0x%jx)",
+	    (uintmax_t) ev.pl_count + 1, ev.pl_offset);
 }
 
 /*
@@ -1597,9 +1603,9 @@ pmcstat_print_log(void)
 			    pmc_name_of_cputype(ev.pl_u.pl_i.pl_arch));
 			if ((ev.pl_u.pl_i.pl_version & 0xFF000000) !=
 			    PMC_VERSION_MAJOR << 24 && args.pa_verbosity > 0)
-				warnx("WARNING: Log version 0x%x != expected "
-				    "version 0x%x.", ev.pl_u.pl_i.pl_version,
-				    PMC_VERSION);
+				warnx(
+"WARNING: Log version 0x%x != expected version 0x%x.",
+				    ev.pl_u.pl_i.pl_version, PMC_VERSION);
 			break;
 		case PMCLOG_TYPE_MAP_IN:
 			PMCSTAT_PRINT_ENTRY("map-in","%d %p \"%s\"",
@@ -1680,8 +1686,8 @@ pmcstat_print_log(void)
 	else if (ev.pl_state ==  PMCLOG_REQUIRE_DATA)
 		return (PMCSTAT_RUNNING);
 
-	errx(EX_DATAERR, "ERROR: event parsing failed "
-	    "(record %jd, offset 0x%jx).",
+	errx(EX_DATAERR,
+	    "ERROR: event parsing failed (record %jd, offset 0x%jx).",
 	    (uintmax_t) ev.pl_count + 1, ev.pl_offset);
 	/*NOTREACHED*/
 }
diff --git a/usr.sbin/portsnap/portsnap/portsnap.sh b/usr.sbin/portsnap/portsnap/portsnap.sh
index e715ce8..c9d2b16 100644
--- a/usr.sbin/portsnap/portsnap/portsnap.sh
+++ b/usr.sbin/portsnap/portsnap/portsnap.sh
@@ -570,14 +570,16 @@ fetch_metadata_sanity() {
 
 # Take a list of ${oldhash}|${newhash} and output a list of needed patches
 fetch_make_patchlist() {
-	grep -vE "^([0-9a-f]{64})\|\1$" | 
-		while read LINE; do
-			X=`echo ${LINE} | cut -f 1 -d '|'`
-			Y=`echo ${LINE} | cut -f 2 -d '|'`
-			if [ -f "files/${Y}.gz" ]; then continue; fi
-			if [ ! -f "files/${X}.gz" ]; then continue; fi
-			echo "${LINE}"
+	IFS='|'
+	echo "" 1>${QUIETREDIR}
+	grep -vE "^([0-9a-f]{64})\|\1$" |
+		while read X Y; do
+			printf "Processing: $X $Y ...\r" 1>${QUIETREDIR}
+			if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi
+			echo "${X}|${Y}"
 		done
+	echo "" 1>${QUIETREDIR}
+	IFS=
 }
 
 # Print user-friendly progress statistics
@@ -692,9 +694,8 @@ fetch_update() {
 
 # Attempt to apply metadata patches
 	echo -n "Applying metadata patches... "
-	while read LINE; do
-		X=`echo ${LINE} | cut -f 1 -d '|'`
-		Y=`echo ${LINE} | cut -f 2 -d '|'`
+	IFS='|'
+	while read X Y; do
 		if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
 		gunzip -c < ${X}-${Y}.gz > diff
 		gunzip -c < files/${X}.gz > OLD
@@ -707,6 +708,7 @@ fetch_update() {
 		fi
 		rm -f diff OLD NEW ${X}-${Y}.gz ptmp
 	done < patchlist 2>${QUIETREDIR}
+	IFS=
 	echo "done."
 
 # Update metadata without patches
@@ -724,16 +726,19 @@ fetch_update() {
 	    2>${QUIETREDIR}
 
 	while read Y; do
+		echo -n "Verifying ${Y}... " 1>${QUIETREDIR}
 		if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
 			mv ${Y}.gz files/${Y}.gz
 		else
 			echo "metadata is corrupt."
 			return 1
 		fi
+		echo "ok." 1>${QUIETREDIR}
 	done < filelist
 	echo "done."
 
 # Extract the index
+	echo -n "Extracting index... " 1>${QUIETREDIR}
 	gunzip -c files/`look INDEX tINDEX.new |
 	    cut -f 2 -d '|'`.gz > INDEX.new
 	fetch_index_sanity || return 1
@@ -754,8 +759,10 @@ fetch_update() {
 	fi
 
 # Generate a list of wanted ports patches
+	echo -n "Generating list of wanted patches..." 1>${QUIETREDIR}
 	join -t '|' -o 1.2,2.2 INDEX INDEX.new |
 	    fetch_make_patchlist > patchlist
+	echo " done." 1>${QUIETREDIR}
 
 # Attempt to fetch ports patches
 	echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
@@ -766,11 +773,18 @@ fetch_update() {
 	echo "done."
 
 # Attempt to apply ports patches
-	echo -n "Applying patches... "
-	while read LINE; do
-		X=`echo ${LINE} | cut -f 1 -d '|'`
-		Y=`echo ${LINE} | cut -f 2 -d '|'`
-		if [ ! -f "${X}-${Y}" ]; then continue; fi
+	PATCHCNT=`wc -l patchlist`
+	echo "Applying patches... "
+	IFS='|'
+	I=0
+	while read X Y; do
+		I=$(($I + 1))
+		F="${X}-${Y}"
+		if [ ! -f "${F}" ]; then
+			printf "  Skipping ${F} (${I} of ${PATCHCNT}).\r"
+			continue;
+		fi
+		echo "  Processing ${F}..." 1>${QUIETREDIR}
 		gunzip -c < files/${X}.gz > OLD
 		${BSPATCH} OLD NEW ${X}-${Y}
 		if [ `${SHA256} -q NEW` = ${Y} ]; then
@@ -779,6 +793,7 @@ fetch_update() {
 		fi
 		rm -f diff OLD NEW ${X}-${Y}
 	done < patchlist 2>${QUIETREDIR}
+	IFS=
 	echo "done."
 
 # Update ports without patches
@@ -795,7 +810,10 @@ fetch_update() {
 	    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
 	    2>${QUIETREDIR}
 
+	I=0
 	while read Y; do
+		I=$(($I + 1))
+		printf "   Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR}
 		if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
 			mv ${Y}.gz files/${Y}.gz
 		else
@@ -877,6 +895,7 @@ extract_metadata() {
 
 # Do the actual work involved in "extract"
 extract_run() {
+	local IFS='|'
 	mkdir -p ${PORTSDIR} || return 1
 
 	if !
@@ -886,7 +905,7 @@ extract_run() {
 			grep -vE "${REFUSE}" ${WORKDIR}/INDEX
 		else
 			cat ${WORKDIR}/INDEX
-		fi | tr '|' ' ' | while read FILE HASH; do
+		fi | while read FILE HASH; do
 		echo ${PORTSDIR}/${FILE}
 		if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
 			echo "files/${HASH}.gz not found -- snapshot corrupt."
@@ -916,35 +935,8 @@ extract_run() {
 	extract_indices
 }
 
-# Do the actual work involved in "update"
-update_run() {
-	if ! [ -z "${INDEXONLY}" ]; then
-		extract_indices >/dev/null || return 1
-		return 0
-	fi
-
-	if sort ${WORKDIR}/INDEX |
-	    cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
-		echo "Ports tree is already up to date."
-		return 0
-	fi
-
-# If we are REFUSEing to touch certain directories, don't remove files
-# from those directories (even if they are out of date)
-	echo -n "Removing old files and directories... "
-	if ! [ -z "${REFUSE}" ]; then 
-		sort ${WORKDIR}/INDEX |
-		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
-		    grep -vE "${REFUSE}" |
-		    lam -s "${PORTSDIR}/" - |
-		    sed -e 's|/$||' | xargs rm -rf
-	else
-		sort ${WORKDIR}/INDEX |
-		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
-		    lam -s "${PORTSDIR}/" - |
-		    sed -e 's|/$||' | xargs rm -rf
-	fi
-	echo "done."
+update_run_extract() {
+	local IFS='|'
 
 # Install new files
 	echo "Extracting new files:"
@@ -955,9 +947,7 @@ update_run() {
 			sort ${WORKDIR}/INDEX
 		fi |
 	    comm -13 ${PORTSDIR}/.portsnap.INDEX - |
-	    while read LINE; do
-		FILE=`echo ${LINE} | cut -f 1 -d '|'`
-		HASH=`echo ${LINE} | cut -f 2 -d '|'`
+	    while read FILE HASH; do
 		echo ${PORTSDIR}/${FILE}
 		if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
 			echo "files/${HASH}.gz not found -- snapshot corrupt."
@@ -977,7 +967,39 @@ update_run() {
 	done; then
 		return 1
 	fi
+}
+
+# Do the actual work involved in "update"
+update_run() {
+	if ! [ -z "${INDEXONLY}" ]; then
+		extract_indices >/dev/null || return 1
+		return 0
+	fi
+
+	if sort ${WORKDIR}/INDEX |
+	    cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
+		echo "Ports tree is already up to date."
+		return 0
+	fi
+
+# If we are REFUSEing to touch certain directories, don't remove files
+# from those directories (even if they are out of date)
+	echo -n "Removing old files and directories... "
+	if ! [ -z "${REFUSE}" ]; then 
+		sort ${WORKDIR}/INDEX |
+		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
+		    grep -vE "${REFUSE}" |
+		    lam -s "${PORTSDIR}/" - |
+		    sed -e 's|/$||' | xargs rm -rf
+	else
+		sort ${WORKDIR}/INDEX |
+		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
+		    lam -s "${PORTSDIR}/" - |
+		    sed -e 's|/$||' | xargs rm -rf
+	fi
+	echo "done."
 
+	update_run_extract || return 1
 	extract_metadata
 	extract_indices
 }
diff --git a/usr.sbin/pwd_mkdb/pwd_mkdb.c b/usr.sbin/pwd_mkdb/pwd_mkdb.c
index 79f91a9..0ba68a5 100644
--- a/usr.sbin/pwd_mkdb/pwd_mkdb.c
+++ b/usr.sbin/pwd_mkdb/pwd_mkdb.c
@@ -68,7 +68,7 @@ __FBSDID("$FreeBSD$");
 #define LEGACY_VERSION(x)  _PW_VERSIONED(x, 3)
 #define CURRENT_VERSION(x) _PW_VERSIONED(x, 4)
 
-HASHINFO openinfo = {
+static HASHINFO openinfo = {
 	4096,		/* bsize */
 	32,		/* ffactor */
 	256,		/* nelem */
diff --git a/usr.sbin/rip6query/rip6query.c b/usr.sbin/rip6query/rip6query.c
index 4426bbc..ea448ba2 100644
--- a/usr.sbin/rip6query/rip6query.c
+++ b/usr.sbin/rip6query/rip6query.c
@@ -56,9 +56,9 @@
 
 #include "route6d.h"
 
-int	s;
-struct sockaddr_in6 sin6;
-struct rip6	*ripbuf;
+static int	s;
+static struct sockaddr_in6 sin6;
+static struct rip6	*ripbuf;
 
 #define	RIPSIZE(n)	(sizeof(struct rip6) + (n-1) * sizeof(struct netinfo6))
 
diff --git a/usr.sbin/rmt/rmt.c b/usr.sbin/rmt/rmt.c
index f5e3cc1..a5c048f 100644
--- a/usr.sbin/rmt/rmt.c
+++ b/usr.sbin/rmt/rmt.c
@@ -54,25 +54,25 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-int	tape = -1;
+static int	tape = -1;
 
-char	*record;
-int	maxrecsize = -1;
+static char	*record;
+static int	maxrecsize = -1;
 
 #define	SSIZE	64
-char	device[SSIZE];
-char	count[SSIZE], mode[SSIZE], pos[SSIZE], op[SSIZE];
+static char	device[SSIZE];
+static char	count[SSIZE], mode[SSIZE], pos[SSIZE], op[SSIZE];
 
-char	resp[BUFSIZ];
+static char	resp[BUFSIZ];
 
-FILE	*debug;
+static FILE	*debug;
 #define	DEBUG(f)	if (debug) fprintf(debug, f)
 #define	DEBUG1(f,a)	if (debug) fprintf(debug, f, a)
 #define	DEBUG2(f,a1,a2)	if (debug) fprintf(debug, f, a1, a2)
 
-char	*checkbuf(char *, int);
-void	 error(int);
-void	 getstring(char *);
+static char	*checkbuf(char *, int);
+static void	 error(int);
+static void	 getstring(char *);
 
 int
 main(int argc, char **argv)
@@ -220,7 +220,7 @@ getstring(char *bp)
 	cp[i] = '\0';
 }
 
-char *
+static char *
 checkbuf(char *rec, int size)
 {
 
@@ -240,7 +240,7 @@ checkbuf(char *rec, int size)
 	return (rec);
 }
 
-void
+static void
 error(int num)
 {
 
diff --git a/usr.sbin/snapinfo/snapinfo.c b/usr.sbin/snapinfo/snapinfo.c
index 3ee66da..45882e0 100644
--- a/usr.sbin/snapinfo/snapinfo.c
+++ b/usr.sbin/snapinfo/snapinfo.c
@@ -39,14 +39,15 @@
 #include <string.h>
 #include <unistd.h>
 
-void	find_inum(char *path);
-void	usage(void);
-int	compare_function(const char *, const struct stat *, int, struct FTW *);
-int	find_snapshot(struct statfs *sfs);
+static void	find_inum(char *path);
+static void	usage(void);
+static int	compare_function(const char *, const struct stat *,
+		    int, struct FTW *);
+static int	find_snapshot(struct statfs *sfs);
 
-int verbose;
-int cont_search;
-uint32_t inode;
+static int	verbose;
+static int	cont_search;
+static uint32_t	inode;
 
 int
 main(int argc, char **argv)
@@ -112,7 +113,7 @@ main(int argc, char **argv)
 	return (0);
 }
 
-int
+static int
 find_snapshot(struct statfs *sfs)
 {
 	struct uufsd disk;
@@ -138,7 +139,7 @@ find_snapshot(struct statfs *sfs)
 	return 0;
 }
 
-int
+static int
 compare_function(const char *path, const struct stat *st, int flags,
     struct FTW * ftwv __unused)
 {
@@ -157,7 +158,7 @@ compare_function(const char *path, const struct stat *st, int flags,
 	return (0);
 }
 
-void
+static void
 find_inum(char *path)
 {
 	int ret;
@@ -169,7 +170,7 @@ find_inum(char *path)
 	}
 }
 
-void
+static void
 usage(void)
 {
 
diff --git a/usr.sbin/spray/spray.c b/usr.sbin/spray/spray.c
index 0e072e2..69691d1 100644
--- a/usr.sbin/spray/spray.c
+++ b/usr.sbin/spray/spray.c
@@ -49,12 +49,12 @@ static void usage(void);
 static void print_xferstats(unsigned int, int, double);
 
 /* spray buffer */
-char spray_buffer[SPRAYMAX];
+static char spray_buffer[SPRAYMAX];
 
 /* RPC timeouts */
-struct timeval NO_DEFAULT = { -1, -1 };
-struct timeval ONE_WAY = { 0, 0 };
-struct timeval TIMEOUT = { 25, 0 };
+static struct timeval NO_DEFAULT = { -1, -1 };
+static struct timeval ONE_WAY = { 0, 0 };
+static struct timeval TIMEOUT = { 25, 0 };
 
 int
 main(int argc, char *argv[])
diff --git a/usr.sbin/timed/timed/readmsg.c b/usr.sbin/timed/timed/readmsg.c
index 0d0d317..e21979a 100644
--- a/usr.sbin/timed/timed/readmsg.c
+++ b/usr.sbin/timed/timed/readmsg.c
@@ -35,10 +35,9 @@ static const char rcsid[] =
   "$FreeBSD$";
 #endif /* not lint */
 
+#define	TSPTYPES
 #include "globals.h"
 
-extern char *tsptype[];
-
 /*
  * LOOKAT checks if the message is of the requested type and comes from
  * the right machine, returning 1 in case of affirmative answer
diff --git a/usr.sbin/timed/timed/timed.c b/usr.sbin/timed/timed/timed.c
index 149255a..63d1a48 100644
--- a/usr.sbin/timed/timed/timed.c
+++ b/usr.sbin/timed/timed/timed.c
@@ -42,7 +42,6 @@ static char sccsid[] = "@(#)timed.c	8.1 (Berkeley) 6/6/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#define TSPTYPES
 #include "globals.h"
 #include <net/if.h>
 #include <sys/file.h>
diff --git a/usr.sbin/tzsetup/tzsetup.c b/usr.sbin/tzsetup/tzsetup.c
index 8251470..5a9ea0a 100644
--- a/usr.sbin/tzsetup/tzsetup.c
+++ b/usr.sbin/tzsetup/tzsetup.c
@@ -565,6 +565,18 @@ install_zoneinfo_file(const char *zoneinfo_file)
 
 				return (DITEM_FAILURE | DITEM_RECREATE);
 			}
+			if (unlink(path_db) < 0 && errno != ENOENT) {
+				snprintf(title, sizeof(title), "Error");
+				snprintf(prompt, sizeof(prompt),
+				     "Could not delete %s: %s", path_db,
+				     strerror(errno));
+				if (usedialog)
+					dialog_mesgbox(title, prompt, 8, 72);
+				else
+					fprintf(stderr, "%s\n", prompt);
+
+				return (DITEM_FAILURE | DITEM_RECREATE);
+			}
 			return (DITEM_LEAVE_MENU);
 		}